; Copyright 2011-2023 - Mersenne Research, Inc.  All rights reserved
; Author:  George Woltman
; Email: woltman@alum.mit.edu
;
; These macros do the second pass of a radix-4/8 two pass FFT using AVX-512 instructions.
;

;;
;; Estimated memory requirements for complex pass 2 macros:
;;
;; Pass 2 size:		48	64	80	192	256	320	768	1024	1280	1536	2048	2560	2304	3072
;; FFT data:		<1K	1K	2K	3K	4K	5K	12K	16K	20K	24K	32K	40K	36K	48K
;; Fixed sin/cos data:	<1K	<1K	<1K	2K	2K	3K	6K	9K	10K	13K	17K	21K	18K	26K
;; Min L2 cache size:	2K	3K	4K	8K	10K	13K	30K	41K	50K	61K	81K	101K	90K	122K
;;
;; Pass 2 size:		3840	4096	5120	6400	4608	6144	7680	8192	10240	12800	9216	12288	15360	16384	20480	25600
;; FFT data:		60K	64K	80K	100K	72K	96K	120K	128K	160K	200K	144K	192K	240K	256K	320K	400K
;; Fixed sin/cos data:	30K	34K	42K	50K	36K	49K	60K	65K	81K	100K	72K	98K	120K	130K	162K	200K
;; Min L2 cache size:	150K	162K	202K	250K	180K	241K	300K	321K	401K	500K	360K	482K	600K	644K	802K	1000K
;;
;; NOTE:  The total size is a minimum estimated efficient L2 cache size.
;; We double the data size as both the current data and next block data must
;; be in the cache at the same time.  The full formula is:
;;	(FFT data size) * 2 + fixed sin/cos data
;; ALSO NOTE:  The real sin/cos data is used only in the first FFT block, so the memory
;; requirements for real data aren't particularly interesting.
;;


;*********************************************************************************************
;			Macros derived from pass2-6-levels
;*********************************************************************************************


; *************** radix4-pass2-64-real macro ******************
; This macro takes 128 real values and performs the final 7 levels of the FFT.
; The results are squared, and the first 7 levels of the inverse FFT are done.

zr4dwpn_pass2_64_real MACRO
	LOCAL	b1b, b2b, b3a, b3b, b3c, b3d, c1b, c2b
	LOCAL	zpass2_64_real_1, zpass2_64_real_2
	LOCAL	zpass2_64_real_3, zpass2_64_real_4
	LOCAL	zpass2_64_real_unfft, zpass2_64_real_done

;; Type 4 FFTs skip the forward FFT process

	start_timer 2
	mov	rsi, DATA_ADDR		;; Load source address
	mov	rbx, DIST_TO_FFTSRCARG
;	mov	rcx, DATA_PREFETCH	;; Prefetch pointer (we need to prefetch 16 cache lines + 0 pad cache lines)
	cmp	ffttype, 4
	je	zpass2_64_real_4

;; Do FFT levels 1,2,3,4,5,6,7 as well as inverse FFT levels 1,2,3,4,5,6,7
;; On input and output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	64	+1	...
;;	8	...
;;	...
;;	56

;; Do 1 hundredtwentyeight_real_fft operation
;;	distance between fft data elements is 1 (8 for swizzling)
;;	do 1 macros each processing 128 data values

;; Execute the proper middle step

	mov	rdx, sincos3		;; Real sin/cos data
	mov	rdi, xsincos_complex	;; Complex sin/cos data
	cmp	ffttype, 2
	jl	zpass2_64_real_1
	jg	zpass2_64_real_3

zpass2_64_real_2:
	mov	r8, DEST2ARG
	zr64f_hundredtwentyeight_real_with_square_preload
	zr64f_hundredtwentyeight_real_with_square rsi, 8*128, 1*128, 2*128, 4*128, rdx, 0*ZMM_SCD4, rdi, 0*ZMM_SCD4, 1
	add	rsi, r8
	jmp	zpass2_64_real_unfft

zpass2_64_real_1:
	zr64f_hundredtwentyeight_real_fft_final_preload
	zr64f_hundredtwentyeight_real_fft_final rsi, 8*128, 1*128, 2*128, 4*128, rdx, 0*ZMM_SCD4, rdi, 0*ZMM_SCD4, 1
	jmp	zpass2_64_real_done

zpass2_64_real_3:
	mov	r8, DEST2ARG
	mov	rbp, DIST_TO_MULSRCARG
	zr64f_hundredtwentyeight_real_with_mult_preload
	zr64f_hundredtwentyeight_real_with_mult rsi, 8*128, 1*128, 2*128, 4*128, rdx, 0*ZMM_SCD4, rdi, 0*ZMM_SCD4, 1
	add	rsi, r8
	jmp	zpass2_64_real_unfft

zpass2_64_real_4:
	mov	rbp, DIST_TO_MULSRCARG
	mov	rdx, sincos3		;; Load sin/cos pointer
	mov	rdi, xsincos_complex	;; Complex sin/cos data
;;	zr64f_hundredtwentyeight_real_with_mulf_preload
;;	zr64f_hundredtwentyeight_real_with_mulf rsi, 8*128, 1*128, 2*128, 4*128, rdx, 0*ZMM_SCD4, rdi, 0*ZMM_SCD4, 1
	CALLP	zreal_mulf

zpass2_64_real_unfft:
zpass2_64_real_done:
	end_timer 2
	ENDM

; *************** radix4-pass2-64-complex macro ******************
; This macro takes 64 complex values and performs the final 6 levels of a standard
; radix-8 FFT, squares the results, and does the first 6 levels of the inverse FFT.

zr4dwpn_pass2_64_complex MACRO
	LOCAL	zpass2_64_complex_1, zpass2_64_complex_2
	LOCAL	zpass2_64_complex_3, zpass2_64_complex_4
	LOCAL	zpass2_64_complex_unfft, zpass2_64_complex_done

;; Type 4 FFTs skip the forward FFT process

	mov	rsi, DATA_ADDR		;; Load address of FFT data
	mov	rbx, DIST_TO_FFTSRCARG
;	mov	rcx, DATA_PREFETCH	;; Prefetch pointer (we need to prefetch 16 cache lines + 0 pad cache lines)
	cmp	ffttype, 4
	je	zpass2_64_complex_4

;; Do FFT levels 1,2,3,4,5,6 as well as inverse FFT levels 1,2,3,4,5,6
;; On input and output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	64	+1	...
;;	8	...
;;	...
;;	56

;; Do 1 sixtyfour_complex_fft operations
;;	distance between fft data elements is 1 (8 for swizzling)
;;	do 1 macros each processing 128 data values

;; Execute the right middle step

	mov	rdi, xsincos_complex	;; The sin/cos data
	cmp	ffttype, 2
	jl	zpass2_64_complex_1
	jg	zpass2_64_complex_3

zpass2_64_complex_2:
	start_timer 9
	mov	r8, DEST2ARG
	zr64f_sixtyfour_complex_with_square_preload
	zr64f_sixtyfour_complex_with_square rsi, 0, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 1
	add	rsi, r8
	end_timer 9
	jmp	zpass2_64_complex_unfft

zpass2_64_complex_1:
	start_timer 9
	zr64f_sixtyfour_complex_fft_final_preload
	zr64f_sixtyfour_complex_fft_final rsi, 0, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 1
	end_timer 9
	jmp	zpass2_64_complex_done

zpass2_64_complex_3:
	start_timer 9
	mov	r8, DEST2ARG
	mov	rbp, DIST_TO_MULSRCARG
	zr64f_sixtyfour_complex_with_mult_preload
	zr64f_sixtyfour_complex_with_mult rsi, 0, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 1
	add	rsi, r8
	end_timer 9
	jmp	zpass2_64_complex_unfft

zpass2_64_complex_4:
	start_timer 9
	mov	rbp, DIST_TO_MULSRCARG
	mov	rdi, xsincos_complex	;; The sin/cos data
	zr64f_sixtyfour_complex_with_mulf_preload
	zr64f_sixtyfour_complex_with_mulf rsi, 0, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 1
	end_timer 9
zpass2_64_complex_unfft:
zpass2_64_complex_done:
	ENDM


;*********************************************************************************************
;			Macros derived from pass2-9-levels
;*********************************************************************************************


; *************** radix8-pass2-320-real macro ******************
; This macro takes 640 real values and performs the final 9.322 levels of the FFT.
; The results are squared, and the first 9.322 levels of the inverse FFT are done.

zr4dwpn_pass2_320_real MACRO
	LOCAL	b1b, b2a, b2b, b2c, b2d, c1b
	LOCAL	zpass2_320_real_1, zpass2_320_real_2
	LOCAL	zpass2_320_real_3, zpass2_320_real_4
	LOCAL	zpass2_320_real_unfft, zpass2_320_real_done

;; Type 4 FFTs skip the forward FFT process

	start_timer 2
	mov	rsi, DATA_ADDR		;; Load source address
	mov	rbx, DIST_TO_FFTSRCARG
	mov	rcx, DATA_PREFETCH	;; Prefetch pointer (we need to prefetch 80 cache lines + 0 pad cache lines)
	cmp	ffttype, 4
	je	zpass2_320_real_4

;; Do FFT levels 1,2,3,4
;;	0	+1	+1	+1	+1	+1	+1	+1	320	+1	...
;;	8	...
;;	...
;;	312
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	64	+1	...
;;	8	...
;;	...
;;	56
;;	128
;;	...

;; Do 64 ten_real_fft operations
;;	distance between fft data elements is 64
;;	do 8 macros each processing 80 data values

	mov	rdx, sincos3		;; Real sin/cos data
	mov	rdi, xsincos_complex	;; Complex sin/cos data
	zloop_init 8			;; 8 iterations
	zloop_set_rw_prefetch ZTRIPLE	;; Prefetch 24 cache lines
	zr5f_2sc_ten_reals_fft_preload
b1b:	zr5f_2sc_ten_reals_fft rsi, 128, 8*128, rdx, ZMM_SCD2, rdi, ZMM_SCD2, 8
	zloop	8, b1b, -8*128		;; Test loop counter, restore source pointer

;; Do FFT levels 5,6,7,8,9,10 as well as inverse FFT levels 5,6,7,8,9,10
;; On input and output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	64	+1	...
;;	8	...
;;	...
;;	56
;;	128
;;	...

;; Do 1 hundredtwentyeight_real_fft operation
;; Do 4 sixtyfour_complex_fft operations
;;	distance between fft data elements is 1 (8 for swizzling)
;;	do 5 macros each processing 128 data values

;; Execute the proper middle step

	cmp	ffttype, 2
	jl	zpass2_320_real_1
	jg	zpass2_320_real_3

zpass2_320_real_2:
	mov	r8, DEST2ARG
;;	zr64_hundredtwentyeight_real_with_square_preload
;;	zr64_hundredtwentyeight_real_with_square rsi, 8*128, 1*128, 2*128, 4*128, rdx, 0*ZMM_SCD4, rdi, 0*ZMM_SCD4, 1
	CALLP	zreal_square
	zloop_init 4			;; 4 more iterations
	zloop_set_rw_prefetch ZOCT	;; Prefetch 32 cache lines
	zr64_sixtyfour_complex_with_square_preload
b2a:	zr64_sixtyfour_complex_with_square rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 4, L1PREFETCH_ALL, 8*128
	zloop	4, b2a			;; Test loop counter
	add	rsi, r8
	jmp	zpass2_320_real_unfft

zpass2_320_real_1:
;;	zr64_hundredtwentyeight_real_fft_final_preload
;;	zr64_hundredtwentyeight_real_fft_final rsi, 8*128, 1*128, 2*128, 4*128, rdx, 0*ZMM_SCD4, rdi, 0*ZMM_SCD4, 1
	CALLP	zreal_fft_final
	zloop_init 4			;; 4 more iterations
	zloop_set_rw_prefetch ZOCT	;; Prefetch 32 cache lines
	zr64_sixtyfour_complex_fft_final_preload
b2b:	zr64_sixtyfour_complex_fft_final rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 4, L1PREFETCH_ALL, 8*128
	zloop	4, b2b			;; Test loop counter
	jmp	zpass2_320_real_done

zpass2_320_real_3:
	mov	r8, DEST2ARG
	mov	rbp, DIST_TO_MULSRCARG
;;	zr64_hundredtwentyeight_real_with_mult_preload
;;	zr64_hundredtwentyeight_real_with_mult rsi, 8*128, 1*128, 2*128, 4*128, rdx, 0*ZMM_SCD4, rdi, 0*ZMM_SCD4, 1
	CALLP	zreal_mult
	zloop_init 4			;; 4 more iterations
	zloop_set_rw_prefetch ZOCT	;; Prefetch 32 cache lines
	zr64_sixtyfour_complex_with_mult_preload
b2c:	zr64_sixtyfour_complex_with_mult rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 4, L1PREFETCH_ALL, 8*128
	zloop	4, b2c			;; Test loop counter
	add	rsi, r8
	jmp	zpass2_320_real_unfft

zpass2_320_real_4:
	mov	rbp, DIST_TO_MULSRCARG
	mov	rdx, sincos3		;; Real sin/cos data
	mov	rdi, xsincos_complex	;; Complex sin/cos data
	bump	rdx, 8*ZMM_SCD2
	bump	rdi, 8*ZMM_SCD2
;;	zr64f_hundredtwentyeight_real_with_mulf_preload
;;	zr64f_hundredtwentyeight_real_with_mulf rsi, 8*128, 1*128, 2*128, 4*128, rdx, 0*ZMM_SCD4, rdi, 0*ZMM_SCD4, 1
	CALLP	zreal_mulf
	zloop_init 4			;; 4 more iterations
	zloop_set_rw_prefetch ZOCT	;; Prefetch 32 cache lines
	zr64f_sixtyfour_complex_with_mulf_preload
b2d:	zr64f_sixtyfour_complex_with_mulf rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 4, L1PREFETCH_ALL, 8*128
	zloop	4, b2d			;; Test loop counter
zpass2_320_real_unfft:
	bump	rsi, -5*8*128		;; Restore source pointer

;; Do inverse FFT levels 1,2,3,4
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	64	+1	...
;;	8	...
;;	...
;;	56
;;	128
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	320	+1	...
;;	8	...
;;	...
;;	312

;; Do 64 ten_real_unfft operations
;;	distance between fft data elements is 64
;;	do 8 macros each processing 80 data values

	bump	rdx, -8*ZMM_SCD2
	bump	rdi, -8*ZMM_SCD2
	zloop_init 8			;; 8 iterations
	zloop_set_rw_prefetch ZTRIPLE	;; Prefetch 24 cache lines
	zr5_2sc_ten_reals_unfft_preload
c1b:	zr5_2sc_ten_reals_unfft rsi, 128, 8*128, rdx, ZMM_SCD2, rdi, ZMM_SCD2, 8
	zloop	8, c1b			;; Test loop counter

zpass2_320_real_done:
	end_timer 2
	ENDM

; *************** radix8-pass2-320-complex macro ******************
; This macro takes 320 complex values and performs the final 8.322 levels of a standard
; radix-8 FFT, squares the results, and does the first 8.322 levels of the inverse FFT.

zr4dwpn_pass2_320_complex MACRO
	LOCAL	b1b, b2a, b2b, b2c, b2d, c1b
	LOCAL	zpass2_320_complex_1, zpass2_320_complex_2
	LOCAL	zpass2_320_complex_3, zpass2_320_complex_4
	LOCAL	zpass2_320_complex_unfft, zpass2_320_complex_done

;; Type 4 FFTs skip the forward FFT process

	mov	rsi, DATA_ADDR		;; Load address of FFT data
	mov	rbx, DIST_TO_FFTSRCARG
	mov	rcx, DATA_PREFETCH	;; Prefetch pointer (we need to prefetch 80 cache lines + 0 pad cache lines)
	cmp	ffttype, 4
	je	zpass2_320_complex_4

;; Do FFT levels 1,2,3
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	320	+1	...
;;	8	...
;;	...
;;	312
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	64	+1	...
;;	8	...
;;	...
;;	56
;;	128
;;	...

;; Do 48 five_complex_fft operations
;;	distance between fft data elements is 64
;;	do 8 macros each processing 80 data values

	start_timer 4
	mov	rdi, xsincos_complex	;; The sin/cos data
	zloop_init 8			;; 8 iterations
	zloop_set_rw_prefetch ZTRIPLE	;; Prefetch 24 cache lines
	zr5f_five_complex_djbfft_preload
b1b:	zr5f_five_complex_djbfft rsi, 128, 8*128, rdi, ZMM_SCD2, 8
	zloop	8, b1b, -8*128		;; Test loop counter, restore source pointer
	end_timer 4

;; Do FFT levels 4,5,6,7,8,9 as well as inverse FFT levels 4,5,6,7,8,9
;; On input and output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	64	+1	...
;;	8	...
;;	...
;;	56
;;	128
;;	...

;; Do 5 sixtyfour_complex_fft operations
;;	distance between fft data elements is 1 (8 for swizzling)
;;	do 5 macros each processing 128 data values

;; Execute the right middle step

	cmp	ffttype, 2
	jl	zpass2_320_complex_1
	jg	zpass2_320_complex_3

zpass2_320_complex_2:
	start_timer 9
	zloop_init 5			;; 5 iterations
	zloop_set_rw_prefetch ZOCT	;; Prefetch 40 cache lines
	mov	r8, DEST2ARG
	zr64_sixtyfour_complex_with_square_preload
b2a:	zr64_sixtyfour_complex_with_square rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 5, L1PREFETCH_ALL, 8*128
	zloop	5, b2a			;; Test loop counter
	add	rsi, r8
	end_timer 9
	jmp	zpass2_320_complex_unfft

zpass2_320_complex_1:
	start_timer 9
	zloop_init 5			;; 5 iterations
	zloop_set_rw_prefetch ZOCT	;; Prefetch 40 cache lines
	zr64_sixtyfour_complex_fft_final_preload
b2b:	zr64_sixtyfour_complex_fft_final rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 5, L1PREFETCH_ALL, 8*128
	zloop	5, b2b			;; Test loop counter
	end_timer 9
	jmp	zpass2_320_complex_done

zpass2_320_complex_3:
	start_timer 9
	zloop_init 5			;; 5 iterations
	zloop_set_rw_prefetch ZOCT	;; Prefetch 40 cache lines
	mov	r8, DEST2ARG
	mov	rbp, DIST_TO_MULSRCARG
	zr64_sixtyfour_complex_with_mult_preload
b2c:	zr64_sixtyfour_complex_with_mult rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 5, L1PREFETCH_ALL, 8*128
	zloop	5, b2c			;; Test loop counter
	add	rsi, r8
	end_timer 9
	jmp	zpass2_320_complex_unfft

zpass2_320_complex_4:
	start_timer 9
	zloop_init 5			;; 5 iterations
	zloop_set_rw_prefetch ZOCT	;; Prefetch 40 cache lines
	mov	rbp, DIST_TO_MULSRCARG
	mov	rdi, xsincos_complex	;; Load sin/cos pointer
	bump	rdi, 8*ZMM_SCD2
	zr64f_sixtyfour_complex_with_mulf_preload
b2d:	zr64f_sixtyfour_complex_with_mulf rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 5, L1PREFETCH_ALL, 8*128
	zloop	5, b2d			;; Test loop counter
	end_timer 9
zpass2_320_complex_unfft:
	bump	rsi, -5*8*128		;; Restore source pointer

;; Do inverse FFT levels 1,2,3
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	64	+1	...
;;	8	...
;;	...
;;	56
;;	128
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	320	+1	...
;;	8	...
;;	...
;;	312

;; Do 64 five_complex_unfft operations
;;	distance between fft data elements is 64
;;	do 8 macros each processing 80 data values

	start_timer 14
	zloop_init 8			;; 8 iterations
	zloop_set_rw_prefetch ZDOUBLE	;; Prefetch 16 cache lines
	bump	rdi, -8*ZMM_SCD2
	zr5_five_complex_djbunfft_preload
c1b:	zr5_five_complex_djbunfft rsi, 128, 8*128, rdi, ZMM_SCD2, 8
	zloop	8, c1b			;; Test loop counter
	end_timer 14

zpass2_320_complex_done:
	ENDM



; *************** radix8-pass2-384-real macro ******************
; This macro takes 768 real values and performs the final 9.585 levels of the FFT.
; The results are squared, and the first 9.585 levels of the inverse FFT are done.

zr4dwpn_pass2_384_real MACRO
	LOCAL	b1b, b2a, b2b, b2c, b2d, c1b
	LOCAL	zpass2_384_real_1, zpass2_384_real_2
	LOCAL	zpass2_384_real_3, zpass2_384_real_4
	LOCAL	zpass2_384_real_unfft, zpass2_384_real_done

;; Type 4 FFTs skip the forward FFT process

	start_timer 2
	mov	rsi, DATA_ADDR		;; Load source address
	mov	rbx, DIST_TO_FFTSRCARG
	mov	rcx, DATA_PREFETCH	;; Prefetch pointer (we need to prefetch 96 cache lines + 0 pad cache lines)
	cmp	ffttype, 4
	je	zpass2_384_real_4

;; Do FFT levels 1,2,3,4
;;	0	+1	+1	+1	+1	+1	+1	+1	384	+1	...
;;	8	...
;;	...
;;	376
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	64	+1	...
;;	8	...
;;	...
;;	56
;;	128
;;	...

;; Do 64 twelve_real_fft operations
;;	distance between fft data elements is 64
;;	do 8 macros each processing 96 data values

	mov	rdx, sincos3		;; Real sin/cos data
	mov	rdi, xsincos_complex	;; Complex sin/cos data
	zloop_init 8			;; 8 iterations
	zloop_set_rw_prefetch ZTRIPLE	;; Prefetch 24 cache lines
	zr6f_2sc_twelve_reals_fft_preload
b1b:	zr6f_2sc_twelve_reals_fft rsi, 128, 8*128, rdx, ZMM_SCD3, rdi, ZMM_SCD3, 8
	zloop	8, b1b, -8*128		;; Test loop counter, restore source pointer

;; Do FFT levels 5,6,7,8,9,10 as well as inverse FFT levels 5,6,7,8,9,10
;; On input and output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	64	+1	...
;;	8	...
;;	...
;;	56
;;	128
;;	...

;; Do 1 hundredtwentyeight_real_fft operation
;; Do 5 sixtyfour_complex_fft operations
;;	distance between fft data elements is 1 (8 for swizzling)
;;	do 6 macros each processing 128 data values

;; Execute the proper middle step

	cmp	ffttype, 2
	jl	zpass2_384_real_1
	jg	zpass2_384_real_3

zpass2_384_real_2:
	mov	r8, DEST2ARG
;;	zr64_hundredtwentyeight_real_with_square_preload
;;	zr64_hundredtwentyeight_real_with_square rsi, 8*128, 1*128, 2*128, 4*128, rdx, 0*ZMM_SCD4, rdi, 0*ZMM_SCD4, 1
	CALLP	zreal_square
	zloop_init 5			;; 5 more iterations
	zloop_set_rw_prefetch ZOCT	;; Prefetch 40 cache lines
	zr64_sixtyfour_complex_with_square_preload
b2a:	zr64_sixtyfour_complex_with_square rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 5, L1PREFETCH_ALL, 8*128
	zloop	5, b2a			;; Test loop counter
	add	rsi, r8
	jmp	zpass2_384_real_unfft

zpass2_384_real_1:
;;	zr64_hundredtwentyeight_real_fft_final_preload
;;	zr64_hundredtwentyeight_real_fft_final rsi, 8*128, 1*128, 2*128, 4*128, rdx, 0*ZMM_SCD4, rdi, 0*ZMM_SCD4, 1
	CALLP	zreal_fft_final
	zloop_init 5			;; 5 more iterations
	zloop_set_rw_prefetch ZOCT	;; Prefetch 40 cache lines
	zr64_sixtyfour_complex_fft_final_preload
b2b:	zr64_sixtyfour_complex_fft_final rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 5, L1PREFETCH_ALL, 8*128
	zloop	5, b2b			;; Test loop counter
	jmp	zpass2_384_real_done

zpass2_384_real_3:
	mov	r8, DEST2ARG
	mov	rbp, DIST_TO_MULSRCARG
;;	zr64_hundredtwentyeight_real_with_mult_preload
;;	zr64_hundredtwentyeight_real_with_mult rsi, 8*128, 1*128, 2*128, 4*128, rdx, 0*ZMM_SCD4, rdi, 0*ZMM_SCD4, 1
	CALLP	zreal_mult
	zloop_init 5			;; 5 more iterations
	zloop_set_rw_prefetch ZOCT	;; Prefetch 40 cache lines
	zr64_sixtyfour_complex_with_mult_preload
b2c:	zr64_sixtyfour_complex_with_mult rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 5, L1PREFETCH_ALL, 8*128
	zloop	5, b2c			;; Test loop counter
	add	rsi, r8
	jmp	zpass2_384_real_unfft

zpass2_384_real_4:
	mov	rbp, DIST_TO_MULSRCARG
	mov	rdx, sincos3		;; Real sin/cos data
	mov	rdi, xsincos_complex	;; Complex sin/cos data
	bump	rdx, 8*ZMM_SCD3
	bump	rdi, 8*ZMM_SCD3
;;	zr64f_hundredtwentyeight_real_with_mulf_preload
;;	zr64f_hundredtwentyeight_real_with_mulf rsi, 8*128, 1*128, 2*128, 4*128, rdx, 0*ZMM_SCD4, rdi, 0*ZMM_SCD4, 1
	CALLP	zreal_mulf
	zloop_init 5			;; 5 more iterations
	zloop_set_rw_prefetch ZOCT	;; Prefetch 40 cache lines
	zr64f_sixtyfour_complex_with_mulf_preload
b2d:	zr64f_sixtyfour_complex_with_mulf rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 5, L1PREFETCH_ALL, 8*128
	zloop	5, b2d			;; Test loop counter
zpass2_384_real_unfft:
	bump	rsi, -6*8*128		;; Restore source pointer

;; Do inverse FFT levels 1,2,3,4
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	64	+1	...
;;	8	...
;;	...
;;	56
;;	128
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	384	+1	...
;;	8	...
;;	...
;;	376

;; Do 64 twelve_real_unfft operations
;;	distance between fft data elements is 64
;;	do 8 macros each processing 96 data values

	bump	rdx, -8*ZMM_SCD3
	bump	rdi, -8*ZMM_SCD3
	zloop_init 8			;; 8 iterations
	zloop_set_rw_prefetch ZQUAD	;; Prefetch 32 cache lines
	zr6_2sc_twelve_reals_unfft_preload
c1b:	zr6_2sc_twelve_reals_unfft rsi, 128, 8*128, rdx, ZMM_SCD3, rdi, ZMM_SCD3, 8
	zloop	8, c1b			;; Test loop counter

zpass2_384_real_done:
	end_timer 2
	ENDM

; *************** radix8-pass2-384-complex macro ******************
; This macro takes 384 complex values and performs the final 8.585 levels of a standard
; radix-8 FFT, squares the results, and does the first 8.585 levels of the inverse FFT.

zr4dwpn_pass2_384_complex MACRO
	LOCAL	b1b, b2a, b2b, b2c, b2d, c1b
	LOCAL	zpass2_384_complex_1, zpass2_384_complex_2
	LOCAL	zpass2_384_complex_3, zpass2_384_complex_4
	LOCAL	zpass2_384_complex_unfft, zpass2_384_complex_done

;; Type 4 FFTs skip the forward FFT process

	mov	rsi, DATA_ADDR		;; Load address of FFT data
	mov	rbx, DIST_TO_FFTSRCARG
	mov	rcx, DATA_PREFETCH	;; Prefetch pointer (we need to prefetch 96 cache lines + 0 pad cache lines)
	cmp	ffttype, 4
	je	zpass2_384_complex_4

;; Do FFT levels 1,2,3
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	384	+1	...
;;	8	...
;;	...
;;	376
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	64	+1	...
;;	8	...
;;	...
;;	56
;;	128
;;	...

;; Do 48 six_complex_fft operations
;;	distance between fft data elements is 64
;;	do 8 macros each processing 96 data values

	start_timer 4
	mov	rdi, xsincos_complex	;; The sin/cos data
	zloop_init 8			;; 8 iterations
	zloop_set_rw_prefetch ZTRIPLE	;; Prefetch 24 cache lines
	zr6f_six_complex_djbfft_preload
b1b:	zr6f_six_complex_djbfft rsi, 128, 8*128, rdi, ZMM_SCD3, 8
	zloop	8, b1b, -8*128		;; Test loop counter, restore source pointer
	end_timer 4

;; Do FFT levels 4,5,6,7,8,9 as well as inverse FFT levels 4,5,6,7,8,9
;; On input and output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	64	+1	...
;;	8	...
;;	...
;;	56
;;	128
;;	...

;; Do 6 sixtyfour_complex_fft operations
;;	distance between fft data elements is 1 (8 for swizzling)
;;	do 6 macros each processing 128 data values

;; Execute the right middle step

	cmp	ffttype, 2
	jl	zpass2_384_complex_1
	jg	zpass2_384_complex_3

zpass2_384_complex_2:
	start_timer 9
	zloop_init 6			;; 6 iterations
	zloop_set_rw_prefetch ZOCT	;; Prefetch 48 cache lines
	mov	r8, DEST2ARG
	zr64_sixtyfour_complex_with_square_preload
b2a:	zr64_sixtyfour_complex_with_square rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 6, L1PREFETCH_ALL, 8*128
	zloop	6, b2a			;; Test loop counter
	add	rsi, r8
	end_timer 9
	jmp	zpass2_384_complex_unfft

zpass2_384_complex_1:
	start_timer 9
	zloop_init 6			;; 6 iterations
	zloop_set_rw_prefetch ZOCT	;; Prefetch 48 cache lines
	zr64_sixtyfour_complex_fft_final_preload
b2b:	zr64_sixtyfour_complex_fft_final rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 6, L1PREFETCH_ALL, 8*128
	zloop	6, b2b			;; Test loop counter
	end_timer 9
	jmp	zpass2_384_complex_done

zpass2_384_complex_3:
	start_timer 9
	zloop_init 6			;; 6 iterations
	zloop_set_rw_prefetch ZOCT	;; Prefetch 48 cache lines
	mov	r8, DEST2ARG
	mov	rbp, DIST_TO_MULSRCARG
	zr64_sixtyfour_complex_with_mult_preload
b2c:	zr64_sixtyfour_complex_with_mult rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 6, L1PREFETCH_ALL, 8*128
	zloop	6, b2c			;; Test loop counter
	add	rsi, r8
	end_timer 9
	jmp	zpass2_384_complex_unfft

zpass2_384_complex_4:
	start_timer 9
	zloop_init 6			;; 6 iterations
	zloop_set_rw_prefetch ZOCT	;; Prefetch 48 cache lines
	mov	rbp, DIST_TO_MULSRCARG
	mov	rdi, xsincos_complex	;; Load sin/cos pointer
	bump	rdi, 8*ZMM_SCD3
	zr64f_sixtyfour_complex_with_mulf_preload
b2d:	zr64f_sixtyfour_complex_with_mulf rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 6, L1PREFETCH_ALL, 8*128
	zloop	6, b2d			;; Test loop counter
	end_timer 9
zpass2_384_complex_unfft:
	bump	rsi, -6*8*128		;; Restore source pointer

;; Do inverse FFT levels 1,2,3
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	64	+1	...
;;	8	...
;;	...
;;	56
;;	128
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	384	+1	...
;;	8	...
;;	...
;;	376

;; Do 64 six_complex_unfft operations
;;	distance between fft data elements is 64
;;	do 8 macros each processing 96 data values

	start_timer 14
	zloop_init 8			;; 8 iterations
	zloop_set_rw_prefetch ZTRIPLE	;; Prefetch 24 cache lines
	bump	rdi, -8*ZMM_SCD3
	zr6_six_complex_djbunfft_preload
c1b:	zr6_six_complex_djbunfft rsi, 128, 8*128, rdi, ZMM_SCD3, 8
	zloop	8, c1b			;; Test loop counter
	end_timer 14

zpass2_384_complex_done:
	ENDM


; *************** radix8-pass2-448-real macro ******************
; This macro takes 896 real values and performs the final 9.807 levels of the FFT.
; The results are squared, and the first 9.807 levels of the inverse FFT are done.

zr4dwpn_pass2_448_real MACRO
	LOCAL	b1b, b2a, b2b, b2c, b2d, c1b
	LOCAL	zpass2_448_real_1, zpass2_448_real_2
	LOCAL	zpass2_448_real_3, zpass2_448_real_4
	LOCAL	zpass2_448_real_unfft, zpass2_448_real_done

;; Type 4 FFTs skip the forward FFT process

	start_timer 2
	mov	rsi, DATA_ADDR		;; Load source address
	mov	rbx, DIST_TO_FFTSRCARG
	mov	rcx, DATA_PREFETCH	;; Prefetch pointer (we need to prefetch 112 cache lines + 0 pad cache lines)
	cmp	ffttype, 4
	je	zpass2_448_real_4

;; Do FFT levels 1,2,3,4
;;	0	+1	+1	+1	+1	+1	+1	+1	448	+1	...
;;	8	...
;;	...
;;	440
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	64	+1	...
;;	8	...
;;	...
;;	56
;;	128
;;	...

;; Do 64 fourteen_real_fft operations
;;	distance between fft data elements is 64
;;	do 8 macros each processing 112 data values

	mov	rdx, sincos3		;; Real sin/cos data
	mov	rdi, xsincos_complex	;; Complex sin/cos data
	zloop_init  8			;; 8 iterations
	zloop_set_rw_prefetch ZQUAD	;; Prefetch 32 cache lines
	zr7f_2sc_fourteen_reals_fft_preload
b1b:	zr7f_2sc_fourteen_reals_fft rsi, 128, 8*128, rdx, ZMM_SCD3, rdi, ZMM_SCD3, 8
	zloop	8, b1b, -8*128		;; Test loop counter, restore source pointer

;; Do FFT levels 5,6,7,8,9,10 as well as inverse FFT levels 5,6,7,8,9,10
;; On input and output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	64	+1	...
;;	8	...
;;	...
;;	56
;;	128
;;	...

;; Do 1 hundredtwentyeight_real_fft operation
;; Do 6 sixtyfour_complex_fft operations
;;	distance between fft data elements is 1 (8 for swizzling)
;;	do 7 macros each processing 128 data values

;; Execute the proper middle step

	cmp	ffttype, 2
	jl	zpass2_448_real_1
	jg	zpass2_448_real_3

zpass2_448_real_2:
	mov	r8, DEST2ARG
;;	zr64_hundredtwentyeight_real_with_square_preload
;;	zr64_hundredtwentyeight_real_with_square rsi, 8*128, 1*128, 2*128, 4*128, rdx, 0*ZMM_SCD4, rdi, 0*ZMM_SCD4, 1
	CALLP	zreal_square
	zloop_init 6			;; 6 more iterations
	zloop_set_rw_prefetch ZOCT	;; Prefetch 48 cache lines
	zr64_sixtyfour_complex_with_square_preload
b2a:	zr64_sixtyfour_complex_with_square rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 6, L1PREFETCH_ALL, 8*128
	zloop	6, b2a			;; Test loop counter
	add	rsi, r8
	jmp	zpass2_448_real_unfft

zpass2_448_real_1:
;;	zr64_hundredtwentyeight_real_fft_final_preload
;;	zr64_hundredtwentyeight_real_fft_final rsi, 8*128, 1*128, 2*128, 4*128, rdx, 0*ZMM_SCD4, rdi, 0*ZMM_SCD4, 1
	CALLP	zreal_fft_final
	zloop_init 6			;; 6 more iterations
	zloop_set_rw_prefetch ZOCT	;; Prefetch 48 cache lines
	zr64_sixtyfour_complex_fft_final_preload
b2b:	zr64_sixtyfour_complex_fft_final rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 6, L1PREFETCH_ALL, 8*128
	zloop	6, b2b			;; Test loop counter
	jmp	zpass2_448_real_done

zpass2_448_real_3:
	mov	r8, DEST2ARG
	mov	rbp, DIST_TO_MULSRCARG
;;	zr64_hundredtwentyeight_real_with_mult_preload
;;	zr64_hundredtwentyeight_real_with_mult rsi, 8*128, 1*128, 2*128, 4*128, rdx, 0*ZMM_SCD4, rdi, 0*ZMM_SCD4, 1
	CALLP	zreal_mult
	zloop_init 6			;; 6 more iterations
	zloop_set_rw_prefetch ZOCT	;; Prefetch 48 cache lines
	zr64_sixtyfour_complex_with_mult_preload
b2c:	zr64_sixtyfour_complex_with_mult rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 6, L1PREFETCH_ALL, 8*128
	zloop	6, b2c			;; Test loop counter
	add	rsi, r8
	jmp	zpass2_448_real_unfft

zpass2_448_real_4:
	mov	rbp, DIST_TO_MULSRCARG
	mov	rdx, sincos3		;; Real sin/cos data
	mov	rdi, xsincos_complex	;; Complex sin/cos data
	bump	rdx, 8*ZMM_SCD3
	bump	rdi, 8*ZMM_SCD3
;;	zr64f_hundredtwentyeight_real_with_mulf_preload
;;	zr64f_hundredtwentyeight_real_with_mulf rsi, 8*128, 1*128, 2*128, 4*128, rdx, 0*ZMM_SCD4, rdi, 0*ZMM_SCD4, 1
	CALLP	zreal_mulf
	zloop_init 6			;; 6 more iterations
	zloop_set_rw_prefetch ZOCT	;; Prefetch 48 cache lines
	zr64f_sixtyfour_complex_with_mulf_preload
b2d:	zr64f_sixtyfour_complex_with_mulf rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 6, L1PREFETCH_ALL, 8*128
	zloop	6, b2d			;; Test loop counter
zpass2_448_real_unfft:
	bump	rsi, -7*8*128		;; Restore source pointer

;; Do inverse FFT levels 1,2,3,4
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	64	+1	...
;;	8	...
;;	...
;;	56
;;	128
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	448	+1	...
;;	8	...
;;	...
;;	440

;; Do 64 fourteen_real_unfft operations
;;	distance between fft data elements is 64
;;	do 8 macros each processing 112 data values

	bump	rdx, -8*ZMM_SCD3
	bump	rdi, -8*ZMM_SCD3
	zloop_init  8			;; 8 iterations
	zloop_set_rw_prefetch ZQUAD	;; Prefetch 32 cache lines
	zr7_2sc_fourteen_reals_unfft_preload
c1b:	zr7_2sc_fourteen_reals_unfft rsi, 128, 8*128, rdx, ZMM_SCD3, rdi, ZMM_SCD3, 8
	zloop	8, c1b			;; Test loop counter

zpass2_448_real_done:
	end_timer 2
	ENDM

; *************** radix8-pass2-448-complex macro ******************
; This macro takes 448 complex values and performs the final 8.807 levels of a standard
; radix-8 FFT, squares the results, and does the first 8.807 levels of the inverse FFT.

zr4dwpn_pass2_448_complex MACRO
	LOCAL	b1b, b2a, b2b, b2c, b2d, c1b
	LOCAL	zpass2_448_complex_1, zpass2_448_complex_2
	LOCAL	zpass2_448_complex_3, zpass2_448_complex_4
	LOCAL	zpass2_448_complex_unfft, zpass2_448_complex_done

;; Type 4 FFTs skip the forward FFT process

	mov	rsi, DATA_ADDR		;; Load address of FFT data
	mov	rbx, DIST_TO_FFTSRCARG
	mov	rcx, DATA_PREFETCH	;; Prefetch pointer (we need to prefetch 112 cache lines + 0 pad cache lines)
	cmp	ffttype, 4
	je	zpass2_448_complex_4

;; Do FFT levels 1,2,3
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	448	+1	...
;;	8	...
;;	...
;;	440
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	64	+1	...
;;	8	...
;;	...
;;	56
;;	128
;;	...

;; Do 48 seven_complex_fft operations
;;	distance between fft data elements is 64
;;	do 8 macros each processing 112 data values

	start_timer 4
	mov	rdi, xsincos_complex	;; The sin/cos data
	zloop_init 8			;; 8 iterations
	zloop_set_rw_prefetch ZQUAD	;; Prefetch 32 cache lines
	zr7f_seven_complex_djbfft_preload
b1b:	zr7f_seven_complex_djbfft rsi, 128, 8*128, rdi, ZMM_SCD3, 8
	zloop	8, b1b, -8*128		;; Test loop counter, restore source pointer
	end_timer 4

;; Do FFT levels 4,5,6,7,8,9 as well as inverse FFT levels 4,5,6,7,8,9
;; On input and output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	64	+1	...
;;	8	...
;;	...
;;	56
;;	128
;;	...

;; Do 7 sixtyfour_complex_fft operations
;;	distance between fft data elements is 1 (8 for swizzling)
;;	do 7 macros each processing 128 data values

;; Execute the right middle step

	cmp	ffttype, 2
	jl	zpass2_448_complex_1
	jg	zpass2_448_complex_3

zpass2_448_complex_2:
	start_timer 9
	zloop_init 7			;; 7 iterations
	zloop_set_rw_prefetch ZOCT	;; Prefetch 56 cache lines
	mov	r8, DEST2ARG
	zr64_sixtyfour_complex_with_square_preload
b2a:	zr64_sixtyfour_complex_with_square rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 7, L1PREFETCH_ALL, 8*128
	zloop	7, b2a			;; Test loop counter
	add	rsi, r8
	end_timer 9
	jmp	zpass2_448_complex_unfft

zpass2_448_complex_1:
	start_timer 9
	zloop_init 7			;; 7 iterations
	zloop_set_rw_prefetch ZOCT	;; Prefetch 56 cache lines
	zr64_sixtyfour_complex_fft_final_preload
b2b:	zr64_sixtyfour_complex_fft_final rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 7, L1PREFETCH_ALL, 8*128
	zloop	7, b2b			;; Test loop counter
	end_timer 9
	jmp	zpass2_448_complex_done

zpass2_448_complex_3:
	start_timer 9
	zloop_init 7			;; 7 iterations
	zloop_set_rw_prefetch ZOCT	;; Prefetch 56 cache lines
	mov	r8, DEST2ARG
	mov	rbp, DIST_TO_MULSRCARG
	zr64_sixtyfour_complex_with_mult_preload
b2c:	zr64_sixtyfour_complex_with_mult rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 7, L1PREFETCH_ALL, 8*128
	zloop	7, b2c			;; Test loop counter
	add	rsi, r8
	end_timer 9
	jmp	zpass2_448_complex_unfft

zpass2_448_complex_4:
	start_timer 9
	zloop_init 7			;; 7 iterations
	zloop_set_rw_prefetch ZOCT	;; Prefetch 56 cache lines
	mov	rbp, DIST_TO_MULSRCARG
	mov	rdi, xsincos_complex	;; Load sin/cos pointer
	bump	rdi, 8*ZMM_SCD3
	zr64f_sixtyfour_complex_with_mulf_preload
b2d:	zr64f_sixtyfour_complex_with_mulf rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 7, L1PREFETCH_ALL, 8*128
	zloop	7, b2d			;; Test loop counter
	end_timer 9
zpass2_448_complex_unfft:
	bump	rsi, -7*8*128		;; Restore source pointer

;; Do inverse FFT levels 1,2,3
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	64	+1	...
;;	8	...
;;	...
;;	56
;;	128
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	448	+1	...
;;	8	...
;;	...
;;	440

;; Do 64 seven_complex_unfft operations
;;	distance between fft data elements is 64
;;	do 8 macros each processing 112 data values

	start_timer 14
	zloop_init 8			;; 8 iterations
	zloop_set_rw_prefetch ZTRIPLE	;; Prefetch 24 cache lines
	bump	rdi, -8*ZMM_SCD3
	zr7_seven_complex_djbunfft_preload
c1b:	zr7_seven_complex_djbunfft rsi, 128, 8*128, rdi, ZMM_SCD3, 8
	zloop	8, c1b			;; Test loop counter
	end_timer 14

zpass2_448_complex_done:
	ENDM


; *************** radix8-pass2-512-real macro ******************
; This macro takes 1024 real values and performs the final 10 levels of the FFT.
; The results are squared, and the first 10 levels of the inverse FFT are done.

zr4dwpn_pass2_512_real MACRO
	LOCAL	b1b, b2a, b2b, b2c, b2d, c1b
	LOCAL	zpass2_512_real_1, zpass2_512_real_2
	LOCAL	zpass2_512_real_3, zpass2_512_real_4
	LOCAL	zpass2_512_real_unfft, zpass2_512_real_done

;; Type 4 FFTs skip the forward FFT process

	start_timer 2
	mov	rsi, DATA_ADDR		;; Load source address
	mov	rbx, DIST_TO_FFTSRCARG
	mov	rcx, DATA_PREFETCH	;; Prefetch pointer (we need to prefetch 128 cache lines + 0 pad cache lines)
	cmp	ffttype, 4
	je	zpass2_512_real_4

;; Do FFT levels 1,2,3,4
;;	0	+1	+1	+1	+1	+1	+1	+1	512	+1	...
;;	8	...
;;	...
;;	504
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	64	+1	...
;;	8	...
;;	...
;;	56
;;	128
;;	...

;; Do 64 sixteen_real_fft operations
;;	distance between fft data elements is 64
;;	do 8 macros each processing 128 data values

	mov	rdx, sincos3		;; Real sin/cos data
	mov	rdi, xsincos_complex	;; Complex sin/cos data
	zloop_init 8			;; 8 iterations
	zloop_set_rw_prefetch ZQUAD	;; Prefetch 32 cache lines
	zr8f_2sc_sixteen_reals_fft_preload
b1b:	zr8f_2sc_sixteen_reals_fft rsi, 128, 8*128, 16*128, 32*128, rdx, ZMM_SCD4, rdi, ZMM_SCD4, 8
	zloop	8, b1b, -8*128		;; Test loop counter, restore source pointer

;; Do FFT levels 5,6,7,8,9,10 as well as inverse FFT levels 5,6,7,8,9,10
;; On input and output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	64	+1	...
;;	8	...
;;	...
;;	56
;;	128
;;	...

;; Do 1 hundredtwentyeight_real_fft operation
;; Do 7 sixtyfour_complex_fft operations
;;	distance between fft data elements is 1 (8 for swizzling)
;;	do 8 macros each processing 128 data values

;; Execute the proper middle step

	cmp	ffttype, 2
	jl	zpass2_512_real_1
	jg	zpass2_512_real_3

zpass2_512_real_2:
	mov	r8, DEST2ARG
;;	zr64_hundredtwentyeight_real_with_square_preload
;;	zr64_hundredtwentyeight_real_with_square rsi, 8*128, 1*128, 2*128, 4*128, rdx, 0*ZMM_SCD4, rdi, 0*ZMM_SCD4, 1
	CALLP	zreal_square
	zloop_init 8, 1, 8		;; 7 more iterations
	zloop_set_rw_prefetch ZOCT	;; Prefetch 56 cache lines
	zr64_sixtyfour_complex_with_square_preload
b2a:	zr64_sixtyfour_complex_with_square rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 1, L1PREFETCH_ALL, 8*128
	zloop	8, b2a			;; Test loop counter
	add	rsi, r8
	jmp	zpass2_512_real_unfft

zpass2_512_real_1:
;;	zr64_hundredtwentyeight_real_fft_final_preload
;;	zr64_hundredtwentyeight_real_fft_final rsi, 8*128, 1*128, 2*128, 4*128, rdx, 0*ZMM_SCD4, rdi, 0*ZMM_SCD4, 1
	CALLP	zreal_fft_final
	zloop_init 8, 1, 8		;; 7 more iterations
	zloop_set_rw_prefetch ZOCT	;; Prefetch 56 cache lines
	zr64_sixtyfour_complex_fft_final_preload
b2b:	zr64_sixtyfour_complex_fft_final rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 1, L1PREFETCH_ALL, 8*128
	zloop	8, b2b			;; Test loop counter
	jmp	zpass2_512_real_done

zpass2_512_real_3:
	mov	r8, DEST2ARG
	mov	rbp, DIST_TO_MULSRCARG
;;	zr64_hundredtwentyeight_real_with_mult_preload
;;	zr64_hundredtwentyeight_real_with_mult rsi, 8*128, 1*128, 2*128, 4*128, rdx, 0*ZMM_SCD4, rdi, 0*ZMM_SCD4, 1
	CALLP	zreal_mult
	zloop_init 8, 1, 8		;; 7 more iterations
	zloop_set_rw_prefetch ZOCT	;; Prefetch 56 cache lines
	zr64_sixtyfour_complex_with_mult_preload
b2c:	zr64_sixtyfour_complex_with_mult rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 1, L1PREFETCH_ALL, 8*128
	zloop	8, b2c			;; Test loop counter
	add	rsi, r8
	jmp	zpass2_512_real_unfft

zpass2_512_real_4:
	mov	rbp, DIST_TO_MULSRCARG
	mov	rdx, sincos3		;; Real sin/cos data
	mov	rdi, xsincos_complex	;; Complex sin/cos data
	bump	rdx, 8*ZMM_SCD4
	bump	rdi, 8*ZMM_SCD4
;;	zr64f_hundredtwentyeight_real_with_mulf_preload
;;	zr64f_hundredtwentyeight_real_with_mulf rsi, 8*128, 1*128, 2*128, 4*128, rdx, 0*ZMM_SCD4, rdi, 0*ZMM_SCD4, 1
	CALLP	zreal_mulf
	zloop_init 8, 1, 8		;; 7 more iterations
	zloop_set_rw_prefetch ZOCT	;; Prefetch 56 cache lines
	zr64f_sixtyfour_complex_with_mulf_preload
b2d:	zr64f_sixtyfour_complex_with_mulf rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 1, L1PREFETCH_ALL, 8*128
	zloop	8, b2d			;; Test loop counter
zpass2_512_real_unfft:
	bump	rsi, -8*8*128		;; Restore source pointer

;; Do inverse FFT levels 1,2,3,4
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	64	+1	...
;;	8	...
;;	...
;;	56
;;	128
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	512	+1	...
;;	8	...
;;	...
;;	504

;; Do 64 sixteen_real_unfft operations
;;	distance between fft data elements is 64
;;	do 8 macros each processing 128 data values

	bump	rdx, -8*ZMM_SCD4
	bump	rdi, -8*ZMM_SCD4
	zloop_init 8			;; 8 iterations
	zloop_set_rw_prefetch ZQUAD	;; Prefetch 32 cache lines
	zr8_2sc_sixteen_reals_unfft_preload
c1b:	zr8_2sc_sixteen_reals_unfft rsi, 128, 8*128, 16*128, 32*128, rdx, ZMM_SCD4, rdi, ZMM_SCD4, 8
	zloop	8, c1b			;; Test loop counter

zpass2_512_real_done:
	end_timer 2
	ENDM

; *************** radix8-pass2-512-complex macro ******************
; This macro takes 512 complex values and performs the final 9 levels of a standard
; radix-8 FFT, squares the results, and does the first 9 levels of the inverse FFT.

zr4dwpn_pass2_512_complex MACRO
	LOCAL	b1b, b2a, b2b, b2c, b2d, c1b
	LOCAL	zpass2_512_complex_1, zpass2_512_complex_2
	LOCAL	zpass2_512_complex_3, zpass2_512_complex_4
	LOCAL	zpass2_512_complex_unfft, zpass2_512_complex_done

;; Type 4 FFTs skip the forward FFT process

	mov	rsi, DATA_ADDR		;; Load address of FFT data
	mov	rbx, DIST_TO_FFTSRCARG
	mov	rcx, DATA_PREFETCH	;; Prefetch pointer (we need to prefetch 128 cache lines + 0 pad cache lines)
	cmp	ffttype, 4
	je	zpass2_512_complex_4

;; Do FFT levels 1,2,3
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	512	+1	...
;;	8	...
;;	...
;;	504
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	64	+1	...
;;	8	...
;;	...
;;	56
;;	128
;;	...

;; Do 64 eight_complex_fft operations
;;	distance between fft data elements is 64
;;	do 8 macros each processing 128 data values

	start_timer 4
	mov	rdi, xsincos_complex	;; The sin/cos data
	zloop_init 8			;; 8 iterations
	zloop_set_rw_prefetch ZQUAD	;; Prefetch 32 cache lines
	zr8f_eight_complex_djbfft_preload
b1b:	zr8f_eight_complex_djbfft rsi, 128, 8*128, 16*128, 32*128, rdi, ZMM_SCD4, 8
	zloop	8, b1b, -8*128		;; Test loop counter, restore source pointer
	end_timer 4

;; Do FFT levels 4,5,6,7,8,9 as well as inverse FFT levels 4,5,6,7,8,9
;; On input and output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	64	+1	...
;;	8	...
;;	...
;;	56
;;	128
;;	...

;; Do 8 sixtyfour_complex_fft operations
;;	distance between fft data elements is 1 (8 for swizzling)
;;	do 8 macros each processing 128 data values

;; Execute the right middle step

	cmp	ffttype, 2
	jl	zpass2_512_complex_1
	jg	zpass2_512_complex_3

zpass2_512_complex_2:
	start_timer 9
	zloop_init 8			;; 8 iterations
	zloop_set_rw_prefetch ZOCT	;; Prefetch 64 cache lines
	mov	r8, DEST2ARG
	zr64_sixtyfour_complex_with_square_preload
b2a:	zr64_sixtyfour_complex_with_square rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 8, L1PREFETCH_ALL, 8*128
	zloop	8, b2a			;; Test loop counter
	add	rsi, r8
	end_timer 9
	jmp	zpass2_512_complex_unfft

zpass2_512_complex_1:
	start_timer 9
	zloop_init 8			;; 8 iterations
	zloop_set_rw_prefetch ZOCT	;; Prefetch 64 cache lines
	zr64_sixtyfour_complex_fft_final_preload
b2b:	zr64_sixtyfour_complex_fft_final rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 8, L1PREFETCH_ALL, 8*128
	zloop	8, b2b			;; Test loop counter
	end_timer 9
	jmp	zpass2_512_complex_done

zpass2_512_complex_3:
	start_timer 9
	zloop_init 8			;; 8 iterations
	zloop_set_rw_prefetch ZOCT	;; Prefetch 64 cache lines
	mov	r8, DEST2ARG
	mov	rbp, DIST_TO_MULSRCARG
	zr64_sixtyfour_complex_with_mult_preload
b2c:	zr64_sixtyfour_complex_with_mult rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 8, L1PREFETCH_ALL, 8*128
	zloop	8, b2c			;; Test loop counter
	add	rsi, r8
	end_timer 9
	jmp	zpass2_512_complex_unfft

zpass2_512_complex_4:
	start_timer 9
	zloop_init 8			;; 8 iterations
	zloop_set_rw_prefetch ZOCT	;; Prefetch 64 cache lines
	mov	rbp, DIST_TO_MULSRCARG
	mov	rdi, xsincos_complex	;; Load sin/cos pointer
	bump	rdi, 8*ZMM_SCD4
	zr64f_sixtyfour_complex_with_mulf_preload
b2d:	zr64f_sixtyfour_complex_with_mulf rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 8, L1PREFETCH_ALL, 8*128
	zloop	8, b2d			;; Test loop counter
	end_timer 9
zpass2_512_complex_unfft:
	bump	rsi, -8*8*128		;; Restore source pointer

;; Do inverse FFT levels 1,2,3
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	64	+1	...
;;	8	...
;;	...
;;	56
;;	128
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	512	+1	...
;;	8	...
;;	...
;;	504

;; Do 64 eight_complex_unfft operations
;;	distance between fft data elements is 64
;;	do 8 macros each processing 128 data values

	start_timer 14
	zloop_init 8			;; 8 iterations
	zloop_set_rw_prefetch ZQUINT	;; Prefetch 40 cache lines
	bump	rdi, -8*ZMM_SCD4
	zr8_eight_complex_djbunfft_preload
c1b:	zr8_eight_complex_djbunfft rsi, 128, 8*128, 16*128, 32*128, rdi, ZMM_SCD4, 8
	zloop	8, c1b			;; Test loop counter
	end_timer 14

zpass2_512_complex_done:
	ENDM


; *************** radix8-pass2-640-real macro ******************
; This macro takes 1280 real values and performs the final 10.322 levels of the FFT.
; The results are squared, and the first 10.322 levels of the inverse FFT are done.

zr4dwpn_pass2_640_real MACRO
	LOCAL	b1b, b2a, b2b, b2c, b2d, c1b
	LOCAL	zpass2_640_real_1, zpass2_640_real_2
	LOCAL	zpass2_640_real_3, zpass2_640_real_4
	LOCAL	zpass2_640_real_unfft, zpass2_640_real_done

;; Type 4 FFTs skip the forward FFT process

	start_timer 2
	mov	rsi, DATA_ADDR		;; Load source address
	mov	rbx, DIST_TO_FFTSRCARG
	mov	rcx, DATA_PREFETCH	;; Prefetch pointer (we need to prefetch 160 cache lines + 0 pad cache lines)
	cmp	ffttype, 4
	je	zpass2_640_real_4

;; Do FFT levels 1,2,3,4
;;	0	+1	+1	+1	+1	+1	+1	+1	640	+1	...
;;	8	...
;;	...
;;	632
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	64	+1	...
;;	8	...
;;	...
;;	56
;;	128
;;	...

;; Do 64 twenty_real_fft operations
;;	distance between fft data elements is 64
;;	do 8 macros each processing 160 data values

	mov	rdx, sincos3		;; Real sin/cos data
	mov	rdi, xsincos_complex	;; Complex sin/cos data
	zloop_init  8			;; 8 iterations
	zloop_set_rw_prefetch ZQUINT	;; Prefetch 40 cache lines
	zr10f_2sc_twenty_reals_fft_preload
b1b:	zr10f_2sc_twenty_reals_fft rsi, 128, 8*128, rdx, ZMM_SCD5, rdi, ZMM_SCD5, 8
	zloop	8, b1b, -8*128		;; Test loop counter, restore source pointer

;; Do FFT levels 5,6,7,8,9,10 as well as inverse FFT levels 5,6,7,8,9,10
;; On input and output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	64	+1	...
;;	8	...
;;	...
;;	56
;;	128
;;	...

;; Do 1 hundredtwentyeight_real_fft operation
;; Do 9 sixtyfour_complex_fft operations
;;	distance between fft data elements is 1 (8 for swizzling)
;;	do 10 macros each processing 128 data values

;; Execute the proper middle step

	cmp	ffttype, 2
	jl	zpass2_640_real_1
	jg	zpass2_640_real_3

zpass2_640_real_2:
	mov	r8, DEST2ARG
;;	zr64_hundredtwentyeight_real_with_square_preload
;;	zr64_hundredtwentyeight_real_with_square rsi, 8*128, 1*128, 2*128, 4*128, rdx, 0*ZMM_SCD4, rdi, 0*ZMM_SCD4, 1
	CALLP	zreal_square
	zloop_init 9			;; 9 more iterations
	zloop_set_rw_prefetch ZOCT	;; Prefetch 72 cache lines
	zr64_sixtyfour_complex_with_square_preload
b2a:	zr64_sixtyfour_complex_with_square rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 9, L1PREFETCH_ALL, 8*128
	zloop	9, b2a			;; Test loop counter
	add	rsi, r8
	jmp	zpass2_640_real_unfft

zpass2_640_real_1:
;;	zr64_hundredtwentyeight_real_fft_final_preload
;;	zr64_hundredtwentyeight_real_fft_final rsi, 8*128, 1*128, 2*128, 4*128, rdx, 0*ZMM_SCD4, rdi, 0*ZMM_SCD4, 1
	CALLP	zreal_fft_final
	zloop_init 9			;; 9 more iterations
	zloop_set_rw_prefetch ZOCT	;; Prefetch 72 cache lines
	zr64_sixtyfour_complex_fft_final_preload
b2b:	zr64_sixtyfour_complex_fft_final rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 9, L1PREFETCH_ALL, 8*128
	zloop	9, b2b			;; Test loop counter
	jmp	zpass2_640_real_done

zpass2_640_real_3:
	mov	r8, DEST2ARG
	mov	rbp, DIST_TO_MULSRCARG
;;	zr64_hundredtwentyeight_real_with_mult_preload
;;	zr64_hundredtwentyeight_real_with_mult rsi, 8*128, 1*128, 2*128, 4*128, rdx, 0*ZMM_SCD4, rdi, 0*ZMM_SCD4, 1
	CALLP	zreal_mult
	zloop_init 9			;; 9 more iterations
	zloop_set_rw_prefetch ZOCT	;; Prefetch 72 cache lines
	zr64_sixtyfour_complex_with_mult_preload
b2c:	zr64_sixtyfour_complex_with_mult rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 9, L1PREFETCH_ALL, 8*128
	zloop	9, b2c			;; Test loop counter
	add	rsi, r8
	jmp	zpass2_640_real_unfft

zpass2_640_real_4:
	mov	rbp, DIST_TO_MULSRCARG
	mov	rdx, sincos3		;; Real sin/cos data
	mov	rdi, xsincos_complex	;; Complex sin/cos data
	bump	rdx, 8*ZMM_SCD5
	bump	rdi, 8*ZMM_SCD5
;;	zr64f_hundredtwentyeight_real_with_mulf_preload
;;	zr64f_hundredtwentyeight_real_with_mulf rsi, 8*128, 1*128, 2*128, 4*128, rdx, 0*ZMM_SCD4, rdi, 0*ZMM_SCD4, 1
	CALLP	zreal_mulf
	zloop_init 9			;; 9 more iterations
	zloop_set_rw_prefetch ZOCT	;; Prefetch 72 cache lines
	zr64f_sixtyfour_complex_with_mulf_preload
b2d:	zr64f_sixtyfour_complex_with_mulf rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 9, L1PREFETCH_ALL, 8*128
	zloop	9, b2d			;; Test loop counter
zpass2_640_real_unfft:
	bump	rsi, -10*8*128		;; Restore source pointer

;; Do inverse FFT levels 1,2,3,4
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	64	+1	...
;;	8	...
;;	...
;;	56
;;	128
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	640	+1	...
;;	8	...
;;	...
;;	632

;; Do 64 twentyfour_real_unfft operations
;;	distance between fft data elements is 64
;;	do 8 macros each processing 160 data values

	bump	rdx, -8*ZMM_SCD5
	bump	rdi, -8*ZMM_SCD5
	zloop_init  8			;; 8 iterations
	zloop_set_rw_prefetch ZSEXT	;; Prefetch 48 cache lines
	zr10_2sc_twenty_reals_unfft_preload
c1b:	zr10_2sc_twenty_reals_unfft rsi, 128, 8*128, rdx, ZMM_SCD5, rdi, ZMM_SCD5, 8
	zloop	8, c1b			;; Test loop counter

zpass2_640_real_done:
	end_timer 2
	ENDM

; *************** radix8-pass2-640-complex macro ******************
; This macro takes 640 complex values and performs the final 9.322 levels of a standard
; radix-8 FFT, squares the results, and does the first 9.322 levels of the inverse FFT.

zr4dwpn_pass2_640_complex MACRO
	LOCAL	b1b, b2a, b2b, b2c, b2d, c1b
	LOCAL	zpass2_640_complex_1, zpass2_640_complex_2
	LOCAL	zpass2_640_complex_3, zpass2_640_complex_4
	LOCAL	zpass2_640_complex_unfft, zpass2_640_complex_done

;; Type 4 FFTs skip the forward FFT process

	mov	rsi, DATA_ADDR		;; Load address of FFT data
	mov	rbx, DIST_TO_FFTSRCARG
	mov	rcx, DATA_PREFETCH	;; Prefetch pointer (we need to prefetch 160 cache lines + 0 pad cache lines)
	cmp	ffttype, 4
	je	zpass2_640_complex_4

;; Do FFT levels 1,2,3
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	640	+1	...
;;	8	...
;;	...
;;	632
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	64	+1	...
;;	8	...
;;	...
;;	56
;;	128
;;	...

;; Do 64 ten_complex_fft operations
;;	distance between fft data elements is 64
;;	do 8 macros each processing 160 data values

	start_timer 4
	mov	rdi, xsincos_complex	;; The sin/cos data
	zloop_init 8			;; 8 iterations
	zloop_set_rw_prefetch ZQUINT	;; Prefetch 40 cache lines
	zr10f_ten_complex_djbfft_preload
b1b:	zr10f_ten_complex_djbfft rsi, 128, 8*128, rdi, ZMM_SCD5, 8
	zloop	8, b1b, -8*128		;; Test loop counter, restore source pointer
	end_timer 4

;; Do FFT levels 4,5,6,7,8,9 as well as inverse FFT levels 4,5,6,7,8,9
;; On input and output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	64	+1	...
;;	8	...
;;	...
;;	56
;;	128
;;	...

;; Do 10 sixtyfour_complex_fft operations
;;	distance between fft data elements is 1 (8 for swizzling)
;;	do 10 macros each processing 128 data values

;; Execute the right middle step

	cmp	ffttype, 2
	jl	zpass2_640_complex_1
	jg	zpass2_640_complex_3

zpass2_640_complex_2:
	start_timer 9
	zloop_init 10			;; 10 iterations
	zloop_set_rw_prefetch ZOCT	;; Prefetch 80 cache lines
	mov	r8, DEST2ARG
	zr64_sixtyfour_complex_with_square_preload
b2a:	zr64_sixtyfour_complex_with_square rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 10, L1PREFETCH_ALL, 8*128
	zloop	10, b2a			;; Test loop counter
	add	rsi, r8
	end_timer 9
	jmp	zpass2_640_complex_unfft

zpass2_640_complex_1:
	start_timer 9
	zloop_init 10			;; 10 iterations
	zloop_set_rw_prefetch ZOCT	;; Prefetch 80 cache lines
	zr64_sixtyfour_complex_fft_final_preload
b2b:	zr64_sixtyfour_complex_fft_final rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 10, L1PREFETCH_ALL, 8*128
	zloop	10, b2b			;; Test loop counter
	end_timer 9
	jmp	zpass2_640_complex_done

zpass2_640_complex_3:
	start_timer 9
	zloop_init 10			;; 10 iterations
	zloop_set_rw_prefetch ZOCT	;; Prefetch 80 cache lines
	mov	r8, DEST2ARG
	mov	rbp, DIST_TO_MULSRCARG
	zr64_sixtyfour_complex_with_mult_preload
b2c:	zr64_sixtyfour_complex_with_mult rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 10, L1PREFETCH_ALL, 8*128
	zloop	10, b2c			;; Test loop counter
	add	rsi, r8
	end_timer 9
	jmp	zpass2_640_complex_unfft

zpass2_640_complex_4:
	start_timer 9
	zloop_init 10			;; 10 iterations
	zloop_set_rw_prefetch ZOCT	;; Prefetch 80 cache lines
	mov	rbp, DIST_TO_MULSRCARG
	mov	rdi, xsincos_complex	;; Load sin/cos pointer
	bump	rdi, 8*ZMM_SCD5
	zr64f_sixtyfour_complex_with_mulf_preload
b2d:	zr64f_sixtyfour_complex_with_mulf rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 10, L1PREFETCH_ALL, 8*128
	zloop	10, b2d			;; Test loop counter
	end_timer 9
zpass2_640_complex_unfft:
	bump	rsi, -10*8*128		;; Restore source pointer

;; Do inverse FFT levels 1,2,3
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	64	+1	...
;;	8	...
;;	...
;;	56
;;	128
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	640	+1	...
;;	8	...
;;	...
;;	632

;; Do 64 ten_complex_unfft operations
;;	distance between fft data elements is 64
;;	do 8 macros each processing 160 data values

	start_timer 14
	zloop_init 8			;; 8 iterations
	zloop_set_rw_prefetch ZQUINT	;; Prefetch 40 cache lines
	bump	rdi, -8*ZMM_SCD5
	zr10_ten_complex_djbunfft_preload
c1b:	zr10_ten_complex_djbunfft rsi, 128, 8*128, rdi, ZMM_SCD5, 8
	zloop	8, c1b			;; Test loop counter
	end_timer 14

zpass2_640_complex_done:
	ENDM



; *************** radix8-pass2-768-real macro ******************
; This macro takes 1536 real values and performs the final 10.535 levels of the FFT.
; The results are squared, and the first 10.535 levels of the inverse FFT are done.

zr4dwpn_pass2_768_real MACRO
	LOCAL	b1b, b2a, b2b, b2c, b2d, c1b
	LOCAL	zpass2_768_real_1, zpass2_768_real_2
	LOCAL	zpass2_768_real_3, zpass2_768_real_4
	LOCAL	zpass2_768_real_unfft, zpass2_768_real_done

;; Type 4 FFTs skip the forward FFT process

	start_timer 2
	mov	rsi, DATA_ADDR		;; Load source address
	mov	rbx, DIST_TO_FFTSRCARG
	mov	rcx, DATA_PREFETCH	;; Prefetch pointer (we need to prefetch 192 cache lines + 0 pad cache lines)
	cmp	ffttype, 4
	je	zpass2_768_real_4

;; Do FFT levels 1,2,3,4
;;	0	+1	+1	+1	+1	+1	+1	+1	768	+1	...
;;	8	...
;;	...
;;	760
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	64	+1	...
;;	8	...
;;	...
;;	56
;;	128
;;	...

;; Do 64 twentyfour_real_fft operations
;;	distance between fft data elements is 64
;;	do 8 macros each processing 192 data values

	mov	rdx, sincos3		;; Real sin/cos data
	mov	rdi, xsincos_complex	;; Complex sin/cos data
	zloop_init  8			;; 8 iterations
	zloop_set_rw_prefetch ZSEXT	;; Prefetch 48 cache lines
	zr12f_2sc_twentyfour_reals_fft_preload
b1b:	zr12f_2sc_twentyfour_reals_fft rsi, 128, 8*128, 16*128, 32*128, rdx, ZMM_SCD6, rdi, ZMM_SCD6, 8
	zloop	8, b1b, -8*128		;; Test loop counter, restore source pointer

;; Do FFT levels 5,6,7,8,9,10 as well as inverse FFT levels 5,6,7,8,9,10
;; On input and output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	64	+1	...
;;	8	...
;;	...
;;	56
;;	128
;;	...

;; Do 1 hundredtwentyeight_real_fft operation
;; Do 11 sixtyfour_complex_fft operations
;;	distance between fft data elements is 1 (8 for swizzling)
;;	do 12 macros each processing 128 data values

;; Execute the proper middle step

	cmp	ffttype, 2
	jl	zpass2_768_real_1
	jg	zpass2_768_real_3

zpass2_768_real_2:
	mov	r8, DEST2ARG
;;	zr64_hundredtwentyeight_real_with_square_preload
;;	zr64_hundredtwentyeight_real_with_square rsi, 8*128, 1*128, 2*128, 4*128, rdx, 0*ZMM_SCD4, rdi, 0*ZMM_SCD4, 1
	CALLP	zreal_square
	zloop_init 11			;; 11 more iterations
	zloop_set_rw_prefetch ZOCT	;; Prefetch 88 cache lines
	zr64_sixtyfour_complex_with_square_preload
b2a:	zr64_sixtyfour_complex_with_square rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 11, L1PREFETCH_ALL, 8*128
	zloop	11, b2a			;; Test loop counter
	add	rsi, r8
	jmp	zpass2_768_real_unfft

zpass2_768_real_1:
;;	zr64_hundredtwentyeight_real_fft_final_preload
;;	zr64_hundredtwentyeight_real_fft_final rsi, 8*128, 1*128, 2*128, 4*128, rdx, 0*ZMM_SCD4, rdi, 0*ZMM_SCD4, 1
	CALLP	zreal_fft_final
	zloop_init 11			;; 11 more iterations
	zloop_set_rw_prefetch ZOCT	;; Prefetch 88 cache lines
	zr64_sixtyfour_complex_fft_final_preload
b2b:	zr64_sixtyfour_complex_fft_final rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 11, L1PREFETCH_ALL, 8*128
	zloop	11, b2b			;; Test loop counter
	jmp	zpass2_768_real_done

zpass2_768_real_3:
	mov	r8, DEST2ARG
	mov	rbp, DIST_TO_MULSRCARG
;;	zr64_hundredtwentyeight_real_with_mult_preload
;;	zr64_hundredtwentyeight_real_with_mult rsi, 8*128, 1*128, 2*128, 4*128, rdx, 0*ZMM_SCD4, rdi, 0*ZMM_SCD4, 1
	CALLP	zreal_mult
	zloop_init 11			;; 11 more iterations
	zloop_set_rw_prefetch ZOCT	;; Prefetch 88 cache lines
	zr64_sixtyfour_complex_with_mult_preload
b2c:	zr64_sixtyfour_complex_with_mult rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 11, L1PREFETCH_ALL, 8*128
	zloop	11, b2c			;; Test loop counter
	add	rsi, r8
	jmp	zpass2_768_real_unfft

zpass2_768_real_4:
	mov	rbp, DIST_TO_MULSRCARG
	mov	rdx, sincos3		;; Real sin/cos data
	mov	rdi, xsincos_complex	;; Complex sin/cos data
	bump	rdx, 8*ZMM_SCD6
	bump	rdi, 8*ZMM_SCD6
;;	zr64f_hundredtwentyeight_real_with_mulf_preload
;;	zr64f_hundredtwentyeight_real_with_mulf rsi, 8*128, 1*128, 2*128, 4*128, rdx, 0*ZMM_SCD4, rdi, 0*ZMM_SCD4, 1
	CALLP	zreal_mulf
	zloop_init 11			;; 11 more iterations
	zloop_set_rw_prefetch ZOCT	;; Prefetch 88 cache lines
	zr64f_sixtyfour_complex_with_mulf_preload
b2d:	zr64f_sixtyfour_complex_with_mulf rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 11, L1PREFETCH_ALL, 8*128
	zloop	11, b2d			;; Test loop counter
zpass2_768_real_unfft:
	bump	rsi, -12*8*128		;; Restore source pointer

;; Do inverse FFT levels 1,2,3,4
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	64	+1	...
;;	8	...
;;	...
;;	56
;;	128
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	768	+1	...
;;	8	...
;;	...
;;	760

;; Do 64 twentyfour_real_unfft operations
;;	distance between fft data elements is 64
;;	do 8 macros each processing 192 data values

	bump	rdx, -8*ZMM_SCD6
	bump	rdi, -8*ZMM_SCD6
	zloop_init  8			;; 8 iterations
	zloop_set_rw_prefetch ZSEXT	;; Prefetch 48 cache lines
	zr12_2sc_twentyfour_reals_unfft_preload
c1b:	zr12_2sc_twentyfour_reals_unfft rsi, 128, 8*128, 16*128, 32*128, rdx, ZMM_SCD6, rdi, ZMM_SCD6, 8
	zloop	8, c1b			;; Test loop counter

zpass2_768_real_done:
	end_timer 2
	ENDM

; *************** radix8-pass2-768-complex macro ******************
; This macro takes 768 complex values and performs the final 9.535 levels of a standard
; radix-8 FFT, squares the results, and does the first 9.535 levels of the inverse FFT.

zr4dwpn_pass2_768_complex MACRO
	LOCAL	b1b, b2a, b2b, b2c, b2d, c1b
	LOCAL	zpass2_768_complex_1, zpass2_768_complex_2
	LOCAL	zpass2_768_complex_3, zpass2_768_complex_4
	LOCAL	zpass2_768_complex_unfft, zpass2_768_complex_done

;; Type 4 FFTs skip the forward FFT process

	mov	rsi, DATA_ADDR		;; Load address of FFT data
	mov	rbx, DIST_TO_FFTSRCARG
	mov	rcx, DATA_PREFETCH	;; Prefetch pointer (we need to prefetch 192 cache lines + 0 pad cache lines)
	cmp	ffttype, 4
	je	zpass2_768_complex_4

;; Do FFT levels 1,2,3
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	768	+1	...
;;	8	...
;;	...
;;	760
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	64	+1	...
;;	8	...
;;	...
;;	56
;;	128
;;	...

;; Do 64 twelve_complex_fft operations
;;	distance between fft data elements is 64
;;	do 8 macros each processing 192 data values

	start_timer 4
	mov	rdi, xsincos_complex	;; The sin/cos data
	zloop_init 8			;; 8 iterations
	zloop_set_rw_prefetch ZSEXT	;; Prefetch 48 cache lines
	zr12f_twelve_complex_djbfft_preload
b1b:	zr12f_twelve_complex_djbfft rsi, 128, 8*128, 16*128, 32*128, rdi, ZMM_SCD6, 8
	zloop	8, b1b, -8*128		;; Test loop counter, restore source pointer
	end_timer 4

;; Do FFT levels 4,5,6,7,8,9 as well as inverse FFT levels 4,5,6,7,8,9
;; On input and output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	64	+1	...
;;	8	...
;;	...
;;	56
;;	128
;;	...

;; Do 12 sixtyfour_complex_fft operations
;;	distance between fft data elements is 1 (8 for swizzling)
;;	do 12 macros each processing 128 data values

;; Execute the right middle step

	cmp	ffttype, 2
	jl	zpass2_768_complex_1
	jg	zpass2_768_complex_3

zpass2_768_complex_2:
	start_timer 9
	zloop_init 12			;; 12 iterations
	zloop_set_rw_prefetch ZOCT	;; Prefetch 96 cache lines
	mov	r8, DEST2ARG
	zr64_sixtyfour_complex_with_square_preload
b2a:	zr64_sixtyfour_complex_with_square rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 12, L1PREFETCH_ALL, 8*128
	zloop	12, b2a			;; Test loop counter
	add	rsi, r8
	end_timer 9
	jmp	zpass2_768_complex_unfft

zpass2_768_complex_1:
	start_timer 9
	zloop_init 12			;; 12 iterations
	zloop_set_rw_prefetch ZOCT	;; Prefetch 96 cache lines
	zr64_sixtyfour_complex_fft_final_preload
b2b:	zr64_sixtyfour_complex_fft_final rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 12, L1PREFETCH_ALL, 8*128
	zloop	12, b2b			;; Test loop counter
	end_timer 9
	jmp	zpass2_768_complex_done

zpass2_768_complex_3:
	start_timer 9
	zloop_init 12			;; 12 iterations
	zloop_set_rw_prefetch ZOCT	;; Prefetch 96 cache lines
	mov	r8, DEST2ARG
	mov	rbp, DIST_TO_MULSRCARG
	zr64_sixtyfour_complex_with_mult_preload
b2c:	zr64_sixtyfour_complex_with_mult rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 12, L1PREFETCH_ALL, 8*128
	zloop	12, b2c			;; Test loop counter
	add	rsi, r8
	end_timer 9
	jmp	zpass2_768_complex_unfft

zpass2_768_complex_4:
	start_timer 9
	zloop_init 12			;; 12 iterations
	zloop_set_rw_prefetch ZOCT	;; Prefetch 96 cache lines
	mov	rbp, DIST_TO_MULSRCARG
	mov	rdi, xsincos_complex	;; Load sin/cos pointer
	bump	rdi, 8*ZMM_SCD6
	zr64f_sixtyfour_complex_with_mulf_preload
b2d:	zr64f_sixtyfour_complex_with_mulf rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 12, L1PREFETCH_ALL, 8*128
	zloop	12, b2d			;; Test loop counter
	end_timer 9
zpass2_768_complex_unfft:
	bump	rsi, -12*8*128		;; Restore source pointer

;; Do inverse FFT levels 1,2,3
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	64	+1	...
;;	8	...
;;	...
;;	56
;;	128
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	768	+1	...
;;	8	...
;;	...
;;	760

;; Do 64 twelve_complex_unfft operations
;;	distance between fft data elements is 64
;;	do 8 macros each processing 192 data values

	start_timer 14
	zloop_init 8			;; 8 iterations
	zloop_set_rw_prefetch ZSEXT	;; Prefetch 48 cache lines
	bump	rdi, -8*ZMM_SCD6
	zr12_twelve_complex_djbunfft_preload
c1b:	zr12_twelve_complex_djbunfft rsi, 128, 8*128, 16*128, 32*128, rdi, ZMM_SCD6, 8
	zloop	8, c1b			;; Test loop counter
	end_timer 14

zpass2_768_complex_done:
	ENDM


; *************** radix8-pass2-1024-real macro ******************
; This macro takes 2048 real values and performs the final 11 levels of the FFT.
; The results are squared, and the first 11 levels of the inverse FFT are done.

zr4dwpn_pass2_1024_real MACRO
	LOCAL	b1b, b2a, b2b, b2c, b2d, c1b
	LOCAL	zpass2_1024_real_1, zpass2_1024_real_2
	LOCAL	zpass2_1024_real_3, zpass2_1024_real_4
	LOCAL	zpass2_1024_real_unfft, zpass2_1024_real_done

;; Type 4 FFTs skip the forward FFT process

	start_timer 2
	mov	rsi, DATA_ADDR		;; Load source address
	mov	rbx, DIST_TO_FFTSRCARG
	mov	rcx, DATA_PREFETCH	;; Prefetch pointer (we need to prefetch 256 cache lines + 0 pad cache lines)
	cmp	ffttype, 4
	je	zpass2_1024_real_4

;; Do FFT levels 1,2,3,4
;;	0	+1	+1	+1	+1	+1	+1	+1	1024	+1	...
;;	8	...
;;	...
;;	1016
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	64	+1	...
;;	8	...
;;	...
;;	56
;;	128
;;	...

;; Do 64 thirtytwo_real_fft operations
;;	distance between fft data elements is 64
;;	do 8 macros each processing 256 data values

	mov	rdx, sincos3		;; Real sin/cos data
	mov	rdi, xsincos_complex	;; Complex sin/cos data
	zloop_init  8			;; 8 iterations
	zloop_set_rw_prefetch ZOCT	;; Prefetch 64 cache lines
	zr16f_2sc_thirtytwo_reals_fft_preload
b1b:	zr16f_2sc_thirtytwo_reals_fft rsi, 128, 8*128, 16*128, 32*128, 64*128, rdx, ZMM_SCD8, rdi, ZMM_SCD8, 8
	zloop	8, b1b, -8*128		;; Test loop counter, restore source pointer

;; Do FFT levels 5,6,7,8,9,10 as well as inverse FFT levels 5,6,7,8,9,10
;; On input and output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	64	+1	...
;;	8	...
;;	...
;;	56
;;	128
;;	...

;; Do 1 hundredtwentyeight_real_fft operation
;; Do 15 sixtyfour_complex_fft operations
;;	distance between fft data elements is 1 (8 for swizzling)
;;	do 16 macros each processing 128 data values

;; Execute the proper middle step

	cmp	ffttype, 2
	jl	zpass2_1024_real_1
	jg	zpass2_1024_real_3

zpass2_1024_real_2:
	mov	r8, DEST2ARG
;;	zr64_hundredtwentyeight_real_with_square_preload
;;	zr64_hundredtwentyeight_real_with_square rsi, 8*128, 1*128, 2*128, 4*128, rdx, 0*ZMM_SCD4, rdi, 0*ZMM_SCD4, 1
	CALLP	zreal_square
	zloop_init 15			;; 15 more iterations
	zloop_set_rw_prefetch ZOCT	;; Prefetch 120 cache lines
	zr64_sixtyfour_complex_with_square_preload
b2a:	zr64_sixtyfour_complex_with_square rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 15, L1PREFETCH_ALL, 8*128
	zloop	15, b2a			;; Test loop counter
	add	rsi, r8
	jmp	zpass2_1024_real_unfft

zpass2_1024_real_1:
;;	zr64_hundredtwentyeight_real_fft_final_preload
;;	zr64_hundredtwentyeight_real_fft_final rsi, 8*128, 1*128, 2*128, 4*128, rdx, 0*ZMM_SCD4, rdi, 0*ZMM_SCD4, 1
	CALLP	zreal_fft_final
	zloop_init 15			;; 15 more iterations
	zloop_set_rw_prefetch ZOCT	;; Prefetch 120 cache lines
	zr64_sixtyfour_complex_fft_final_preload
b2b:	zr64_sixtyfour_complex_fft_final rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 15, L1PREFETCH_ALL, 8*128
	zloop	15, b2b			;; Test loop counter
	jmp	zpass2_1024_real_done

zpass2_1024_real_3:
	mov	r8, DEST2ARG
	mov	rbp, DIST_TO_MULSRCARG
;;	zr64_hundredtwentyeight_real_with_mult_preload
;;	zr64_hundredtwentyeight_real_with_mult rsi, 8*128, 1*128, 2*128, 4*128, rdx, 0*ZMM_SCD4, rdi, 0*ZMM_SCD4, 1
	CALLP	zreal_mult
	zloop_init 15			;; 15 more iterations
	zloop_set_rw_prefetch ZOCT	;; Prefetch 120 cache lines
	zr64_sixtyfour_complex_with_mult_preload
b2c:	zr64_sixtyfour_complex_with_mult rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 15, L1PREFETCH_ALL, 8*128
	zloop	15, b2c			;; Test loop counter
	add	rsi, r8
	jmp	zpass2_1024_real_unfft

zpass2_1024_real_4:
	mov	rbp, DIST_TO_MULSRCARG
	mov	rdx, sincos3		;; Real sin/cos data
	mov	rdi, xsincos_complex	;; Complex sin/cos data
	bump	rdx, 8*ZMM_SCD8
	bump	rdi, 8*ZMM_SCD8
;;	zr64f_hundredtwentyeight_real_with_mulf_preload
;;	zr64f_hundredtwentyeight_real_with_mulf rsi, 8*128, 1*128, 2*128, 4*128, rdx, 0*ZMM_SCD4, rdi, 0*ZMM_SCD4, 1
	CALLP	zreal_mulf
	zloop_init 15			;; 15 more iterations
	zloop_set_rw_prefetch ZOCT	;; Prefetch 120 cache lines
	zr64f_sixtyfour_complex_with_mulf_preload
b2d:	zr64f_sixtyfour_complex_with_mulf rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 15, L1PREFETCH_ALL, 8*128
	zloop	15, b2d			;; Test loop counter
zpass2_1024_real_unfft:
	bump	rsi, -16*8*128		;; Restore source pointer

;; Do inverse FFT levels 1,2,3,4
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	64	+1	...
;;	8	...
;;	...
;;	56
;;	128
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	1024	+1	...
;;	8	...
;;	...
;;	1016

;; Do 64 thirtytwo_real_unfft operations
;;	distance between fft data elements is 64
;;	do 8 macros each processing 256 data values

	bump	rdx, -8*ZMM_SCD8
	bump	rdi, -8*ZMM_SCD8
	zloop_init  8			;; 8 iterations
	zloop_set_rw_prefetch ZOCT	;; Prefetch 64 cache lines
	zr16_2sc_thirtytwo_reals_unfft_preload
c1b:	zr16_2sc_thirtytwo_reals_unfft rsi, 128, 8*128, 16*128, 32*128, 64*128, rdx, ZMM_SCD8, rdi, ZMM_SCD8, 8
	zloop	8, c1b			;; Test loop counter

zpass2_1024_real_done:
	end_timer 2
	ENDM

; *************** radix8-pass2-1024-complex macro ******************
; This macro takes 1024 complex values and performs the final 10 levels of a standard
; radix-8 FFT, squares the results, and does the first 10 levels of the inverse FFT.

zr4dwpn_pass2_1024_complex MACRO
	LOCAL	b1b, b2a, b2b, b2c, b2d, c1b
	LOCAL	zpass2_1024_complex_1, zpass2_1024_complex_2
	LOCAL	zpass2_1024_complex_3, zpass2_1024_complex_4
	LOCAL	zpass2_1024_complex_unfft, zpass2_1024_complex_done

;; Type 4 FFTs skip the forward FFT process

	mov	rsi, DATA_ADDR		;; Load address of FFT data
	mov	rbx, DIST_TO_FFTSRCARG
	mov	rcx, DATA_PREFETCH	;; Prefetch pointer (we need to prefetch 256 cache lines + 0 pad cache lines)
	cmp	ffttype, 4
	je	zpass2_1024_complex_4

;; Do FFT levels 1,2,3
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	1024	+1	...
;;	8	...
;;	...
;;	1016
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	64	+1	...
;;	8	...
;;	...
;;	56
;;	128
;;	...

;; Do 64 sixteen_complex_fft operations
;;	distance between fft data elements is 64
;;	do 8 macros each processing 256 data values

	start_timer 4
	mov	rdi, xsincos_complex	;; The sin/cos data
	zloop_init 8			;; 8 iterations
	zloop_set_rw_prefetch ZOCT	;; Prefetch 64 cache lines
	zr16f_sixteen_complex_djbfft_preload
b1b:	zr16f_sixteen_complex_djbfft rsi, 128, 8*128, 16*128, 32*128, 64*128, rdi, ZMM_SCD8, 8
	zloop	8, b1b, -8*128		;; Test loop counter, restore source pointer
	end_timer 4

;; Do FFT levels 4,5,6,7,8,9 as well as inverse FFT levels 4,5,6,7,8,9
;; On input and output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	64	+1	...
;;	8	...
;;	...
;;	56
;;	128
;;	...

;; Do 16 sixtyfour_complex_fft operations
;;	distance between fft data elements is 1 (8 for swizzling)
;;	do 16 macros each processing 128 data values

;; Execute the right middle step

	cmp	ffttype, 2
	jl	zpass2_1024_complex_1
	jg	zpass2_1024_complex_3

zpass2_1024_complex_2:
	start_timer 9
	zloop_init 16			;; 16 iterations
	zloop_set_rw_prefetch ZOCT	;; Prefetch 128 cache lines
	mov	r8, DEST2ARG
	zr64_sixtyfour_complex_with_square_preload
b2a:	zr64_sixtyfour_complex_with_square rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 16, L1PREFETCH_ALL, 8*128
	zloop	16, b2a			;; Test loop counter
	add	rsi, r8
	end_timer 9
	jmp	zpass2_1024_complex_unfft

zpass2_1024_complex_1:
	start_timer 9
	zloop_init 16			;; 16 iterations
	zloop_set_rw_prefetch ZOCT	;; Prefetch 128 cache lines
	zr64_sixtyfour_complex_fft_final_preload
b2b:	zr64_sixtyfour_complex_fft_final rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 16, L1PREFETCH_ALL, 8*128
	zloop	16, b2b			;; Test loop counter
	end_timer 9
	jmp	zpass2_1024_complex_done

zpass2_1024_complex_3:
	start_timer 9
	zloop_init 16			;; 16 iterations
	zloop_set_rw_prefetch ZOCT	;; Prefetch 128 cache lines
	mov	r8, DEST2ARG
	mov	rbp, DIST_TO_MULSRCARG
	zr64_sixtyfour_complex_with_mult_preload
b2c:	zr64_sixtyfour_complex_with_mult rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 16, L1PREFETCH_ALL, 8*128
	zloop	16, b2c			;; Test loop counter
	add	rsi, r8
	end_timer 9
	jmp	zpass2_1024_complex_unfft

zpass2_1024_complex_4:
	start_timer 9
	zloop_init 16			;; 16 iterations
	zloop_set_rw_prefetch ZOCT	;; Prefetch 128 cache lines
	mov	rbp, DIST_TO_MULSRCARG
	mov	rdi, xsincos_complex	;; Load sin/cos pointer
	bump	rdi, 8*ZMM_SCD8
	zr64f_sixtyfour_complex_with_mulf_preload
b2d:	zr64f_sixtyfour_complex_with_mulf rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 16, L1PREFETCH_ALL, 8*128
	zloop	16, b2d			;; Test loop counter
	end_timer 9
zpass2_1024_complex_unfft:
	bump	rsi, -16*8*128		;; Restore source pointer

;; Do inverse FFT levels 1,2,3
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	64	+1	...
;;	8	...
;;	...
;;	56
;;	128
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	1024	+1	...
;;	8	...
;;	...
;;	1026

;; Do 64 sixteen_complex_unfft operations
;;	distance between fft data elements is 64
;;	do 8 macros each processing 256 data values

	start_timer 14
	zloop_init 8			;; 8 iterations
	zloop_set_rw_prefetch ZOCT	;; Prefetch 64 cache lines
	bump	rdi, -8*ZMM_SCD8
	zr16_sixteen_complex_djbunfft_preload
c1b:	zr16_sixteen_complex_djbunfft rsi, 128, 8*128, 16*128, 32*128, 64*128, rdi, ZMM_SCD8, 8
	zloop	8, c1b			;; Test loop counter
	end_timer 14

zpass2_1024_complex_done:
	ENDM


;*********************************************************************************************
;			Macros derived from pass2-12-levels
;*********************************************************************************************


; *************** radix8-pass2-1600-real macro ******************
; This macro takes 3200 real values and performs the final 11.644 levels of the FFT.
; The results are squared, and the first 11.644 levels of the inverse FFT are done.

zr4dwpn_pass2_1600_real MACRO
	LOCAL	b1b, b2a, b2b, b3a, b3b, b3c, b3d, c1b, c2a, c2b
	LOCAL	zpass2_1600_real_1, zpass2_1600_real_2
	LOCAL	zpass2_1600_real_3, zpass2_1600_real_4
	LOCAL	zpass2_1600_real_unfft, zpass2_1600_real_done

;; Type 4 FFTs skip the forward FFT process

	start_timer 2
	mov	rsi, DATA_ADDR		;; Load source address
	mov	rbx, DIST_TO_FFTSRCARG
	mov	rcx, DATA_PREFETCH	;; Prefetch pointer (we need to prefetch 400 cache lines + ?? pad cache lines)
	cmp	ffttype, 4
	je	zpass2_1600_real_4

;; Do FFT levels 1,2,3,4
;;	0	+1	+1	+1	+1	+1	+1	+1	1600	+1	...
;;	8	...
;;	...
;;	1592
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	320	+1	...
;;	8	...
;;	...
;;	312
;;	640
;;	...

;; Do 320 ten_real_fft operations
;;	distance between fft data elements is 320
;;	do 40 macros each processing 80 data values

	mov	rdx, sincos3		;; Real sin/cos data
	mov	rdi, xsincos_complex	;; Complex sin/cos data
	zloop_init 40			;; 40 iterations
	zloop_set_rw_prefetch ZDOUBLE	;; Prefetch 80 cache lines
	zr5f_2sc_ten_reals_fft_preload
b1b:	zr5f_2sc_ten_reals_fft rsi, 128, 40*128, rdx, ZMM_SCD2, rdi, ZMM_SCD2, 40
	zloop	40, b1b, -40*128	;; Test loop counter, restore source pointer

;; Do FFT levels 5,6,7
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	320	+1	...
;;	8	...
;;	...
;;	312
;;	640
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	64	+1	...
;;	8	...
;;	...
;;	56
;;	128
;;	...

;; Do 40 ten_reals_fft operations
;; Do 280 five_complex_fft operations
;;	distance between fft data elements is 64
;;	do 40 macros each processing 80 data values

	zloop_init 40			;; 5 iterations of 8
	zloop_set_rw_prefetch ZDOUBLE	;; Prefetch 80 cache lines
	zr5_2sc_ten_reals_fft_preload
b2a:	zr5_2sc_ten_reals_fft rsi, 128, 8*128, rdx, ZMM_SCD2, rdi, ZMM_SCD2, 8
	zloop	8, b2a, -8*128+40*128, rdi, -8*ZMM_SCD2 ;; Test loop counter, next source pointer, restore sin/cos pointer
	loops_reset			;; 4 more iters of 8
	zr5_five_complex_djbfft_preload
b2b:	zr5_five_complex_djbfft rsi, 128, 8*128, rdi, ZMM_SCD2, 8
	zloop	8, b2b, -8*128+40*128, rdi, -8*ZMM_SCD2 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	5, b2b, -5*40*128, rdi, 8*ZMM_SCD2 ;; Test loop counter, restore source pointer, next sin/cos pointer

;; Do FFT levels 8,9,10,11,12,13 as well as inverse FFT levels 8,9,10,11,12,13
;; On input and output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	64	+1	...
;;	8	...
;;	...
;;	56
;;	128
;;	...

;; Do 1 hundredtwentyeight_reals_fft operation
;; Do 24 sixtyfour_complex_fft operations
;;	distance between fft data elements is 1 (8 for swizzling)
;;	do 25 macros each processing 128 data values

;; Execute the proper middle step

pfing	bump	rcx, -16*64		;; We will prefetch 16 cache lines too many
	cmp	ffttype, 2
	jl	zpass2_1600_real_1
	jg	zpass2_1600_real_3

zpass2_1600_real_2:
	mov	r8, DEST2ARG
;;	zr64_hundredtwentyeight_real_with_square_preload
;;	zr64_hundredtwentyeight_real_with_square rsi, 8*128, 1*128, 2*128, 4*128, rdx, 0*ZMM_SCD4, rdi, 0*ZMM_SCD4, 1
	CALLP	zreal_square
	zloop_init 24			;; 24 more iterations
	zloop_set_rw_prefetch ZQUAD	;; Prefetch 96 cache lines
	zr64_sixtyfour_complex_with_square_preload
b3a:	zr64_sixtyfour_complex_with_square rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 1, L1PREFETCH_ALL, 8*128
	zloop	24, b3a			;; Test loop counter
	add	rsi, r8
	jmp	zpass2_1600_real_unfft

zpass2_1600_real_1:
;;	zr64_hundredtwentyeight_real_fft_final_preload
;;	zr64_hundredtwentyeight_real_fft_final rsi, 8*128, 1*128, 2*128, 4*128, rdx, 0*ZMM_SCD4, rdi, 0*ZMM_SCD4, 1
	CALLP	zreal_fft_final
	zloop_init 24			;; 24 more iterations
	zloop_set_rw_prefetch ZQUAD	;; Prefetch 96 cache lines
	zr64_sixtyfour_complex_fft_final_preload
b3b:	zr64_sixtyfour_complex_fft_final rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 1, L1PREFETCH_ALL, 8*128
	zloop	24, b3b			;; Test loop counter
	jmp	zpass2_1600_real_done

zpass2_1600_real_3:
	mov	r8, DEST2ARG
	mov	rbp, DIST_TO_MULSRCARG
;;	zr64_hundredtwentyeight_real_with_mult_preload
;;	zr64_hundredtwentyeight_real_with_mult rsi, 8*128, 1*128, 2*128, 4*128, rdx, 0*ZMM_SCD4, rdi, 0*ZMM_SCD4, 1
	CALLP	zreal_mult
	zloop_init 24			;; 24 more iterations
	zloop_set_rw_prefetch ZQUAD	;; Prefetch 96 cache lines
	zr64_sixtyfour_complex_with_mult_preload
b3c:	zr64_sixtyfour_complex_with_mult rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 1, L1PREFETCH_ALL, 8*128
	zloop	24, b3c			;; Test loop counter
	add	rsi, r8
	jmp	zpass2_1600_real_unfft

zpass2_1600_real_4:
	mov	rbp, DIST_TO_MULSRCARG
	mov	rdx, sincos3		;; Real sin/cos data
	mov	rdi, xsincos_complex	;; Complex sin/cos data
	bump	rdx, 40*ZMM_SCD2+8*ZMM_SCD2
	bump	rdi, 40*ZMM_SCD2+8*ZMM_SCD2
;;	zr64f_hundredtwentyeight_real_with_mulf_preload
;;	zr64f_hundredtwentyeight_real_with_mulf rsi, 8*128, 1*128, 2*128, 4*128, rdx, 0*ZMM_SCD4, rdi, 0*ZMM_SCD4, 1
	CALLP	zreal_mulf
	zloop_init 24			;; 24 more iterations
	zloop_set_rw_prefetch ZQUAD	;; Prefetch 96 cache lines
	zr64f_sixtyfour_complex_with_mulf_preload
b3d:	zr64f_sixtyfour_complex_with_mulf rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 1, L1PREFETCH_ALL, 8*128
	zloop	24, b3d			;; Test loop counter
zpass2_1600_real_unfft:
	bump	rsi, -25*8*128		;; Restore source pointer

;; Do inverse FFT levels 5,6,7
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	64	+1	...
;;	8	...
;;	...
;;	56
;;	128
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	320	+1	...
;;	8	...
;;	...
;;	312
;;	640
;;	...

;; Do 40 ten_reals_unfft operations
;; Do 280 five_complex_unfft operations
;;	distance between fft data elements is 64
;;	do 40 macros each processing 80 data values

	zloop_init 40			;; 5 iterations of 8
	zloop_set_rw_prefetch ZDOUBLE	;; Prefetch 80 cache lines
	bump	rdx, -8*ZMM_SCD2
	bump	rdi, -8*ZMM_SCD2
	zr5_2sc_ten_reals_unfft_preload
c2a:	zr5_2sc_ten_reals_unfft rsi, 128, 8*128, rdx, ZMM_SCD2, rdi, ZMM_SCD2, 8
	zloop	8, c2a, -8*128+40*128, rdi, -8*ZMM_SCD2 ;; Test loop counter, next source pointer, restore sin/cos pointer
	loops_reset			;; 4 more iters of 8
	zr5_five_complex_djbunfft_preload
c2b:	zr5_five_complex_djbunfft rsi, 128, 8*128, rdi, ZMM_SCD2, 8
	zloop	8, c2b, -8*128+40*128, rdi, -8*ZMM_SCD2 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	5, c2b, -5*40*128	;; Test loop counter, restore source pointer

;; Do inverse FFT levels 1,2,3,4
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	320	+1	...
;;	8	...
;;	...
;;	312
;;	640
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	1600	+1	...
;;	8	...
;;	...
;;	1592

;; Do 320 ten_reals_last_unfft operations
;;	distance between fft data elements is 320
;;	do 40 macros each processing 80 data values

	bump	rdx, -8*ZMM_SCD2-40*ZMM_SCD2
	bump	rdi, -40*ZMM_SCD2
	zloop_init 40			;; 40 iterations
	zloop_set_rw_prefetch ZDOUBLE	;; Prefetch 80 cache lines
	zr5_2sc_ten_reals_unfft_preload
c1b:	zr5_2sc_ten_reals_unfft rsi, 128, 40*128, rdx, ZMM_SCD2, rdi, ZMM_SCD2, 40
	zloop	40, c1b			;; Test loop counter

zpass2_1600_real_done:
	end_timer 2
	ENDM

; *************** radix8-pass2-1600-complex macro ******************
; This macro takes 1600 complex values and performs the final 10.644 levels of a standard
; radix-8 FFT, squares the results, and does the first 10.644 levels of the inverse FFT.

zr4dwpn_pass2_1600_complex MACRO
	LOCAL	b1b, b2b, b3a, b3b, b3c, b3d, c1b, c2b
	LOCAL	zpass2_1600_complex_1, zpass2_1600_complex_2
	LOCAL	zpass2_1600_complex_3, zpass2_1600_complex_4
	LOCAL	zpass2_1600_complex_unfft, zpass2_1600_complex_done

;; Type 4 FFTs skip the forward FFT process

	mov	rsi, DATA_ADDR		;; Load address of FFT data
	mov	rbx, DIST_TO_FFTSRCARG
	mov	rcx, DATA_PREFETCH	;; Prefetch pointer (we need to prefetch 400 cache lines + ?? pad cache lines)
	cmp	ffttype, 4
	je	zpass2_1600_complex_4

;; Do FFT levels 1,2,3
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	1600	+1	...
;;	8	...
;;	...
;;	1592
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	320	+1	...
;;	8	...
;;	...
;;	312
;;	640
;;	...

;; Do 320 five_complex_fft operations
;;	distance between fft data elements is 320
;;	do 40 macros each processing 80 data values

	start_timer 4
	mov	rdi, xsincos_complex	;; The sin/cos data
	zloop_init 40			;; 40 iterations
	zloop_set_rw_prefetch ZDOUBLE	;; Prefetch 80 cache lines
	zr5f_five_complex_djbfft_preload
b1b:	zr5f_five_complex_djbfft rsi, 128, 40*128, rdi, ZMM_SCD2, 40, L1PREFETCH_ALL, 128
	zloop	40, b1b, -40*128	;; Test loop counter, restore source pointer
	end_timer 4

;; Do FFT levels 4,5,6
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	384	+1	...
;;	8	...
;;	...
;;	376
;;	768
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	64	+1	...
;;	8	...
;;	...
;;	56
;;	128
;;	...

;; Do 320 five_complex_fft operations
;;	distance between fft data elements is 64
;;	do 40 macros each processing 80 data values

	start_timer 5
	zloop_init 40			;; 5 iterations of 8
	zloop_set_rw_prefetch ZDOUBLE	;; Prefetch 80 cache lines
	zr5_five_complex_djbfft_preload
b2b:	zr5_five_complex_djbfft rsi, 128, 8*128, rdi, ZMM_SCD2, 8, L1PREFETCH_ALL, 128
	zloop	8, b2b, -8*128+40*128, rdi, -8*ZMM_SCD2	;; Test loop counter, restore source pointer, restore sin/cos pointer
	zloop	5, b2b, -5*40*128, rdi, 8*ZMM_SCD2 ;; Test loop counter, restore source pointer, next sin/cos pointer
	end_timer 5

;; Do FFT levels 7,8,9,10,11,12 as well as inverse FFT levels 7,8,9,10,11,12
;; On input and output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	64	+1	...
;;	8	...
;;	...
;;	56
;;	128
;;	...

;; Do 25 sixtyfour_complex_fft operations
;;	distance between fft data elements is 1 (8 for swizzling)
;;	do 25 macros each processing 128 data values

;; Execute the right middle step

pfing	bump	rcx, -20*64		;; We will prefetch 20 cache lines too many
	cmp	ffttype, 2
	jl	zpass2_1600_complex_1
	jg	zpass2_1600_complex_3

zpass2_1600_complex_2:
	start_timer 9
	zloop_init 25			;; 25 iterations
	zloop_set_rw_prefetch ZQUAD	;; Prefetch 100 cache lines
	mov	r8, DEST2ARG
	zr64_sixtyfour_complex_with_square_preload
b3a:	zr64_sixtyfour_complex_with_square rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 25, L1PREFETCH_ALL, 8*128
	zloop	25, b3a			;; Test loop counter
	add	rsi, r8
	end_timer 9
	jmp	zpass2_1600_complex_unfft

zpass2_1600_complex_1:
	start_timer 9
	zloop_init 25			;; 25 iterations
	zloop_set_rw_prefetch ZQUAD	;; Prefetch 100 cache lines
	zr64_sixtyfour_complex_fft_final_preload
b3b:	zr64_sixtyfour_complex_fft_final rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 25, L1PREFETCH_ALL, 8*128
	zloop	25, b3b			;; Test loop counter
	end_timer 9
	jmp	zpass2_1600_complex_done

zpass2_1600_complex_3:
	start_timer 9
	zloop_init 25			;; 25 iterations
	zloop_set_rw_prefetch ZQUAD	;; Prefetch 100 cache lines
	mov	r8, DEST2ARG
	mov	rbp, DIST_TO_MULSRCARG
	zr64_sixtyfour_complex_with_mult_preload
b3c:	zr64_sixtyfour_complex_with_mult rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 25, L1PREFETCH_ALL, 8*128
	zloop	25, b3c			;; Test loop counter
	add	rsi, r8
	end_timer 9
	jmp	zpass2_1600_complex_unfft

zpass2_1600_complex_4:
	start_timer 9
	zloop_init 25			;; 25 iterations
	zloop_set_rw_prefetch ZQUAD	;; Prefetch 100 cache lines
	mov	rbp, DIST_TO_MULSRCARG
	mov	rdi, xsincos_complex	;; The sin/cos data
	bump	rdi, 40*ZMM_SCD2+8*ZMM_SCD2
	zr64f_sixtyfour_complex_with_mulf_preload
b3d:	zr64f_sixtyfour_complex_with_mulf rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 25, L1PREFETCH_ALL, 8*128
	zloop	25, b3d			;; Test loop counter
	end_timer 9
zpass2_1600_complex_unfft:
	bump	rsi, -25*8*128		;; Restore source pointer

;; Do inverse FFT levels 4,5,6
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	64	+1	...
;;	8	...
;;	...
;;	56
;;	128
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	320	+1	...
;;	8	...
;;	...
;;	312
;;	640
;;	...

;; Do 320 five_complex_unfft operations
;;	distance between fft data elements is 64
;;	do 40 macros each processing 80 data values

	start_timer 13
	bump	rdi, -8*ZMM_SCD2
	zloop_init 40			;; 5 iterations of 8
	zloop_set_rw_prefetch ZDOUBLE	;; Prefetch 80 cache lines
	zr5_five_complex_djbunfft_preload
c2b:	zr5_five_complex_djbunfft rsi, 128, 8*128, rdi, ZMM_SCD2, 8, L1PREFETCH_ALL, 128
	zloop	8, c2b, -8*128+40*128, rdi, -8*ZMM_SCD2 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	5, c2b, -5*40*128	;; Test loop counter, restore source pointer
	end_timer 13

;; Do inverse FFT levels 1,2,3
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	320	+1	...
;;	8	...
;;	...
;;	312
;;	640
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	1600	+1	...
;;	8	...
;;	...
;;	1592

;; Do 320 five_complex_unfft operations
;;	distance between fft data elements is 320
;;	do 40 macros each processing 80 data values

	start_timer 14
	bump	rdi, -40*ZMM_SCD2
	zloop_init 40			;; 40 iterations
	zloop_set_rw_prefetch ZDOUBLE	;; Prefetch 80 cache lines
	zr5_five_complex_djbunfft_preload
c1b:	zr5_five_complex_djbunfft rsi, 128, 40*128, rdi, ZMM_SCD2, 40, L1PREFETCH_ALL, 128
	zloop	40, c1b			;; Test loop counter
	end_timer 14

zpass2_1600_complex_done:
	ENDM



; *************** radix8-pass2-1920-real macro ******************
; This macro takes 3840 real values and performs the final 11.907 levels of the FFT.
; The results are squared, and the first 11.907 levels of the inverse FFT are done.

zr4dwpn_pass2_1920_real MACRO
	LOCAL	b1b, b2a, b2b, b3a, b3b, b3c, b3d, c1b, c2a, c2b
	LOCAL	zpass2_1920_real_1, zpass2_1920_real_2
	LOCAL	zpass2_1920_real_3, zpass2_1920_real_4
	LOCAL	zpass2_1920_real_unfft, zpass2_1920_real_done

;; Type 4 FFTs skip the forward FFT process

	start_timer 2
	mov	rsi, DATA_ADDR		;; Load source address
	mov	rbx, DIST_TO_FFTSRCARG
	mov	rcx, DATA_PREFETCH	;; Prefetch pointer (we need to prefetch 480 cache lines + ?? pad cache lines)
	cmp	ffttype, 4
	je	zpass2_1920_real_4

;; Do FFT levels 1,2,3,4
;;	0	+1	+1	+1	+1	+1	+1	+1	1920	+1	...
;;	8	...
;;	...
;;	1912
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	384	+1	...
;;	8	...
;;	...
;;	376
;;	768
;;	...

;; Do 384 ten_real_fft operations
;;	distance between fft data elements is 384
;;	do 48 macros each processing 80 data values

	mov	rdx, sincos3		;; Real sin/cos data
	mov	rdi, xsincos_complex	;; Complex sin/cos data
	zloop_init 48			;; 48 iterations
	zloop_set_rw_prefetch ZDOUBLE	;; Prefetch 96 cache lines
	zr5f_2sc_ten_reals_fft_preload
b1b:	zr5f_2sc_ten_reals_fft rsi, 128, 48*128, rdx, ZMM_SCD2, rdi, ZMM_SCD2, 48
	zloop	48, b1b, -48*128	;; Test loop counter, restore source pointer

;; Do FFT levels 5,6,7
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	384	+1	...
;;	8	...
;;	...
;;	376
;;	768
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	64	+1	...
;;	8	...
;;	...
;;	56
;;	128
;;	...

;; Do 40 twelve_reals_fft operations
;; Do 280 six_complex_fft operations
;;	distance between fft data elements is 64
;;	do 40 macros each processing 96 data values

	zloop_init 40			;; 5 iterations of 8
	zloop_set_rw_prefetch ZDOUBLE	;; Prefetch 80 cache lines
	zr6_2sc_twelve_reals_fft_preload
b2a:	zr6_2sc_twelve_reals_fft rsi, 128, 8*128, rdx, ZMM_SCD3, rdi, ZMM_SCD3, 8
	zloop	8, b2a, -8*128+48*128, rdi, -8*ZMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer
	loops_reset			;; 4 more iters of 8
	zr6_six_complex_djbfft_preload
b2b:	zr6_six_complex_djbfft rsi, 128, 8*128, rdi, ZMM_SCD3, 8
	zloop	8, b2b, -8*128+48*128, rdi, -8*ZMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	5, b2b, -5*48*128, rdi, 8*ZMM_SCD3 ;; Test loop counter, restore source pointer, next sin/cos pointer

;; Do FFT levels 8,9,10,11,12,13 as well as inverse FFT levels 8,9,10,11,12,13
;; On input and output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	64	+1	...
;;	8	...
;;	...
;;	56
;;	128
;;	...

;; Do 1 hundredtwentyeight_reals_fft operation
;; Do 29 sixtyfour_complex_fft operations
;;	distance between fft data elements is 1 (8 for swizzling)
;;	do 30 macros each processing 128 data values

;; Execute the proper middle step

pfing	bump	rcx, -17*64		;; We will prefetch 17 cache lines too many
	cmp	ffttype, 2
	jl	zpass2_1920_real_1
	jg	zpass2_1920_real_3

zpass2_1920_real_2:
	mov	r8, DEST2ARG
;;	zr64_hundredtwentyeight_real_with_square_preload
;;	zr64_hundredtwentyeight_real_with_square rsi, 8*128, 1*128, 2*128, 4*128, rdx, 0*ZMM_SCD4, rdi, 0*ZMM_SCD4, 1
	CALLP	zreal_square
	zloop_init 29			;; 29 more iterations
	zloop_set_rw_prefetch ZQUINT	;; Prefetch 145 cache lines
	zr64_sixtyfour_complex_with_square_preload
b3a:	zr64_sixtyfour_complex_with_square rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 1, L1PREFETCH_ALL, 8*128
	zloop	29, b3a			;; Test loop counter
	add	rsi, r8
	jmp	zpass2_1920_real_unfft

zpass2_1920_real_1:
;;	zr64_hundredtwentyeight_real_fft_final_preload
;;	zr64_hundredtwentyeight_real_fft_final rsi, 8*128, 1*128, 2*128, 4*128, rdx, 0*ZMM_SCD4, rdi, 0*ZMM_SCD4, 1
	CALLP	zreal_fft_final
	zloop_init 29			;; 29 more iterations
	zloop_set_rw_prefetch ZQUINT	;; Prefetch 145 cache lines
	zr64_sixtyfour_complex_fft_final_preload
b3b:	zr64_sixtyfour_complex_fft_final rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 1, L1PREFETCH_ALL, 8*128
	zloop	29, b3b			;; Test loop counter
	jmp	zpass2_1920_real_done

zpass2_1920_real_3:
	mov	r8, DEST2ARG
	mov	rbp, DIST_TO_MULSRCARG
;;	zr64_hundredtwentyeight_real_with_mult_preload
;;	zr64_hundredtwentyeight_real_with_mult rsi, 8*128, 1*128, 2*128, 4*128, rdx, 0*ZMM_SCD4, rdi, 0*ZMM_SCD4, 1
	CALLP	zreal_mult
	zloop_init 29			;; 29 more iterations
	zloop_set_rw_prefetch ZQUINT	;; Prefetch 145 cache lines
	zr64_sixtyfour_complex_with_mult_preload
b3c:	zr64_sixtyfour_complex_with_mult rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 1, L1PREFETCH_ALL, 8*128
	zloop	29, b3c			;; Test loop counter
	add	rsi, r8
	jmp	zpass2_1920_real_unfft

zpass2_1920_real_4:
	mov	rbp, DIST_TO_MULSRCARG
	mov	rdx, sincos3		;; Real sin/cos data
	mov	rdi, xsincos_complex	;; Complex sin/cos data
	bump	rdx, 48*ZMM_SCD2+8*ZMM_SCD3
	bump	rdi, 48*ZMM_SCD2+8*ZMM_SCD3
;;	zr64f_hundredtwentyeight_real_with_mulf_preload
;;	zr64f_hundredtwentyeight_real_with_mulf rsi, 8*128, 1*128, 2*128, 4*128, rdx, 0*ZMM_SCD4, rdi, 0*ZMM_SCD4, 1
	CALLP	zreal_mulf
	zloop_init 29			;; 29 more iterations
	zloop_set_rw_prefetch ZQUINT	;; Prefetch 145 cache lines
	zr64f_sixtyfour_complex_with_mulf_preload
b3d:	zr64f_sixtyfour_complex_with_mulf rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 1, L1PREFETCH_ALL, 8*128
	zloop	29, b3d			;; Test loop counter
zpass2_1920_real_unfft:
	bump	rsi, -30*8*128		;; Restore source pointer

;; Do inverse FFT levels 5,6,7
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	64	+1	...
;;	8	...
;;	...
;;	56
;;	128
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	384	+1	...
;;	8	...
;;	...
;;	376
;;	768
;;	...

;; Do 40 twelve_reals_unfft operations
;; Do 280 six_complex_unfft operations
;;	distance between fft data elements is 64
;;	do 40 macros each processing 96 data values

	zloop_init 40			;; 5 iterations of 8
	zloop_set_rw_prefetch ZDOUBLE	;; Prefetch 80 cache lines
	bump	rdx, -8*ZMM_SCD3
	bump	rdi, -8*ZMM_SCD3
	zr6_2sc_twelve_reals_unfft_preload
c2a:	zr6_2sc_twelve_reals_unfft rsi, 128, 8*128, rdx, ZMM_SCD3, rdi, ZMM_SCD3, 8
	zloop	8, c2a, -8*128+48*128, rdi, -8*ZMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer
	loops_reset			;; 4 more iters of 8
	zr6_six_complex_djbunfft_preload
c2b:	zr6_six_complex_djbunfft rsi, 128, 8*128, rdi, ZMM_SCD3, 8
	zloop	8, c2b, -8*128+48*128, rdi, -8*ZMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	5, c2b, -5*48*128	;; Test loop counter, restore source pointer

;; Do inverse FFT levels 1,2,3,4
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	384	+1	...
;;	8	...
;;	...
;;	376
;;	768
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	1920	+1	...
;;	8	...
;;	...
;;	1912

;; Do 384 ten_reals_last_unfft operations
;;	distance between fft data elements is 384
;;	do 48 macros each processing 80 data values

	bump	rdx, -8*ZMM_SCD3-48*ZMM_SCD2
	bump	rdi, -48*ZMM_SCD2
	zloop_init 48			;; 48 iterations
	zloop_set_rw_prefetch ZDOUBLE	;; Prefetch 96 cache lines
	zr5_2sc_ten_reals_unfft_preload
c1b:	zr5_2sc_ten_reals_unfft rsi, 128, 48*128, rdx, ZMM_SCD2, rdi, ZMM_SCD2, 48
	zloop	48, c1b			;; Test loop counter

zpass2_1920_real_done:
	end_timer 2
	ENDM

; *************** radix8-pass2-1920-complex macro ******************
; This macro takes 1920 complex values and performs the final 10.907 levels of a standard
; radix-8 FFT, squares the results, and does the first 10.907 levels of the inverse FFT.

zr4dwpn_pass2_1920_complex MACRO
	LOCAL	b1b, b2b, b3a, b3b, b3c, b3d, c1b, c2b
	LOCAL	zpass2_1920_complex_1, zpass2_1920_complex_2
	LOCAL	zpass2_1920_complex_3, zpass2_1920_complex_4
	LOCAL	zpass2_1920_complex_unfft, zpass2_1920_complex_done

;; Type 4 FFTs skip the forward FFT process

	mov	rsi, DATA_ADDR		;; Load address of FFT data
	mov	rbx, DIST_TO_FFTSRCARG
	mov	rcx, DATA_PREFETCH	;; Prefetch pointer (we need to prefetch 480 cache lines + ?? pad cache lines)
	cmp	ffttype, 4
	je	zpass2_1920_complex_4

;; Do FFT levels 1,2,3
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	1920	+1	...
;;	8	...
;;	...
;;	1912
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	384	+1	...
;;	8	...
;;	...
;;	376
;;	768
;;	...

;; Do 384 five_complex_fft operations
;;	distance between fft data elements is 384
;;	do 48 macros each processing 80 data values

	start_timer 4
	mov	rdi, xsincos_complex	;; The sin/cos data
	zloop_init 48			;; 48 iterations
	zloop_set_rw_prefetch ZDOUBLE	;; Prefetch 96 cache lines
	zr5f_five_complex_djbfft_preload
b1b:	zr5f_five_complex_djbfft rsi, 128, 48*128, rdi, ZMM_SCD2, 48, L1PREFETCH_ALL, 128
	zloop	48, b1b, -48*128	;; Test loop counter, restore source pointer
	end_timer 4

;; Do FFT levels 4,5,6
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	384	+1	...
;;	8	...
;;	...
;;	376
;;	768
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	64	+1	...
;;	8	...
;;	...
;;	56
;;	128
;;	...

;; Do 320 six_complex_fft operations
;;	distance between fft data elements is 64
;;	do 40 macros each processing 96 data values

	start_timer 5
	zloop_init 40			;; 5 iterations of 8
	zloop_set_rw_prefetch ZDOUBLE	;; Prefetch 80 cache lines
	zr6_six_complex_djbfft_preload
b2b:	zr6_six_complex_djbfft rsi, 128, 8*128, rdi, ZMM_SCD3, 8, L1PREFETCH_ALL, 128
	zloop	8, b2b, -8*128+48*128, rdi, -8*ZMM_SCD3	;; Test loop counter, restore source pointer, restore sin/cos pointer
	zloop	5, b2b, -5*48*128, rdi, 8*ZMM_SCD3 ;; Test loop counter, restore source pointer, next sin/cos pointer
	end_timer 5

;; Do FFT levels 7,8,9,10,11,12 as well as inverse FFT levels 7,8,9,10,11,12
;; On input and output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	64	+1	...
;;	8	...
;;	...
;;	56
;;	128
;;	...

;; Do 30 sixtyfour_complex_fft operations
;;	distance between fft data elements is 1 (8 for swizzling)
;;	do 30 macros each processing 128 data values

;; Execute the right middle step

pfing	bump	rcx, -22*64		;; We will prefetch 22 cache lines too many
	cmp	ffttype, 2
	jl	zpass2_1920_complex_1
	jg	zpass2_1920_complex_3

zpass2_1920_complex_2:
	start_timer 9
	zloop_init 30			;; 30 iterations
	zloop_set_rw_prefetch ZQUINT	;; Prefetch 150 cache lines
	mov	r8, DEST2ARG
	zr64_sixtyfour_complex_with_square_preload
b3a:	zr64_sixtyfour_complex_with_square rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 30, L1PREFETCH_ALL, 8*128
	zloop	30, b3a			;; Test loop counter
	add	rsi, r8
	end_timer 9
	jmp	zpass2_1920_complex_unfft

zpass2_1920_complex_1:
	start_timer 9
	zloop_init 30			;; 30 iterations
	zloop_set_rw_prefetch ZQUINT	;; Prefetch 150 cache lines
	zr64_sixtyfour_complex_fft_final_preload
b3b:	zr64_sixtyfour_complex_fft_final rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 30, L1PREFETCH_ALL, 8*128
	zloop	30, b3b			;; Test loop counter
	end_timer 9
	jmp	zpass2_1920_complex_done

zpass2_1920_complex_3:
	start_timer 9
	zloop_init 30			;; 30 iterations
	zloop_set_rw_prefetch ZQUINT	;; Prefetch 150 cache lines
	mov	r8, DEST2ARG
	mov	rbp, DIST_TO_MULSRCARG
	zr64_sixtyfour_complex_with_mult_preload
b3c:	zr64_sixtyfour_complex_with_mult rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 30, L1PREFETCH_ALL, 8*128
	zloop	30, b3c			;; Test loop counter
	add	rsi, r8
	end_timer 9
	jmp	zpass2_1920_complex_unfft

zpass2_1920_complex_4:
	start_timer 9
	zloop_init 30			;; 30 iterations
	zloop_set_rw_prefetch ZQUINT	;; Prefetch 150 cache lines
	mov	rbp, DIST_TO_MULSRCARG
	mov	rdi, xsincos_complex	;; The sin/cos data
	bump	rdi, 48*ZMM_SCD2+8*ZMM_SCD3
	zr64f_sixtyfour_complex_with_mulf_preload
b3d:	zr64f_sixtyfour_complex_with_mulf rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 30, L1PREFETCH_ALL, 8*128
	zloop	30, b3d			;; Test loop counter
	end_timer 9
zpass2_1920_complex_unfft:
	bump	rsi, -30*8*128		;; Restore source pointer

;; Do inverse FFT levels 4,5,6
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	64	+1	...
;;	8	...
;;	...
;;	56
;;	128
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	384	+1	...
;;	8	...
;;	...
;;	376
;;	768
;;	...

;; Do 320 six_complex_unfft operations
;;	distance between fft data elements is 64
;;	do 40 macros each processing 96 data values

	start_timer 13
	bump	rdi, -8*ZMM_SCD3
	zloop_init 40			;; 5 iterations of 8
	zloop_set_rw_prefetch ZDOUBLE	;; Prefetch 80 cache lines
	zr6_six_complex_djbunfft_preload
c2b:	zr6_six_complex_djbunfft rsi, 128, 8*128, rdi, ZMM_SCD3, 8, L1PREFETCH_ALL, 128
	zloop	8, c2b, -8*128+48*128, rdi, -8*ZMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	5, c2b, -5*48*128	;; Test loop counter, restore source pointer
	end_timer 13

;; Do inverse FFT levels 1,2,3
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	384	+1	...
;;	8	...
;;	...
;;	376
;;	768
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	1920	+1	...
;;	8	...
;;	...
;;	1912

;; Do 384 five_complex_unfft operations
;;	distance between fft data elements is 384
;;	do 48 macros each processing 80 data values

	start_timer 14
	bump	rdi, -48*ZMM_SCD2
	zloop_init 48			;; 48 iterations
	zloop_set_rw_prefetch ZDOUBLE	;; Prefetch 96 cache lines
	zr5_five_complex_djbunfft_preload
c1b:	zr5_five_complex_djbunfft rsi, 128, 48*128, rdi, ZMM_SCD2, 48, L1PREFETCH_ALL, 128
	zloop	48, c1b			;; Test loop counter
	end_timer 14

zpass2_1920_complex_done:
	ENDM


; *************** radix8-pass2-2240-real macro ******************
; This macro takes 4480 real values and performs the final 12.129 levels of the FFT.
; The results are squared, and the first 12.129 levels of the inverse FFT are done.

zr4dwpn_pass2_2240_real MACRO
	LOCAL	b1b, b2a, b2b, b3a, b3b, b3c, b3d, c1b, c2a, c2b
	LOCAL	zpass2_2240_real_1, zpass2_2240_real_2
	LOCAL	zpass2_2240_real_3, zpass2_2240_real_4
	LOCAL	zpass2_2240_real_unfft, zpass2_2240_real_done

;; Type 4 FFTs skip the forward FFT process

	start_timer 2
	mov	rsi, DATA_ADDR		;; Load source address
	mov	rbx, DIST_TO_FFTSRCARG
	mov	rcx, DATA_PREFETCH	;; Prefetch pointer (we need to prefetch 560 cache lines + 0 pad cache lines)
	cmp	ffttype, 4
	je	zpass2_2240_real_4

;; Do FFT levels 1,2,3,4
;;	0	+1	+1	+1	+1	+1	+1	+1	2240	+1	...
;;	8	...
;;	...
;;	2232
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	448	+1	...
;;	8	...
;;	...
;;	440
;;	896
;;	...

;; Do 448 ten_real_fft operations
;;	distance between fft data elements is 448
;;	do 56 macros each processing 80 data values

	mov	rdx, sincos3		;; Real sin/cos data
	mov	rdi, xsincos_complex	;; Complex sin/cos data
	zloop_init 56			;; 56 iterations
	zloop_set_rw_prefetch ZDOUBLE	;; Prefetch 112 cache lines
	zr5f_2sc_ten_reals_fft_preload
b1b:	zr5f_2sc_ten_reals_fft rsi, 128, 56*128, rdx, ZMM_SCD2, rdi, ZMM_SCD2, 56
	zloop	56, b1b, -56*128	;; Test loop counter, restore source pointer

;; Do FFT levels 5,6,7
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	448	+1	...
;;	8	...
;;	...
;;	440
;;	896
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	64	+1	...
;;	8	...
;;	...
;;	56
;;	128
;;	...

;; Do 40 fourteen_reals_fft operations
;; Do 280 seven_complex_fft operations
;;	distance between fft data elements is 64
;;	do 40 macros each processing 112 data values

	zloop_init 40			;; 5 iterations of 8
	zloop_set_rw_prefetch ZDOUBLE	;; Prefetch 80 cache lines
	zr7_2sc_fourteen_reals_fft_preload
b2a:	zr7_2sc_fourteen_reals_fft rsi, 128, 8*128, rdx, ZMM_SCD3, rdi, ZMM_SCD3, 8
	zloop	8, b2a, -8*128+56*128, rdi, -8*ZMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer
	loops_reset			;; 4 more iters of 8
	zr7_seven_complex_djbfft_preload
b2b:	zr7_seven_complex_djbfft rsi, 128, 8*128, rdi, ZMM_SCD3, 8
	zloop	8, b2b, -8*128+56*128, rdi, -8*ZMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	5, b2b, -5*56*128, rdi, 8*ZMM_SCD3 ;; Test loop counter, restore source pointer, next sin/cos pointer

;; Do FFT levels 8,9,10,11,12,13 as well as inverse FFT levels 8,9,10,11,12,13
;; On input and output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	64	+1	...
;;	8	...
;;	...
;;	56
;;	128
;;	...

;; Do 1 hundredtwentyeight_reals_fft operation
;; Do 34 sixtyfour_complex_fft operations
;;	distance between fft data elements is 1 (8 for swizzling)
;;	do 35 macros each processing 128 data values

;; Execute the proper middle step

pfing	bump	rcx, -4*64		;; We will prefetch 4 cache lines too many
	cmp	ffttype, 2
	jl	zpass2_2240_real_1
	jg	zpass2_2240_real_3

zpass2_2240_real_2:
	mov	r8, DEST2ARG
;;	zr64_hundredtwentyeight_real_with_square_preload
;;	zr64_hundredtwentyeight_real_with_square rsi, 8*128, 1*128, 2*128, 4*128, rdx, 0*ZMM_SCD4, rdi, 0*ZMM_SCD4, 1
	CALLP	zreal_square
	zloop_init 34			;; 34 more iterations
	zloop_set_rw_prefetch ZQUINT	;; Prefetch 170 cache lines
	zr64_sixtyfour_complex_with_square_preload
b3a:	zr64_sixtyfour_complex_with_square rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 1, L1PREFETCH_ALL, 8*128
	zloop	34, b3a			;; Test loop counter
	add	rsi, r8
	jmp	zpass2_2240_real_unfft

zpass2_2240_real_1:
;;	zr64_hundredtwentyeight_real_fft_final_preload
;;	zr64_hundredtwentyeight_real_fft_final rsi, 8*128, 1*128, 2*128, 4*128, rdx, 0*ZMM_SCD4, rdi, 0*ZMM_SCD4, 1
	CALLP	zreal_fft_final
	zloop_init 34			;; 34 more iterations
	zloop_set_rw_prefetch ZQUINT	;; Prefetch 170 cache lines
	zr64_sixtyfour_complex_fft_final_preload
b3b:	zr64_sixtyfour_complex_fft_final rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 1, L1PREFETCH_ALL, 8*128
	zloop	34, b3b			;; Test loop counter
	jmp	zpass2_2240_real_done

zpass2_2240_real_3:
	mov	r8, DEST2ARG
	mov	rbp, DIST_TO_MULSRCARG
;;	zr64_hundredtwentyeight_real_with_mult_preload
;;	zr64_hundredtwentyeight_real_with_mult rsi, 8*128, 1*128, 2*128, 4*128, rdx, 0*ZMM_SCD4, rdi, 0*ZMM_SCD4, 1
	CALLP	zreal_mult
	zloop_init 34			;; 34 more iterations
	zloop_set_rw_prefetch ZQUINT	;; Prefetch 170 cache lines
	zr64_sixtyfour_complex_with_mult_preload
b3c:	zr64_sixtyfour_complex_with_mult rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 1, L1PREFETCH_ALL, 8*128
	zloop	34, b3c			;; Test loop counter
	add	rsi, r8
	jmp	zpass2_2240_real_unfft

zpass2_2240_real_4:
	mov	rbp, DIST_TO_MULSRCARG
	mov	rdx, sincos3		;; Real sin/cos data
	mov	rdi, xsincos_complex	;; Complex sin/cos data
	bump	rdx, 56*ZMM_SCD2+8*ZMM_SCD3
	bump	rdi, 56*ZMM_SCD2+8*ZMM_SCD3
;;	zr64f_hundredtwentyeight_real_with_mulf_preload
;;	zr64f_hundredtwentyeight_real_with_mulf rsi, 8*128, 1*128, 2*128, 4*128, rdx, 0*ZMM_SCD4, rdi, 0*ZMM_SCD4, 1
	CALLP	zreal_mulf
	zloop_init 34			;; 34 more iterations
	zloop_set_rw_prefetch ZQUINT	;; Prefetch 170 cache lines
	zr64f_sixtyfour_complex_with_mulf_preload
b3d:	zr64f_sixtyfour_complex_with_mulf rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 1, L1PREFETCH_ALL, 8*128
	zloop	34, b3d			;; Test loop counter
zpass2_2240_real_unfft:
	bump	rsi, -35*8*128		;; Restore source pointer

;; Do inverse FFT levels 5,6,7
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	64	+1	...
;;	8	...
;;	...
;;	56
;;	128
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	448	+1	...
;;	8	...
;;	...
;;	440
;;	896
;;	...

;; Do 40 fourteen_reals_unfft operations
;; Do 280 seven_complex_unfft operations
;;	distance between fft data elements is 64
;;	do 40 macros each processing 112 data values

	zloop_init 40			;; 5 iterations of 8
	zloop_set_rw_prefetch ZDOUBLE	;; Prefetch 80 cache lines
	bump	rdx, -8*ZMM_SCD3
	bump	rdi, -8*ZMM_SCD3
	zr7_2sc_fourteen_reals_unfft_preload
c2a:	zr7_2sc_fourteen_reals_unfft rsi, 128, 8*128, rdx, ZMM_SCD3, rdi, ZMM_SCD3, 8
	zloop	8, c2a, -8*128+56*128, rdi, -8*ZMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer
	loops_reset			;; 4 more iters of 8
	zr7_seven_complex_djbunfft_preload
c2b:	zr7_seven_complex_djbunfft rsi, 128, 8*128, rdi, ZMM_SCD3, 8
	zloop	8, c2b, -8*128+56*128, rdi, -8*ZMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	5, c2b, -5*56*128	;; Test loop counter, restore source pointer

;; Do inverse FFT levels 1,2,3,4
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	448	+1	...
;;	8	...
;;	...
;;	440
;;	896
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	2240	+1	...
;;	8	...
;;	...
;;	2232

;; Do 448 ten_reals_last_unfft operations
;;	distance between fft data elements is 448
;;	do 56 macros each processing 80 data values

	bump	rdx, -8*ZMM_SCD3-56*ZMM_SCD2
	bump	rdi, -56*ZMM_SCD2
	zloop_init 56			;; 56 iterations
	zloop_set_rw_prefetch ZDOUBLE	;; Prefetch 112 cache lines
	zr5_2sc_ten_reals_unfft_preload
c1b:	zr5_2sc_ten_reals_unfft rsi, 128, 56*128, rdx, ZMM_SCD2, rdi, ZMM_SCD2, 56
	zloop	56, c1b			;; Test loop counter

zpass2_2240_real_done:
	end_timer 2
	ENDM

; *************** radix8-pass2-2240-complex macro ******************
; This macro takes 2240 complex values and performs the final 11.129 levels of a standard
; radix-8 FFT, squares the results, and does the first 11.129 levels of the inverse FFT.

zr4dwpn_pass2_2240_complex MACRO
	LOCAL	b1b, b2b, b3a, b3b, b3c, b3d, c1b, c2b
	LOCAL	zpass2_2240_complex_1, zpass2_2240_complex_2
	LOCAL	zpass2_2240_complex_3, zpass2_2240_complex_4
	LOCAL	zpass2_2240_complex_unfft, zpass2_2240_complex_done

;; Type 4 FFTs skip the forward FFT process

	mov	rsi, DATA_ADDR		;; Load address of FFT data
	mov	rbx, DIST_TO_FFTSRCARG
	mov	rcx, DATA_PREFETCH	;; Prefetch pointer (we need to prefetch 560 cache lines + 0 pad cache lines)
	cmp	ffttype, 4
	je	zpass2_2240_complex_4

;; Do FFT levels 1,2,3
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	2240	+1	...
;;	8	...
;;	...
;;	2232
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	448	+1	...
;;	8	...
;;	...
;;	440
;;	896
;;	...

;; Do 448 five_complex_fft operations
;;	distance between fft data elements is 448
;;	do 56 macros each processing 80 data values

	start_timer 4
	mov	rdi, xsincos_complex	;; The sin/cos data
	zloop_init 56			;; 56 iterations
	zloop_set_rw_prefetch ZDOUBLE	;; Prefetch 112 cache lines
	zr5f_five_complex_djbfft_preload
b1b:	zr5f_five_complex_djbfft rsi, 128, 56*128, rdi, ZMM_SCD2, 56, L1PREFETCH_ALL, 128
	zloop	56, b1b, -56*128	;; Test loop counter, restore source pointer
	end_timer 4

;; Do FFT levels 4,5,6
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	448	+1	...
;;	8	...
;;	...
;;	440
;;	896
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	64	+1	...
;;	8	...
;;	...
;;	56
;;	128
;;	...

;; Do 320 seven_complex_fft operations
;;	distance between fft data elements is 64
;;	do 40 macros each processing 112 data values

	start_timer 5
	zloop_init 40			;; 5 iterations of 8
	zloop_set_rw_prefetch ZDOUBLE	;; Prefetch 80 cache lines
	zr7_seven_complex_djbfft_preload
b2b:	zr7_seven_complex_djbfft rsi, 128, 8*128, rdi, ZMM_SCD3, 8, L1PREFETCH_ALL, 128
	zloop	8, b2b, -8*128+56*128, rdi, -8*ZMM_SCD3	;; Test loop counter, restore source pointer, restore sin/cos pointer
	zloop	5, b2b, -5*56*128, rdi, 8*ZMM_SCD3 ;; Test loop counter, restore source pointer, next sin/cos pointer
	end_timer 5

;; Do FFT levels 7,8,9,10,11,12 as well as inverse FFT levels 7,8,9,10,11,12
;; On input and output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	64	+1	...
;;	8	...
;;	...
;;	56
;;	128
;;	...

;; Do 35 sixtyfour_complex_fft operations
;;	distance between fft data elements is 1 (8 for swizzling)
;;	do 35 macros each processing 128 data values

;; Execute the right middle step

pfing	bump	rcx, -9*64		;; We will prefetch 9 cache lines too many
	cmp	ffttype, 2
	jl	zpass2_2240_complex_1
	jg	zpass2_2240_complex_3

zpass2_2240_complex_2:
	start_timer 9
	zloop_init 35			;; 35 iterations
	zloop_set_rw_prefetch ZQUINT	;; Prefetch 175 cache lines
	mov	r8, DEST2ARG
	zr64_sixtyfour_complex_with_square_preload
b3a:	zr64_sixtyfour_complex_with_square rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 35, L1PREFETCH_ALL, 8*128
	zloop	35, b3a			;; Test loop counter
	add	rsi, r8
	end_timer 9
	jmp	zpass2_2240_complex_unfft

zpass2_2240_complex_1:
	start_timer 9
	zloop_init 35			;; 35 iterations
	zloop_set_rw_prefetch ZQUINT	;; Prefetch 175 cache lines
	zr64_sixtyfour_complex_fft_final_preload
b3b:	zr64_sixtyfour_complex_fft_final rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 35, L1PREFETCH_ALL, 8*128
	zloop	35, b3b			;; Test loop counter
	end_timer 9
	jmp	zpass2_2240_complex_done

zpass2_2240_complex_3:
	start_timer 9
	zloop_init 35			;; 35 iterations
	zloop_set_rw_prefetch ZQUINT	;; Prefetch 175 cache lines
	mov	r8, DEST2ARG
	mov	rbp, DIST_TO_MULSRCARG
	zr64_sixtyfour_complex_with_mult_preload
b3c:	zr64_sixtyfour_complex_with_mult rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 35, L1PREFETCH_ALL, 8*128
	zloop	35, b3c			;; Test loop counter
	add	rsi, r8
	end_timer 9
	jmp	zpass2_2240_complex_unfft

zpass2_2240_complex_4:
	start_timer 9
	zloop_init 35			;; 35 iterations
	zloop_set_rw_prefetch ZQUINT	;; Prefetch 175 cache lines
	mov	rbp, DIST_TO_MULSRCARG
	mov	rdi, xsincos_complex	;; The sin/cos data
	bump	rdi, 56*ZMM_SCD2+8*ZMM_SCD3
	zr64f_sixtyfour_complex_with_mulf_preload
b3d:	zr64f_sixtyfour_complex_with_mulf rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 35, L1PREFETCH_ALL, 8*128
	zloop	35, b3d			;; Test loop counter
	end_timer 9
zpass2_2240_complex_unfft:
	bump	rsi, -35*8*128		;; Restore source pointer

;; Do inverse FFT levels 4,5,6
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	64	+1	...
;;	8	...
;;	...
;;	56
;;	128
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	448	+1	...
;;	8	...
;;	...
;;	440
;;	896
;;	...

;; Do 320 seven_complex_unfft operations
;;	distance between fft data elements is 64
;;	do 40 macros each processing 112 data values

	start_timer 13
	bump	rdi, -8*ZMM_SCD3
	zloop_init 40			;; 5 iterations of 8
	zloop_set_rw_prefetch ZDOUBLE	;; Prefetch 80 cache lines
	zr7_seven_complex_djbunfft_preload
c2b:	zr7_seven_complex_djbunfft rsi, 128, 8*128, rdi, ZMM_SCD3, 8, L1PREFETCH_ALL, 128
	zloop	8, c2b, -8*128+56*128, rdi, -8*ZMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	5, c2b, -5*56*128	;; Test loop counter, restore source pointer
	end_timer 13

;; Do inverse FFT levels 1,2,3
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	448	+1	...
;;	8	...
;;	...
;;	440
;;	896
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	2240	+1	...
;;	8	...
;;	...
;;	2232

;; Do 448 five_complex_unfft operations
;;	distance between fft data elements is 448
;;	do 56 macros each processing 80 data values

	start_timer 14
	bump	rdi, -56*ZMM_SCD2
	zloop_init 56			;; 56 iterations
	zloop_set_rw_prefetch ZDOUBLE	;; Prefetch 112 cache lines
	zr5_five_complex_djbunfft_preload
c1b:	zr5_five_complex_djbunfft rsi, 128, 56*128, rdi, ZMM_SCD2, 56, L1PREFETCH_ALL, 128
	zloop	56, c1b			;; Test loop counter
	end_timer 14

zpass2_2240_complex_done:
	ENDM


; *************** radix8-pass2-2304-real macro ******************
; This macro takes 4608 real values and performs the final 12.17 levels of the FFT.
; The results are squared, and the first 12.17 levels of the inverse FFT are done.

zr4dwpn_pass2_2304_real MACRO
	LOCAL	b1b, b2a, b2b, b3a, b3b, b3c, b3d, c1b, c2a, c2b
	LOCAL	zpass2_2304_real_1, zpass2_2304_real_2
	LOCAL	zpass2_2304_real_3, zpass2_2304_real_4
	LOCAL	zpass2_2304_real_unfft, zpass2_2304_real_done

;; Type 4 FFTs skip the forward FFT process

	start_timer 2
	mov	rsi, DATA_ADDR		;; Load source address
	mov	rbx, DIST_TO_FFTSRCARG
	mov	rcx, DATA_PREFETCH	;; Prefetch pointer (we need to prefetch 576 cache lines + ?? pad cache lines)
	cmp	ffttype, 4
	je	zpass2_2304_real_4

;; Do FFT levels 1,2,3,4
;;	0	+1	+1	+1	+1	+1	+1	+1	2304	+1	...
;;	8	...
;;	...
;;	2296
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	384	+1	...
;;	8	...
;;	...
;;	376
;;	768
;;	...

;; Do 384 twelve_real_fft operations
;;	distance between fft data elements is 384
;;	do 48 macros each processing 96 data values

	mov	rdx, sincos3		;; Real sin/cos data
	mov	rdi, xsincos_complex	;; Complex sin/cos data
	zloop_init 48			;; 48 iterations
	zloop_set_rw_prefetch ZDOUBLE	;; Prefetch 96 cache lines
	zr6f_2sc_twelve_reals_fft_preload
b1b:	zr6f_2sc_twelve_reals_fft rsi, 128, 48*128, rdx, ZMM_SCD3, rdi, ZMM_SCD3, 48
	zloop	48, b1b, -48*128	;; Test loop counter, restore source pointer

;; Do FFT levels 5,6,7
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	384	+1	...
;;	8	...
;;	...
;;	376
;;	768
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	64	+1	...
;;	8	...
;;	...
;;	56
;;	128
;;	...

;; Do 48 twelve_reals_fft operations
;; Do 336 six_complex_fft operations
;;	distance between fft data elements is 64
;;	do 48 macros each processing 96 data values

	zloop_init 48			;; 6 iterations of 8
	zloop_set_rw_prefetch ZDOUBLE	;; Prefetch 96 cache lines
	zr6_2sc_twelve_reals_fft_preload
b2a:	zr6_2sc_twelve_reals_fft rsi, 128, 8*128, rdx, ZMM_SCD3, rdi, ZMM_SCD3, 8
	zloop	8, b2a, -8*128+48*128, rdi, -8*ZMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer
	loops_reset			;; 5 more iters of 8
	zr6_six_complex_djbfft_preload
b2b:	zr6_six_complex_djbfft rsi, 128, 8*128, rdi, ZMM_SCD3, 8
	zloop	8, b2b, -8*128+48*128, rdi, -8*ZMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	6, b2b, -6*48*128, rdi, 8*ZMM_SCD3 ;; Test loop counter, restore source pointer, next sin/cos pointer

;; Do FFT levels 8,9,10,11,12,13 as well as inverse FFT levels 8,9,10,11,12,13
;; On input and output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	64	+1	...
;;	8	...
;;	...
;;	56
;;	128
;;	...

;; Do 1 hundredtwentyeight_reals_fft operation
;; Do 35 sixtyfour_complex_fft operations
;;	distance between fft data elements is 1 (8 for swizzling)
;;	do 36 macros each processing 128 data values

;; Execute the proper middle step

pfing	bump	rcx, -18*64		;; We will prefetch 18 cache lines too many
	cmp	ffttype, 2
	jl	zpass2_2304_real_1
	jg	zpass2_2304_real_3

zpass2_2304_real_2:
	mov	r8, DEST2ARG
;;	zr64_hundredtwentyeight_real_with_square_preload
;;	zr64_hundredtwentyeight_real_with_square rsi, 8*128, 1*128, 2*128, 4*128, rdx, 0*ZMM_SCD4, rdi, 0*ZMM_SCD4, 1
	CALLP	zreal_square
	zloop_init 35			;; 35 more iterations
	zloop_set_rw_prefetch ZSEXT	;; Prefetch 210 cache lines
	zr64_sixtyfour_complex_with_square_preload
b3a:	zr64_sixtyfour_complex_with_square rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 1, L1PREFETCH_ALL, 8*128
	zloop	35, b3a			;; Test loop counter
	add	rsi, r8
	jmp	zpass2_2304_real_unfft

zpass2_2304_real_1:
;;	zr64_hundredtwentyeight_real_fft_final_preload
;;	zr64_hundredtwentyeight_real_fft_final rsi, 8*128, 1*128, 2*128, 4*128, rdx, 0*ZMM_SCD4, rdi, 0*ZMM_SCD4, 1
	CALLP	zreal_fft_final
	zloop_init 35			;; 35 more iterations
	zloop_set_rw_prefetch ZSEXT	;; Prefetch 210 cache lines
	zr64_sixtyfour_complex_fft_final_preload
b3b:	zr64_sixtyfour_complex_fft_final rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 1, L1PREFETCH_ALL, 8*128
	zloop	35, b3b			;; Test loop counter
	jmp	zpass2_2304_real_done

zpass2_2304_real_3:
	mov	r8, DEST2ARG
	mov	rbp, DIST_TO_MULSRCARG
;;	zr64_hundredtwentyeight_real_with_mult_preload
;;	zr64_hundredtwentyeight_real_with_mult rsi, 8*128, 1*128, 2*128, 4*128, rdx, 0*ZMM_SCD4, rdi, 0*ZMM_SCD4, 1
	CALLP	zreal_mult
	zloop_init 35			;; 35 more iterations
	zloop_set_rw_prefetch ZSEXT	;; Prefetch 210 cache lines
	zr64_sixtyfour_complex_with_mult_preload
b3c:	zr64_sixtyfour_complex_with_mult rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 1, L1PREFETCH_ALL, 8*128
	zloop	35, b3c			;; Test loop counter
	add	rsi, r8
	jmp	zpass2_2304_real_unfft

zpass2_2304_real_4:
	mov	rbp, DIST_TO_MULSRCARG
	mov	rdx, sincos3		;; Real sin/cos data
	mov	rdi, xsincos_complex	;; Complex sin/cos data
	bump	rdx, 48*ZMM_SCD3+8*ZMM_SCD3
	bump	rdi, 48*ZMM_SCD3+8*ZMM_SCD3
;;	zr64f_hundredtwentyeight_real_with_mulf_preload
;;	zr64f_hundredtwentyeight_real_with_mulf rsi, 8*128, 1*128, 2*128, 4*128, rdx, 0*ZMM_SCD4, rdi, 0*ZMM_SCD4, 1
	CALLP	zreal_mulf
	zloop_init 35			;; 35 more iterations
	zloop_set_rw_prefetch ZSEXT	;; Prefetch 210 cache lines
	zr64f_sixtyfour_complex_with_mulf_preload
b3d:	zr64f_sixtyfour_complex_with_mulf rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 1, L1PREFETCH_ALL, 8*128
	zloop	35, b3d			;; Test loop counter
zpass2_2304_real_unfft:
	bump	rsi, -36*8*128		;; Restore source pointer

;; Do inverse FFT levels 5,6,7
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	64	+1	...
;;	8	...
;;	...
;;	56
;;	128
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	384	+1	...
;;	8	...
;;	...
;;	376
;;	768
;;	...

;; Do 48 twelve_reals_unfft operations
;; Do 336 six_complex_unfft operations
;;	distance between fft data elements is 64
;;	do 48 macros each processing 96 data values

	zloop_init 48			;; 6 iterations of 8
	zloop_set_rw_prefetch ZDOUBLE	;; Prefetch 96 cache lines
	bump	rdx, -8*ZMM_SCD3
	bump	rdi, -8*ZMM_SCD3
	zr6_2sc_twelve_reals_unfft_preload
c2a:	zr6_2sc_twelve_reals_unfft rsi, 128, 8*128, rdx, ZMM_SCD3, rdi, ZMM_SCD3, 8
	zloop	8, c2a, -8*128+48*128, rdi, -8*ZMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer
	loops_reset			;; 5 more iters of 8
	zr6_six_complex_djbunfft_preload
c2b:	zr6_six_complex_djbunfft rsi, 128, 8*128, rdi, ZMM_SCD3, 8
	zloop	8, c2b, -8*128+48*128, rdi, -8*ZMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	6, c2b, -6*48*128	;; Test loop counter, restore source pointer

;; Do inverse FFT levels 1,2,3,4
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	384	+1	...
;;	8	...
;;	...
;;	376
;;	768
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	2304	+1	...
;;	8	...
;;	...
;;	2296

;; Do 384 twelve_reals_last_unfft operations
;;	distance between fft data elements is 384
;;	do 48 macros each processing 96 data values

	bump	rdx, -8*ZMM_SCD3-48*ZMM_SCD3
	bump	rdi, -48*ZMM_SCD3
	zloop_init 48			;; 48 iterations
	zloop_set_rw_prefetch ZDOUBLE	;; Prefetch 96 cache lines
	zr6_2sc_twelve_reals_unfft_preload
c1b:	zr6_2sc_twelve_reals_unfft rsi, 128, 48*128, rdx, ZMM_SCD3, rdi, ZMM_SCD3, 48
	zloop	48, c1b			;; Test loop counter

zpass2_2304_real_done:
	end_timer 2
	ENDM

; *************** radix8-pass2-2304-complex macro ******************
; This macro takes 2304 complex values and performs the final 11.17 levels of a standard
; radix-8 FFT, squares the results, and does the first 11.17 levels of the inverse FFT.

zr4dwpn_pass2_2304_complex MACRO
	LOCAL	b1b, b2b, b3a, b3b, b3c, b3d, c1b, c2b
	LOCAL	zpass2_2304_complex_1, zpass2_2304_complex_2
	LOCAL	zpass2_2304_complex_3, zpass2_2304_complex_4
	LOCAL	zpass2_2304_complex_unfft, zpass2_2304_complex_done

;; Type 4 FFTs skip the forward FFT process

	mov	rsi, DATA_ADDR		;; Load address of FFT data
	mov	rbx, DIST_TO_FFTSRCARG
	mov	rcx, DATA_PREFETCH	;; Prefetch pointer (we need to prefetch 576 cache lines + ?? pad cache lines)
	cmp	ffttype, 4
	je	zpass2_2304_complex_4

;; Do FFT levels 1,2,3
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	2304	+1	...
;;	8	...
;;	...
;;	2296
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	384	+1	...
;;	8	...
;;	...
;;	376
;;	768
;;	...

;; Do 384 six_complex_fft operations
;;	distance between fft data elements is 384
;;	do 48 macros each processing 96 data values

	start_timer 4
	mov	rdi, xsincos_complex	;; The sin/cos data
	zloop_init 48			;; 48 iterations
	zloop_set_rw_prefetch ZDOUBLE	;; Prefetch 96 cache lines
	zr6f_six_complex_djbfft_preload
b1b:	zr6f_six_complex_djbfft rsi, 128, 48*128, rdi, ZMM_SCD3, 48, L1PREFETCH_ALL, 128
	zloop	48, b1b, -48*128	;; Test loop counter, restore source pointer
	end_timer 4

;; Do FFT levels 4,5,6
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	384	+1	...
;;	8	...
;;	...
;;	376
;;	768
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	64	+1	...
;;	8	...
;;	...
;;	56
;;	128
;;	...

;; Do 384 six_complex_fft operations
;;	distance between fft data elements is 64
;;	do 48 macros each processing 96 data values

	start_timer 5
	zloop_init 48			;; 6 iterations of 8
	zloop_set_rw_prefetch ZDOUBLE	;; Prefetch 96 cache lines
	zr6_six_complex_djbfft_preload
b2b:	zr6_six_complex_djbfft rsi, 128, 8*128, rdi, ZMM_SCD3, 8, L1PREFETCH_ALL, 128
	zloop	8, b2b, -8*128+48*128, rdi, -8*ZMM_SCD3	;; Test loop counter, restore source pointer, restore sin/cos pointer
	zloop	6, b2b, -6*48*128, rdi, 8*ZMM_SCD3 ;; Test loop counter, restore source pointer, next sin/cos pointer
	end_timer 5

;; Do FFT levels 7,8,9,10,11,12 as well as inverse FFT levels 7,8,9,10,11,12
;; On input and output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	64	+1	...
;;	8	...
;;	...
;;	56
;;	128
;;	...

;; Do 36 sixtyfour_complex_fft operations
;;	distance between fft data elements is 1 (8 for swizzling)
;;	do 36 macros each processing 128 data values

;; Execute the right middle step

pfing	bump	rcx, -24*64		;; We will prefetch 24 cache lines too many
	cmp	ffttype, 2
	jl	zpass2_2304_complex_1
	jg	zpass2_2304_complex_3

zpass2_2304_complex_2:
	start_timer 9
	zloop_init 36			;; 36 iterations
	zloop_set_rw_prefetch ZSEXT	;; Prefetch 216 cache lines
	mov	r8, DEST2ARG
	zr64_sixtyfour_complex_with_square_preload
b3a:	zr64_sixtyfour_complex_with_square rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 36, L1PREFETCH_ALL, 8*128
	zloop	36, b3a			;; Test loop counter
	add	rsi, r8
	end_timer 9
	jmp	zpass2_2304_complex_unfft

zpass2_2304_complex_1:
	start_timer 9
	zloop_init 36			;; 36 iterations
	zloop_set_rw_prefetch ZSEXT	;; Prefetch 216 cache lines
	zr64_sixtyfour_complex_fft_final_preload
b3b:	zr64_sixtyfour_complex_fft_final rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 36, L1PREFETCH_ALL, 8*128
	zloop	36, b3b			;; Test loop counter
	end_timer 9
	jmp	zpass2_2304_complex_done

zpass2_2304_complex_3:
	start_timer 9
	zloop_init 36			;; 36 iterations
	zloop_set_rw_prefetch ZSEXT	;; Prefetch 216 cache lines
	mov	r8, DEST2ARG
	mov	rbp, DIST_TO_MULSRCARG
	zr64_sixtyfour_complex_with_mult_preload
b3c:	zr64_sixtyfour_complex_with_mult rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 36, L1PREFETCH_ALL, 8*128
	zloop	36, b3c			;; Test loop counter
	add	rsi, r8
	end_timer 9
	jmp	zpass2_2304_complex_unfft

zpass2_2304_complex_4:
	start_timer 9
	zloop_init 36			;; 36 iterations
	zloop_set_rw_prefetch ZSEXT	;; Prefetch 216 cache lines
	mov	rbp, DIST_TO_MULSRCARG
	mov	rdi, xsincos_complex	;; The sin/cos data
	bump	rdi, 48*ZMM_SCD3+8*ZMM_SCD3
	zr64f_sixtyfour_complex_with_mulf_preload
b3d:	zr64f_sixtyfour_complex_with_mulf rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 36, L1PREFETCH_ALL, 8*128
	zloop	36, b3d			;; Test loop counter
	end_timer 9
zpass2_2304_complex_unfft:
	bump	rsi, -36*8*128		;; Restore source pointer

;; Do inverse FFT levels 4,5,6
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	64	+1	...
;;	8	...
;;	...
;;	56
;;	128
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	384	+1	...
;;	8	...
;;	...
;;	376
;;	768
;;	...

;; Do 384 six_complex_unfft operations
;;	distance between fft data elements is 64
;;	do 48 macros each processing 96 data values

	start_timer 13
	bump	rdi, -8*ZMM_SCD3
	zloop_init 48			;; 6 iterations of 8
	zloop_set_rw_prefetch ZDOUBLE	;; Prefetch 96 cache lines
	zr6_six_complex_djbunfft_preload
c2b:	zr6_six_complex_djbunfft rsi, 128, 8*128, rdi, ZMM_SCD3, 8, L1PREFETCH_ALL, 128
	zloop	8, c2b, -8*128+48*128, rdi, -8*ZMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	6, c2b, -6*48*128	;; Test loop counter, restore source pointer
	end_timer 13

;; Do inverse FFT levels 1,2,3
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	384	+1	...
;;	8	...
;;	...
;;	376
;;	768
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	2304	+1	...
;;	8	...
;;	...
;;	2296

;; Do 384 six_complex_unfft operations
;;	distance between fft data elements is 384
;;	do 64 macros each processing 96 data values

	start_timer 14
	bump	rdi, -48*ZMM_SCD3
	zloop_init 48			;; 48 iterations
	zloop_set_rw_prefetch ZDOUBLE	;; Prefetch 96 cache lines
	zr6_six_complex_djbunfft_preload
c1b:	zr6_six_complex_djbunfft rsi, 128, 48*128, rdi, ZMM_SCD3, 48, L1PREFETCH_ALL, 128
	zloop	48, c1b			;; Test loop counter
	end_timer 14

zpass2_2304_complex_done:
	ENDM



; *************** radix8-pass2-2560-real macro ******************
; This macro takes 5120 real values and performs the final 12.322 levels of the FFT.
; The results are squared, and the first 12.322 levels of the inverse FFT are done.

zr4dwpn_pass2_2560_real MACRO
	LOCAL	b1b, b2a, b2b, b3a, b3b, b3c, b3d, c1b, c2a, c2b
	LOCAL	zpass2_2560_real_1, zpass2_2560_real_2
	LOCAL	zpass2_2560_real_3, zpass2_2560_real_4
	LOCAL	zpass2_2560_real_unfft, zpass2_2560_real_done

;; Type 4 FFTs skip the forward FFT process

	start_timer 2
	mov	rsi, DATA_ADDR		;; Load source address
	mov	rbx, DIST_TO_FFTSRCARG
	mov	rcx, DATA_PREFETCH	;; Prefetch pointer (we need to prefetch 640 cache lines + ?? pad cache lines)
	cmp	ffttype, 4
	je	zpass2_2560_real_4

;; Do FFT levels 1,2,3,4
;;	0	+1	+1	+1	+1	+1	+1	+1	2560	+1	...
;;	8	...
;;	...
;;	2552
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	512	+1	...
;;	8	...
;;	...
;;	504
;;	1K
;;	...

;; Do 512 ten_real_fft operations
;;	distance between fft data elements is 512
;;	do 64 macros each processing 80 data values

	mov	rdx, sincos3		;; Real sin/cos data
	mov	rdi, xsincos_complex	;; Complex sin/cos data
	zloop_init 64			;; 64 iterations
	zloop_set_rw_prefetch ZDOUBLE	;; Prefetch 128 cache lines
	zr5f_2sc_ten_reals_fft_preload
b1b:	zr5f_2sc_ten_reals_fft rsi, 128, 64*128, rdx, ZMM_SCD2, rdi, ZMM_SCD2, 64
	zloop	64, b1b, -64*128	;; Test loop counter, restore source pointer

;; Do FFT levels 5,6,7
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	512	+1	...
;;	8	...
;;	...
;;	504
;;	1K
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	64	+1	...
;;	8	...
;;	...
;;	56
;;	128
;;	...

;; Do 40 sixteen_reals_fft operations
;; Do 280 eight_complex_fft operations
;;	distance between fft data elements is 64
;;	do 40 macros each processing 128 data values

	zloop_init 40			;; 5 iterations of 8
	zloop_set_rw_prefetch ZDOUBLE	;; Prefetch 80 cache lines
	zr8_2sc_sixteen_reals_fft_preload
b2a:	zr8_2sc_sixteen_reals_fft rsi, 128, 8*128, 16*128, 32*128, rdx, ZMM_SCD4, rdi, ZMM_SCD4, 8
	zloop	8, b2a, -8*128+64*128, rdi, -8*ZMM_SCD4 ;; Test loop counter, next source pointer, restore sin/cos pointer
	loops_reset			;; 4 more iters of 8
	zr8_eight_complex_djbfft_preload
b2b:	zr8_eight_complex_djbfft rsi, 128, 8*128, 16*128, 32*128, rdi, ZMM_SCD4, 8
	zloop	8, b2b, -8*128+64*128, rdi, -8*ZMM_SCD4 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	5, b2b, -5*64*128, rdi, 8*ZMM_SCD4 ;; Test loop counter, restore source pointer, next sin/cos pointer

;; Do FFT levels 8,9,10,11,12,13 as well as inverse FFT levels 8,9,10,11,12,13
;; On input and output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	64	+1	...
;;	8	...
;;	...
;;	56
;;	128
;;	...

;; Do 1 hundredtwentyeight_reals_fft operation
;; Do 39 sixtyfour_complex_fft operations
;;	distance between fft data elements is 1 (8 for swizzling)
;;	do 40 macros each processing 128 data values

;; Execute the proper middle step

pfing	bump	rcx, -10*64		;; We will prefetch 10 cache lines too many
	cmp	ffttype, 2
	jl	zpass2_2560_real_1
	jg	zpass2_2560_real_3

zpass2_2560_real_2:
	mov	r8, DEST2ARG
;;	zr64_hundredtwentyeight_real_with_square_preload
;;	zr64_hundredtwentyeight_real_with_square rsi, 8*128, 1*128, 2*128, 4*128, rdx, 0*ZMM_SCD4, rdi, 0*ZMM_SCD4, 1
	CALLP	zreal_square
	zloop_init 40;, 1, 40		;; 39 more iterations
	zloop_adjust 1
	zloop_set_rw_prefetch ZSEXT	;; Prefetch 240 cache lines
	zr64_sixtyfour_complex_with_square_preload
b3a:	zr64_sixtyfour_complex_with_square rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 1, L1PREFETCH_ALL, 8*128
	zloop	40, b3a			;; Test loop counter
	add	rsi, r8
	jmp	zpass2_2560_real_unfft

zpass2_2560_real_1:
;;	zr64_hundredtwentyeight_real_fft_final_preload
;;	zr64_hundredtwentyeight_real_fft_final rsi, 8*128, 1*128, 2*128, 4*128, rdx, 0*ZMM_SCD4, rdi, 0*ZMM_SCD4, 1
	CALLP	zreal_fft_final
	zloop_init 40;, 1, 40		;; 39 more iterations
	zloop_adjust 1
	zloop_set_rw_prefetch ZSEXT	;; Prefetch 240 cache lines
	zr64_sixtyfour_complex_fft_final_preload
b3b:	zr64_sixtyfour_complex_fft_final rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 1, L1PREFETCH_ALL, 8*128
	zloop	40, b3b			;; Test loop counter
	jmp	zpass2_2560_real_done

zpass2_2560_real_3:
	mov	r8, DEST2ARG
	mov	rbp, DIST_TO_MULSRCARG
;;	zr64_hundredtwentyeight_real_with_mult_preload
;;	zr64_hundredtwentyeight_real_with_mult rsi, 8*128, 1*128, 2*128, 4*128, rdx, 0*ZMM_SCD4, rdi, 0*ZMM_SCD4, 1
	CALLP	zreal_mult
	zloop_init 40;, 1, 40		;; 39 more iterations
	zloop_adjust 1
	zloop_set_rw_prefetch ZSEXT	;; Prefetch 240 cache lines
	zr64_sixtyfour_complex_with_mult_preload
b3c:	zr64_sixtyfour_complex_with_mult rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 1, L1PREFETCH_ALL, 8*128
	zloop	40, b3c			;; Test loop counter
	add	rsi, r8
	jmp	zpass2_2560_real_unfft

zpass2_2560_real_4:
	mov	rbp, DIST_TO_MULSRCARG
	mov	rdx, sincos3		;; Real sin/cos data
	mov	rdi, xsincos_complex	;; Complex sin/cos data
	bump	rdx, 64*ZMM_SCD2+8*ZMM_SCD4
	bump	rdi, 64*ZMM_SCD2+8*ZMM_SCD4
;;	zr64f_hundredtwentyeight_real_with_mulf_preload
;;	zr64f_hundredtwentyeight_real_with_mulf rsi, 8*128, 1*128, 2*128, 4*128, rdx, 0*ZMM_SCD4, rdi, 0*ZMM_SCD4, 1
	CALLP	zreal_mulf
	zloop_init 40;, 1, 40		;; 39 more iterations
	zloop_adjust 1
	zloop_set_rw_prefetch ZSEXT	;; Prefetch 240 cache lines
	zr64f_sixtyfour_complex_with_mulf_preload
b3d:	zr64f_sixtyfour_complex_with_mulf rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 1, L1PREFETCH_ALL, 8*128
	zloop	40, b3d			;; Test loop counter
zpass2_2560_real_unfft:
	bump	rsi, -40*8*128		;; Restore source pointer

;; Do inverse FFT levels 5,6,7
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	64	+1	...
;;	8	...
;;	...
;;	56
;;	128
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	512	+1	...
;;	8	...
;;	...
;;	504
;;	1K
;;	...

;; Do 40 sixteen_reals_unfft operations
;; Do 280 eight_complex_unfft operations
;;	distance between fft data elements is 64
;;	do 40 macros each processing 128 data values

	zloop_init 40			;; 5 iterations of 8
	zloop_set_rw_prefetch ZDOUBLE	;; Prefetch 80 cache lines
	bump	rdx, -8*ZMM_SCD4
	bump	rdi, -8*ZMM_SCD4
	zr8_2sc_sixteen_reals_unfft_preload
c2a:	zr8_2sc_sixteen_reals_unfft rsi, 128, 8*128, 16*128, 32*128, rdx, ZMM_SCD4, rdi, ZMM_SCD4, 8
	zloop	8, c2a, -8*128+64*128, rdi, -8*ZMM_SCD4 ;; Test loop counter, next source pointer, restore sin/cos pointer
	loops_reset			;; 4 more iters of 8
	zr8_eight_complex_djbunfft_preload
c2b:	zr8_eight_complex_djbunfft rsi, 128, 8*128, 16*128, 32*128, rdi, ZMM_SCD4, 8
	zloop	8, c2b, -8*128+64*128, rdi, -8*ZMM_SCD4 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	5, c2b, -5*64*128	;; Test loop counter, restore source pointer

;; Do inverse FFT levels 1,2,3,4
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	512	+1	...
;;	8	...
;;	...
;;	504
;;	1K
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	2560	+1	...
;;	8	...
;;	...
;;	2552

;; Do 512 ten_reals_last_unfft operations
;;	distance between fft data elements is 512
;;	do 64 macros each processing 80 data values

	bump	rdx, -8*ZMM_SCD4-64*ZMM_SCD2
	bump	rdi, -64*ZMM_SCD2
	zloop_init 64			;; 64 iterations
	zloop_set_rw_prefetch ZDOUBLE	;; Prefetch 128 cache lines
	zr5_2sc_ten_reals_unfft_preload
c1b:	zr5_2sc_ten_reals_unfft rsi, 128, 64*128, rdx, ZMM_SCD2, rdi, ZMM_SCD2, 64
	zloop	64, c1b			;; Test loop counter

zpass2_2560_real_done:
	end_timer 2
	ENDM

; *************** radix8-pass2-2560-complex macro ******************
; This macro takes 2560 complex values and performs the final 11.322 levels of a standard
; radix-8 FFT, squares the results, and does the first 11.322 levels of the inverse FFT.

zr4dwpn_pass2_2560_complex MACRO
	LOCAL	b1b, b2b, b3a, b3b, b3c, b3d, c1b, c2b
	LOCAL	zpass2_2560_complex_1, zpass2_2560_complex_2
	LOCAL	zpass2_2560_complex_3, zpass2_2560_complex_4
	LOCAL	zpass2_2560_complex_unfft, zpass2_2560_complex_done

;; Type 4 FFTs skip the forward FFT process

	mov	rsi, DATA_ADDR		;; Load address of FFT data
	mov	rbx, DIST_TO_FFTSRCARG
	mov	rcx, DATA_PREFETCH	;; Prefetch pointer (we need to prefetch 640 cache lines + ?? pad cache lines)
	cmp	ffttype, 4
	je	zpass2_2560_complex_4

;; Do FFT levels 1,2,3
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	2560	+1	...
;;	8	...
;;	...
;;	2552
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	512	+1	...
;;	8	...
;;	...
;;	504
;;	1K
;;	...

;; Do 512 five_complex_fft operations
;;	distance between fft data elements is 512
;;	do 64 macros each processing 80 data values

	start_timer 4
	mov	rdi, xsincos_complex	;; The sin/cos data
	zloop_init 64			;; 64 iterations
	zloop_set_rw_prefetch ZDOUBLE	;; Prefetch 128 cache lines
	zr5f_five_complex_djbfft_preload
b1b:	zr5f_five_complex_djbfft rsi, 128, 64*128, rdi, ZMM_SCD2, 64, L1PREFETCH_ALL, 128
	zloop	64, b1b, -64*128	;; Test loop counter, restore source pointer
	end_timer 4

;; Do FFT levels 4,5,6
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	512	+1	...
;;	8	...
;;	...
;;	504
;;	1K
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	64	+1	...
;;	8	...
;;	...
;;	56
;;	128
;;	...

;; Do 320 eight_complex_fft operations
;;	distance between fft data elements is 64
;;	do 40 macros each processing 128 data values

	start_timer 5
	zloop_init 40			;; 5 iterations of 8
	zloop_set_rw_prefetch ZDOUBLE	;; Prefetch 80 cache lines
	zr8_eight_complex_djbfft_preload
b2b:	zr8_eight_complex_djbfft rsi, 128, 8*128, 16*128, 32*128, rdi, ZMM_SCD4, 8, L1PREFETCH_ALL, 128
	zloop	8, b2b, -8*128+64*128, rdi, -8*ZMM_SCD4	;; Test loop counter, restore source pointer, restore sin/cos pointer
	zloop	5, b2b, -5*64*128, rdi, 8*ZMM_SCD4 ;; Test loop counter, restore source pointer, next sin/cos pointer
	end_timer 5

;; Do FFT levels 7,8,9,10,11,12 as well as inverse FFT levels 7,8,9,10,11,12
;; On input and output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	64	+1	...
;;	8	...
;;	...
;;	56
;;	128
;;	...

;; Do 40 sixtyfour_complex_fft operations
;;	distance between fft data elements is 1 (8 for swizzling)
;;	do 40 macros each processing 128 data values

;; Execute the right middle step

pfing	bump	rcx, -16*64		;; We will prefetch 16 cache lines too many
	cmp	ffttype, 2
	jl	zpass2_2560_complex_1
	jg	zpass2_2560_complex_3

zpass2_2560_complex_2:
	start_timer 9
	zloop_init 40			;; 40 iterations
	zloop_set_rw_prefetch ZSEXT	;; Prefetch 240 cache lines
	mov	r8, DEST2ARG
	zr64_sixtyfour_complex_with_square_preload
b3a:	zr64_sixtyfour_complex_with_square rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 40, L1PREFETCH_ALL, 8*128
	zloop	40, b3a			;; Test loop counter
	add	rsi, r8
	end_timer 9
	jmp	zpass2_2560_complex_unfft

zpass2_2560_complex_1:
	start_timer 9
	zloop_init 40			;; 40 iterations
	zloop_set_rw_prefetch ZSEXT	;; Prefetch 240 cache lines
	zr64_sixtyfour_complex_fft_final_preload
b3b:	zr64_sixtyfour_complex_fft_final rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 40, L1PREFETCH_ALL, 8*128
	zloop	40, b3b			;; Test loop counter
	end_timer 9
	jmp	zpass2_2560_complex_done

zpass2_2560_complex_3:
	start_timer 9
	zloop_init 40			;; 40 iterations
	zloop_set_rw_prefetch ZSEXT	;; Prefetch 240 cache lines
	mov	r8, DEST2ARG
	mov	rbp, DIST_TO_MULSRCARG
	zr64_sixtyfour_complex_with_mult_preload
b3c:	zr64_sixtyfour_complex_with_mult rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 40, L1PREFETCH_ALL, 8*128
	zloop	40, b3c			;; Test loop counter
	add	rsi, r8
	end_timer 9
	jmp	zpass2_2560_complex_unfft

zpass2_2560_complex_4:
	start_timer 9
	zloop_init 40			;; 40 iterations
	zloop_set_rw_prefetch ZSEXT	;; Prefetch 240 cache lines
	mov	rbp, DIST_TO_MULSRCARG
	mov	rdi, xsincos_complex	;; The sin/cos data
	bump	rdi, 64*ZMM_SCD2+8*ZMM_SCD4
	zr64f_sixtyfour_complex_with_mulf_preload
b3d:	zr64f_sixtyfour_complex_with_mulf rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 40, L1PREFETCH_ALL, 8*128
	zloop	40, b3d			;; Test loop counter
	end_timer 9
zpass2_2560_complex_unfft:
	bump	rsi, -40*8*128		;; Restore source pointer

;; Do inverse FFT levels 4,5,6
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	64	+1	...
;;	8	...
;;	...
;;	56
;;	128
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	512	+1	...
;;	8	...
;;	...
;;	504
;;	1K
;;	...

;; Do 320 eight_complex_unfft operations
;;	distance between fft data elements is 64
;;	do 40 macros each processing 128 data values

	start_timer 13
	bump	rdi, -8*ZMM_SCD4
	zloop_init 40			;; 5 iterations of 8
	zloop_set_rw_prefetch ZDOUBLE	;; Prefetch 80 cache lines
	zr8_eight_complex_djbunfft_preload
c2b:	zr8_eight_complex_djbunfft rsi, 128, 8*128, 16*128, 32*128, rdi, ZMM_SCD4, 8, L1PREFETCH_ALL, 128
	zloop	8, c2b, -8*128+64*128, rdi, -8*ZMM_SCD4 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	5, c2b, -5*64*128	;; Test loop counter, restore source pointer
	end_timer 13

;; Do inverse FFT levels 1,2,3
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	512	+1	...
;;	8	...
;;	...
;;	504
;;	1K
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	2560	+1	...
;;	8	...
;;	...
;;	2552

;; Do 512 five_complex_unfft operations
;;	distance between fft data elements is 512
;;	do 64 macros each processing 80 data values

	start_timer 14
	bump	rdi, -64*ZMM_SCD2
	zloop_init 64			;; 64 iterations
	zloop_set_rw_prefetch ZDOUBLE	;; Prefetch 128 cache lines
	zr5_five_complex_djbunfft_preload
c1b:	zr5_five_complex_djbunfft rsi, 128, 64*128, rdi, ZMM_SCD2, 64, L1PREFETCH_ALL, 128
	zloop	64, c1b			;; Test loop counter
	end_timer 14

zpass2_2560_complex_done:
	ENDM


; *************** radix8-pass2-2688-real macro ******************
; This macro takes 5376 real values and performs the final 12.392 levels of the FFT.
; The results are squared, and the first 12.392 levels of the inverse FFT are done.

zr4dwpn_pass2_2688_real MACRO
	LOCAL	b1b, b2a, b2b, b3a, b3b, b3c, b3d, c1b, c2a, c2b
	LOCAL	zpass2_2688_real_1, zpass2_2688_real_2
	LOCAL	zpass2_2688_real_3, zpass2_2688_real_4
	LOCAL	zpass2_2688_real_unfft, zpass2_2688_real_done

;; Type 4 FFTs skip the forward FFT process

	start_timer 2
	mov	rsi, DATA_ADDR		;; Load source address
	mov	rbx, DIST_TO_FFTSRCARG
	mov	rcx, DATA_PREFETCH	;; Prefetch pointer (we need to prefetch 672 cache lines + ?? pad cache lines)
	cmp	ffttype, 4
	je	zpass2_2688_real_4

;; Do FFT levels 1,2,3,4
;;	0	+1	+1	+1	+1	+1	+1	+1	2688	+1	...
;;	8	...
;;	...
;;	2680
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	384	+1	...
;;	8	...
;;	...
;;	376
;;	768
;;	...

;; Do 384 fourteen_real_fft operations
;;	distance between fft data elements is 384
;;	do 48 macros each processing 112 data values

	mov	rdx, sincos3		;; Real sin/cos data
	mov	rdi, xsincos_complex	;; Complex sin/cos data
	zloop_init 48			;; 48 iterations
	zloop_set_rw_prefetch ZDOUBLE	;; Prefetch 96 cache lines
	zr7f_2sc_fourteen_reals_fft_preload
b1b:	zr7f_2sc_fourteen_reals_fft rsi, 128, 48*128, rdx, ZMM_SCD3, rdi, ZMM_SCD3, 48
	zloop	48, b1b, -48*128	;; Test loop counter, restore source pointer

;; Do FFT levels 5,6,7
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	384	+1	...
;;	8	...
;;	...
;;	380
;;	768
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	64	+1	...
;;	8	...
;;	...
;;	56
;;	128
;;	...

;; Do 64 twelve_reals_fft operations
;; Do 384 six_complex_fft operations
;;	distance between fft data elements is 64
;;	do 56 macros each processing 96 data values

	zloop_init 56			;; 7 iterations of 8
	zloop_set_rw_prefetch ZDOUBLE	;; Prefetch 112 cache lines
	zr6_2sc_twelve_reals_fft_preload
b2a:	zr6_2sc_twelve_reals_fft rsi, 128, 8*128, rdx, ZMM_SCD3, rdi, ZMM_SCD3, 8
	zloop	8, b2a, -8*128+48*128, rdi, -8*ZMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer
	loops_reset			;; 6 more iters of 8
	zr6_six_complex_djbfft_preload
b2b:	zr6_six_complex_djbfft rsi, 128, 8*128, rdi, ZMM_SCD3, 8
	zloop	8, b2b, -8*128+48*128, rdi, -8*ZMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	7, b2b, -7*48*128, rdi, 8*ZMM_SCD3 ;; Test loop counter, restore source pointer, next sin/cos pointer

;; Do FFT levels 8,9,10,11,12,13 as well as inverse FFT levels 8,9,10,11,12,13
;; On input and output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	64	+1	...
;;	8	...
;;	...
;;	56
;;	128
;;	...

;; Do 1 hundredtwentyeight_reals_fft operation
;; Do 41 sixtyfour_complex_fft operations
;;	distance between fft data elements is 1 (8 for swizzling)
;;	do 42 macros each processing 128 data values

;; Execute the proper middle step

;;pfing	bump	rcx, -3*64		;; We will prefetch 10 cache lines too few
	cmp	ffttype, 2
	jl	zpass2_2688_real_1
	jg	zpass2_2688_real_3

zpass2_2688_real_2:
	mov	r8, DEST2ARG
;;	zr64_hundredtwentyeight_real_with_square_preload
;;	zr64_hundredtwentyeight_real_with_square rsi, 8*128, 1*128, 2*128, 4*128, rdx, 0*ZMM_SCD4, rdi, 0*ZMM_SCD4, 1
	CALLP	zreal_square
	zloop_init 41			;; 41 more iterations
	zloop_set_rw_prefetch ZSEXT	;; Prefetch 246 cache lines
	zr64_sixtyfour_complex_with_square_preload
b3a:	zr64_sixtyfour_complex_with_square rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 1, L1PREFETCH_ALL, 8*128
	zloop	41, b3a			;; Test loop counter
	add	rsi, r8
	jmp	zpass2_2688_real_unfft

zpass2_2688_real_1:
;;	zr64_hundredtwentyeight_real_fft_final_preload
;;	zr64_hundredtwentyeight_real_fft_final rsi, 8*128, 1*128, 2*128, 4*128, rdx, 0*ZMM_SCD4, rdi, 0*ZMM_SCD4, 1
	CALLP	zreal_fft_final
	zloop_init 41			;; 41 more iterations
	zloop_set_rw_prefetch ZSEXT	;; Prefetch 246 cache lines
	zr64_sixtyfour_complex_fft_final_preload
b3b:	zr64_sixtyfour_complex_fft_final rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 1, L1PREFETCH_ALL, 8*128
	zloop	41, b3b			;; Test loop counter
	jmp	zpass2_2688_real_done

zpass2_2688_real_3:
	mov	r8, DEST2ARG
	mov	rbp, DIST_TO_MULSRCARG
;;	zr64_hundredtwentyeight_real_with_mult_preload
;;	zr64_hundredtwentyeight_real_with_mult rsi, 8*128, 1*128, 2*128, 4*128, rdx, 0*ZMM_SCD4, rdi, 0*ZMM_SCD4, 1
	CALLP	zreal_mult
	zloop_init 41			;; 41 more iterations
	zloop_set_rw_prefetch ZSEXT	;; Prefetch 246 cache lines
	zr64_sixtyfour_complex_with_mult_preload
b3c:	zr64_sixtyfour_complex_with_mult rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 1, L1PREFETCH_ALL, 8*128
	zloop	41, b3c			;; Test loop counter
	add	rsi, r8
	jmp	zpass2_2688_real_unfft

zpass2_2688_real_4:
	mov	rbp, DIST_TO_MULSRCARG
	mov	rdx, sincos3		;; Real sin/cos data
	mov	rdi, xsincos_complex	;; Complex sin/cos data
	bump	rdx, 48*ZMM_SCD3+8*ZMM_SCD3
	bump	rdi, 48*ZMM_SCD3+8*ZMM_SCD3
;;	zr64f_hundredtwentyeight_real_with_mulf_preload
;;	zr64f_hundredtwentyeight_real_with_mulf rsi, 8*128, 1*128, 2*128, 4*128, rdx, 0*ZMM_SCD4, rdi, 0*ZMM_SCD4, 1
	CALLP	zreal_mulf
	zloop_init 41			;; 41 more iterations
	zloop_set_rw_prefetch ZSEXT	;; Prefetch 246 cache lines
	zr64f_sixtyfour_complex_with_mulf_preload
b3d:	zr64f_sixtyfour_complex_with_mulf rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 1, L1PREFETCH_ALL, 8*128
	zloop	41, b3d			;; Test loop counter
zpass2_2688_real_unfft:
	bump	rsi, -42*8*128		;; Restore source pointer

;; Do inverse FFT levels 5,6,7
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	64	+1	...
;;	8	...
;;	...
;;	56
;;	128
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	384	+1	...
;;	8	...
;;	...
;;	376
;;	768
;;	...

;; Do 64 twelve_reals_unfft operations
;; Do 384 six_complex_unfft operations
;;	distance between fft data elements is 64
;;	do 56 macros each processing 96 data values

	zloop_init 56			;; 7 iterations of 8
	zloop_set_rw_prefetch ZDOUBLE	;; Prefetch 112 cache lines
	bump	rdx, -8*ZMM_SCD3
	bump	rdi, -8*ZMM_SCD3
	zr6_2sc_twelve_reals_unfft_preload
c2a:	zr6_2sc_twelve_reals_unfft rsi, 128, 8*128, rdx, ZMM_SCD3, rdi, ZMM_SCD3, 8
	zloop	8, c2a, -8*128+48*128, rdi, -8*ZMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer
	loops_reset			;; 6 more iters of 8
	zr6_six_complex_djbunfft_preload
c2b:	zr6_six_complex_djbunfft rsi, 128, 8*128, rdi, ZMM_SCD3, 8
	zloop	8, c2b, -8*128+48*128, rdi, -8*ZMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	7, c2b, -7*48*128	;; Test loop counter, restore source pointer

;; Do inverse FFT levels 1,2,3,4
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	384	+1	...
;;	8	...
;;	...
;;	376
;;	768
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	2688	+1	...
;;	8	...
;;	...
;;	2680

;; Do 384 fourteen_reals_last_unfft operations
;;	distance between fft data elements is 384
;;	do 48 macros each processing 112 data values

	bump	rdx, -8*ZMM_SCD3-48*ZMM_SCD3
	bump	rdi, -48*ZMM_SCD3
	zloop_init 48			;; 48 iterations
	zloop_set_rw_prefetch ZDOUBLE	;; Prefetch 96 cache lines
	zr7_2sc_fourteen_reals_unfft_preload
c1b:	zr7_2sc_fourteen_reals_unfft rsi, 128, 48*128, rdx, ZMM_SCD3, rdi, ZMM_SCD3, 48
	zloop	48, c1b			;; Test loop counter

zpass2_2688_real_done:
	end_timer 2
	ENDM

; *************** radix8-pass2-2688-complex macro ******************
; This macro takes 2688 complex values and performs the final 11.392 levels of a standard
; radix-8 FFT, squares the results, and does the first 11.392 levels of the inverse FFT.

zr4dwpn_pass2_2688_complex MACRO
	LOCAL	b1b, b2b, b3a, b3b, b3c, b3d, c1b, c2b
	LOCAL	zpass2_2688_complex_1, zpass2_2688_complex_2
	LOCAL	zpass2_2688_complex_3, zpass2_2688_complex_4
	LOCAL	zpass2_2688_complex_unfft, zpass2_2688_complex_done

;; Type 4 FFTs skip the forward FFT process

	mov	rsi, DATA_ADDR		;; Load address of FFT data
	mov	rbx, DIST_TO_FFTSRCARG
	mov	rcx, DATA_PREFETCH	;; Prefetch pointer (we need to prefetch 672 cache lines + ?? pad cache lines)
	cmp	ffttype, 4
	je	zpass2_2688_complex_4

;; Do FFT levels 1,2,3
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	2688	+1	...
;;	8	...
;;	...
;;	2680
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	384	+1	...
;;	8	...
;;	...
;;	376
;;	768
;;	...

;; Do 284 seven_complex_fft operations
;;	distance between fft data elements is 384
;;	do 48 macros each processing 112 data values

	start_timer 4
	mov	rdi, xsincos_complex	;; The sin/cos data
	zloop_init 48			;; 48 iterations
	zloop_set_rw_prefetch ZDOUBLE	;; Prefetch 96 cache lines
	zr7f_seven_complex_djbfft_preload
b1b:	zr7f_seven_complex_djbfft rsi, 128, 48*128, rdi, ZMM_SCD3, 48, L1PREFETCH_ALL, 128
	zloop	48, b1b, -48*128	;; Test loop counter, restore source pointer
	end_timer 4

;; Do FFT levels 4,5,6
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	384	+1	...
;;	8	...
;;	...
;;	376
;;	768
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	64	+1	...
;;	8	...
;;	...
;;	56
;;	128
;;	...

;; Do 448 six_complex_fft operations
;;	distance between fft data elements is 64
;;	do 56 macros each processing 96 data values

	start_timer 5
	zloop_init 56			;; 7 iterations of 8
	zloop_set_rw_prefetch ZDOUBLE	;; Prefetch 112 cache lines
	zr6_six_complex_djbfft_preload
b2b:	zr6_six_complex_djbfft rsi, 128, 8*128, rdi, ZMM_SCD3, 8, L1PREFETCH_ALL, 128
	zloop	8, b2b, -8*128+48*128, rdi, -8*ZMM_SCD3	;; Test loop counter, restore source pointer, restore sin/cos pointer
	zloop	7, b2b, -7*48*128, rdi, 8*ZMM_SCD3 ;; Test loop counter, restore source pointer, next sin/cos pointer
	end_timer 5

;; Do FFT levels 7,8,9,10,11,12 as well as inverse FFT levels 7,8,9,10,11,12
;; On input and output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	64	+1	...
;;	8	...
;;	...
;;	56
;;	128
;;	...

;; Do 42 sixtyfour_complex_fft operations
;;	distance between fft data elements is 1 (8 for swizzling)
;;	do 42 macros each processing 128 data values

;; Execute the right middle step

;;pfing	bump	rcx, -3*64		;; We will prefetch 4 cache lines too few
	cmp	ffttype, 2
	jl	zpass2_2688_complex_1
	jg	zpass2_2688_complex_3

zpass2_2688_complex_2:
	start_timer 9
	zloop_init 42			;; 42 iterations
	zloop_set_rw_prefetch ZSEXT	;; Prefetch 252 cache lines
	mov	r8, DEST2ARG
	zr64_sixtyfour_complex_with_square_preload
b3a:	zr64_sixtyfour_complex_with_square rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 42, L1PREFETCH_ALL, 8*128
	zloop	42, b3a			;; Test loop counter
	add	rsi, r8
	end_timer 9
	jmp	zpass2_2688_complex_unfft

zpass2_2688_complex_1:
	start_timer 9
	zloop_init 42			;; 42 iterations
	zloop_set_rw_prefetch ZSEXT	;; Prefetch 252 cache lines
	zr64_sixtyfour_complex_fft_final_preload
b3b:	zr64_sixtyfour_complex_fft_final rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 42, L1PREFETCH_ALL, 8*128
	zloop	42, b3b			;; Test loop counter
	end_timer 9
	jmp	zpass2_2688_complex_done

zpass2_2688_complex_3:
	start_timer 9
	zloop_init 42			;; 42 iterations
	zloop_set_rw_prefetch ZSEXT	;; Prefetch 252 cache lines
	mov	r8, DEST2ARG
	mov	rbp, DIST_TO_MULSRCARG
	zr64_sixtyfour_complex_with_mult_preload
b3c:	zr64_sixtyfour_complex_with_mult rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 42, L1PREFETCH_ALL, 8*128
	zloop	42, b3c			;; Test loop counter
	add	rsi, r8
	end_timer 9
	jmp	zpass2_2688_complex_unfft

zpass2_2688_complex_4:
	start_timer 9
	zloop_init 42			;; 42 iterations
	zloop_set_rw_prefetch ZSEXT	;; Prefetch 252 cache lines
	mov	rbp, DIST_TO_MULSRCARG
	mov	rdi, xsincos_complex	;; The sin/cos data
	bump	rdi, 48*ZMM_SCD3+8*ZMM_SCD3
	zr64f_sixtyfour_complex_with_mulf_preload
b3d:	zr64f_sixtyfour_complex_with_mulf rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 42, L1PREFETCH_ALL, 8*128
	zloop	42, b3d			;; Test loop counter
	end_timer 9
zpass2_2688_complex_unfft:
	bump	rsi, -42*8*128		;; Restore source pointer

;; Do inverse FFT levels 4,5,6
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	64	+1	...
;;	8	...
;;	...
;;	56
;;	128
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	384	+1	...
;;	8	...
;;	...
;;	376
;;	768
;;	...

;; Do 448 six_complex_unfft operations
;;	distance between fft data elements is 64
;;	do 56 macros each processing 96 data values

	start_timer 13
	bump	rdi, -8*ZMM_SCD3
	zloop_init 56			;; 7 iterations of 8
	zloop_set_rw_prefetch ZDOUBLE	;; Prefetch 112 cache lines
	zr6_six_complex_djbunfft_preload
c2b:	zr6_six_complex_djbunfft rsi, 128, 8*128, rdi, ZMM_SCD3, 8, L1PREFETCH_ALL, 128
	zloop	8, c2b, -8*128+48*128, rdi, -8*ZMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	7, c2b, -7*48*128	;; Test loop counter, restore source pointer
	end_timer 13

;; Do inverse FFT levels 1,2,3
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	384	+1	...
;;	8	...
;;	...
;;	376
;;	768
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	2688	+1	...
;;	8	...
;;	...
;;	2680

;; Do 384 seven_complex_unfft operations
;;	distance between fft data elements is 384
;;	do 64 macros each processing 112 data values

	start_timer 14
	bump	rdi, -48*ZMM_SCD3
	zloop_init 48			;; 48 iterations
	zloop_set_rw_prefetch ZDOUBLE	;; Prefetch 96 cache lines
	zr7_seven_complex_djbunfft_preload
c1b:	zr7_seven_complex_djbunfft rsi, 128, 48*128, rdi, ZMM_SCD3, 48, L1PREFETCH_ALL, 128
	zloop	48, c1b			;; Test loop counter
	end_timer 14

zpass2_2688_complex_done:
	ENDM


; *************** radix8-pass2-3072-real macro ******************
; This macro takes 6144 real values and performs the final 12.585 levels of the FFT.
; The results are squared, and the first 12.585 levels of the inverse FFT are done.

zr4dwpn_pass2_3072_real MACRO
	LOCAL	b1b, b2a, b2b, b3a, b3b, b3c, b3d, c1b, c2a, c2b
	LOCAL	zpass2_3072_real_1, zpass2_3072_real_2
	LOCAL	zpass2_3072_real_3, zpass2_3072_real_4
	LOCAL	zpass2_3072_real_unfft, zpass2_3072_real_done

;; Type 4 FFTs skip the forward FFT process

	start_timer 2
	mov	rsi, DATA_ADDR		;; Load source address
	mov	rbx, DIST_TO_FFTSRCARG
	mov	rcx, DATA_PREFETCH	;; Prefetch pointer (we need to prefetch 768 cache lines + 11 pad cache lines)
	cmp	ffttype, 4
	je	zpass2_3072_real_4

;; Do FFT levels 1,2,3,4
;;	0	+1	+1	+1	+1	+1	+1	+1	3072	+1	...
;;	8	...
;;	...
;;	3064
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	512	+1	...
;;	8	...
;;	...
;;	504
;;	1K
;;	...

;; Do 512 twelve_real_fft operations
;;	distance between fft data elements is 512
;;	do 64 macros each processing 96 data values

	mov	rdx, sincos3		;; Real sin/cos data
	mov	rdi, xsincos_complex	;; Complex sin/cos data
	zloop_init 64			;; 64 iterations
	zloop_set_rw_prefetch ZDOUBLE	;; Prefetch 128 cache lines
	zr6f_2sc_twelve_reals_fft_preload
b1b:	zr6f_2sc_twelve_reals_fft rsi, 128, 2*dist32, rdx, ZMM_SCD3, rdi, ZMM_SCD3, 32
	zloop	32, b1b, -32*128+dist32	;; Test loop counter, next source pointer
	zloop	2, b1b, -2*dist32	;; Test loop counter, restore source pointer

;; Do FFT levels 5,6,7
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	512	+1	...
;;	8	...
;;	...
;;	504
;;	1K
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	64	+1	...
;;	8	...
;;	...
;;	56
;;	128
;;	...

;; Do 64 sixteen_reals_fft operations
;; Do 320 eight_complex_fft operations
;;	distance between fft data elements is 64
;;	do 48 macros each processing 128 data values

	zloop_init 48			;; 6 iterations of 8
	zloop_set_rw_prefetch ZDOUBLE	;; Prefetch 96 cache lines
	zr8_2sc_sixteen_reals_fft_preload
b2a:	zr8_2sc_sixteen_reals_fft rsi, 128, 8*128, 16*128, dist32, rdx, ZMM_SCD4, rdi, ZMM_SCD4, 8
	zloop	8, b2a, -8*128+2*dist32, rdi, -8*ZMM_SCD4 ;; Test loop counter, next source pointer, restore sin/cos pointer
	loops_reset			;; 5 more iters of 8
	zr8_eight_complex_djbfft_preload
b2b:	zr8_eight_complex_djbfft rsi, 128, 8*128, 16*128, dist32, rdi, ZMM_SCD4, 8
	zloop	8, b2b, -8*128+2*dist32, rdi, -8*ZMM_SCD4 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	6, b2b, -6*2*dist32, rdi, 8*ZMM_SCD4 ;; Test loop counter, restore source pointer, next sin/cos pointer

;; Do FFT levels 8,9,10,11,12,13 as well as inverse FFT levels 8,9,10,11,12,13
;; On input and output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	64	+1	...
;;	8	...
;;	...
;;	56
;;	128
;;	...

;; Do 1 hundredtwentyeight_reals_fft operation
;; Do 47 sixtyfour_complex_fft operations
;;	distance between fft data elements is 1 (8 for swizzling)
;;	do 48 macros each processing 128 data values

;; Execute the proper middle step

;pfing	bump	rcx, -5*64		;; We will prefetch 2 cache lines too few
	cmp	ffttype, 2
	jl	zpass2_3072_real_1
	jg	zpass2_3072_real_3

zpass2_3072_real_2:
	mov	r8, DEST2ARG
;;	zr64_hundredtwentyeight_real_with_square_preload
;;	zr64_hundredtwentyeight_real_with_square rsi, 8*128, 1*128, 2*128, 4*128, rdx, 0*ZMM_SCD4, rdi, 0*ZMM_SCD4, 1
	CALLP	zreal_square
	zloop_init 48, 1, 4		;; 47 more iterations
	zloop_set_rw_prefetch ZSEPT	;; Prefetch 329 cache lines
	zr64_sixtyfour_complex_with_square_preload
b3a:	zr64_sixtyfour_complex_with_square rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 1, L1PREFETCH_ALL, 8*128
	zloop	4, b3a, -4*8*128+dist32	;; Test loop counter, next source pointer
	zloop	12, b3a			;; Test loop counter
	add	rsi, r8
	jmp	zpass2_3072_real_unfft

zpass2_3072_real_1:
;;	zr64_hundredtwentyeight_real_fft_final_preload
;;	zr64_hundredtwentyeight_real_fft_final rsi, 8*128, 1*128, 2*128, 4*128, rdx, 0*ZMM_SCD4, rdi, 0*ZMM_SCD4, 1
	CALLP	zreal_fft_final
	zloop_init 48, 1, 4		;; 47 more iterations
	zloop_set_rw_prefetch ZSEPT	;; Prefetch 329 cache lines
	zr64_sixtyfour_complex_fft_final_preload
b3b:	zr64_sixtyfour_complex_fft_final rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 1, L1PREFETCH_ALL, 8*128
	zloop	4, b3b, -4*8*128+dist32	;; Test loop counter, next source pointer
	zloop	12, b3b			;; Test loop counter
	jmp	zpass2_3072_real_done

zpass2_3072_real_3:
	mov	r8, DEST2ARG
	mov	rbp, DIST_TO_MULSRCARG
;;	zr64_hundredtwentyeight_real_with_mult_preload
;;	zr64_hundredtwentyeight_real_with_mult rsi, 8*128, 1*128, 2*128, 4*128, rdx, 0*ZMM_SCD4, rdi, 0*ZMM_SCD4, 1
	CALLP	zreal_mult
	zloop_init 48, 1, 4		;; 47 more iterations
	zloop_set_rw_prefetch ZSEPT	;; Prefetch 329 cache lines
	zr64_sixtyfour_complex_with_mult_preload
b3c:	zr64_sixtyfour_complex_with_mult rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 1, L1PREFETCH_ALL, 8*128
	zloop	4, b3c, -4*8*128+dist32	;; Test loop counter, next source pointer
	zloop	12, b3c			;; Test loop counter
	add	rsi, r8
	jmp	zpass2_3072_real_unfft

zpass2_3072_real_4:
	mov	rbp, DIST_TO_MULSRCARG
	mov	rdx, sincos3		;; Real sin/cos data
	mov	rdi, xsincos_complex	;; Complex sin/cos data
	bump	rdx, 64*ZMM_SCD3+8*ZMM_SCD4
	bump	rdi, 64*ZMM_SCD3+8*ZMM_SCD4
;;	zr64f_hundredtwentyeight_real_with_mulf_preload
;;	zr64f_hundredtwentyeight_real_with_mulf rsi, 8*128, 1*128, 2*128, 4*128, rdx, 0*ZMM_SCD4, rdi, 0*ZMM_SCD4, 1
	CALLP	zreal_mulf
	zloop_init 48, 1, 4		;; 47 more iterations
	zloop_set_rw_prefetch ZSEPT	;; Prefetch 329 cache lines
	zr64f_sixtyfour_complex_with_mulf_preload
b3d:	zr64f_sixtyfour_complex_with_mulf rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 1, L1PREFETCH_ALL, 8*128
	zloop	4, b3d, -4*8*128+dist32	;; Test loop counter, next source pointer
	zloop	12, b3d			;; Test loop counter
zpass2_3072_real_unfft:
	bump	rsi, -12*dist32		;; Restore source pointer

;; Do inverse FFT levels 5,6,7
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	64	+1	...
;;	8	...
;;	...
;;	56
;;	128
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	512	+1	...
;;	8	...
;;	...
;;	504
;;	1K
;;	...

;; Do 64 sixteen_reals_unfft operations
;; Do 320 eight_complex_unfft operations
;;	distance between fft data elements is 64
;;	do 48 macros each processing 128 data values

	zloop_init 48			;; 6 iterations of 8
	zloop_set_rw_prefetch ZDOUBLE	;; Prefetch 96 cache lines
	bump	rdx, -8*ZMM_SCD4
	bump	rdi, -8*ZMM_SCD4
	zr8_2sc_sixteen_reals_unfft_preload
c2a:	zr8_2sc_sixteen_reals_unfft rsi, 128, 8*128, 16*128, dist32, rdx, ZMM_SCD4, rdi, ZMM_SCD4, 8
	zloop	8, c2a, -8*128+2*dist32, rdi, -8*ZMM_SCD4 ;; Test loop counter, next source pointer, restore sin/cos pointer
	loops_reset			;; 5 more iters of 8
	zr8_eight_complex_djbunfft_preload
c2b:	zr8_eight_complex_djbunfft rsi, 128, 8*128, 16*128, dist32, rdi, ZMM_SCD4, 8
	zloop	8, c2b, -8*128+2*dist32, rdi, -8*ZMM_SCD4 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	6, c2b, -6*2*dist32	;; Test loop counter, restore source pointer

;; Do inverse FFT levels 1,2,3,4
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	512	+1	...
;;	8	...
;;	...
;;	504
;;	1K
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	3072	+1	...
;;	8	...
;;	...
;;	3064

;; Do 512 twelve_reals_last_unfft operations
;;	distance between fft data elements is 512
;;	do 64 macros each processing 96 data values

	bump	rdx, -8*ZMM_SCD4-64*ZMM_SCD3
	bump	rdi, -64*ZMM_SCD3
	zloop_init 64			;; 64 iterations
	zloop_set_rw_prefetch ZDOUBLE	;; Prefetch 128 cache lines
	zr6_2sc_twelve_reals_unfft_preload
c1b:	zr6_2sc_twelve_reals_unfft rsi, 128, 2*dist32, rdx, ZMM_SCD3, rdi, ZMM_SCD3, 32
	zloop	32, c1b, -32*128+dist32	;; Test loop counter, next source pointer
	zloop	2, c1b			;; Test loop counter

zpass2_3072_real_done:
	end_timer 2
	ENDM

; *************** radix8-pass2-3072-complex macro ******************
; This macro takes 3072 complex values and performs the final 11.585 levels of a standard
; radix-8 FFT, squares the results, and does the first 11.585 levels of the inverse FFT.

zr4dwpn_pass2_3072_complex MACRO
	LOCAL	b1b, b2b, b3a, b3b, b3c, b3d, c1b, c2b
	LOCAL	zpass2_3072_complex_1, zpass2_3072_complex_2
	LOCAL	zpass2_3072_complex_3, zpass2_3072_complex_4
	LOCAL	zpass2_3072_complex_unfft, zpass2_3072_complex_done

;; Type 4 FFTs skip the forward FFT process

	mov	rsi, DATA_ADDR		;; Load address of FFT data
	mov	rbx, DIST_TO_FFTSRCARG
	mov	rcx, DATA_PREFETCH	;; Prefetch pointer (we need to prefetch 768 cache lines + 11 pad cache lines)
	cmp	ffttype, 4
	je	zpass2_3072_complex_4

;; Do FFT levels 1,2,3
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	3072	+1	...
;;	8	...
;;	...
;;	3064
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	512	+1	...
;;	8	...
;;	...
;;	504
;;	1K
;;	...

;; Do 512 six_complex_fft operations
;;	distance between fft data elements is 512
;;	do 64 macros each processing 96 data values

	start_timer 4
	mov	rdi, xsincos_complex	;; The sin/cos data
	zloop_init 64			;; 64 iterations
	zloop_set_rw_prefetch ZDOUBLE	;; Prefetch 128 cache lines
	zr6f_six_complex_djbfft_preload
b1b:	zr6f_six_complex_djbfft rsi, 128, 2*dist32, rdi, ZMM_SCD3, 32, L1PREFETCH_ALL, 128
	zloop	32, b1b, -32*128+dist32	;; Test loop counter, next source pointer
	zloop	2, b1b, -2*dist32	;; Test loop counter, restore source pointer
	end_timer 4

;; Do FFT levels 4,5,6
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	512	+1	...
;;	8	...
;;	...
;;	504
;;	1K
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	64	+1	...
;;	8	...
;;	...
;;	56
;;	128
;;	...

;; Do 384 eight_complex_fft operations
;;	distance between fft data elements is 64
;;	do 48 macros each processing 128 data values

	start_timer 5
	zloop_init 48			;; 6 iterations of 8
	zloop_set_rw_prefetch ZDOUBLE	;; Prefetch 96 cache lines
	zr8_eight_complex_djbfft_preload
b2b:	zr8_eight_complex_djbfft rsi, 128, 8*128, 16*128, dist32, rdi, ZMM_SCD4, 8, L1PREFETCH_ALL, 128
	zloop	8, b2b, -8*128+2*dist32, rdi, -8*ZMM_SCD4	;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	6, b2b, -6*2*dist32, rdi, 8*ZMM_SCD4 ;; Test loop counter, restore source pointer, next sin/cos pointer
	end_timer 5

;; Do FFT levels 7,8,9,10,11,12 as well as inverse FFT levels 7,8,9,10,11,12
;; On input and output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	64	+1	...
;;	8	...
;;	...
;;	56
;;	128
;;	...

;; Do 48 sixtyfour_complex_fft operations
;;	distance between fft data elements is 1 (8 for swizzling)
;;	do 48 macros each processing 128 data values

;; Execute the right middle step

pfing	bump	rcx, -5*64		;; We will prefetch 5 cache lines too many
	cmp	ffttype, 2
	jl	zpass2_3072_complex_1
	jg	zpass2_3072_complex_3

zpass2_3072_complex_2:
	start_timer 9
	zloop_init 48			;; 48 iterations
	zloop_set_rw_prefetch ZSEPT	;; Prefetch 336 cache lines
	mov	r8, DEST2ARG
	zr64_sixtyfour_complex_with_square_preload
b3a:	zr64_sixtyfour_complex_with_square rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 4, L1PREFETCH_ALL, 8*128
	zloop	4, b3a, -4*8*128+dist32	;; Test loop counter, next source pointer
	zloop	12, b3a			;; Test loop counter
	add	rsi, r8
	end_timer 9
	jmp	zpass2_3072_complex_unfft

zpass2_3072_complex_1:
	start_timer 9
	zloop_init 48			;; 48 iterations
	zloop_set_rw_prefetch ZSEPT	;; Prefetch 336 cache lines
	zr64_sixtyfour_complex_fft_final_preload
b3b:	zr64_sixtyfour_complex_fft_final rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 4, L1PREFETCH_ALL, 8*128
	zloop	4, b3b, -4*8*128+dist32	;; Test loop counter, next source pointer
	zloop	12, b3b			;; Test loop counter
	end_timer 9
	jmp	zpass2_3072_complex_done

zpass2_3072_complex_3:
	start_timer 9
	zloop_init 48			;; 48 iterations
	zloop_set_rw_prefetch ZSEPT	;; Prefetch 336 cache lines
	mov	r8, DEST2ARG
	mov	rbp, DIST_TO_MULSRCARG
	zr64_sixtyfour_complex_with_mult_preload
b3c:	zr64_sixtyfour_complex_with_mult rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 4, L1PREFETCH_ALL, 8*128
	zloop	4, b3c, -4*8*128+dist32	;; Test loop counter, next source pointer
	zloop	12, b3c			;; Test loop counter
	add	rsi, r8
	end_timer 9
	jmp	zpass2_3072_complex_unfft

zpass2_3072_complex_4:
	start_timer 9
	zloop_init 48			;; 48 iterations
	zloop_set_rw_prefetch ZSEPT	;; Prefetch 336 cache lines
	mov	rbp, DIST_TO_MULSRCARG
	mov	rdi, xsincos_complex	;; The sin/cos data
	bump	rdi, 64*ZMM_SCD3+8*ZMM_SCD4
	zr64f_sixtyfour_complex_with_mulf_preload
b3d:	zr64f_sixtyfour_complex_with_mulf rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 4, L1PREFETCH_ALL, 8*128
	zloop	4, b3d, -4*8*128+dist32	;; Test loop counter, next source pointer
	zloop	12, b3d			;; Test loop counter
	end_timer 9
zpass2_3072_complex_unfft:
	bump	rsi, -12*dist32		;; Restore source pointer

;; Do inverse FFT levels 4,5,6
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	64	+1	...
;;	8	...
;;	...
;;	56
;;	128
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	512	+1	...
;;	8	...
;;	...
;;	504
;;	1K
;;	...

;; Do 384 eight_complex_unfft operations
;;	distance between fft data elements is 64
;;	do 48 macros each processing 128 data values

	start_timer 13
	bump	rdi, -8*ZMM_SCD4
	zloop_init 48			;; 6 iterations of 8
	zloop_set_rw_prefetch ZDOUBLE	;; Prefetch 96 cache lines
	zr8_eight_complex_djbunfft_preload
c2b:	zr8_eight_complex_djbunfft rsi, 128, 8*128, 16*128, dist32, rdi, ZMM_SCD4, 8, L1PREFETCH_ALL, 128
	zloop	8, c2b, -8*128+2*dist32, rdi, -8*ZMM_SCD4 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	6, c2b, -6*2*dist32	;; Test loop counter, restore source pointer
	end_timer 13

;; Do inverse FFT levels 1,2,3
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	512	+1	...
;;	8	...
;;	...
;;	504
;;	1K
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	3072	+1	...
;;	8	...
;;	...
;;	3064

;; Do 512 six_complex_unfft operations
;;	distance between fft data elements is 512
;;	do 64 macros each processing 96 data values

	start_timer 14
	bump	rdi, -64*ZMM_SCD3
	zloop_init 64			;; 64 iterations
	zloop_set_rw_prefetch ZDOUBLE	;; Prefetch 128 cache lines
	zr6_six_complex_djbunfft_preload
c1b:	zr6_six_complex_djbunfft rsi, 128, 2*dist32, rdi, ZMM_SCD3, 32, L1PREFETCH_ALL, 128
	zloop	32, c1b, -32*128+dist32	;; Test loop counter, next source pointer
	zloop	2, c1b			;; Test loop counter
	end_timer 14

zpass2_3072_complex_done:
	ENDM


; *************** radix8-pass2-3136-real macro ******************
; This macro takes 6272 real values and performs the final 12.615 levels of the FFT.
; The results are squared, and the first 12.615 levels of the inverse FFT are done.

zr4dwpn_pass2_3136_real MACRO
	LOCAL	b1b, b2a, b2b, b3a, b3b, b3c, b3d, c1b, c2a, c2b
	LOCAL	zpass2_3136_real_1, zpass2_3136_real_2
	LOCAL	zpass2_3136_real_3, zpass2_3136_real_4
	LOCAL	zpass2_3136_real_unfft, zpass2_3136_real_done

;; Type 4 FFTs skip the forward FFT process

	start_timer 2
	mov	rsi, DATA_ADDR		;; Load source address
	mov	rbx, DIST_TO_FFTSRCARG
	mov	rcx, DATA_PREFETCH	;; Prefetch pointer (we need to prefetch 784 cache lines + ?? pad cache lines)
	cmp	ffttype, 4
	je	zpass2_3136_real_4

;; Do FFT levels 1,2,3,4
;;	0	+1	+1	+1	+1	+1	+1	+1	3136	+1	...
;;	8	...
;;	...
;;	3128
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	448	+1	...
;;	8	...
;;	...
;;	440
;;	896
;;	...

;; Do 448 fourteen_real_fft operations
;;	distance between fft data elements is 448
;;	do 56 macros each processing 112 data values

	mov	rdx, sincos3		;; Real sin/cos data
	mov	rdi, xsincos_complex	;; Complex sin/cos data
	zloop_init 56			;; 56 iterations
	zloop_set_rw_prefetch ZDOUBLE	;; Prefetch 112 cache lines
	zr7f_2sc_fourteen_reals_fft_preload
b1b:	zr7f_2sc_fourteen_reals_fft rsi, 128, 56*128, rdx, ZMM_SCD3, rdi, ZMM_SCD3, 56
	zloop	56, b1b, -56*128	;; Test loop counter, restore source pointer

;; Do FFT levels 5,6,7
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	448	+1	...
;;	8	...
;;	...
;;	440
;;	896
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	64	+1	...
;;	8	...
;;	...
;;	56
;;	128
;;	...

;; Do 64 fourteen_reals_fft operations
;; Do 384 seven_complex_fft operations
;;	distance between fft data elements is 64
;;	do 56 macros each processing 112 data values

	zloop_init 56			;; 7 iterations of 8
	zloop_set_rw_prefetch ZDOUBLE	;; Prefetch 112 cache lines
	zr7_2sc_fourteen_reals_fft_preload
b2a:	zr7_2sc_fourteen_reals_fft rsi, 128, 8*128, rdx, ZMM_SCD3, rdi, ZMM_SCD3, 8
	zloop	8, b2a, -8*128+56*128, rdi, -8*ZMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer
	loops_reset			;; 6 more iters of 8
	zr7_seven_complex_djbfft_preload
b2b:	zr7_seven_complex_djbfft rsi, 128, 8*128, rdi, ZMM_SCD3, 8
	zloop	8, b2b, -8*128+56*128, rdi, -8*ZMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	7, b2b, -7*56*128, rdi, 8*ZMM_SCD3 ;; Test loop counter, restore source pointer, next sin/cos pointer

;; Do FFT levels 8,9,10,11,12,13 as well as inverse FFT levels 8,9,10,11,12,13
;; On input and output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	64	+1	...
;;	8	...
;;	...
;;	56
;;	128
;;	...

;; Do 1 hundredtwentyeight_reals_fft operation
;; Do 48 sixtyfour_complex_fft operations
;;	distance between fft data elements is 1 (8 for swizzling)
;;	do 49 macros each processing 128 data values

;; Execute the proper middle step

;pfing	bump	rcx, -7*64		;; We will prefetch 0 cache lines too many
	cmp	ffttype, 2
	jl	zpass2_3136_real_1
	jg	zpass2_3136_real_3

zpass2_3136_real_2:
	mov	r8, DEST2ARG
;;	zr64_hundredtwentyeight_real_with_square_preload
;;	zr64_hundredtwentyeight_real_with_square rsi, 8*128, 1*128, 2*128, 4*128, rdx, 0*ZMM_SCD4, rdi, 0*ZMM_SCD4, 1
	CALLP	zreal_square
	zloop_init 48			;; 48 more iterations
	zloop_set_rw_prefetch ZSEPT	;; Prefetch 336 cache lines
	zr64_sixtyfour_complex_with_square_preload
b3a:	zr64_sixtyfour_complex_with_square rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 1, L1PREFETCH_ALL, 8*128
	zloop	48, b3a			;; Test loop counter
	add	rsi, r8
	jmp	zpass2_3136_real_unfft

zpass2_3136_real_1:
;;	zr64_hundredtwentyeight_real_fft_final_preload
;;	zr64_hundredtwentyeight_real_fft_final rsi, 8*128, 1*128, 2*128, 4*128, rdx, 0*ZMM_SCD4, rdi, 0*ZMM_SCD4, 1
	CALLP	zreal_fft_final
	zloop_init 48			;; 48 more iterations
	zloop_set_rw_prefetch ZSEPT	;; Prefetch 336 cache lines
	zr64_sixtyfour_complex_fft_final_preload
b3b:	zr64_sixtyfour_complex_fft_final rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 1, L1PREFETCH_ALL, 8*128
	zloop	48, b3b			;; Test loop counter
	jmp	zpass2_3136_real_done

zpass2_3136_real_3:
	mov	r8, DEST2ARG
	mov	rbp, DIST_TO_MULSRCARG
;;	zr64_hundredtwentyeight_real_with_mult_preload
;;	zr64_hundredtwentyeight_real_with_mult rsi, 8*128, 1*128, 2*128, 4*128, rdx, 0*ZMM_SCD4, rdi, 0*ZMM_SCD4, 1
	CALLP	zreal_mult
	zloop_init 48			;; 48 more iterations
	zloop_set_rw_prefetch ZSEPT	;; Prefetch 336 cache lines
	zr64_sixtyfour_complex_with_mult_preload
b3c:	zr64_sixtyfour_complex_with_mult rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 1, L1PREFETCH_ALL, 8*128
	zloop	48, b3c			;; Test loop counter
	add	rsi, r8
	jmp	zpass2_3136_real_unfft

zpass2_3136_real_4:
	mov	rbp, DIST_TO_MULSRCARG
	mov	rdx, sincos3		;; Real sin/cos data
	mov	rdi, xsincos_complex	;; Complex sin/cos data
	bump	rdx, 56*ZMM_SCD3+8*ZMM_SCD3
	bump	rdi, 56*ZMM_SCD3+8*ZMM_SCD3
;;	zr64f_hundredtwentyeight_real_with_mulf_preload
;;	zr64f_hundredtwentyeight_real_with_mulf rsi, 8*128, 1*128, 2*128, 4*128, rdx, 0*ZMM_SCD4, rdi, 0*ZMM_SCD4, 1
	CALLP	zreal_mulf
	zloop_init 48			;; 48 more iterations
	zloop_set_rw_prefetch ZSEPT	;; Prefetch 336 cache lines
	zr64f_sixtyfour_complex_with_mulf_preload
b3d:	zr64f_sixtyfour_complex_with_mulf rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 1, L1PREFETCH_ALL, 8*128
	zloop	48, b3d			;; Test loop counter
zpass2_3136_real_unfft:
	bump	rsi, -49*8*128		;; Restore source pointer

;; Do inverse FFT levels 5,6,7
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	64	+1	...
;;	8	...
;;	...
;;	56
;;	128
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	448	+1	...
;;	8	...
;;	...
;;	440
;;	896
;;	...

;; Do 64 fourteen_reals_unfft operations
;; Do 384 seven_complex_unfft operations
;;	distance between fft data elements is 64
;;	do 56 macros each processing 112 data values

	zloop_init 56			;; 7 iterations of 8
	zloop_set_rw_prefetch ZDOUBLE	;; Prefetch 112 cache lines
	bump	rdx, -8*ZMM_SCD3
	bump	rdi, -8*ZMM_SCD3
	zr7_2sc_fourteen_reals_unfft_preload
c2a:	zr7_2sc_fourteen_reals_unfft rsi, 128, 8*128, rdx, ZMM_SCD3, rdi, ZMM_SCD3, 8
	zloop	8, c2a, -8*128+56*128, rdi, -8*ZMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer
	loops_reset			;; 6 more iters of 8
	zr7_seven_complex_djbunfft_preload
c2b:	zr7_seven_complex_djbunfft rsi, 128, 8*128, rdi, ZMM_SCD3, 8
	zloop	8, c2b, -8*128+56*128, rdi, -8*ZMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	7, c2b, -7*56*128	;; Test loop counter, restore source pointer

;; Do inverse FFT levels 1,2,3,4
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	448	+1	...
;;	8	...
;;	...
;;	440
;;	896
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	3136	+1	...
;;	8	...
;;	...
;;	3128

;; Do 448 fourteen_reals_last_unfft operations
;;	distance between fft data elements is 448
;;	do 56 macros each processing 112 data values

	bump	rdx, -8*ZMM_SCD3-56*ZMM_SCD3
	bump	rdi, -56*ZMM_SCD3
	zloop_init 56			;; 56 iterations
	zloop_set_rw_prefetch ZDOUBLE	;; Prefetch 112 cache lines
	zr7_2sc_fourteen_reals_unfft_preload
c1b:	zr7_2sc_fourteen_reals_unfft rsi, 128, 56*128, rdx, ZMM_SCD3, rdi, ZMM_SCD3, 56
	zloop	56, c1b			;; Test loop counter

zpass2_3136_real_done:
	end_timer 2
	ENDM

; *************** radix8-pass2-3136-complex macro ******************
; This macro takes 3136 complex values and performs the final 11.615 levels of a standard
; radix-8 FFT, squares the results, and does the first 11.615 levels of the inverse FFT.

zr4dwpn_pass2_3136_complex MACRO
	LOCAL	b1b, b2b, b3a, b3b, b3c, b3d, c1b, c2b
	LOCAL	zpass2_3136_complex_1, zpass2_3136_complex_2
	LOCAL	zpass2_3136_complex_3, zpass2_3136_complex_4
	LOCAL	zpass2_3136_complex_unfft, zpass2_3136_complex_done

;; Type 4 FFTs skip the forward FFT process

	mov	rsi, DATA_ADDR		;; Load address of FFT data
	mov	rbx, DIST_TO_FFTSRCARG
	mov	rcx, DATA_PREFETCH	;; Prefetch pointer (we need to prefetch 784 cache lines + ?? pad cache lines)
	cmp	ffttype, 4
	je	zpass2_3136_complex_4

;; Do FFT levels 1,2,3
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	3136	+1	...
;;	8	...
;;	...
;;	3128
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	448	+1	...
;;	8	...
;;	...
;;	440
;;	896
;;	...

;; Do 448 seven_complex_fft operations
;;	distance between fft data elements is 448
;;	do 56 macros each processing 112 data values

	start_timer 4
	mov	rdi, xsincos_complex	;; The sin/cos data
	zloop_init 56			;; 56 iterations
	zloop_set_rw_prefetch ZDOUBLE	;; Prefetch 112 cache lines
	zr7f_seven_complex_djbfft_preload
b1b:	zr7f_seven_complex_djbfft rsi, 128, 56*128, rdi, ZMM_SCD3, 56, L1PREFETCH_ALL, 128
	zloop	56, b1b, -56*128	;; Test loop counter, restore source pointer
	end_timer 4

;; Do FFT levels 4,5,6
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	448	+1	...
;;	8	...
;;	...
;;	440
;;	896
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	64	+1	...
;;	8	...
;;	...
;;	56
;;	128
;;	...

;; Do 448 seven_complex_fft operations
;;	distance between fft data elements is 64
;;	do 56 macros each processing 112 data values

	start_timer 5
	zloop_init 56			;; 7 iterations of 8
	zloop_set_rw_prefetch ZDOUBLE	;; Prefetch 112 cache lines
	zr7_seven_complex_djbfft_preload
b2b:	zr7_seven_complex_djbfft rsi, 128, 8*128, rdi, ZMM_SCD3, 8, L1PREFETCH_ALL, 128
	zloop	8, b2b, -8*128+56*128, rdi, -8*ZMM_SCD3	;; Test loop counter, restore source pointer, restore sin/cos pointer
	zloop	7, b2b, -7*56*128, rdi, 8*ZMM_SCD3 ;; Test loop counter, restore source pointer, next sin/cos pointer
	end_timer 5

;; Do FFT levels 7,8,9,10,11,12 as well as inverse FFT levels 7,8,9,10,11,12
;; On input and output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	64	+1	...
;;	8	...
;;	...
;;	56
;;	128
;;	...

;; Do 49 sixtyfour_complex_fft operations
;;	distance between fft data elements is 1 (8 for swizzling)
;;	do 49 macros each processing 128 data values

;; Execute the right middle step

pfing	bump	rcx, -7*64		;; We will prefetch 7 cache lines too many
	cmp	ffttype, 2
	jl	zpass2_3136_complex_1
	jg	zpass2_3136_complex_3

zpass2_3136_complex_2:
	start_timer 9
	zloop_init 49			;; 49 iterations
	zloop_set_rw_prefetch ZSEPT	;; Prefetch 343 cache lines
	mov	r8, DEST2ARG
	zr64_sixtyfour_complex_with_square_preload
b3a:	zr64_sixtyfour_complex_with_square rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 49, L1PREFETCH_ALL, 8*128
	zloop	49, b3a			;; Test loop counter
	add	rsi, r8
	end_timer 9
	jmp	zpass2_3136_complex_unfft

zpass2_3136_complex_1:
	start_timer 9
	zloop_init 49			;; 49 iterations
	zloop_set_rw_prefetch ZSEPT	;; Prefetch 343 cache lines
	zr64_sixtyfour_complex_fft_final_preload
b3b:	zr64_sixtyfour_complex_fft_final rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 49, L1PREFETCH_ALL, 8*128
	zloop	49, b3b			;; Test loop counter
	end_timer 9
	jmp	zpass2_3136_complex_done

zpass2_3136_complex_3:
	start_timer 9
	zloop_init 49			;; 49 iterations
	zloop_set_rw_prefetch ZSEPT	;; Prefetch 343 cache lines
	mov	r8, DEST2ARG
	mov	rbp, DIST_TO_MULSRCARG
	zr64_sixtyfour_complex_with_mult_preload
b3c:	zr64_sixtyfour_complex_with_mult rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 49, L1PREFETCH_ALL, 8*128
	zloop	49, b3c			;; Test loop counter
	add	rsi, r8
	end_timer 9
	jmp	zpass2_3136_complex_unfft

zpass2_3136_complex_4:
	start_timer 9
	zloop_init 49			;; 49 iterations
	zloop_set_rw_prefetch ZSEPT	;; Prefetch 343 cache lines
	mov	rbp, DIST_TO_MULSRCARG
	mov	rdi, xsincos_complex	;; The sin/cos data
	bump	rdi, 56*ZMM_SCD3+8*ZMM_SCD3
	zr64f_sixtyfour_complex_with_mulf_preload
b3d:	zr64f_sixtyfour_complex_with_mulf rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 49, L1PREFETCH_ALL, 8*128
	zloop	49, b3d			;; Test loop counter
	end_timer 9
zpass2_3136_complex_unfft:
	bump	rsi, -49*8*128		;; Restore source pointer

;; Do inverse FFT levels 4,5,6
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	64	+1	...
;;	8	...
;;	...
;;	56
;;	128
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	448	+1	...
;;	8	...
;;	...
;;	440
;;	896
;;	...

;; Do 448 seven_complex_unfft operations
;;	distance between fft data elements is 64
;;	do 56 macros each processing 112 data values

	start_timer 13
	bump	rdi, -8*ZMM_SCD3
	zloop_init 56			;; 7 iterations of 8
	zloop_set_rw_prefetch ZDOUBLE	;; Prefetch 112 cache lines
	zr7_seven_complex_djbunfft_preload
c2b:	zr7_seven_complex_djbunfft rsi, 128, 8*128, rdi, ZMM_SCD3, 8, L1PREFETCH_ALL, 128
	zloop	8, c2b, -8*128+56*128, rdi, -8*ZMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	7, c2b, -7*56*128	;; Test loop counter, restore source pointer
	end_timer 13

;; Do inverse FFT levels 1,2,3
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	448	+1	...
;;	8	...
;;	...
;;	440
;;	896
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	3136	+1	...
;;	8	...
;;	...
;;	3128

;; Do 448 seven_complex_unfft operations
;;	distance between fft data elements is 448
;;	do 56 macros each processing 112 data values

	start_timer 14
	bump	rdi, -56*ZMM_SCD3
	zloop_init 56			;; 56 iterations
	zloop_set_rw_prefetch ZDOUBLE	;; Prefetch 112 cache lines
	zr7_seven_complex_djbunfft_preload
c1b:	zr7_seven_complex_djbunfft rsi, 128, 56*128, rdi, ZMM_SCD3, 56, L1PREFETCH_ALL, 128
	zloop	56, c1b			;; Test loop counter
	end_timer 14

zpass2_3136_complex_done:
	ENDM


; *************** radix8-pass2-3200-real macro ******************
; This macro takes 3200 real values and performs the final 12.644 levels of the FFT.
; The results are squared, and the first 12.644 levels of the inverse FFT are done.

zr4dwpn_pass2_3200_real MACRO
	LOCAL	b1b, b2a, b2b, b3a, b3b, b3c, b3d, c1b, c2a, c2b
	LOCAL	zpass2_3200_real_1, zpass2_3200_real_2
	LOCAL	zpass2_3200_real_3, zpass2_3200_real_4
	LOCAL	zpass2_3200_real_unfft, zpass2_3200_real_done

;; Type 4 FFTs skip the forward FFT process

	start_timer 2
	mov	rsi, DATA_ADDR		;; Load source address
	mov	rbx, DIST_TO_FFTSRCARG
	mov	rcx, DATA_PREFETCH	;; Prefetch pointer (we need to prefetch 800 cache lines + ?? pad cache lines)
	cmp	ffttype, 4
	je	zpass2_3200_real_4

;; Do FFT levels 1,2,3,4
;;	0	+1	+1	+1	+1	+1	+1	+1	3200	+1	...
;;	8	...
;;	...
;;	3192
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	640	+1	...
;;	8	...
;;	...
;;	632
;;	1280
;;	...

;; Do 640 ten_real_fft operations
;;	distance between fft data elements is 640
;;	do 80 macros each processing 80 data values

	mov	rdx, sincos3		;; Real sin/cos data
	mov	rdi, xsincos_complex	;; Complex sin/cos data
	zloop_init 80			;; 80 iterations
	zloop_set_rw_prefetch ZDOUBLE	;; Prefetch 160 cache lines
	zr5f_2sc_ten_reals_fft_preload
b1b:	zr5f_2sc_ten_reals_fft rsi, 128, 80*128, rdx, ZMM_SCD2, rdi, ZMM_SCD2, 80
	zloop	80, b1b, -80*128	;; Test loop counter, restore source pointer

;; Do FFT levels 5,6,7
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	640	+1	...
;;	8	...
;;	...
;;	632
;;	1280
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	64	+1	...
;;	8	...
;;	...
;;	56
;;	128
;;	...

;; Do 40 twenty_reals_fft operations
;; Do 280 ten_complex_fft operations
;;	distance between fft data elements is 64
;;	do 40 macros each processing 160 data values

	zloop_init 40			;; 5 iterations of 8
	zloop_set_rw_prefetch ZDOUBLE	;; Prefetch 80 cache lines
	zr10_2sc_twenty_reals_fft_preload
b2a:	zr10_2sc_twenty_reals_fft rsi, 128, 8*128, rdx, ZMM_SCD5, rdi, ZMM_SCD5, 8
	zloop	8, b2a, -8*128+80*128, rdi, -8*ZMM_SCD5 ;; Test loop counter, next source pointer, restore sin/cos pointer
	loops_reset			;; 4 more iters of 8
	zr10_ten_complex_djbfft_preload
b2b:	zr10_ten_complex_djbfft rsi, 128, 8*128, rdi, ZMM_SCD5, 8
	zloop	8, b2b, -8*128+80*128, rdi, -8*ZMM_SCD5 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	5, b2b, -5*80*128, rdi, 8*ZMM_SCD5 ;; Test loop counter, restore source pointer, next sin/cos pointer

;; Do FFT levels 8,9,10,11,12,13 as well as inverse FFT levels 8,9,10,11,12,13
;; On input and output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	64	+1	...
;;	8	...
;;	...
;;	56
;;	128
;;	...

;; Do 1 hundredtwentyeight_reals_fft operation
;; Do 49 sixtyfour_complex_fft operations
;;	distance between fft data elements is 1 (8 for swizzling)
;;	do 50 macros each processing 128 data values

;; Execute the proper middle step

pfing	bump	rcx, -23*64		;; We will prefetch 23 cache lines too many
	cmp	ffttype, 2
	jl	zpass2_3200_real_1
	jg	zpass2_3200_real_3

zpass2_3200_real_2:
	mov	r8, DEST2ARG
;;	zr64_hundredtwentyeight_real_with_square_preload
;;	zr64_hundredtwentyeight_real_with_square rsi, 8*128, 1*128, 2*128, 4*128, rdx, 0*ZMM_SCD4, rdi, 0*ZMM_SCD4, 1
	CALLP	zreal_square
	zloop_init 49			;; 49 more iterations
	zloop_set_rw_prefetch ZSEPT	;; Prefetch 343 cache lines
	zr64_sixtyfour_complex_with_square_preload
b3a:	zr64_sixtyfour_complex_with_square rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 49, L1PREFETCH_ALL, 8*128
	zloop	49, b3a			;; Test loop counter
	add	rsi, r8
	jmp	zpass2_3200_real_unfft

zpass2_3200_real_1:
;;	zr64_hundredtwentyeight_real_fft_final_preload
;;	zr64_hundredtwentyeight_real_fft_final rsi, 8*128, 1*128, 2*128, 4*128, rdx, 0*ZMM_SCD4, rdi, 0*ZMM_SCD4, 1
	CALLP	zreal_fft_final
	zloop_init 49			;; 49 more iterations
	zloop_set_rw_prefetch ZSEPT	;; Prefetch 343 cache lines
	zr64_sixtyfour_complex_fft_final_preload
b3b:	zr64_sixtyfour_complex_fft_final rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 49, L1PREFETCH_ALL, 8*128
	zloop	49, b3b			;; Test loop counter
	jmp	zpass2_3200_real_done

zpass2_3200_real_3:
	mov	r8, DEST2ARG
	mov	rbp, DIST_TO_MULSRCARG
;;	zr64_hundredtwentyeight_real_with_mult_preload
;;	zr64_hundredtwentyeight_real_with_mult rsi, 8*128, 1*128, 2*128, 4*128, rdx, 0*ZMM_SCD4, rdi, 0*ZMM_SCD4, 1
	CALLP	zreal_mult
	zloop_init 49			;; 49 more iterations
	zloop_set_rw_prefetch ZSEPT	;; Prefetch 343 cache lines
	zr64_sixtyfour_complex_with_mult_preload
b3c:	zr64_sixtyfour_complex_with_mult rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 49, L1PREFETCH_ALL, 8*128
	zloop	49, b3c			;; Test loop counter
	add	rsi, r8
	jmp	zpass2_3200_real_unfft

zpass2_3200_real_4:
	mov	rbp, DIST_TO_MULSRCARG
	mov	rdx, sincos3		;; Real sin/cos data
	mov	rdi, xsincos_complex	;; Complex sin/cos data
	bump	rdx, 80*ZMM_SCD2+8*ZMM_SCD5
	bump	rdi, 80*ZMM_SCD2+8*ZMM_SCD5
;;	zr64f_hundredtwentyeight_real_with_mulf_preload
;;	zr64f_hundredtwentyeight_real_with_mulf rsi, 8*128, 1*128, 2*128, 4*128, rdx, 0*ZMM_SCD4, rdi, 0*ZMM_SCD4, 1
	CALLP	zreal_mulf
	zloop_init 49			;; 49 more iterations
	zloop_set_rw_prefetch ZSEPT	;; Prefetch 343 cache lines
	zr64f_sixtyfour_complex_with_mulf_preload
b3d:	zr64f_sixtyfour_complex_with_mulf rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 49, L1PREFETCH_ALL, 8*128
	zloop	49, b3d			;; Test loop counter
zpass2_3200_real_unfft:
	bump	rsi, -50*8*128		;; Restore source pointer

;; Do inverse FFT levels 5,6,7
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	64	+1	...
;;	8	...
;;	...
;;	56
;;	128
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	640	+1	...
;;	8	...
;;	...
;;	632
;;	1280
;;	...

;; Do 40 twenty_reals_unfft operations
;; Do 280 ten_complex_unfft operations
;;	distance between fft data elements is 64
;;	do 40 macros each processing 160 data values

	zloop_init 40			;; 5 iterations of 8
	zloop_set_rw_prefetch ZDOUBLE	;; Prefetch 80 cache lines
	bump	rdx, -8*ZMM_SCD5
	bump	rdi, -8*ZMM_SCD5
	zr10_2sc_twenty_reals_unfft_preload
c2a:	zr10_2sc_twenty_reals_unfft rsi, 128, 8*128, rdx, ZMM_SCD5, rdi, ZMM_SCD5, 8
	zloop	8, c2a, -8*128+80*128, rdi, -8*ZMM_SCD5 ;; Test loop counter, next source pointer, restore sin/cos pointer
	loops_reset			;; 4 more iters of 8
	zr10_ten_complex_djbunfft_preload
c2b:	zr10_ten_complex_djbunfft rsi, 128, 8*128, rdi, ZMM_SCD5, 8
	zloop	8, c2b, -8*128+80*128, rdi, -8*ZMM_SCD5 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	5, c2b, -5*80*128	;; Test loop counter, restore source pointer

;; Do inverse FFT levels 1,2,3,4
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	640	+1	...
;;	8	...
;;	...
;;	632
;;	1280
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	3200	+1	...
;;	8	...
;;	...
;;	3192

;; Do 640 ten_reals_last_unfft operations
;;	distance between fft data elements is 640
;;	do 80 macros each processing 80 data values

	bump	rdx, -8*ZMM_SCD5-80*ZMM_SCD2
	bump	rdi, -80*ZMM_SCD2
	zloop_init 80			;; 80 iterations
	zloop_set_rw_prefetch ZDOUBLE	;; Prefetch 160 cache lines
	zr5_2sc_ten_reals_unfft_preload
c1b:	zr5_2sc_ten_reals_unfft rsi, 128, 80*128, rdx, ZMM_SCD2, rdi, ZMM_SCD2, 80
	zloop	80, c1b			;; Test loop counter

zpass2_3200_real_done:
	end_timer 2
	ENDM

; *************** radix8-pass2-3200-complex macro ******************
; This macro takes 3200 complex values and performs the final 11.644 levels of a standard
; radix-8 FFT, squares the results, and does the first 11.644 levels of the inverse FFT.

zr4dwpn_pass2_3200_complex MACRO
	LOCAL	b1b, b2b, b3a, b3b, b3c, b3d, c1b, c2b
	LOCAL	zpass2_3200_complex_1, zpass2_3200_complex_2
	LOCAL	zpass2_3200_complex_3, zpass2_3200_complex_4
	LOCAL	zpass2_3200_complex_unfft, zpass2_3200_complex_done

;; Type 4 FFTs skip the forward FFT process

	mov	rsi, DATA_ADDR		;; Load address of FFT data
	mov	rbx, DIST_TO_FFTSRCARG
	mov	rcx, DATA_PREFETCH	;; Prefetch pointer (we need to prefetch 800 cache lines + ?? pad cache lines)
	cmp	ffttype, 4
	je	zpass2_3200_complex_4

;; Do FFT levels 1,2,3
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	3200	+1	...
;;	8	...
;;	...
;;	3192
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	640	+1	...
;;	8	...
;;	...
;;	632
;;	1280
;;	...

;; Do 640 five_complex_fft operations
;;	distance between fft data elements is 640
;;	do 80 macros each processing 80 data values

	start_timer 4
	mov	rdi, xsincos_complex	;; The sin/cos data
	zloop_init 80			;; 80 iterations
	zloop_set_rw_prefetch ZDOUBLE	;; Prefetch 160 cache lines
	zr5f_five_complex_djbfft_preload
b1b:	zr5f_five_complex_djbfft rsi, 128, 80*128, rdi, ZMM_SCD2, 80, L1PREFETCH_ALL, 128
	zloop	80, b1b, -80*128	;; Test loop counter, restore source pointer
	end_timer 4

;; Do FFT levels 4,5,6
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	640	+1	...
;;	8	...
;;	...
;;	632
;;	1280
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	64	+1	...
;;	8	...
;;	...
;;	56
;;	128
;;	...

;; Do 320 ten_complex_fft operations
;;	distance between fft data elements is 64
;;	do 40 macros each processing 160 data values

	start_timer 5
	zloop_init 40			;; 5 iterations of 8
	zloop_set_rw_prefetch ZDOUBLE	;; Prefetch 80 cache lines
	zr10_ten_complex_djbfft_preload
b2b:	zr10_ten_complex_djbfft rsi, 128, 8*128, rdi, ZMM_SCD5, 8, L1PREFETCH_ALL, 128
	zloop	8, b2b, -8*128+80*128, rdi, -8*ZMM_SCD5	;; Test loop counter, restore source pointer, restore sin/cos pointer
	zloop	5, b2b, -5*80*128, rdi, 8*ZMM_SCD5 ;; Test loop counter, restore source pointer, next sin/cos pointer
	end_timer 5

;; Do FFT levels 7,8,9,10,11,12 as well as inverse FFT levels 7,8,9,10,11,12
;; On input and output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	64	+1	...
;;	8	...
;;	...
;;	56
;;	128
;;	...

;; Do 50 sixtyfour_complex_fft operations
;;	distance between fft data elements is 1 (8 for swizzling)
;;	do 50 macros each processing 128 data values

;; Execute the right middle step

pfing	bump	rcx, -30*64		;; We will prefetch 30 cache lines too many
	cmp	ffttype, 2
	jl	zpass2_3200_complex_1
	jg	zpass2_3200_complex_3

zpass2_3200_complex_2:
	start_timer 9
	zloop_init 50			;; 50 iterations
	zloop_set_rw_prefetch ZSEPT	;; Prefetch 350 cache lines
	mov	r8, DEST2ARG
	zr64_sixtyfour_complex_with_square_preload
b3a:	zr64_sixtyfour_complex_with_square rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 50, L1PREFETCH_ALL, 8*128
	zloop	50, b3a			;; Test loop counter
	add	rsi, r8
	end_timer 9
	jmp	zpass2_3200_complex_unfft

zpass2_3200_complex_1:
	start_timer 9
	zloop_init 50			;; 50 iterations
	zloop_set_rw_prefetch ZSEPT	;; Prefetch 350 cache lines
	zr64_sixtyfour_complex_fft_final_preload
b3b:	zr64_sixtyfour_complex_fft_final rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 50, L1PREFETCH_ALL, 8*128
	zloop	50, b3b			;; Test loop counter
	end_timer 9
	jmp	zpass2_3200_complex_done

zpass2_3200_complex_3:
	start_timer 9
	zloop_init 50			;; 50 iterations
	zloop_set_rw_prefetch ZSEPT	;; Prefetch 350 cache lines
	mov	r8, DEST2ARG
	mov	rbp, DIST_TO_MULSRCARG
	zr64_sixtyfour_complex_with_mult_preload
b3c:	zr64_sixtyfour_complex_with_mult rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 50, L1PREFETCH_ALL, 8*128
	zloop	50, b3c			;; Test loop counter
	add	rsi, r8
	end_timer 9
	jmp	zpass2_3200_complex_unfft

zpass2_3200_complex_4:
	start_timer 9
	zloop_init 50			;; 50 iterations
	zloop_set_rw_prefetch ZSEPT	;; Prefetch 350 cache lines
	mov	rbp, DIST_TO_MULSRCARG
	mov	rdi, xsincos_complex	;; The sin/cos data
	bump	rdi, 80*ZMM_SCD2+8*ZMM_SCD5
	zr64f_sixtyfour_complex_with_mulf_preload
b3d:	zr64f_sixtyfour_complex_with_mulf rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 50, L1PREFETCH_ALL, 8*128
	zloop	50, b3d			;; Test loop counter
	end_timer 9
zpass2_3200_complex_unfft:
	bump	rsi, -50*8*128		;; Restore source pointer

;; Do inverse FFT levels 4,5,6
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	64	+1	...
;;	8	...
;;	...
;;	56
;;	128
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	640	+1	...
;;	8	...
;;	...
;;	632
;;	1280
;;	...

;; Do 320 ten_complex_unfft operations
;;	distance between fft data elements is 64
;;	do 40 macros each processing 160 data values

	start_timer 13
	bump	rdi, -8*ZMM_SCD5
	zloop_init 40			;; 5 iterations of 8
	zloop_set_rw_prefetch ZDOUBLE	;; Prefetch 80 cache lines
	zr10_ten_complex_djbunfft_preload
c2b:	zr10_ten_complex_djbunfft rsi, 128, 8*128, rdi, ZMM_SCD5, 8, L1PREFETCH_ALL, 128
	zloop	8, c2b, -8*128+80*128, rdi, -8*ZMM_SCD5 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	5, c2b, -5*80*128	;; Test loop counter, restore source pointer
	end_timer 13

;; Do inverse FFT levels 1,2,3
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	640	+1	...
;;	8	...
;;	...
;;	632
;;	1280
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	3200	+1	...
;;	8	...
;;	...
;;	3192

;; Do 640 five_complex_unfft operations
;;	distance between fft data elements is 640
;;	do 80 macros each processing 80 data values

	start_timer 14
	bump	rdi, -80*ZMM_SCD2
	zloop_init 80			;; 80 iterations
	zloop_set_rw_prefetch ZDOUBLE	;; Prefetch 160 cache lines
	zr5_five_complex_djbunfft_preload
c1b:	zr5_five_complex_djbunfft rsi, 128, 80*128, rdi, ZMM_SCD2, 80, L1PREFETCH_ALL, 128
	zloop	80, c1b			;; Test loop counter
	end_timer 14

zpass2_3200_complex_done:
	ENDM


; *************** radix8-pass2-3584-real macro ******************
; This macro takes 7168 real values and performs the final 12.807 levels of the FFT.
; The results are squared, and the first 12.807 levels of the inverse FFT are done.

zr4dwpn_pass2_3584_real MACRO
	LOCAL	b1b, b2a, b2b, b3a, b3b, b3c, b3d, c1b, c2a, c2b
	LOCAL	zpass2_3584_real_1, zpass2_3584_real_2
	LOCAL	zpass2_3584_real_3, zpass2_3584_real_4
	LOCAL	zpass2_3584_real_unfft, zpass2_3584_real_done

;; Type 4 FFTs skip the forward FFT process

	start_timer 2
	mov	rsi, DATA_ADDR		;; Load source address
	mov	rbx, DIST_TO_FFTSRCARG
	mov	rcx, DATA_PREFETCH	;; Prefetch pointer (we need to prefetch 896 cache lines + 13 pad cache lines)
	cmp	ffttype, 4
	je	zpass2_3584_real_4

;; Do FFT levels 1,2,3,4
;;	0	+1	+1	+1	+1	+1	+1	+1	3584	+1	...
;;	8	...
;;	...
;;	3576
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	512	+1	...
;;	8	...
;;	...
;;	504
;;	1K
;;	...

;; Do 512 fourteen_real_fft operations
;;	distance between fft data elements is 512
;;	do 64 macros each processing 112 data values

	mov	rdx, sincos3		;; Real sin/cos data
	mov	rdi, xsincos_complex	;; Complex sin/cos data
	zloop_init 64			;; 64 iterations
	zloop_set_rw_prefetch ZDOUBLE	;; Prefetch 128 cache lines
	zr7f_2sc_fourteen_reals_fft_preload
b1b:	zr7f_2sc_fourteen_reals_fft rsi, 128, 2*dist32, rdx, ZMM_SCD3, rdi, ZMM_SCD3, 32
	zloop	32, b1b, -32*128+dist32	;; Test loop counter, next source pointer
	zloop	2, b1b, -2*dist32	;; Test loop counter, restore source pointer

;; Do FFT levels 5,6,7
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	512	+1	...
;;	8	...
;;	...
;;	504
;;	1K
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	64	+1	...
;;	8	...
;;	...
;;	56
;;	128
;;	...

;; Do 64 sixteen_reals_fft operations
;; Do 384 eight_complex_fft operations
;;	distance between fft data elements is 64
;;	do 56 macros each processing 128 data values

	zloop_init 56			;; 7 iterations of 8
	zloop_set_rw_prefetch ZDOUBLE	;; Prefetch 112 cache lines
	zr8_2sc_sixteen_reals_fft_preload
b2a:	zr8_2sc_sixteen_reals_fft rsi, 128, 8*128, 16*128, dist32, rdx, ZMM_SCD4, rdi, ZMM_SCD4, 8
	zloop	8, b2a, -8*128+2*dist32, rdi, -8*ZMM_SCD4 ;; Test loop counter, next source pointer, restore sin/cos pointer
	loops_reset			;; 6 more iters of 8
	zr8_eight_complex_djbfft_preload
b2b:	zr8_eight_complex_djbfft rsi, 128, 8*128, 16*128, dist32, rdi, ZMM_SCD4, 8
	zloop	8, b2b, -8*128+2*dist32, rdi, -8*ZMM_SCD4 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	7, b2b, -7*2*dist32, rdi, 8*ZMM_SCD4 ;; Test loop counter, restore source pointer, next sin/cos pointer

;; Do FFT levels 8,9,10,11,12,13 as well as inverse FFT levels 8,9,10,11,12,13
;; On input and output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	64	+1	...
;;	8	...
;;	...
;;	56
;;	128
;;	...

;; Do 1 hundredtwentyeight_reals_fft operation
;; Do 55 sixtyfour_complex_fft operations
;;	distance between fft data elements is 1 (8 for swizzling)
;;	do 56 macros each processing 128 data values

;; Execute the proper middle step

pfing	bump	rcx, -11*64		;; We will prefetch 11 cache lines too many
	cmp	ffttype, 2
	jl	zpass2_3584_real_1
	jg	zpass2_3584_real_3

zpass2_3584_real_2:
	mov	r8, DEST2ARG
;;	zr64_hundredtwentyeight_real_with_square_preload
;;	zr64_hundredtwentyeight_real_with_square rsi, 8*128, 1*128, 2*128, 4*128, rdx, 0*ZMM_SCD4, rdi, 0*ZMM_SCD4, 1
	CALLP	zreal_square
	zloop_init 56, 1, 4		;; 55 more iterations
	zloop_set_rw_prefetch ZOCT	;; Prefetch 440 cache lines
	zr64_sixtyfour_complex_with_square_preload
b3a:	zr64_sixtyfour_complex_with_square rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 1, L1PREFETCH_ALL, 8*128
	zloop	4, b3a, -4*8*128+dist32	;; Test loop counter
	zloop	14, b3a			;; Test loop counter
	add	rsi, r8
	jmp	zpass2_3584_real_unfft

zpass2_3584_real_1:
;;	zr64_hundredtwentyeight_real_fft_final_preload
;;	zr64_hundredtwentyeight_real_fft_final rsi, 8*128, 1*128, 2*128, 4*128, rdx, 0*ZMM_SCD4, rdi, 0*ZMM_SCD4, 1
	CALLP	zreal_fft_final
	zloop_init 56, 1, 4		;; 55 more iterations
	zloop_set_rw_prefetch ZOCT	;; Prefetch 440 cache lines
	zr64_sixtyfour_complex_fft_final_preload
b3b:	zr64_sixtyfour_complex_fft_final rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 1, L1PREFETCH_ALL, 8*128
	zloop	4, b3b, -4*8*128+dist32	;; Test loop counter
	zloop	14, b3b			;; Test loop counter
	jmp	zpass2_3584_real_done

zpass2_3584_real_3:
	mov	r8, DEST2ARG
	mov	rbp, DIST_TO_MULSRCARG
;;	zr64_hundredtwentyeight_real_with_mult_preload
;;	zr64_hundredtwentyeight_real_with_mult rsi, 8*128, 1*128, 2*128, 4*128, rdx, 0*ZMM_SCD4, rdi, 0*ZMM_SCD4, 1
	CALLP	zreal_mult
	zloop_init 56, 1, 4		;; 55 more iterations
	zloop_set_rw_prefetch ZOCT	;; Prefetch 440 cache lines
	zr64_sixtyfour_complex_with_mult_preload
b3c:	zr64_sixtyfour_complex_with_mult rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 1, L1PREFETCH_ALL, 8*128
	zloop	4, b3c, -4*8*128+dist32	;; Test loop counter
	zloop	14, b3c			;; Test loop counter
	add	rsi, r8
	jmp	zpass2_3584_real_unfft

zpass2_3584_real_4:
	mov	rbp, DIST_TO_MULSRCARG
	mov	rdx, sincos3		;; Real sin/cos data
	mov	rdi, xsincos_complex	;; Complex sin/cos data
	bump	rdx, 64*ZMM_SCD3+8*ZMM_SCD4
	bump	rdi, 64*ZMM_SCD3+8*ZMM_SCD4
;;	zr64f_hundredtwentyeight_real_with_mulf_preload
;;	zr64f_hundredtwentyeight_real_with_mulf rsi, 8*128, 1*128, 2*128, 4*128, rdx, 0*ZMM_SCD4, rdi, 0*ZMM_SCD4, 1
	CALLP	zreal_mulf
	zloop_init 56, 1, 4		;; 55 more iterations
	zloop_set_rw_prefetch ZOCT	;; Prefetch 440 cache lines
	zr64f_sixtyfour_complex_with_mulf_preload
b3d:	zr64f_sixtyfour_complex_with_mulf rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 1, L1PREFETCH_ALL, 8*128
	zloop	4, b3d, -4*8*128+dist32	;; Test loop counter
	zloop	14, b3d			;; Test loop counter
zpass2_3584_real_unfft:
	bump	rsi, -14*dist32		;; Restore source pointer

;; Do inverse FFT levels 5,6,7
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	64	+1	...
;;	8	...
;;	...
;;	56
;;	128
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	512	+1	...
;;	8	...
;;	...
;;	504
;;	1K
;;	...

;; Do 64 sixteen_reals_unfft operations
;; Do 384 eight_complex_unfft operations
;;	distance between fft data elements is 64
;;	do 56 macros each processing 128 data values

	zloop_init 56			;; 7 iterations of 8
	zloop_set_rw_prefetch ZDOUBLE	;; Prefetch 112 cache lines
	bump	rdx, -8*ZMM_SCD4
	bump	rdi, -8*ZMM_SCD4
	zr8_2sc_sixteen_reals_unfft_preload
c2a:	zr8_2sc_sixteen_reals_unfft rsi, 128, 8*128, 16*128, dist32, rdx, ZMM_SCD4, rdi, ZMM_SCD4, 8
	zloop	8, c2a, -8*128+2*dist32, rdi, -8*ZMM_SCD4 ;; Test loop counter, next source pointer, restore sin/cos pointer
	loops_reset			;; 6 more iters of 8
	zr8_eight_complex_djbunfft_preload
c2b:	zr8_eight_complex_djbunfft rsi, 128, 8*128, 16*128, dist32, rdi, ZMM_SCD4, 8
	zloop	8, c2b, -8*128+2*dist32, rdi, -8*ZMM_SCD4 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	7, c2b, -7*2*dist32	;; Test loop counter, restore source pointer

;; Do inverse FFT levels 1,2,3,4
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	512	+1	...
;;	8	...
;;	...
;;	504
;;	1K
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	3584	+1	...
;;	8	...
;;	...
;;	3576

;; Do 512 fourteen_reals_last_unfft operations
;;	distance between fft data elements is 512
;;	do 64 macros each processing 112 data values

	bump	rdx, -8*ZMM_SCD4-64*ZMM_SCD3
	bump	rdi, -64*ZMM_SCD3
	zloop_init 64			;; 64 iterations
	zloop_set_rw_prefetch ZDOUBLE	;; Prefetch 128 cache lines
	zr7_2sc_fourteen_reals_unfft_preload
c1b:	zr7_2sc_fourteen_reals_unfft rsi, 128, 2*dist32, rdx, ZMM_SCD3, rdi, ZMM_SCD3, 32
	zloop	32, c1b, -32*128+dist32	;; Test loop counter, next source pointer
	zloop	2, c1b			;; Test loop counter

zpass2_3584_real_done:
	end_timer 2
	ENDM

; *************** radix8-pass2-3584-complex macro ******************
; This macro takes 3584 complex values and performs the final 11.807 levels of a standard
; radix-8 FFT, squares the results, and does the first 11.807 levels of the inverse FFT.

zr4dwpn_pass2_3584_complex MACRO
	LOCAL	b1b, b2b, b3a, b3b, b3c, b3d, c1b, c2b
	LOCAL	zpass2_3584_complex_1, zpass2_3584_complex_2
	LOCAL	zpass2_3584_complex_3, zpass2_3584_complex_4
	LOCAL	zpass2_3584_complex_unfft, zpass2_3584_complex_done

;; Type 4 FFTs skip the forward FFT process

	mov	rsi, DATA_ADDR		;; Load address of FFT data
	mov	rbx, DIST_TO_FFTSRCARG
	mov	rcx, DATA_PREFETCH	;; Prefetch pointer (we need to prefetch 896 cache lines + 13 pad cache lines)
	cmp	ffttype, 4
	je	zpass2_3584_complex_4

;; Do FFT levels 1,2,3
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	3584	+1	...
;;	8	...
;;	...
;;	3576
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	512	+1	...
;;	8	...
;;	...
;;	504
;;	1K
;;	...

;; Do 512 seven_complex_fft operations
;;	distance between fft data elements is 512
;;	do 64 macros each processing 112 data values

	start_timer 4
	mov	rdi, xsincos_complex	;; The sin/cos data
	zloop_init 64			;; 64 iterations
	zloop_set_rw_prefetch ZDOUBLE	;; Prefetch 128 cache lines
	zr7f_seven_complex_djbfft_preload
b1b:	zr7f_seven_complex_djbfft rsi, 128, 2*dist32, rdi, ZMM_SCD3, 32, L1PREFETCH_ALL, 128
	zloop	32, b1b, -32*128+dist32	;; Test loop counter, next source pointer
	zloop	2, b1b, -2*dist32	;; Test loop counter, restore source pointer
	end_timer 4

;; Do FFT levels 4,5,6
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	512	+1	...
;;	8	...
;;	...
;;	504
;;	1K
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	64	+1	...
;;	8	...
;;	...
;;	56
;;	128
;;	...

;; Do 448 eight_complex_fft operations
;;	distance between fft data elements is 64
;;	do 56 macros each processing 128 data values

	start_timer 5
	zloop_init 56			;; 7 iterations of 8
	zloop_set_rw_prefetch ZDOUBLE	;; Prefetch 112 cache lines
	zr8_eight_complex_djbfft_preload
b2b:	zr8_eight_complex_djbfft rsi, 128, 8*128, 16*128, dist32, rdi, ZMM_SCD4, 8, L1PREFETCH_ALL, 128
	zloop	8, b2b, -8*128+2*dist32, rdi, -8*ZMM_SCD4	;; Test loop counter, restore source pointer, restore sin/cos pointer
	zloop	7, b2b, -7*2*dist32, rdi, 8*ZMM_SCD4 ;; Test loop counter, restore source pointer, next sin/cos pointer
	end_timer 5

;; Do FFT levels 7,8,9,10,11,12 as well as inverse FFT levels 7,8,9,10,11,12
;; On input and output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	64	+1	...
;;	8	...
;;	...
;;	56
;;	128
;;	...

;; Do 56 sixtyfour_complex_fft operations
;;	distance between fft data elements is 1 (8 for swizzling)
;;	do 56 macros each processing 128 data values

;; Execute the right middle step

pfing	bump	rcx, -19*64		;; We will prefetch 19 cache lines too many
	cmp	ffttype, 2
	jl	zpass2_3584_complex_1
	jg	zpass2_3584_complex_3

zpass2_3584_complex_2:
	start_timer 9
	zloop_init 56			;; 56 iterations
	zloop_set_rw_prefetch ZOCT	;; Prefetch 448 cache lines
	mov	r8, DEST2ARG
	zr64_sixtyfour_complex_with_square_preload
b3a:	zr64_sixtyfour_complex_with_square rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 4, L1PREFETCH_ALL, 8*128
	zloop	4, b3a, -4*8*128+dist32	;; Test loop counter
	zloop	14, b3a			;; Test loop counter
	add	rsi, r8
	end_timer 9
	jmp	zpass2_3584_complex_unfft

zpass2_3584_complex_1:
	start_timer 9
	zloop_init 56			;; 56 iterations
	zloop_set_rw_prefetch ZOCT	;; Prefetch 448 cache lines
	zr64_sixtyfour_complex_fft_final_preload
b3b:	zr64_sixtyfour_complex_fft_final rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 4, L1PREFETCH_ALL, 8*128
	zloop	4, b3b, -4*8*128+dist32	;; Test loop counter
	zloop	14, b3b			;; Test loop counter
	end_timer 9
	jmp	zpass2_3584_complex_done

zpass2_3584_complex_3:
	start_timer 9
	zloop_init 56			;; 56 iterations
	zloop_set_rw_prefetch ZOCT	;; Prefetch 448 cache lines
	mov	r8, DEST2ARG
	mov	rbp, DIST_TO_MULSRCARG
	zr64_sixtyfour_complex_with_mult_preload
b3c:	zr64_sixtyfour_complex_with_mult rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 4, L1PREFETCH_ALL, 8*128
	zloop	4, b3c, -4*8*128+dist32	;; Test loop counter
	zloop	14, b3c			;; Test loop counter
	add	rsi, r8
	end_timer 9
	jmp	zpass2_3584_complex_unfft

zpass2_3584_complex_4:
	start_timer 9
	zloop_init 56			;; 56 iterations
	zloop_set_rw_prefetch ZOCT	;; Prefetch 448 cache lines
	mov	rbp, DIST_TO_MULSRCARG
	mov	rdi, xsincos_complex	;; The sin/cos data
	bump	rdi, 64*ZMM_SCD3+8*ZMM_SCD4
	zr64f_sixtyfour_complex_with_mulf_preload
b3d:	zr64f_sixtyfour_complex_with_mulf rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 4, L1PREFETCH_ALL, 8*128
	zloop	4, b3d, -4*8*128+dist32	;; Test loop counter
	zloop	14, b3d			;; Test loop counter
	end_timer 9
zpass2_3584_complex_unfft:
	bump	rsi, -14*dist32		;; Restore source pointer

;; Do inverse FFT levels 4,5,6
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	64	+1	...
;;	8	...
;;	...
;;	56
;;	128
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	512	+1	...
;;	8	...
;;	...
;;	504
;;	1K
;;	...

;; Do 448 eight_complex_unfft operations
;;	distance between fft data elements is 64
;;	do 56 macros each processing 128 data values

	start_timer 13
	bump	rdi, -8*ZMM_SCD4
	zloop_init 56			;; 7 iterations of 8
	zloop_set_rw_prefetch ZDOUBLE	;; Prefetch 112 cache lines
	zr8_eight_complex_djbunfft_preload
c2b:	zr8_eight_complex_djbunfft rsi, 128, 8*128, 16*128, dist32, rdi, ZMM_SCD4, 8, L1PREFETCH_ALL, 128
	zloop	8, c2b, -8*128+2*dist32, rdi, -8*ZMM_SCD4 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	7, c2b, -7*2*dist32	;; Test loop counter, restore source pointer
	end_timer 13

;; Do inverse FFT levels 1,2,3
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	512	+1	...
;;	8	...
;;	...
;;	504
;;	1K
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	3584	+1	...
;;	8	...
;;	...
;;	3576

;; Do 512 seven_complex_unfft operations
;;	distance between fft data elements is 512
;;	do 64 macros each processing 112 data values

	start_timer 14
	bump	rdi, -64*ZMM_SCD3
	zloop_init 64			;; 64 iterations
	zloop_set_rw_prefetch ZDOUBLE	;; Prefetch 128 cache lines
	zr7_seven_complex_djbunfft_preload
c1b:	zr7_seven_complex_djbunfft rsi, 128, 2*dist32, rdi, ZMM_SCD3, 32, L1PREFETCH_ALL, 128
	zloop	32, c1b, -32*128+dist32	;; Test loop counter, next source pointer
	zloop	2, c1b			;; Test loop counter
	end_timer 14

zpass2_3584_complex_done:
	ENDM


; *************** radix8-pass2-3840-real macro ******************
; This macro takes 7680 real values and performs the final 12.907 levels of the FFT.
; The results are squared, and the first 12.907 levels of the inverse FFT are done.

zr4dwpn_pass2_3840_real MACRO
	LOCAL	b1b, b2a, b2b, b3a, b3b, b3c, b3d, c1b, c2a, c2b
	LOCAL	zpass2_3840_real_1, zpass2_3840_real_2
	LOCAL	zpass2_3840_real_3, zpass2_3840_real_4
	LOCAL	zpass2_3840_real_unfft, zpass2_3840_real_done

;; Type 4 FFTs skip the forward FFT process

	start_timer 2
	mov	rsi, DATA_ADDR		;; Load source address
	mov	rbx, DIST_TO_FFTSRCARG
	mov	rcx, DATA_PREFETCH	;; Prefetch pointer (we need to prefetch 960 cache lines + 14 pad cache lines)
	cmp	ffttype, 4
	je	zpass2_3840_real_4

;; Do FFT levels 1,2,3,4
;;	0	+1	+1	+1	+1	+1	+1	+1	3840	+1	...
;;	8	...
;;	...
;;	3832
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	768	+1	...
;;	8	...
;;	...
;;	760
;;	1536
;;	...

;; Do 768 ten_real_fft operations
;;	distance between fft data elements is 768
;;	do 96 macros each processing 80 data values

	mov	rdx, sincos3		;; Real sin/cos data
	mov	rdi, xsincos_complex	;; Complex sin/cos data
	zloop_init 96			;; 96 iterations
	zloop_set_rw_prefetch ZDOUBLE	;; Prefetch 192 cache lines
	zr5f_2sc_ten_reals_fft_preload
b1b:	zr5f_2sc_ten_reals_fft rsi, 128, 3*dist32, rdx, ZMM_SCD2, rdi, ZMM_SCD2, 32
	zloop	32, b1b, -32*128+dist32	;; Test loop counter, next source pointer
	zloop	3, b1b, -3*dist32	;; Test loop counter, restore source pointer

;; Do FFT levels 5,6,7
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	768	+1	...
;;	8	...
;;	...
;;	760
;;	1536
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	64	+1	...
;;	8	...
;;	...
;;	56
;;	128
;;	...

;; Do 40 twentyfour_reals_fft operations
;; Do 280 twelve_complex_fft operations
;;	distance between fft data elements is 64
;;	do 40 macros each processing 192 data values

	zloop_init 40			;; 5 iterations of 8
	zloop_set_rw_prefetch ZTRIPLE	;; Prefetch 120 cache lines
	zr12_2sc_twentyfour_reals_fft_preload
b2a:	zr12_2sc_twentyfour_reals_fft rsi, 128, 8*128, 16*128, dist32, rdx, ZMM_SCD6, rdi, ZMM_SCD6, 8
	zloop	8, b2a, -8*128+3*dist32, rdi, -8*ZMM_SCD6 ;; Test loop counter, next source pointer, restore sin/cos pointer
	loops_reset			;; 4 more iters of 8
	zr12_twelve_complex_djbfft_preload
b2b:	zr12_twelve_complex_djbfft rsi, 128, 8*128, 16*128, dist32, rdi, ZMM_SCD6, 8
	zloop	8, b2b, -8*128+3*dist32, rdi, -8*ZMM_SCD6 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	5, b2b, -5*3*dist32, rdi, 8*ZMM_SCD6 ;; Test loop counter, restore source pointer, next sin/cos pointer

;; Do FFT levels 8,9,10,11,12,13 as well as inverse FFT levels 8,9,10,11,12,13
;; On input and output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	64	+1	...
;;	8	...
;;	...
;;	56
;;	128
;;	...

;; Do 1 hundredtwentyeight_reals_fft operation
;; Do 59 sixtyfour_complex_fft operations
;;	distance between fft data elements is 1 (8 for swizzling)
;;	do 60 macros each processing 128 data values

;; Execute the proper middle step

pfing	bump	rcx, -7*64		;; We will prefetch 7 cache lines too many
	cmp	ffttype, 2
	jl	zpass2_3840_real_1
	jg	zpass2_3840_real_3

zpass2_3840_real_2:
	mov	r8, DEST2ARG
;;	zr64_hundredtwentyeight_real_with_square_preload
;;	zr64_hundredtwentyeight_real_with_square rsi, 8*128, 1*128, 2*128, 4*128, rdx, 0*ZMM_SCD4, rdi, 0*ZMM_SCD4, 1
	CALLP	zreal_square
	zloop_init 60, 1, 4		;; 59 more iterations
	zloop_set_rw_prefetch ZSEPT	;; Prefetch 413 cache lines
	zr64_sixtyfour_complex_with_square_preload
b3a:	zr64_sixtyfour_complex_with_square rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 1, L1PREFETCH_ALL, 8*128
	zloop	4, b3a, -4*8*128+dist32	;; Test loop counter
	zloop	15, b3a			;; Test loop counter
	add	rsi, r8
	jmp	zpass2_3840_real_unfft

zpass2_3840_real_1:
;;	zr64_hundredtwentyeight_real_fft_final_preload
;;	zr64_hundredtwentyeight_real_fft_final rsi, 8*128, 1*128, 2*128, 4*128, rdx, 0*ZMM_SCD4, rdi, 0*ZMM_SCD4, 1
	CALLP	zreal_fft_final
	zloop_init 60, 1, 4		;; 59 more iterations
	zloop_set_rw_prefetch ZSEPT	;; Prefetch 413 cache lines
	zr64_sixtyfour_complex_fft_final_preload
b3b:	zr64_sixtyfour_complex_fft_final rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 1, L1PREFETCH_ALL, 8*128
	zloop	4, b3b, -4*8*128+dist32	;; Test loop counter
	zloop	15, b3b			;; Test loop counter
	jmp	zpass2_3840_real_done

zpass2_3840_real_3:
	mov	r8, DEST2ARG
	mov	rbp, DIST_TO_MULSRCARG
;;	zr64_hundredtwentyeight_real_with_mult_preload
;;	zr64_hundredtwentyeight_real_with_mult rsi, 8*128, 1*128, 2*128, 4*128, rdx, 0*ZMM_SCD4, rdi, 0*ZMM_SCD4, 1
	CALLP	zreal_mult
	zloop_init 60, 1, 4		;; 59 more iterations
	zloop_set_rw_prefetch ZSEPT	;; Prefetch 413 cache lines
	zr64_sixtyfour_complex_with_mult_preload
b3c:	zr64_sixtyfour_complex_with_mult rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 1, L1PREFETCH_ALL, 8*128
	zloop	4, b3c, -4*8*128+dist32	;; Test loop counter
	zloop	15, b3c			;; Test loop counter
	add	rsi, r8
	jmp	zpass2_3840_real_unfft

zpass2_3840_real_4:
	mov	rbp, DIST_TO_MULSRCARG
	mov	rdx, sincos3		;; Real sin/cos data
	mov	rdi, xsincos_complex	;; Complex sin/cos data
	bump	rdx, 96*ZMM_SCD2+8*ZMM_SCD6
	bump	rdi, 96*ZMM_SCD2+8*ZMM_SCD6
;;	zr64f_hundredtwentyeight_real_with_mulf_preload
;;	zr64f_hundredtwentyeight_real_with_mulf rsi, 8*128, 1*128, 2*128, 4*128, rdx, 0*ZMM_SCD4, rdi, 0*ZMM_SCD4, 1
	CALLP	zreal_mulf
	zloop_init 60, 1, 4		;; 59 more iterations
	zloop_set_rw_prefetch ZSEPT	;; Prefetch 413 cache lines
	zr64f_sixtyfour_complex_with_mulf_preload
b3d:	zr64f_sixtyfour_complex_with_mulf rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 1, L1PREFETCH_ALL, 8*128
	zloop	4, b3d, -4*8*128+dist32	;; Test loop counter
	zloop	15, b3d			;; Test loop counter
zpass2_3840_real_unfft:
	bump	rsi, -15*dist32		;; Restore source pointer

;; Do inverse FFT levels 5,6,7
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	64	+1	...
;;	8	...
;;	...
;;	56
;;	128
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	768	+1	...
;;	8	...
;;	...
;;	760
;;	1536
;;	...

;; Do 40 twentyfour_reals_unfft operations
;; Do 280 twelve_complex_unfft operations
;;	distance between fft data elements is 64
;;	do 40 macros each processing 192 data values

	zloop_init 40			;; 5 iterations of 8
	zloop_set_rw_prefetch ZQUAD	;; Prefetch 160 cache lines
	bump	rdx, -8*ZMM_SCD6
	bump	rdi, -8*ZMM_SCD6
	zr12_2sc_twentyfour_reals_unfft_preload
c2a:	zr12_2sc_twentyfour_reals_unfft rsi, 128, 8*128, 16*128, dist32, rdx, ZMM_SCD6, rdi, ZMM_SCD6, 8
	zloop	8, c2a, -8*128+3*dist32, rdi, -8*ZMM_SCD6 ;; Test loop counter, next source pointer, restore sin/cos pointer
	loops_reset			;; 4 more iters of 8
	zr12_twelve_complex_djbunfft_preload
c2b:	zr12_twelve_complex_djbunfft rsi, 128, 8*128, 16*128, dist32, rdi, ZMM_SCD6, 8
	zloop	8, c2b, -8*128+3*dist32, rdi, -8*ZMM_SCD6 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	5, c2b, -5*3*dist32	;; Test loop counter, restore source pointer

;; Do inverse FFT levels 1,2,3,4
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	768	+1	...
;;	8	...
;;	...
;;	760
;;	1536
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	3840	+1	...
;;	8	...
;;	...
;;	3832

;; Do 768 ten_reals_last_unfft operations
;;	distance between fft data elements is 768
;;	do 96 macros each processing 80 data values

	bump	rdx, -8*ZMM_SCD6-96*ZMM_SCD2
	bump	rdi, -96*ZMM_SCD2
	zloop_init 96			;; 96 iterations
	zloop_set_rw_prefetch ZSINGLE	;; Prefetch 96 cache lines
	zr5_2sc_ten_reals_unfft_preload
c1b:	zr5_2sc_ten_reals_unfft rsi, 128, 3*dist32, rdx, ZMM_SCD2, rdi, ZMM_SCD2, 32
	zloop	32, c1b, -32*128+dist32	;; Test loop counter, next source pointer
	zloop	3, c1b			;; Test loop counter

zpass2_3840_real_done:
	end_timer 2
	ENDM

; *************** radix8-pass2-3840-complex macro ******************
; This macro takes 3840 complex values and performs the final 11.907 levels of a standard
; radix-8 FFT, squares the results, and does the first 11.907 levels of the inverse FFT.

zr4dwpn_pass2_3840_complex MACRO
	LOCAL	b1b, b2b, b3a, b3b, b3c, b3d, c1b, c2b
	LOCAL	zpass2_3840_complex_1, zpass2_3840_complex_2
	LOCAL	zpass2_3840_complex_3, zpass2_3840_complex_4
	LOCAL	zpass2_3840_complex_unfft, zpass2_3840_complex_done

;; Type 4 FFTs skip the forward FFT process

	mov	rsi, DATA_ADDR		;; Load address of FFT data
	mov	rbx, DIST_TO_FFTSRCARG
	mov	rcx, DATA_PREFETCH	;; Prefetch pointer (we need to prefetch 960 cache lines + 14 pad cache lines)
	cmp	ffttype, 4
	je	zpass2_3840_complex_4

;; Do FFT levels 1,2,3
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	3840	+1	...
;;	8	...
;;	...
;;	3832
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	768	+1	...
;;	8	...
;;	...
;;	760
;;	1536
;;	...

;; Do 768 five_complex_fft operations
;;	distance between fft data elements is 768
;;	do 96 macros each processing 80 data values

	start_timer 4
	mov	rdi, xsincos_complex	;; The sin/cos data
	zloop_init 96			;; 96 iterations
	zloop_set_rw_prefetch ZDOUBLE	;; Prefetch 192 cache lines
	zr5f_five_complex_djbfft_preload
b1b:	zr5f_five_complex_djbfft rsi, 128, 3*dist32, rdi, ZMM_SCD2, 32, L1PREFETCH_ALL, 128
	zloop	32, b1b, -32*128+dist32	;; Test loop counter, next source pointer
	zloop	3, b1b, -3*dist32	;; Test loop counter, restore source pointer
	end_timer 4

;; Do FFT levels 4,5,6
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	768	+1	...
;;	8	...
;;	...
;;	760
;;	1536
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	64	+1	...
;;	8	...
;;	...
;;	56
;;	128
;;	...

;; Do 320 twelve_complex_fft operations
;;	distance between fft data elements is 64
;;	do 40 macros each processing 192 data values

	start_timer 5
	zloop_init 40			;; 5 iterations of 8
	zloop_set_rw_prefetch ZTRIPLE	;; Prefetch 120 cache lines
	zr12_twelve_complex_djbfft_preload
b2b:	zr12_twelve_complex_djbfft rsi, 128, 8*128, 16*128, dist32, rdi, ZMM_SCD6, 8, L1PREFETCH_ALL, 128
	zloop	8, b2b, -8*128+3*dist32, rdi, -8*ZMM_SCD6 ;; Test loop counter, restore source pointer, restore sin/cos pointer
	zloop	5, b2b, -5*3*dist32, rdi, 8*ZMM_SCD6 ;; Test loop counter, restore source pointer, next sin/cos pointer
	end_timer 5

;; Do FFT levels 7,8,9,10,11,12 as well as inverse FFT levels 7,8,9,10,11,12
;; On input and output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	64	+1	...
;;	8	...
;;	...
;;	56
;;	128
;;	...

;; Do 60 sixtyfour_complex_fft operations
;;	distance between fft data elements is 1 (8 for swizzling)
;;	do 60 macros each processing 128 data values

;; Execute the right middle step

pfing	bump	rcx, -14*64		;; We will prefetch 14 cache lines too many
	cmp	ffttype, 2
	jl	zpass2_3840_complex_1
	jg	zpass2_3840_complex_3

zpass2_3840_complex_2:
	start_timer 9
	zloop_init 60			;; 60 iterations
	zloop_set_rw_prefetch ZSEPT	;; Prefetch 420 cache lines
	mov	r8, DEST2ARG
	zr64_sixtyfour_complex_with_square_preload
b3a:	zr64_sixtyfour_complex_with_square rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 4, L1PREFETCH_ALL, 8*128
	zloop	4, b3a, -4*8*128+dist32	;; Test loop counter
	zloop	15, b3a			;; Test loop counter
	add	rsi, r8
	end_timer 9
	jmp	zpass2_3840_complex_unfft

zpass2_3840_complex_1:
	start_timer 9
	zloop_init 60			;; 60 iterations
	zloop_set_rw_prefetch ZSEPT	;; Prefetch 420 cache lines
	zr64_sixtyfour_complex_fft_final_preload
b3b:	zr64_sixtyfour_complex_fft_final rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 4, L1PREFETCH_ALL, 8*128
	zloop	4, b3b, -4*8*128+dist32	;; Test loop counter
	zloop	15, b3b			;; Test loop counter
	end_timer 9
	jmp	zpass2_3840_complex_done

zpass2_3840_complex_3:
	start_timer 9
	zloop_init 60			;; 60 iterations
	zloop_set_rw_prefetch ZSEPT	;; Prefetch 420 cache lines
	mov	r8, DEST2ARG
	mov	rbp, DIST_TO_MULSRCARG
	zr64_sixtyfour_complex_with_mult_preload
b3c:	zr64_sixtyfour_complex_with_mult rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 4, L1PREFETCH_ALL, 8*128
	zloop	4, b3c, -4*8*128+dist32	;; Test loop counter
	zloop	15, b3c			;; Test loop counter
	add	rsi, r8
	end_timer 9
	jmp	zpass2_3840_complex_unfft

zpass2_3840_complex_4:
	start_timer 9
	zloop_init 60			;; 60 iterations
	zloop_set_rw_prefetch ZSEPT	;; Prefetch 420 cache lines
	mov	rbp, DIST_TO_MULSRCARG
	mov	rdi, xsincos_complex	;; The sin/cos data
	bump	rdi, 96*ZMM_SCD2+8*ZMM_SCD6
	zr64f_sixtyfour_complex_with_mulf_preload
b3d:	zr64f_sixtyfour_complex_with_mulf rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 4, L1PREFETCH_ALL, 8*128
	zloop	4, b3d, -4*8*128+dist32	;; Test loop counter
	zloop	15, b3d			;; Test loop counter
	end_timer 9
zpass2_3840_complex_unfft:
	bump	rsi, -15*dist32		;; Restore source pointer

;; Do inverse FFT levels 4,5,6
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	64	+1	...
;;	8	...
;;	...
;;	56
;;	128
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	768	+1	...
;;	8	...
;;	...
;;	760
;;	1536
;;	...

;; Do 320 twelve_complex_unfft operations
;;	distance between fft data elements is 64
;;	do 40 macros each processing 192 data values

	start_timer 13
	bump	rdi, -8*ZMM_SCD6
	zloop_init 40			;; 5 iterations of 8
	zloop_set_rw_prefetch ZQUAD	;; Prefetch 160 cache lines
	zr12_twelve_complex_djbunfft_preload
c2b:	zr12_twelve_complex_djbunfft rsi, 128, 8*128, 16*128, dist32, rdi, ZMM_SCD6, 8, L1PREFETCH_ALL, 128
	zloop	8, c2b, -8*128+3*dist32, rdi, -8*ZMM_SCD6 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	5, c2b, -5*3*dist32	;; Test loop counter, restore source pointer
	end_timer 13

;; Do inverse FFT levels 1,2,3
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	768	+1	...
;;	8	...
;;	...
;;	760
;;	1536
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	3840	+1	...
;;	8	...
;;	...
;;	3832

;; Do 768 five_complex_unfft operations
;;	distance between fft data elements is 768
;;	do 96 macros each processing 80 data values

	start_timer 14
	bump	rdi, -96*ZMM_SCD2
	zloop_init 96			;; 96 iterations
	zloop_set_rw_prefetch ZSINGLE	;; Prefetch 96 cache lines
	zr5_five_complex_djbunfft_preload
c1b:	zr5_five_complex_djbunfft rsi, 128, 3*dist32, rdi, ZMM_SCD2, 32, L1PREFETCH_ALL, 128
	zloop	32, c1b, -32*128+dist32	;; Test loop counter, next source pointer
	zloop	3, c1b			;; Test loop counter
	end_timer 14

zpass2_3840_complex_done:
	ENDM


; *************** radix8-pass2-4096-real macro ******************
; This macro takes 8192 real values and performs the final 13 levels of the FFT.
; The results are squared, and the first 13 levels of the inverse FFT are done.

zr4dwpn_pass2_4096_real MACRO
	LOCAL	b1b, b2a, b2b, b3a, b3b, b3c, b3d, c1b, c2a, c2b
	LOCAL	zpass2_4096_real_1, zpass2_4096_real_2
	LOCAL	zpass2_4096_real_3, zpass2_4096_real_4
	LOCAL	zpass2_4096_real_unfft, zpass2_4096_real_done

;; Type 4 FFTs skip the forward FFT process

	start_timer 2
	mov	rsi, DATA_ADDR		;; Load source address
	mov	rbx, DIST_TO_FFTSRCARG
	mov	rcx, DATA_PREFETCH	;; Prefetch pointer (we need to prefetch 1024 cache lines + 15 pad cache lines)
	cmp	ffttype, 4
	je	zpass2_4096_real_4

;; Do FFT levels 1,2,3,4
;;	0	+1	+1	+1	+1	+1	+1	+1	4096	+1	...
;;	8	...
;;	...
;;	4088
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	512	+1	...
;;	8	...
;;	...
;;	504
;;	1K
;;	...

;; Do 512 sixteen_real_fft operations
;;	distance between fft data elements is 512
;;	do 64 macros each processing 128 data values

	mov	rdx, sincos3		;; Real sin/cos data
	mov	rdi, xsincos_complex	;; Complex sin/cos data
	zloop_init 64			;; 64 iterations
	zloop_set_rw_prefetch ZDOUBLE	;; Prefetch 128 cache lines
	zr8f_2sc_sixteen_reals_fft_preload
b1b:	zr8f_2sc_sixteen_reals_fft rsi, 128, 2*dist32, 4*dist32, 8*dist32, rdx, ZMM_SCD4, rdi, ZMM_SCD4, 32
	zloop	32, b1b, -32*128+dist32	;; Test loop counter, next source pointer
	zloop	2, b1b, -2*dist32	;; Test loop counter, restore source pointer

;; Do FFT levels 5,6,7
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	512	+1	...
;;	8	...
;;	...
;;	504
;;	1K
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	64	+1	...
;;	8	...
;;	...
;;	56
;;	128
;;	...

;; Do 64 sixteen_reals_fft operations
;; Do 448 eight_complex_fft operations
;;	distance between fft data elements is 64
;;	do 64 macros each processing 128 data values

	zloop_init 64			;; 8 iterations of 8
	zloop_set_rw_prefetch ZTRIPLE	;; Prefetch 192 cache lines
	zr8_2sc_sixteen_reals_fft_preload
b2a:	zr8_2sc_sixteen_reals_fft rsi, 128, 8*128, 16*128, dist32, rdx, ZMM_SCD4, rdi, ZMM_SCD4, 8
	zloop	8, b2a, -8*128+2*dist32, rdi, -8*ZMM_SCD4 ;; Test loop counter, next source pointer, restore sin/cos pointer
	loops_reset			;; 7 more iters of 8
	zr8_eight_complex_djbfft_preload
b2b:	zr8_eight_complex_djbfft rsi, 128, 8*128, 16*128, dist32, rdi, ZMM_SCD4, 8
	zloop	8, b2b, -8*128+2*dist32, rdi, -8*ZMM_SCD4 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	8, b2b, -8*2*dist32, rdi, 8*ZMM_SCD4 ;; Test loop counter, restore source pointer, next sin/cos pointer

;; Do FFT levels 8,9,10,11,12,13 as well as inverse FFT levels 8,9,10,11,12,13
;; On input and output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	64	+1	...
;;	8	...
;;	...
;;	56
;;	128
;;	...

;; Do 1 hundredtwentyeight_reals_fft operation
;; Do 63 sixtyfour_complex_fft operations
;;	distance between fft data elements is 1 (8 for swizzling)
;;	do 64 macros each processing 128 data values

;; Execute the proper middle step

pfing	bump	rcx, -41*64		;; We will prefetch 41 cache lines too many
	cmp	ffttype, 2
	jl	zpass2_4096_real_1
	jg	zpass2_4096_real_3

zpass2_4096_real_2:
	mov	r8, DEST2ARG
;;	zr64_hundredtwentyeight_real_with_square_preload
;;	zr64_hundredtwentyeight_real_with_square rsi, 8*128, 1*128, 2*128, 4*128, rdx, 0*ZMM_SCD4, rdi, 0*ZMM_SCD4, 1
	CALLP	zreal_square
	zloop_init 64, 1, 4		;; 63 more iterations
	zloop_set_rw_prefetch ZOCT	;; Prefetch 504 cache lines
	zr64_sixtyfour_complex_with_square_preload
b3a:	zr64_sixtyfour_complex_with_square rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 1, L1PREFETCH_ALL, 8*128
	zloop	4, b3a, -4*8*128+dist32	;; Test loop counter
	zloop	16, b3a			;; Test loop counter
	add	rsi, r8
	jmp	zpass2_4096_real_unfft

zpass2_4096_real_1:
;;	zr64_hundredtwentyeight_real_fft_final_preload
;;	zr64_hundredtwentyeight_real_fft_final rsi, 8*128, 1*128, 2*128, 4*128, rdx, 0*ZMM_SCD4, rdi, 0*ZMM_SCD4, 1
	CALLP	zreal_fft_final
	zloop_init 64, 1, 4		;; 63 more iterations
	zloop_set_rw_prefetch ZOCT	;; Prefetch 504 cache lines
	zr64_sixtyfour_complex_fft_final_preload
b3b:	zr64_sixtyfour_complex_fft_final rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 1, L1PREFETCH_ALL, 8*128
	zloop	4, b3b, -4*8*128+dist32	;; Test loop counter
	zloop	16, b3b			;; Test loop counter
	jmp	zpass2_4096_real_done

zpass2_4096_real_3:
	mov	r8, DEST2ARG
	mov	rbp, DIST_TO_MULSRCARG
;;	zr64_hundredtwentyeight_real_with_mult_preload
;;	zr64_hundredtwentyeight_real_with_mult rsi, 8*128, 1*128, 2*128, 4*128, rdx, 0*ZMM_SCD4, rdi, 0*ZMM_SCD4, 1
	CALLP	zreal_mult
	zloop_init 64, 1, 4		;; 63 more iterations
	zloop_set_rw_prefetch ZOCT	;; Prefetch 504 cache lines
	zr64_sixtyfour_complex_with_mult_preload
b3c:	zr64_sixtyfour_complex_with_mult rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 1, L1PREFETCH_ALL, 8*128
	zloop	4, b3c, -4*8*128+dist32	;; Test loop counter
	zloop	16, b3c			;; Test loop counter
	add	rsi, r8
	jmp	zpass2_4096_real_unfft

zpass2_4096_real_4:
	mov	rbp, DIST_TO_MULSRCARG
	mov	rdx, sincos3		;; Real sin/cos data
	mov	rdi, xsincos_complex	;; Complex sin/cos data
	bump	rdx, 64*ZMM_SCD4+8*ZMM_SCD4
	bump	rdi, 64*ZMM_SCD4+8*ZMM_SCD4
;;	zr64f_hundredtwentyeight_real_with_mulf_preload
;;	zr64f_hundredtwentyeight_real_with_mulf rsi, 8*128, 1*128, 2*128, 4*128, rdx, 0*ZMM_SCD4, rdi, 0*ZMM_SCD4, 1
	CALLP	zreal_mulf
	zloop_init 64, 1, 4		;; 63 more iterations
	zloop_set_rw_prefetch ZOCT	;; Prefetch 504 cache lines
	zr64f_sixtyfour_complex_with_mulf_preload
b3d:	zr64f_sixtyfour_complex_with_mulf rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 1, L1PREFETCH_ALL, 8*128
	zloop	4, b3d, -4*8*128+dist32	;; Test loop counter
	zloop	16, b3d			;; Test loop counter
zpass2_4096_real_unfft:
	bump	rsi, -16*dist32		;; Restore source pointer

;; Do inverse FFT levels 5,6,7
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	64	+1	...
;;	8	...
;;	...
;;	56
;;	128
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	512	+1	...
;;	8	...
;;	...
;;	504
;;	1K
;;	...

;; Do 64 sixteen_reals_unfft operations
;; Do 448 eight_complex_unfft operations
;;	distance between fft data elements is 64
;;	do 64 macros each processing 128 data values

	zloop_init 64			;; 8 iterations of 8
	zloop_set_rw_prefetch ZDOUBLE	;; Prefetch 128 cache lines
	bump	rdx, -8*ZMM_SCD4
	bump	rdi, -8*ZMM_SCD4
	zr8_2sc_sixteen_reals_unfft_preload
c2a:	zr8_2sc_sixteen_reals_unfft rsi, 128, 8*128, 16*128, dist32, rdx, ZMM_SCD4, rdi, ZMM_SCD4, 8
	zloop	8, c2a, -8*128+2*dist32, rdi, -8*ZMM_SCD4 ;; Test loop counter, next source pointer, restore sin/cos pointer
	loops_reset			;; 7 more iters of 8
	zr8_eight_complex_djbunfft_preload
c2b:	zr8_eight_complex_djbunfft rsi, 128, 8*128, 16*128, dist32, rdi, ZMM_SCD4, 8
	zloop	8, c2b, -8*128+2*dist32, rdi, -8*ZMM_SCD4 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	8, c2b, -8*2*dist32	;; Test loop counter, restore source pointer

;; Do inverse FFT levels 1,2,3,4
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	512	+1	...
;;	8	...
;;	...
;;	504
;;	1K
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	4096	+1	...
;;	8	...
;;	...
;;	4088

;; Do 512 sixteen_reals_last_unfft operations
;;	distance between fft data elements is 512
;;	do 64 macros each processing 128 data values

	bump	rdx, -8*ZMM_SCD4-64*ZMM_SCD4
	bump	rdi, -64*ZMM_SCD4
	zloop_init 64			;; 64 iterations
	zloop_set_rw_prefetch ZDOUBLE	;; Prefetch 128 cache lines
	zr8_2sc_sixteen_reals_unfft_preload
c1b:	zr8_2sc_sixteen_reals_unfft rsi, 128, 2*dist32, 4*dist32, 8*dist32, rdx, ZMM_SCD4, rdi, ZMM_SCD4, 32
	zloop	32, c1b, -32*128+dist32	;; Test loop counter, next source pointer
	zloop	2, c1b			;; Test loop counter

zpass2_4096_real_done:
	end_timer 2
	ENDM

; *************** radix8-pass2-4096-complex macro ******************
; This macro takes 4096 complex values and performs the final 12 levels of a standard
; radix-8 FFT, squares the results, and does the first 12 levels of the inverse FFT.

zr4dwpn_pass2_4096_complex MACRO
	LOCAL	b1b, b2b, b3a, b3b, b3c, b3d, c1b, c2b
	LOCAL	zpass2_4096_complex_1, zpass2_4096_complex_2
	LOCAL	zpass2_4096_complex_3, zpass2_4096_complex_4
	LOCAL	zpass2_4096_complex_unfft, zpass2_4096_complex_done

;; Type 4 FFTs skip the forward FFT process

	mov	rsi, DATA_ADDR		;; Load address of FFT data
	mov	rbx, DIST_TO_FFTSRCARG
	mov	rcx, DATA_PREFETCH	;; Prefetch pointer (we need to prefetch 1024 cache lines + 15 pad cache lines)
	cmp	ffttype, 4
	je	zpass2_4096_complex_4

;; Do FFT levels 1,2,3
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	4096	+1	...
;;	8	...
;;	...
;;	4088
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	512	+1	...
;;	8	...
;;	...
;;	504
;;	1K
;;	...

;; Do 512 eight_complex_fft operations
;;	distance between fft data elements is 512
;;	do 64 macros each processing 128 data values

	start_timer 4
	mov	rdi, xsincos_complex	;; The sin/cos data
	zloop_init 64			;; 64 iterations
	zloop_set_rw_prefetch ZDOUBLE	;; Prefetch 128 cache lines
	zr8f_eight_complex_djbfft_preload
b1b:	zr8f_eight_complex_djbfft rsi, 128, 2*dist32, 4*dist32, 8*dist32, rdi, ZMM_SCD4, 32, L1PREFETCH_ALL, 128
	zloop	32, b1b, -32*128+dist32	;; Test loop counter, next source pointer
	zloop	2, b1b, -2*dist32	;; Test loop counter, restore source pointer
	end_timer 4

;; Do FFT levels 4,5,6
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	512	+1	...
;;	8	...
;;	...
;;	504
;;	1K
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	64	+1	...
;;	8	...
;;	...
;;	56
;;	128
;;	...

;; Do 512 eight_complex_fft operations
;;	distance between fft data elements is 64
;;	do 64 macros each processing 128 data values

	start_timer 5
	zloop_init 64			;; 8 iterations of 8
	zloop_set_rw_prefetch ZTRIPLE	;; Prefetch 192 cache lines
	zr8_eight_complex_djbfft_preload
b2b:	zr8_eight_complex_djbfft rsi, 128, 8*128, 16*128, dist32, rdi, ZMM_SCD4, 8, L1PREFETCH_ALL, 128
	zloop	8, b2b, -8*128+2*dist32, rdi, -8*ZMM_SCD4 ;; Test loop counter, restore source pointer, restore sin/cos pointer
	zloop	8, b2b, -8*2*dist32, rdi, 8*ZMM_SCD4 ;; Test loop counter, restore source pointer, next sin/cos pointer
	end_timer 5

;; Do FFT levels 7,8,9,10,11,12 as well as inverse FFT levels 7,8,9,10,11,12
;; On input and output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	64	+1	...
;;	8	...
;;	...
;;	56
;;	128
;;	...

;; Do 64 sixtyfour_complex_fft operations
;;	distance between fft data elements is 1 (8 for swizzling)
;;	do 64 macros each processing 128 data values

;; Execute the right middle step

pfing	bump	rcx, -49*64		;; We will prefetch 49 cache lines too many
	cmp	ffttype, 2
	jl	zpass2_4096_complex_1
	jg	zpass2_4096_complex_3

zpass2_4096_complex_2:
	start_timer 9
	zloop_init 64			;; 64 iterations
	zloop_set_rw_prefetch ZOCT	;; Prefetch 512 cache lines
	mov	r8, DEST2ARG
	zr64_sixtyfour_complex_with_square_preload
b3a:	zr64_sixtyfour_complex_with_square rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 4, L1PREFETCH_ALL, 8*128
	zloop	4, b3a, -4*8*128+dist32	;; Test loop counter
	zloop	16, b3a			;; Test loop counter
	add	rsi, r8
	end_timer 9
	jmp	zpass2_4096_complex_unfft

zpass2_4096_complex_1:
	start_timer 9
	zloop_init 64			;; 64 iterations
	zloop_set_rw_prefetch ZOCT	;; Prefetch 512 cache lines
	zr64_sixtyfour_complex_fft_final_preload
b3b:	zr64_sixtyfour_complex_fft_final rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 4, L1PREFETCH_ALL, 8*128
	zloop	4, b3b, -4*8*128+dist32	;; Test loop counter
	zloop	16, b3b			;; Test loop counter
	end_timer 9
	jmp	zpass2_4096_complex_done

zpass2_4096_complex_3:
	start_timer 9
	zloop_init 64			;; 64 iterations
	zloop_set_rw_prefetch ZOCT	;; Prefetch 512 cache lines
	mov	r8, DEST2ARG
	mov	rbp, DIST_TO_MULSRCARG
	zr64_sixtyfour_complex_with_mult_preload
b3c:	zr64_sixtyfour_complex_with_mult rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 4, L1PREFETCH_ALL, 8*128
	zloop	4, b3c, -4*8*128+dist32	;; Test loop counter
	zloop	16, b3c			;; Test loop counter
	add	rsi, r8
	end_timer 9
	jmp	zpass2_4096_complex_unfft

zpass2_4096_complex_4:
	start_timer 9
	zloop_init 64			;; 64 iterations
	zloop_set_rw_prefetch ZOCT	;; Prefetch 512 cache lines
	mov	rbp, DIST_TO_MULSRCARG
	mov	rdi, xsincos_complex	;; The sin/cos data
	bump	rdi, 64*ZMM_SCD4+8*ZMM_SCD4
	zr64f_sixtyfour_complex_with_mulf_preload
b3d:	zr64f_sixtyfour_complex_with_mulf rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 4, L1PREFETCH_ALL, 8*128
	zloop	4, b3d, -4*8*128+dist32	;; Test loop counter
	zloop	16, b3d			;; Test loop counter
	end_timer 9
zpass2_4096_complex_unfft:
	bump	rsi, -16*dist32		;; Restore source pointer

;; Do inverse FFT levels 4,5,6
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	64	+1	...
;;	8	...
;;	...
;;	56
;;	128
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	512	+1	...
;;	8	...
;;	...
;;	504
;;	1K
;;	...

;; Do 512 eight_complex_unfft operations
;;	distance between fft data elements is 64
;;	do 64 macros each processing 128 data values

	start_timer 13
	bump	rdi, -8*ZMM_SCD4
	zloop_init 64			;; 8 iterations of 8
	zloop_set_rw_prefetch ZDOUBLE	;; Prefetch 128 cache lines
	zr8_eight_complex_djbunfft_preload
c2b:	zr8_eight_complex_djbunfft rsi, 128, 8*128, 16*128, dist32, rdi, ZMM_SCD4, 8, L1PREFETCH_ALL, 128
	zloop	8, c2b, -8*128+2*dist32, rdi, -8*ZMM_SCD4 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	8, c2b, -8*2*dist32	;; Test loop counter, restore source pointer
	end_timer 13

;; Do inverse FFT levels 1,2,3
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	512	+1	...
;;	8	...
;;	...
;;	504
;;	1K
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	4096	+1	...
;;	8	...
;;	...
;;	4088

;; Do 512 eight_complex_unfft operations
;;	distance between fft data elements is 512
;;	do 64 macros each processing 128 data values

	start_timer 14
	bump	rdi, -64*ZMM_SCD4
	zloop_init 64			;; 64 iterations
	zloop_set_rw_prefetch ZDOUBLE	;; Prefetch 128 cache lines
	zr8_eight_complex_djbunfft_preload
c1b:	zr8_eight_complex_djbunfft rsi, 128, 2*dist32, 4*dist32, 8*dist32, rdi, ZMM_SCD4, 32, L1PREFETCH_ALL, 128
	zloop	32, c1b, -32*128+dist32	;; Test loop counter, next source pointer
	zloop	2, c1b			;; Test loop counter
	end_timer 14

zpass2_4096_complex_done:
	ENDM


; *************** radix8-pass2-4480-real macro ******************
; This macro takes 8960 real values and performs the final 13.129 levels of the FFT.
; The results are squared, and the first 13.129 levels of the inverse FFT are done.

zr4dwpn_pass2_4480_real MACRO
	LOCAL	b1b, b2a, b2b, b3a, b3b, b3c, b3d, c1b, c2a, c2b
	LOCAL	zpass2_4480_real_1, zpass2_4480_real_2
	LOCAL	zpass2_4480_real_3, zpass2_4480_real_4
	LOCAL	zpass2_4480_real_unfft, zpass2_4480_real_done

;; Type 4 FFTs skip the forward FFT process

	start_timer 2
	mov	rsi, DATA_ADDR		;; Load source address
	mov	rbx, DIST_TO_FFTSRCARG
	mov	rcx, DATA_PREFETCH	;; Prefetch pointer (we need to prefetch 1120 cache lines + ?? pad cache lines)
	cmp	ffttype, 4
	je	zpass2_4480_real_4

;; Do FFT levels 1,2,3,4
;;	0	+1	+1	+1	+1	+1	+1	+1	4480	+1	...
;;	8	...
;;	...
;;	4472
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	640	+1	...
;;	8	...
;;	...
;;	632
;;	1280
;;	...

;; Do 640 fourteen_real_fft operations
;;	distance between fft data elements is 640
;;	do 80 macros each processing 112 data values

	mov	rdx, sincos3		;; Real sin/cos data
	mov	rdi, xsincos_complex	;; Complex sin/cos data
	zloop_init 80			;; 80 iterations
	zloop_set_rw_prefetch ZDOUBLE	;; Prefetch 160 cache lines
	zr7f_2sc_fourteen_reals_fft_preload
b1b:	zr7f_2sc_fourteen_reals_fft rsi, 128, 80*128, rdx, ZMM_SCD3, rdi, ZMM_SCD3, 80
	zloop	80, b1b, -80*128	;; Test loop counter, restore source pointer

;; Do FFT levels 5,6,7
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	640	+1	...
;;	8	...
;;	...
;;	632
;;	1280
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	64	+1	...
;;	8	...
;;	...
;;	56
;;	128
;;	...

;; Do 64 twenty_reals_fft operations
;; Do 384 ten_complex_fft operations
;;	distance between fft data elements is 64
;;	do 56 macros each processing 160 data values

	zloop_init 56			;; 7 iterations of 8
	zloop_set_rw_prefetch ZTRIPLE	;; Prefetch 168 cache lines
	zr10_2sc_twenty_reals_fft_preload
b2a:	zr10_2sc_twenty_reals_fft rsi, 128, 8*128, rdx, ZMM_SCD5, rdi, ZMM_SCD5, 8
	zloop	8, b2a, -8*128+80*128, rdi, -8*ZMM_SCD5 ;; Test loop counter, next source pointer, restore sin/cos pointer
	loops_reset			;; 6 more iters of 8
	zr10_ten_complex_djbfft_preload
b2b:	zr10_ten_complex_djbfft rsi, 128, 8*128, rdi, ZMM_SCD5, 8
	zloop	8, b2b, -8*128+80*128, rdi, -8*ZMM_SCD5 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	7, b2b, -7*80*128, rdi, 8*ZMM_SCD5 ;; Test loop counter, restore source pointer, next sin/cos pointer

;; Do FFT levels 8,9,10,11,12,13 as well as inverse FFT levels 8,9,10,11,12,13
;; On input and output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	64	+1	...
;;	8	...
;;	...
;;	56
;;	128
;;	...

;; Do 1 hundredtwentyeight_reals_fft operation
;; Do 69 sixtyfour_complex_fft operations
;;	distance between fft data elements is 1 (8 for swizzling)
;;	do 70 macros each processing 128 data values

;; Execute the proper middle step

pfing	bump	rcx, -32*64		;; We will prefetch 32 cache lines too many
	cmp	ffttype, 2
	jl	zpass2_4480_real_1
	jg	zpass2_4480_real_3

zpass2_4480_real_2:
	mov	r8, DEST2ARG
;;	zr64_hundredtwentyeight_real_with_square_preload
;;	zr64_hundredtwentyeight_real_with_square rsi, 8*128, 1*128, 2*128, 4*128, rdx, 0*ZMM_SCD4, rdi, 0*ZMM_SCD4, 1
	CALLP	zreal_square
	zloop_init 69			;; 69 more iterations
	zloop_set_rw_prefetch ZOCT	;; Prefetch 552 cache lines
	zr64_sixtyfour_complex_with_square_preload
b3a:	zr64_sixtyfour_complex_with_square rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 69, L1PREFETCH_ALL, 8*128
	zloop	69, b3a			;; Test loop counter
	add	rsi, r8
	jmp	zpass2_4480_real_unfft

zpass2_4480_real_1:
;;	zr64_hundredtwentyeight_real_fft_final_preload
;;	zr64_hundredtwentyeight_real_fft_final rsi, 8*128, 1*128, 2*128, 4*128, rdx, 0*ZMM_SCD4, rdi, 0*ZMM_SCD4, 1
	CALLP	zreal_fft_final
	zloop_init 69			;; 69 more iterations
	zloop_set_rw_prefetch ZOCT	;; Prefetch 552 cache lines
	zr64_sixtyfour_complex_fft_final_preload
b3b:	zr64_sixtyfour_complex_fft_final rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 69, L1PREFETCH_ALL, 8*128
	zloop	69, b3b			;; Test loop counter
	jmp	zpass2_4480_real_done

zpass2_4480_real_3:
	mov	r8, DEST2ARG
	mov	rbp, DIST_TO_MULSRCARG
;;	zr64_hundredtwentyeight_real_with_mult_preload
;;	zr64_hundredtwentyeight_real_with_mult rsi, 8*128, 1*128, 2*128, 4*128, rdx, 0*ZMM_SCD4, rdi, 0*ZMM_SCD4, 1
	CALLP	zreal_mult
	zloop_init 69			;; 69 more iterations
	zloop_set_rw_prefetch ZOCT	;; Prefetch 552 cache lines
	zr64_sixtyfour_complex_with_mult_preload
b3c:	zr64_sixtyfour_complex_with_mult rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 69, L1PREFETCH_ALL, 8*128
	zloop	69, b3c			;; Test loop counter
	add	rsi, r8
	jmp	zpass2_4480_real_unfft

zpass2_4480_real_4:
	mov	rbp, DIST_TO_MULSRCARG
	mov	rdx, sincos3		;; Real sin/cos data
	mov	rdi, xsincos_complex	;; Complex sin/cos data
	bump	rdx, 80*ZMM_SCD3+8*ZMM_SCD5
	bump	rdi, 80*ZMM_SCD3+8*ZMM_SCD5
;;	zr64f_hundredtwentyeight_real_with_mulf_preload
;;	zr64f_hundredtwentyeight_real_with_mulf rsi, 8*128, 1*128, 2*128, 4*128, rdx, 0*ZMM_SCD4, rdi, 0*ZMM_SCD4, 1
	CALLP	zreal_mulf
	zloop_init 69			;; 69 more iterations
	zloop_set_rw_prefetch ZOCT	;; Prefetch 552 cache lines
	zr64f_sixtyfour_complex_with_mulf_preload
b3d:	zr64f_sixtyfour_complex_with_mulf rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 69, L1PREFETCH_ALL, 8*128
	zloop	69, b3d			;; Test loop counter
zpass2_4480_real_unfft:
	bump	rsi, -70*8*128		;; Restore source pointer

;; Do inverse FFT levels 5,6,7
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	64	+1	...
;;	8	...
;;	...
;;	56
;;	128
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	640	+1	...
;;	8	...
;;	...
;;	632
;;	1280
;;	...

;; Do 64 twenty_reals_unfft operations
;; Do 384 ten_complex_unfft operations
;;	distance between fft data elements is 64
;;	do 56 macros each processing 160 data values

	zloop_init 56			;; 7 iterations of 8
	zloop_set_rw_prefetch ZDOUBLE	;; Prefetch 112 cache lines
	bump	rdx, -8*ZMM_SCD5
	bump	rdi, -8*ZMM_SCD5
	zr10_2sc_twenty_reals_unfft_preload
c2a:	zr10_2sc_twenty_reals_unfft rsi, 128, 8*128, rdx, ZMM_SCD5, rdi, ZMM_SCD5, 8
	zloop	8, c2a, -8*128+80*128, rdi, -8*ZMM_SCD5 ;; Test loop counter, next source pointer, restore sin/cos pointer
	loops_reset			;; 6 more iters of 8
	zr10_ten_complex_djbunfft_preload
c2b:	zr10_ten_complex_djbunfft rsi, 128, 8*128, rdi, ZMM_SCD5, 8
	zloop	8, c2b, -8*128+80*128, rdi, -8*ZMM_SCD5 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	7, c2b, -7*80*128	;; Test loop counter, restore source pointer

;; Do inverse FFT levels 1,2,3,4
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	640	+1	...
;;	8	...
;;	...
;;	632
;;	1280
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	4480	+1	...
;;	8	...
;;	...
;;	4472

;; Do 640 fourteen_reals_last_unfft operations
;;	distance between fft data elements is 640
;;	do 80 macros each processing 112 data values

	bump	rdx, -8*ZMM_SCD5-80*ZMM_SCD3
	bump	rdi, -80*ZMM_SCD3
	zloop_init 80			;; 80 iterations
	zloop_set_rw_prefetch ZDOUBLE	;; Prefetch 160 cache lines
	zr7_2sc_fourteen_reals_unfft_preload
c1b:	zr7_2sc_fourteen_reals_unfft rsi, 128, 80*128, rdx, ZMM_SCD3, rdi, ZMM_SCD3, 80
	zloop	80, c1b			;; Test loop counter

zpass2_4480_real_done:
	end_timer 2
	ENDM

; *************** radix8-pass2-4480-complex macro ******************
; This macro takes 4480 complex values and performs the final 12.129 levels of a standard
; radix-8 FFT, squares the results, and does the first 12.129 levels of the inverse FFT.

zr4dwpn_pass2_4480_complex MACRO
	LOCAL	b1b, b2b, b3a, b3b, b3c, b3d, c1b, c2b
	LOCAL	zpass2_4480_complex_1, zpass2_4480_complex_2
	LOCAL	zpass2_4480_complex_3, zpass2_4480_complex_4
	LOCAL	zpass2_4480_complex_unfft, zpass2_4480_complex_done

;; Type 4 FFTs skip the forward FFT process

	mov	rsi, DATA_ADDR		;; Load address of FFT data
	mov	rbx, DIST_TO_FFTSRCARG
	mov	rcx, DATA_PREFETCH	;; Prefetch pointer (we need to prefetch 1120 cache lines + ?? pad cache lines)
	cmp	ffttype, 4
	je	zpass2_4480_complex_4

;; Do FFT levels 1,2,3
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	4480	+1	...
;;	8	...
;;	...
;;	5368
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	640	+1	...
;;	8	...
;;	...
;;	632
;;	1280
;;	...

;; Do 640 seven_complex_fft operations
;;	distance between fft data elements is 640
;;	do 80 macros each processing 112 data values

	start_timer 4
	mov	rdi, xsincos_complex	;; The sin/cos data
	zloop_init 80			;; 80 iterations
	zloop_set_rw_prefetch ZDOUBLE	;; Prefetch 160 cache lines
	zr7f_seven_complex_djbfft_preload
b1b:	zr7f_seven_complex_djbfft rsi, 128, 80*128, rdi, ZMM_SCD3, 80, L1PREFETCH_ALL, 128
	zloop	80, b1b, -80*128	;; Test loop counter, restore source pointer
	end_timer 4

;; Do FFT levels 4,5,6
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	640	+1	...
;;	8	...
;;	...
;;	632
;;	1280
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	64	+1	...
;;	8	...
;;	...
;;	56
;;	128
;;	...

;; Do 448 ten_complex_fft operations
;;	distance between fft data elements is 64
;;	do 56 macros each processing 160 data values

	start_timer 5
	zloop_init 56			;; 7 iterations of 8
	zloop_set_rw_prefetch ZTRIPLE	;; Prefetch 168 cache lines
	zr10_ten_complex_djbfft_preload
b2b:	zr10_ten_complex_djbfft rsi, 128, 8*128, rdi, ZMM_SCD5, 8, L1PREFETCH_ALL, 128
	zloop	8, b2b, -8*128+80*128, rdi, -8*ZMM_SCD5	;; Test loop counter, restore source pointer, restore sin/cos pointer
	zloop	7, b2b, -7*80*128, rdi, 8*ZMM_SCD5 ;; Test loop counter, restore source pointer, next sin/cos pointer
	end_timer 5

;; Do FFT levels 7,8,9,10,11,12 as well as inverse FFT levels 7,8,9,10,11,12
;; On input and output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	64	+1	...
;;	8	...
;;	...
;;	56
;;	128
;;	...

;; Do 70 sixtyfour_complex_fft operations
;;	distance between fft data elements is 1 (8 for swizzling)
;;	do 70 macros each processing 128 data values

;; Execute the right middle step

pfing	bump	rcx, -40*64		;; We will prefetch 40 cache lines too many
	cmp	ffttype, 2
	jl	zpass2_4480_complex_1
	jg	zpass2_4480_complex_3

zpass2_4480_complex_2:
	start_timer 9
	zloop_init 70			;; 70 iterations
	zloop_set_rw_prefetch ZOCT	;; Prefetch 560 cache lines
	mov	r8, DEST2ARG
	zr64_sixtyfour_complex_with_square_preload
b3a:	zr64_sixtyfour_complex_with_square rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 70, L1PREFETCH_ALL, 8*128
	zloop	70, b3a			;; Test loop counter
	add	rsi, r8
	end_timer 9
	jmp	zpass2_4480_complex_unfft

zpass2_4480_complex_1:
	start_timer 9
	zloop_init 70			;; 70 iterations
	zloop_set_rw_prefetch ZOCT	;; Prefetch 560 cache lines
	zr64_sixtyfour_complex_fft_final_preload
b3b:	zr64_sixtyfour_complex_fft_final rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 70, L1PREFETCH_ALL, 8*128
	zloop	70, b3b			;; Test loop counter
	end_timer 9
	jmp	zpass2_4480_complex_done

zpass2_4480_complex_3:
	start_timer 9
	zloop_init 70			;; 70 iterations
	zloop_set_rw_prefetch ZOCT	;; Prefetch 560 cache lines
	mov	r8, DEST2ARG
	mov	rbp, DIST_TO_MULSRCARG
	zr64_sixtyfour_complex_with_mult_preload
b3c:	zr64_sixtyfour_complex_with_mult rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 70, L1PREFETCH_ALL, 8*128
	zloop	70, b3c			;; Test loop counter
	add	rsi, r8
	end_timer 9
	jmp	zpass2_4480_complex_unfft

zpass2_4480_complex_4:
	start_timer 9
	zloop_init 70			;; 70 iterations
	zloop_set_rw_prefetch ZOCT	;; Prefetch 560 cache lines
	mov	rbp, DIST_TO_MULSRCARG
	mov	rdi, xsincos_complex	;; The sin/cos data
	bump	rdi, 80*ZMM_SCD3+8*ZMM_SCD5
	zr64f_sixtyfour_complex_with_mulf_preload
b3d:	zr64f_sixtyfour_complex_with_mulf rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 70, L1PREFETCH_ALL, 8*128
	zloop	70, b3d			;; Test loop counter
	end_timer 9
zpass2_4480_complex_unfft:
	bump	rsi, -70*8*128		;; Restore source pointer

;; Do inverse FFT levels 4,5,6
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	64	+1	...
;;	8	...
;;	...
;;	56
;;	128
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	640	+1	...
;;	8	...
;;	...
;;	632
;;	1280
;;	...

;; Do 448 ten_complex_unfft operations
;;	distance between fft data elements is 64
;;	do 56 macros each processing 160 data values

	start_timer 13
	bump	rdi, -8*ZMM_SCD5
	zloop_init 56			;; 7 iterations of 8
	zloop_set_rw_prefetch ZDOUBLE	;; Prefetch 112 cache lines
	zr10_ten_complex_djbunfft_preload
c2b:	zr10_ten_complex_djbunfft rsi, 128, 8*128, rdi, ZMM_SCD5, 8, L1PREFETCH_ALL, 128
	zloop	8, c2b, -8*128+80*128, rdi, -8*ZMM_SCD5 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	7, c2b, -7*80*128	;; Test loop counter, restore source pointer
	end_timer 13

;; Do inverse FFT levels 1,2,3
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	640	+1	...
;;	8	...
;;	...
;;	632
;;	1280
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	4480	+1	...
;;	8	...
;;	...
;;	4472

;; Do 640 seven_complex_unfft operations
;;	distance between fft data elements is 640
;;	do 80 macros each processing 112 data values

	start_timer 14
	bump	rdi, -80*ZMM_SCD3
	zloop_init 80			;; 80 iterations
	zloop_set_rw_prefetch ZDOUBLE	;; Prefetch 160 cache lines
	zr7_seven_complex_djbunfft_preload
c1b:	zr7_seven_complex_djbunfft rsi, 128, 80*128, rdi, ZMM_SCD3, 80, L1PREFETCH_ALL, 128
	zloop	80, c1b			;; Test loop counter
	end_timer 14

zpass2_4480_complex_done:
	ENDM


; *************** radix8-pass2-4608-real macro ******************
; This macro takes 9216 real values and performs the final 13.17 levels of the FFT.
; The results are squared, and the first 13.17 levels of the inverse FFT are done.

zr4dwpn_pass2_4608_real MACRO
	LOCAL	b1b, b2a, b2b, b3a, b3b, b3c, b3d, c1b, c2a, c2b
	LOCAL	zpass2_4608_real_1, zpass2_4608_real_2
	LOCAL	zpass2_4608_real_3, zpass2_4608_real_4
	LOCAL	zpass2_4608_real_unfft, zpass2_4608_real_done

;; Type 4 FFTs skip the forward FFT process

	start_timer 2
	mov	rsi, DATA_ADDR		;; Load source address
	mov	rbx, DIST_TO_FFTSRCARG
	mov	rcx, DATA_PREFETCH	;; Prefetch pointer (we need to prefetch 1172 cache lines + ?? pad cache lines)
	cmp	ffttype, 4
	je	zpass2_4608_real_4

;; Do FFT levels 1,2,3,4
;;	0	+1	+1	+1	+1	+1	+1	+1	4608	+1	...
;;	8	...
;;	...
;;	4600
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	384	+1	...
;;	8	...
;;	...
;;	376
;;	768
;;	...

;; Do 384 twentyfour_real_fft operations
;;	distance between fft data elements is 384
;;	do 48 macros each processing 192 data values

	mov	rdx, sincos3		;; Real sin/cos data
	mov	rdi, xsincos_complex	;; Complex sin/cos data
	zloop_init 48			;; 48 iterations
	zloop_set_rw_prefetch ZDOUBLE	;; Prefetch 96 cache lines
	zr12f_2sc_twentyfour_reals_fft_preload
b1b:	zr12f_2sc_twentyfour_reals_fft rsi, 128, 48*128, 96*128, 192*128, rdx, ZMM_SCD6, rdi, ZMM_SCD6, 48
	zloop	48, b1b, -48*128	;; Test loop counter, restore source pointer

;; Do FFT levels 5,6,7
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	384	+1	...
;;	8	...
;;	...
;;	376
;;	768
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	64	+1	...
;;	8	...
;;	...
;;	56
;;	128
;;	...

;; Do 48 twelve_reals_fft operations
;; Do 720 six_complex_fft operations
;;	distance between fft data elements is 64
;;	do 96 macros each processing 96 data values

	zloop_init 96			;; 12 iterations of 8
	zloop_set_rw_prefetch ZDOUBLE	;; Prefetch 192 cache lines
	zr6_2sc_twelve_reals_fft_preload
b2a:	zr6_2sc_twelve_reals_fft rsi, 128, 8*128, rdx, ZMM_SCD3, rdi, ZMM_SCD3, 8
	zloop	8, b2a, -8*128+48*128, rdi, -8*ZMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer
	loops_reset			;; 5 more iters of 8
	zr6_six_complex_djbfft_preload
b2b:	zr6_six_complex_djbfft rsi, 128, 8*128, rdi, ZMM_SCD3, 8
	zloop	8, b2b, -8*128+48*128, rdi, -8*ZMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	12, b2b, -12*48*128, rdi, 8*ZMM_SCD3 ;; Test loop counter, restore source pointer, next sin/cos pointer

;; Do FFT levels 8,9,10,11,12,13 as well as inverse FFT levels 8,9,10,11,12,13
;; On input and output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	64	+1	...
;;	8	...
;;	...
;;	56
;;	128
;;	...

;; Do 1 hundredtwentyeight_reals_fft operation
;; Do 71 sixtyfour_complex_fft operations
;;	distance between fft data elements is 1 (8 for swizzling)
;;	do 72 macros each processing 128 data values

;; Execute the proper middle step

	cmp	ffttype, 2
	jl	zpass2_4608_real_1
	jg	zpass2_4608_real_3

zpass2_4608_real_2:
	mov	r8, DEST2ARG
;;	zr64_hundredtwentyeight_real_with_square_preload
;;	zr64_hundredtwentyeight_real_with_square rsi, 8*128, 1*128, 2*128, 4*128, rdx, 0*ZMM_SCD4, rdi, 0*ZMM_SCD4, 1
	CALLP	zreal_square
	zloop_init 71			;; 71 more iterations
	zloop_set_rw_prefetch ZOCT	;; Prefetch 568 cache lines
	zr64_sixtyfour_complex_with_square_preload
b3a:	zr64_sixtyfour_complex_with_square rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 1, L1PREFETCH_ALL, 8*128
	zloop	71, b3a			;; Test loop counter
	add	rsi, r8
	jmp	zpass2_4608_real_unfft

zpass2_4608_real_1:
;;	zr64_hundredtwentyeight_real_fft_final_preload
;;	zr64_hundredtwentyeight_real_fft_final rsi, 8*128, 1*128, 2*128, 4*128, rdx, 0*ZMM_SCD4, rdi, 0*ZMM_SCD4, 1
	CALLP	zreal_fft_final
	zloop_init 71			;; 71 more iterations
	zloop_set_rw_prefetch ZOCT	;; Prefetch 568 cache lines
	zr64_sixtyfour_complex_fft_final_preload
b3b:	zr64_sixtyfour_complex_fft_final rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 1, L1PREFETCH_ALL, 8*128
	zloop	71, b3b			;; Test loop counter
	jmp	zpass2_4608_real_done

zpass2_4608_real_3:
	mov	r8, DEST2ARG
	mov	rbp, DIST_TO_MULSRCARG
;;	zr64_hundredtwentyeight_real_with_mult_preload
;;	zr64_hundredtwentyeight_real_with_mult rsi, 8*128, 1*128, 2*128, 4*128, rdx, 0*ZMM_SCD4, rdi, 0*ZMM_SCD4, 1
	CALLP	zreal_mult
	zloop_init 71			;; 71 more iterations
	zloop_set_rw_prefetch ZOCT	;; Prefetch 568 cache lines
	zr64_sixtyfour_complex_with_mult_preload
b3c:	zr64_sixtyfour_complex_with_mult rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 1, L1PREFETCH_ALL, 8*128
	zloop	71, b3c			;; Test loop counter
	add	rsi, r8
	jmp	zpass2_4608_real_unfft

zpass2_4608_real_4:
	mov	rbp, DIST_TO_MULSRCARG
	mov	rdx, sincos3		;; Real sin/cos data
	mov	rdi, xsincos_complex	;; Complex sin/cos data
	bump	rdx, 48*ZMM_SCD6+8*ZMM_SCD3
	bump	rdi, 48*ZMM_SCD6+8*ZMM_SCD3
;;	zr64f_hundredtwentyeight_real_with_mulf_preload
;;	zr64f_hundredtwentyeight_real_with_mulf rsi, 8*128, 1*128, 2*128, 4*128, rdx, 0*ZMM_SCD4, rdi, 0*ZMM_SCD4, 1
	CALLP	zreal_mulf
	zloop_init 71			;; 71 more iterations
	zloop_set_rw_prefetch ZOCT	;; Prefetch 568 cache lines
	zr64f_sixtyfour_complex_with_mulf_preload
b3d:	zr64f_sixtyfour_complex_with_mulf rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 1, L1PREFETCH_ALL, 8*128
	zloop	71, b3d			;; Test loop counter
zpass2_4608_real_unfft:
	bump	rsi, -72*8*128		;; Restore source pointer

;; Do inverse FFT levels 5,6,7
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	64	+1	...
;;	8	...
;;	...
;;	56
;;	128
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	384	+1	...
;;	8	...
;;	...
;;	376
;;	768
;;	...

;; Do 48 twelve_reals_unfft operations
;; Do 720 six_complex_unfft operations
;;	distance between fft data elements is 64
;;	do 96 macros each processing 96 data values

	zloop_init 96			;; 12 iterations of 8
	zloop_set_rw_prefetch ZDOUBLE	;; Prefetch 192 cache lines
	bump	rdx, -8*ZMM_SCD3
	bump	rdi, -8*ZMM_SCD3
	zr6_2sc_twelve_reals_unfft_preload
c2a:	zr6_2sc_twelve_reals_unfft rsi, 128, 8*128, rdx, ZMM_SCD3, rdi, ZMM_SCD3, 8
	zloop	8, c2a, -8*128+48*128, rdi, -8*ZMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer
	loops_reset			;; 5 more iters of 8
	zr6_six_complex_djbunfft_preload
c2b:	zr6_six_complex_djbunfft rsi, 128, 8*128, rdi, ZMM_SCD3, 8
	zloop	8, c2b, -8*128+48*128, rdi, -8*ZMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	12, c2b, -12*48*128	;; Test loop counter, restore source pointer

;; Do inverse FFT levels 1,2,3,4
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	384	+1	...
;;	8	...
;;	...
;;	376
;;	768
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	4608	+1	...
;;	8	...
;;	...
;;	4600

;; Do 384 twentyfour_reals_last_unfft operations
;;	distance between fft data elements is 384
;;	do 48 macros each processing 192 data values

	bump	rdx, -8*ZMM_SCD3-48*ZMM_SCD6
	bump	rdi, -48*ZMM_SCD6
	zloop_init 48			;; 48 iterations
	zloop_set_rw_prefetch ZDOUBLE	;; Prefetch 96 cache lines
	zr12_2sc_twentyfour_reals_unfft_preload
c1b:	zr12_2sc_twentyfour_reals_unfft rsi, 128, 48*128, 96*128, 192*128, rdx, ZMM_SCD6, rdi, ZMM_SCD6, 48
	zloop	48, c1b			;; Test loop counter

zpass2_4608_real_done:
	end_timer 2
	ENDM

; *************** radix8-pass2-4608-complex macro ******************
; This macro takes 4608 complex values and performs the final 12.17 levels of a standard
; radix-8 FFT, squares the results, and does the first 12.17 levels of the inverse FFT.

zr4dwpn_pass2_4608_complex MACRO
	LOCAL	b1b, b2b, b3a, b3b, b3c, b3d, c1b, c2b
	LOCAL	zpass2_4608_complex_1, zpass2_4608_complex_2
	LOCAL	zpass2_4608_complex_3, zpass2_4608_complex_4
	LOCAL	zpass2_4608_complex_unfft, zpass2_4608_complex_done

;; Type 4 FFTs skip the forward FFT process

	mov	rsi, DATA_ADDR		;; Load address of FFT data
	mov	rbx, DIST_TO_FFTSRCARG
	mov	rcx, DATA_PREFETCH	;; Prefetch pointer (we need to prefetch 1152 cache lines + 0 pad cache lines)
	cmp	ffttype, 4
	je	zpass2_4608_complex_4

;; Do FFT levels 1,2,3
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	4608	+1	...
;;	8	...
;;	...
;;	4600
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	384	+1	...
;;	8	...
;;	...
;;	376
;;	768
;;	...

;; Do 384 twelve_complex_fft operations
;;	distance between fft data elements is 384
;;	do 48 macros each processing 192 data values

	start_timer 4
	mov	rdi, xsincos_complex	;; The sin/cos data
	zloop_init 48			;; 48 iterations
	zloop_set_rw_prefetch ZDOUBLE	;; Prefetch 96 cache lines
	zr12f_twelve_complex_djbfft_preload
b1b:	zr12f_twelve_complex_djbfft rsi, 128, 48*128, 96*128, 192*128, rdi, ZMM_SCD6, 48, L1PREFETCH_ALL, 128
	zloop	48, b1b, -48*128	;; Test loop counter, restore source pointer
	end_timer 4

;; Do FFT levels 4,5,6
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	384	+1	...
;;	8	...
;;	...
;;	376
;;	768
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	64	+1	...
;;	8	...
;;	...
;;	56
;;	128
;;	...

;; Do 768 six_complex_fft operations
;;	distance between fft data elements is 64
;;	do 96 macros each processing 96 data values

	start_timer 5
	zloop_init 96			;; 12 iterations of 8
	zloop_set_rw_prefetch ZDOUBLE	;; Prefetch 192 cache lines
	zr6_six_complex_djbfft_preload
b2b:	zr6_six_complex_djbfft rsi, 128, 8*128, rdi, ZMM_SCD3, 8, L1PREFETCH_ALL, 128
	zloop	8, b2b, -8*128+48*128, rdi, -8*ZMM_SCD3	;; Test loop counter, restore source pointer, restore sin/cos pointer
	zloop	12, b2b, -12*48*128, rdi, 8*ZMM_SCD3 ;; Test loop counter, restore source pointer, next sin/cos pointer
	end_timer 5

;; Do FFT levels 7,8,9,10,11,12 as well as inverse FFT levels 7,8,9,10,11,12
;; On input and output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	64	+1	...
;;	8	...
;;	...
;;	56
;;	128
;;	...

;; Do 72 sixtyfour_complex_fft operations
;;	distance between fft data elements is 1 (8 for swizzling)
;;	do 72 macros each processing 128 data values

;; Execute the right middle step

	cmp	ffttype, 2
	jl	zpass2_4608_complex_1
	jg	zpass2_4608_complex_3

zpass2_4608_complex_2:
	start_timer 9
	zloop_init 72			;; 72 iterations
	zloop_set_rw_prefetch ZOCT	;; Prefetch 576 cache lines
	mov	r8, DEST2ARG
	zr64_sixtyfour_complex_with_square_preload
b3a:	zr64_sixtyfour_complex_with_square rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 72, L1PREFETCH_ALL, 8*128
	zloop	72, b3a			;; Test loop counter
	add	rsi, r8
	end_timer 9
	jmp	zpass2_4608_complex_unfft

zpass2_4608_complex_1:
	start_timer 9
	zloop_init 72			;; 72 iterations
	zloop_set_rw_prefetch ZOCT	;; Prefetch 576 cache lines
	zr64_sixtyfour_complex_fft_final_preload
b3b:	zr64_sixtyfour_complex_fft_final rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 72, L1PREFETCH_ALL, 8*128
	zloop	72, b3b			;; Test loop counter
	end_timer 9
	jmp	zpass2_4608_complex_done

zpass2_4608_complex_3:
	start_timer 9
	zloop_init 72			;; 72 iterations
	zloop_set_rw_prefetch ZOCT	;; Prefetch 576 cache lines
	mov	r8, DEST2ARG
	mov	rbp, DIST_TO_MULSRCARG
	zr64_sixtyfour_complex_with_mult_preload
b3c:	zr64_sixtyfour_complex_with_mult rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 72, L1PREFETCH_ALL, 8*128
	zloop	72, b3c			;; Test loop counter
	add	rsi, r8
	end_timer 9
	jmp	zpass2_4608_complex_unfft

zpass2_4608_complex_4:
	start_timer 9
	zloop_init 72			;; 72 iterations
	zloop_set_rw_prefetch ZOCT	;; Prefetch 576 cache lines
	mov	rbp, DIST_TO_MULSRCARG
	mov	rdi, xsincos_complex	;; The sin/cos data
	bump	rdi, 48*ZMM_SCD6+8*ZMM_SCD3
	zr64f_sixtyfour_complex_with_mulf_preload
b3d:	zr64f_sixtyfour_complex_with_mulf rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 72, L1PREFETCH_ALL, 8*128
	zloop	72, b3d			;; Test loop counter
	end_timer 9
zpass2_4608_complex_unfft:
	bump	rsi, -72*8*128		;; Restore source pointer

;; Do inverse FFT levels 4,5,6
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	64	+1	...
;;	8	...
;;	...
;;	56
;;	128
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	384	+1	...
;;	8	...
;;	...
;;	376
;;	768
;;	...

;; Do 768 six_complex_unfft operations
;;	distance between fft data elements is 64
;;	do 96 macros each processing 96 data values

	start_timer 13
	bump	rdi, -8*ZMM_SCD3
	zloop_init 96			;; 12 iterations of 8
	zloop_set_rw_prefetch ZDOUBLE	;; Prefetch 192 cache lines
	zr6_six_complex_djbunfft_preload
c2b:	zr6_six_complex_djbunfft rsi, 128, 8*128, rdi, ZMM_SCD3, 8, L1PREFETCH_ALL, 128
	zloop	8, c2b, -8*128+48*128, rdi, -8*ZMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	12, c2b, -12*48*128	;; Test loop counter, restore source pointer
	end_timer 13

;; Do inverse FFT levels 1,2,3
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	384	+1	...
;;	8	...
;;	...
;;	376
;;	768
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	4608	+1	...
;;	8	...
;;	...
;;	4600

;; Do 384 twelve_complex_unfft operations
;;	distance between fft data elements is 384
;;	do 64 macros each processing 192 data values

	start_timer 14
	bump	rdi, -48*ZMM_SCD6
	zloop_init 48			;; 48 iterations
	zloop_set_rw_prefetch ZDOUBLE	;; Prefetch 96 cache lines
	zr12_twelve_complex_djbunfft_preload
c1b:	zr12_twelve_complex_djbunfft rsi, 128, 48*128, 96*128, 192*128, rdi, ZMM_SCD6, 48, L1PREFETCH_ALL, 128
	zloop	48, c1b			;; Test loop counter
	end_timer 14

zpass2_4608_complex_done:
	ENDM


; *************** radix8-pass2-5120-levels-real macro ******************
; This macro takes 10240 real values and performs the final 13.322 levels of the FFT.
; The results are squared, and the first 13.322 levels of the inverse FFT are done.

zr4dwpn_pass2_5120_real MACRO
	LOCAL	b1b, b2a, b2b, b3a, b3b, b3c, b3d, c1b, c2a, c2b
	LOCAL	zpass2_5120_levels_real_1, zpass2_5120_levels_real_2
	LOCAL	zpass2_5120_levels_real_3, zpass2_5120_levels_real_4
	LOCAL	zpass2_5120_levels_real_unfft, zpass2_5120_levels_real_done

;; Type 4 FFTs skip the forward FFT process

	start_timer 2
	mov	rsi, DATA_ADDR		;; Load source address
	mov	rbx, DIST_TO_FFTSRCARG
	mov	rcx, DATA_PREFETCH	;; Prefetch pointer (we need to prefetch 1280 cache lines + 19 pad cache lines)
	cmp	ffttype, 4
	je	zpass2_5120_levels_real_4

;; Do FFT levels 1,2,3,4
;;	0	+1	+1	+1	+1	+1	+1	+1	5K	+1	...
;;	8	...
;;	...
;;	5K-8
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	1K	+1	...
;;	8	...
;;	...
;;	1K-8
;;	2K
;;	...

;; Do 1024 ten_real_fft operations
;;	distance between fft data elements is 1024
;;	do 128 macros each processing 80 data values

	mov	rdx, sincos3		;; Real sin/cos data
	mov	rdi, xsincos_complex	;; Complex sin/cos data
	zloop_init 128			;; 128 iterations
	zloop_set_rw_prefetch ZSINGLE	;; Prefetch 128 cache lines
	zr5f_2sc_ten_reals_fft_preload
b1b:	zr5f_2sc_ten_reals_fft rsi, 128, 4*dist32, rdx, ZMM_SCD2, rdi, ZMM_SCD2, 32
	zloop	32, b1b, -32*128+dist32	;; Test loop counter, next source pointer
	zloop	4, b1b, -4*dist32	;; Test loop counter, restore source pointer

;; Do FFT levels 5,6,7
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	1K	+1	...
;;	8	...
;;	...
;;	1K-8
;;	2K
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	64	+1	...
;;	8	...
;;	...
;;	56
;;	128
;;	...

;; Do 64 thirtytwo_reals_fft operations
;; Do 256 sixteen_complex_fft operations
;;	distance between fft data elements is 64
;;	do 40 macros each processing 256 data values

	zloop_init 40			;; 5 iterations of 8
	zloop_set_rw_prefetch ZQUINT	;; Prefetch 200 cache lines
	zr16_2sc_thirtytwo_reals_fft_preload
b2a:	zr16_2sc_thirtytwo_reals_fft rsi, 128, 8*128, 16*128, dist32, 2*dist32, rdx, ZMM_SCD8, rdi, ZMM_SCD8, 8
	zloop	8, b2a, -8*128+4*dist32, rdi, -8*ZMM_SCD8 ;; Test loop counter, next source pointer, restore sin/cos pointer
	loops_reset			;; 4 more iters of 8
	zr16_sixteen_complex_djbfft_preload
b2b:	zr16_sixteen_complex_djbfft rsi, 128, 8*128, 16*128, dist32, 2*dist32, rdi, ZMM_SCD8, 8
	zloop	8, b2b, -8*128+4*dist32, rdi, -8*ZMM_SCD8 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	5, b2b, -5*4*dist32, rdi, 8*ZMM_SCD8 ;; Test loop counter, restore source pointer, next sin/cos pointer

;; Do FFT levels 8,9,10,11,12,13 as well as inverse FFT levels 8,9,10,11,12,13
;; On input and output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	64	+1	...
;;	8	...
;;	...
;;	56
;;	128
;;	...

;; Do 1 hundredtwentyeight_reals_fft operation
;; Do 79 sixtyfour_complex_fft operations
;;	distance between fft data elements is 1 (8 for swizzling)
;;	do 112 macros each processing 128 data values

;; Execute the proper middle step

pfing	bump	rcx, -29*64		;; We will prefetch 29 cache lines too many
	cmp	ffttype, 2
	jl	zpass2_5120_levels_real_1
	jg	zpass2_5120_levels_real_3

zpass2_5120_levels_real_2:
	mov	r8, DEST2ARG
;;	zr64_hundredtwentyeight_real_with_square_preload
;;	zr64_hundredtwentyeight_real_with_square rsi, 8*128, 1*128, 2*128, 4*128, rdx, 0*ZMM_SCD4, rdi, 0*ZMM_SCD4, 1
	CALLP	zreal_square
	zloop_init 80, 1, 4		;; 79 more iterations
	zloop_set_rw_prefetch ZOCT	;; Prefetch 632 cache lines
	zr64_sixtyfour_complex_with_square_preload
b3a:	zr64_sixtyfour_complex_with_square rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 1, L1PREFETCH_ALL, 8*128
	zloop	4, b3a, -4*8*128+dist32	;; Test loop counter, next source pointer
	zloop	20, b3a			;; Test loop counter
	add	rsi, r8
	jmp	zpass2_5120_levels_real_unfft

zpass2_5120_levels_real_1:
;;	zr64_hundredtwentyeight_real_fft_final_preload
;;	zr64_hundredtwentyeight_real_fft_final rsi, 8*128, 1*128, 2*128, 4*128, rdx, 0*ZMM_SCD4, rdi, 0*ZMM_SCD4, 1
	CALLP	zreal_fft_final
	zloop_init 80, 1, 4		;; 79 more iterations
	zloop_set_rw_prefetch ZOCT	;; Prefetch 632 cache lines
	zr64_sixtyfour_complex_fft_final_preload
b3b:	zr64_sixtyfour_complex_fft_final rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 1, L1PREFETCH_ALL, 8*128
	zloop	4, b3b, -4*8*128+dist32	;; Test loop counter, next source pointer
	zloop	20, b3b			;; Test loop counter
	jmp	zpass2_5120_levels_real_done

zpass2_5120_levels_real_3:
	mov	r8, DEST2ARG
	mov	rbp, DIST_TO_MULSRCARG
;;	zr64_hundredtwentyeight_real_with_mult_preload
;;	zr64_hundredtwentyeight_real_with_mult rsi, 8*128, 1*128, 2*128, 4*128, rdx, 0*ZMM_SCD4, rdi, 0*ZMM_SCD4, 1
	CALLP	zreal_mult
	zloop_init 80, 1, 4		;; 79 more iterations
	zloop_set_rw_prefetch ZOCT	;; Prefetch 632 cache lines
	zr64_sixtyfour_complex_with_mult_preload
b3c:	zr64_sixtyfour_complex_with_mult rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 1, L1PREFETCH_ALL, 8*128
	zloop	4, b3c, -4*8*128+dist32	;; Test loop counter, next source pointer
	zloop	20, b3c			;; Test loop counter
	add	rsi, r8
	jmp	zpass2_5120_levels_real_unfft

zpass2_5120_levels_real_4:
	mov	rbp, DIST_TO_MULSRCARG
	mov	rdx, sincos3		;; Real sin/cos data
	mov	rdi, xsincos_complex	;; Complex sin/cos data
	bump	rdx, 128*ZMM_SCD2+8*ZMM_SCD8
	bump	rdi, 128*ZMM_SCD2+8*ZMM_SCD8
;;	zr64f_hundredtwentyeight_real_with_mulf_preload
;;	zr64f_hundredtwentyeight_real_with_mulf rsi, 8*128, 1*128, 2*128, 4*128, rdx, 0*ZMM_SCD4, rdi, 0*ZMM_SCD4, 1
	CALLP	zreal_mulf
	zloop_init 80, 1, 4		;; 79 more iterations
	zloop_set_rw_prefetch ZOCT	;; Prefetch 632 cache lines
	zr64f_sixtyfour_complex_with_mulf_preload
b3d:	zr64f_sixtyfour_complex_with_mulf rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 1, L1PREFETCH_ALL, 8*128
	zloop	4, b3d, -4*8*128+dist32	;; Test loop counter, next source pointer
	zloop	20, b3d			;; Test loop counter
zpass2_5120_levels_real_unfft:
	bump	rsi, -20*dist32		;; Restore source pointer

;; Do inverse FFT levels 5,6,7
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	64	+1	...
;;	8	...
;;	...
;;	56
;;	128
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	1K	+1	...
;;	8	...
;;	...
;;	1K-8
;;	2K
;;	...

;; Do 64 thirtytwo_reals_unfft operations
;; Do 256 sixteen_complex_unfft operations
;;	distance between fft data elements is 64
;;	do 40 macros each processing 256 data values

	zloop_init 40			;; 5 iterations of 8
	zloop_set_rw_prefetch ZSEXT	;; Prefetch 240 cache lines
	bump	rdx, -8*ZMM_SCD8
	bump	rdi, -8*ZMM_SCD8
	zr16_2sc_thirtytwo_reals_unfft_preload
c2a:	zr16_2sc_thirtytwo_reals_unfft rsi, 128, 8*128, 16*128, dist32, 2*dist32, rdx, ZMM_SCD8, rdi, ZMM_SCD8, 8
	zloop	8, c2a, -8*128+4*dist32, rdi, -8*ZMM_SCD8 ;; Test loop counter, next source pointer, restore sin/cos pointer
	loops_reset			;; 4 more iters of 8
	zr16_sixteen_complex_djbunfft_preload
c2b:	zr16_sixteen_complex_djbunfft rsi, 128, 8*128, 16*128, dist32, 2*dist32, rdi, ZMM_SCD8, 8
	zloop	8, c2b, -8*128+4*dist32, rdi, -8*ZMM_SCD8 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	5, c2b, -5*4*dist32	;; Test loop counter, restore source pointer

;; Do inverse FFT levels 1,2,3,4
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	1K	+1	...
;;	8	...
;;	...
;;	1K-8
;;	2K
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	5K	+1	...
;;	8	...
;;	...
;;	5K-8

;; Do 1024 ten_reals_last_unfft operations
;;	distance between fft data elements is 1024
;;	do 128 macros each processing 80 data values

	bump	rdx, -8*ZMM_SCD8-128*ZMM_SCD2
	bump	rdi, -128*ZMM_SCD2
	zloop_init 128			;; 128 iterations
	zloop_set_rw_prefetch ZSINGLE	;; Prefetch 128 cache lines
	zr5_2sc_ten_reals_unfft_preload
c1b:	zr5_2sc_ten_reals_unfft rsi, 128, 4*dist32, rdx, ZMM_SCD2, rdi, ZMM_SCD2, 32
	zloop	32, c1b, -32*128+dist32		;; Test loop counter, next source pointer
	zloop	4, c1b				;; Test loop counter

zpass2_5120_levels_real_done:
	end_timer 2
	ENDM

; *************** radix8-pass2-5120-levels-complex macro ******************
; This macro takes 5120 complex values and performs the final 12.322 levels of a standard
; radix-8 FFT, squares the results, and does the first 12.322 levels of the inverse FFT.

zr4dwpn_pass2_5120_complex MACRO
	LOCAL	b1b, b2b, b3a, b3b, b3c, b3d, c1b, c2b
	LOCAL	zpass2_5120_levels_complex_1, zpass2_5120_levels_complex_2
	LOCAL	zpass2_5120_levels_complex_3, zpass2_5120_levels_complex_4
	LOCAL	zpass2_5120_levels_complex_unfft, zpass2_5120_levels_complex_done

;; Type 4 FFTs skip the forward FFT process

	mov	rsi, DATA_ADDR		;; Load address of FFT data
	mov	rbx, DIST_TO_FFTSRCARG
	mov	rcx, DATA_PREFETCH	;; Prefetch pointer (we need to prefetch 1280 cache lines + 19 pad cache lines)
	cmp	ffttype, 4
	je	zpass2_5120_levels_complex_4

;; Do FFT levels 1,2,3
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	5K	+1	...
;;	8	...
;;	...
;;	5K-8
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	1K	+1	...
;;	8	...
;;	...
;;	1K-8
;;	2K
;;	...

;; Do 1024 five_complex_fft operations
;;	distance between fft data elements is 1024
;;	do 128 macros each processing 80 data values

	start_timer 4
	mov	rdi, xsincos_complex	;; The sin/cos data
	zloop_init 128			;; 128 iterations
	zloop_set_rw_prefetch ZSINGLE	;; Prefetch 128 cache lines
	zr5f_five_complex_djbfft_preload
b1b:	zr5f_five_complex_djbfft rsi, 128, 4*dist32, rdi, ZMM_SCD2, 32, L1PREFETCH_ALL, 128
	zloop	32, b1b, -32*128+dist32	;; Test loop counter, next source pointer
	zloop	4, b1b, -4*dist32	;; Test loop counter, restore source pointer
	end_timer 4

;; Do FFT levels 4,5,6
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	1K	+1	...
;;	8	...
;;	...
;;	1K-8
;;	2K
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	64	+1	...
;;	8	...
;;	...
;;	56
;;	128
;;	...

;; Do 320 sixteen_complex_fft operations
;;	distance between fft data elements is 64
;;	do 40 macros each processing 256 data values

	start_timer 5
	zloop_init 40			;; 5 iterations of 8
	zloop_set_rw_prefetch ZQUINT	;; Prefetch 200 cache lines
	zr16_sixteen_complex_djbfft_preload
b2b:	zr16_sixteen_complex_djbfft rsi, 128, 8*128, 16*128, dist32, 2*dist32, rdi, ZMM_SCD8, 8, L1PREFETCH_ALL, 128
	zloop	8, b2b, -8*128+4*dist32, rdi, -8*ZMM_SCD8 ;; Test loop counter, restore source pointer, restore sin/cos pointer
	zloop	5, b2b, -5*4*dist32, rdi, 8*ZMM_SCD8 ;; Test loop counter, restore source pointer, next sin/cos pointer
	end_timer 5

;; Do FFT levels 7,8,9,10,11,12 as well as inverse FFT levels 7,8,9,10,11,12
;; On input and output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	64	+1	...
;;	8	...
;;	...
;;	56
;;	128
;;	...

;; Do 80 sixtyfour_complex_fft operations
;;	distance between fft data elements is 1 (8 for swizzling)
;;	do 80 macros each processing 128 data values

;; Execute the right middle step

pfing	bump	rcx, -37*64		;; We will prefetch 37 cache lines too many
	cmp	ffttype, 2
	jl	zpass2_5120_levels_complex_1
	jg	zpass2_5120_levels_complex_3

zpass2_5120_levels_complex_2:
	start_timer 9
	zloop_init 80			;; 80 iterations
	zloop_set_rw_prefetch ZOCT	;; Prefetch 640 cache lines
	mov	r8, DEST2ARG
	zr64_sixtyfour_complex_with_square_preload
b3a:	zr64_sixtyfour_complex_with_square rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 4, L1PREFETCH_ALL, 8*128
	zloop	4, b3a, -4*8*128+dist32	;; Test loop counter, next source pointer
	zloop	20, b3a			;; Test loop counter
	add	rsi, r8
	end_timer 9
	jmp	zpass2_5120_levels_complex_unfft

zpass2_5120_levels_complex_1:
	start_timer 9
	zloop_init 80			;; 80 iterations
	zloop_set_rw_prefetch ZOCT	;; Prefetch 640 cache lines
	zr64_sixtyfour_complex_fft_final_preload
b3b:	zr64_sixtyfour_complex_fft_final rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 4, L1PREFETCH_ALL, 8*128
	zloop	4, b3b, -4*8*128+dist32	;; Test loop counter, next source pointer
	zloop	20, b3b			;; Test loop counter
	end_timer 9
	jmp	zpass2_5120_levels_complex_done

zpass2_5120_levels_complex_3:
	start_timer 9
	zloop_init 80			;; 80 iterations
	zloop_set_rw_prefetch ZOCT	;; Prefetch 640 cache lines
	mov	r8, DEST2ARG
	mov	rbp, DIST_TO_MULSRCARG
	zr64_sixtyfour_complex_with_mult_preload
b3c:	zr64_sixtyfour_complex_with_mult rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 4, L1PREFETCH_ALL, 8*128
	zloop	4, b3c, -4*8*128+dist32	;; Test loop counter, next source pointer
	zloop	20, b3c			;; Test loop counter
	add	rsi, r8
	end_timer 9
	jmp	zpass2_5120_levels_complex_unfft

zpass2_5120_levels_complex_4:
	start_timer 9
	zloop_init 80			;; 80 iterations
	zloop_set_rw_prefetch ZOCT	;; Prefetch 640 cache lines
	mov	rbp, DIST_TO_MULSRCARG
	mov	rdi, xsincos_complex	;; The sin/cos data
	bump	rdi, 128*ZMM_SCD2+8*ZMM_SCD8
	zr64f_sixtyfour_complex_with_mulf_preload
b3d:	zr64f_sixtyfour_complex_with_mulf rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 4, L1PREFETCH_ALL, 8*128
	zloop	4, b3d, -4*8*128+dist32	;; Test loop counter, next source pointer
	zloop	20, b3d			;; Test loop counter
	end_timer 9
zpass2_5120_levels_complex_unfft:
	bump	rsi, -20*dist32		;; Restore source pointer

;; Do inverse FFT levels 4,5,6
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	64	+1	...
;;	8	...
;;	...
;;	56
;;	128
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	1K	+1	...
;;	8	...
;;	...
;;	1K-8
;;	2K
;;	...

;; Do 320 sixteen_complex_unfft operations
;;	distance between fft data elements is 64
;;	do 40 macros each processing 256 data values

	start_timer 13
	bump	rdi, -8*ZMM_SCD8
	zloop_init 40			;; 5 iterations of 8
	zloop_set_rw_prefetch ZSEXT	;; Prefetch 240 cache lines
	zr16_sixteen_complex_djbunfft_preload
c2b:	zr16_sixteen_complex_djbunfft rsi, 128, 8*128, 16*128, dist32, 2*dist32, rdi, ZMM_SCD8, 8, L1PREFETCH_ALL, 128
	zloop	8, c2b, -8*128+4*dist32, rdi, -8*ZMM_SCD8 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	5, c2b, -5*4*dist32	;; Test loop counter, restore source pointer
	end_timer 13

;; Do inverse FFT levels 1,2,3
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	1K	+1	...
;;	8	...
;;	...
;;	1K-8
;;	2K
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	5K	+1	...
;;	8	...
;;	...
;;	5K-8

;; Do 1024 five_complex_unfft operations
;;	distance between fft data elements is 1024
;;	do 128 macros each processing 80 data values

	start_timer 14
	bump	rdi, -128*ZMM_SCD2
	zloop_init 128			;; 128 iterations
	zloop_set_rw_prefetch ZSINGLE	;; Prefetch 128 cache lines
	zr5_five_complex_djbunfft_preload
c1b:	zr5_five_complex_djbunfft rsi, 128, 4*dist32, rdi, ZMM_SCD2, 32, L1PREFETCH_ALL, 128
	zloop	32, c1b, -32*128+dist32	;; Test loop counter, next source pointer
	zloop	4, c1b			;; Test loop counter
	end_timer 14

zpass2_5120_levels_complex_done:
	ENDM


; *************** radix8-pass2-5376-real macro ******************
; This macro takes 10752 real values and performs the final 13.392 levels of the FFT.
; The results are squared, and the first 13.392 levels of the inverse FFT are done.

zr4dwpn_pass2_5376_real MACRO
	LOCAL	b1b, b2a, b2b, b3a, b3b, b3c, b3d, c1b, c2a, c2b
	LOCAL	zpass2_5376_real_1, zpass2_5376_real_2
	LOCAL	zpass2_5376_real_3, zpass2_5376_real_4
	LOCAL	zpass2_5376_real_unfft, zpass2_5376_real_done

;; Type 4 FFTs skip the forward FFT process

	start_timer 2
	mov	rsi, DATA_ADDR		;; Load source address
	mov	rbx, DIST_TO_FFTSRCARG
	mov	rcx, DATA_PREFETCH	;; Prefetch pointer (we need to prefetch 1344 cache lines + 20 pad cache lines)
	cmp	ffttype, 4
	je	zpass2_5376_real_4

;; Do FFT levels 1,2,3,4
;;	0	+1	+1	+1	+1	+1	+1	+1	5376	+1	...
;;	8	...
;;	...
;;	5368
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	768	+1	...
;;	8	...
;;	...
;;	760
;;	1536
;;	...

;; Do 768 fourteen_real_fft operations
;;	distance between fft data elements is 768
;;	do 96 macros each processing 112 data values

	mov	rdx, sincos3		;; Real sin/cos data
	mov	rdi, xsincos_complex	;; Complex sin/cos data
	zloop_init 96			;; 96 iterations
	zloop_set_rw_prefetch ZDOUBLE	;; Prefetch 192 cache lines
	zr7f_2sc_fourteen_reals_fft_preload
b1b:	zr7f_2sc_fourteen_reals_fft rsi, 128, 3*dist32, rdx, ZMM_SCD3, rdi, ZMM_SCD3, 32
	zloop	32, b1b, -32*128+dist32	;; Test loop counter, next source pointer
	zloop	3, b1b, -3*dist32	;; Test loop counter, restore source pointer

;; Do FFT levels 5,6,7
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	768	+1	...
;;	8	...
;;	...
;;	760
;;	1536
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	64	+1	...
;;	8	...
;;	...
;;	56
;;	128
;;	...

;; Do 64 twentyfour_reals_fft operations
;; Do 384 twelve_complex_fft operations
;;	distance between fft data elements is 64
;;	do 56 macros each processing 192 data values

	zloop_init 56			;; 7 iterations of 8
	zloop_set_rw_prefetch ZTRIPLE	;; Prefetch 168 cache lines
	zr12_2sc_twentyfour_reals_fft_preload
b2a:	zr12_2sc_twentyfour_reals_fft rsi, 128, 8*128, 16*128, dist32, rdx, ZMM_SCD6, rdi, ZMM_SCD6, 8
	zloop	8, b2a, -8*128+3*dist32, rdi, -8*ZMM_SCD6 ;; Test loop counter, next source pointer, restore sin/cos pointer
	loops_reset			;; 6 more iters of 8
	zr12_twelve_complex_djbfft_preload
b2b:	zr12_twelve_complex_djbfft rsi, 128, 8*128, 16*128, dist32, rdi, ZMM_SCD6, 8
	zloop	8, b2b, -8*128+3*dist32, rdi, -8*ZMM_SCD6 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	7, b2b, -7*3*dist32, rdi, 8*ZMM_SCD6 ;; Test loop counter, restore source pointer, next sin/cos pointer

;; Do FFT levels 8,9,10,11,12,13 as well as inverse FFT levels 8,9,10,11,12,13
;; On input and output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	64	+1	...
;;	8	...
;;	...
;;	56
;;	128
;;	...

;; Do 1 hundredtwentyeight_reals_fft operation
;; Do 83 sixtyfour_complex_fft operations
;;	distance between fft data elements is 1 (8 for swizzling)
;;	do 84 macros each processing 128 data values

;; Execute the proper middle step

pfing	bump	rcx, -20*64		;; We will prefetch 20 cache lines too many
	cmp	ffttype, 2
	jl	zpass2_5376_real_1
	jg	zpass2_5376_real_3

zpass2_5376_real_2:
	mov	r8, DEST2ARG
;;	zr64_hundredtwentyeight_real_with_square_preload
;;	zr64_hundredtwentyeight_real_with_square rsi, 8*128, 1*128, 2*128, 4*128, rdx, 0*ZMM_SCD4, rdi, 0*ZMM_SCD4, 1
	CALLP	zreal_square
	zloop_init 84, 1, 4		;; 83 more iterations
	zloop_set_rw_prefetch ZOCT	;; Prefetch 664 cache lines
	zr64_sixtyfour_complex_with_square_preload
b3a:	zr64_sixtyfour_complex_with_square rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 1, L1PREFETCH_ALL, 8*128
	zloop	4, b3a, -4*8*128+dist32	;; Test loop counter
	zloop	21, b3a			;; Test loop counter
	add	rsi, r8
	jmp	zpass2_5376_real_unfft

zpass2_5376_real_1:
;;	zr64_hundredtwentyeight_real_fft_final_preload
;;	zr64_hundredtwentyeight_real_fft_final rsi, 8*128, 1*128, 2*128, 4*128, rdx, 0*ZMM_SCD4, rdi, 0*ZMM_SCD4, 1
	CALLP	zreal_fft_final
	zloop_init 84, 1, 4		;; 83 more iterations
	zloop_set_rw_prefetch ZOCT	;; Prefetch 664 cache lines
	zr64_sixtyfour_complex_fft_final_preload
b3b:	zr64_sixtyfour_complex_fft_final rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 1, L1PREFETCH_ALL, 8*128
	zloop	4, b3b, -4*8*128+dist32	;; Test loop counter
	zloop	21, b3b			;; Test loop counter
	jmp	zpass2_5376_real_done

zpass2_5376_real_3:
	mov	r8, DEST2ARG
	mov	rbp, DIST_TO_MULSRCARG
;;	zr64_hundredtwentyeight_real_with_mult_preload
;;	zr64_hundredtwentyeight_real_with_mult rsi, 8*128, 1*128, 2*128, 4*128, rdx, 0*ZMM_SCD4, rdi, 0*ZMM_SCD4, 1
	CALLP	zreal_mult
	zloop_init 84, 1, 4		;; 83 more iterations
	zloop_set_rw_prefetch ZOCT	;; Prefetch 664 cache lines
	zr64_sixtyfour_complex_with_mult_preload
b3c:	zr64_sixtyfour_complex_with_mult rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 1, L1PREFETCH_ALL, 8*128
	zloop	4, b3c, -4*8*128+dist32	;; Test loop counter
	zloop	21, b3c			;; Test loop counter
	add	rsi, r8
	jmp	zpass2_5376_real_unfft

zpass2_5376_real_4:
	mov	rbp, DIST_TO_MULSRCARG
	mov	rdx, sincos3		;; Real sin/cos data
	mov	rdi, xsincos_complex	;; Complex sin/cos data
	bump	rdx, 96*ZMM_SCD3+8*ZMM_SCD6
	bump	rdi, 96*ZMM_SCD3+8*ZMM_SCD6
;;	zr64f_hundredtwentyeight_real_with_mulf_preload
;;	zr64f_hundredtwentyeight_real_with_mulf rsi, 8*128, 1*128, 2*128, 4*128, rdx, 0*ZMM_SCD4, rdi, 0*ZMM_SCD4, 1
	CALLP	zreal_mulf
	zloop_init 84, 1, 4		;; 83 more iterations
	zloop_set_rw_prefetch ZOCT	;; Prefetch 664 cache lines
	zr64f_sixtyfour_complex_with_mulf_preload
b3d:	zr64f_sixtyfour_complex_with_mulf rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 1, L1PREFETCH_ALL, 8*128
	zloop	4, b3d, -4*8*128+dist32	;; Test loop counter
	zloop	21, b3d			;; Test loop counter
zpass2_5376_real_unfft:
	bump	rsi, -21*dist32		;; Restore source pointer

;; Do inverse FFT levels 5,6,7
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	64	+1	...
;;	8	...
;;	...
;;	56
;;	128
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	768	+1	...
;;	8	...
;;	...
;;	760
;;	1536
;;	...

;; Do 64 twentyfour_reals_unfft operations
;; Do 384 twelve_complex_unfft operations
;;	distance between fft data elements is 64
;;	do 56 macros each processing 192 data values

	zloop_init 56			;; 7 iterations of 8
	zloop_set_rw_prefetch ZTRIPLE	;; Prefetch 168 cache lines
	bump	rdx, -8*ZMM_SCD6
	bump	rdi, -8*ZMM_SCD6
	zr12_2sc_twentyfour_reals_unfft_preload
c2a:	zr12_2sc_twentyfour_reals_unfft rsi, 128, 8*128, 16*128, dist32, rdx, ZMM_SCD6, rdi, ZMM_SCD6, 8
	zloop	8, c2a, -8*128+3*dist32, rdi, -8*ZMM_SCD6 ;; Test loop counter, next source pointer, restore sin/cos pointer
	loops_reset			;; 6 more iters of 8
	zr12_twelve_complex_djbunfft_preload
c2b:	zr12_twelve_complex_djbunfft rsi, 128, 8*128, 16*128, dist32, rdi, ZMM_SCD6, 8
	zloop	8, c2b, -8*128+3*dist32, rdi, -8*ZMM_SCD6 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	7, c2b, -7*3*dist32	;; Test loop counter, restore source pointer

;; Do inverse FFT levels 1,2,3,4
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	768	+1	...
;;	8	...
;;	...
;;	760
;;	1536
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	5376	+1	...
;;	8	...
;;	...
;;	5368

;; Do 768 fourteen_reals_last_unfft operations
;;	distance between fft data elements is 768
;;	do 96 macros each processing 112 data values

	bump	rdx, -8*ZMM_SCD6-96*ZMM_SCD3
	bump	rdi, -96*ZMM_SCD3
	zloop_init 96			;; 96 iterations
	zloop_set_rw_prefetch ZDOUBLE	;; Prefetch 192 cache lines
	zr7_2sc_fourteen_reals_unfft_preload
c1b:	zr7_2sc_fourteen_reals_unfft rsi, 128, 3*dist32, rdx, ZMM_SCD3, rdi, ZMM_SCD3, 96
	zloop	32, c1b, -32*128+dist32	;; Test loop counter, next source pointer
	zloop	3, c1b			;; Test loop counter

zpass2_5376_real_done:
	end_timer 2
	ENDM

; *************** radix8-pass2-5376-complex macro ******************
; This macro takes 5376 complex values and performs the final 12.392 levels of a standard
; radix-8 FFT, squares the results, and does the first 12.392 levels of the inverse FFT.

zr4dwpn_pass2_5376_complex MACRO
	LOCAL	b1b, b2b, b3a, b3b, b3c, b3d, c1b, c2b
	LOCAL	zpass2_5376_complex_1, zpass2_5376_complex_2
	LOCAL	zpass2_5376_complex_3, zpass2_5376_complex_4
	LOCAL	zpass2_5376_complex_unfft, zpass2_5376_complex_done

;; Type 4 FFTs skip the forward FFT process

	mov	rsi, DATA_ADDR		;; Load address of FFT data
	mov	rbx, DIST_TO_FFTSRCARG
	mov	rcx, DATA_PREFETCH	;; Prefetch pointer (we need to prefetch 1344 cache lines + 20 pad cache lines)
	cmp	ffttype, 4
	je	zpass2_5376_complex_4

;; Do FFT levels 1,2,3
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	5376	+1	...
;;	8	...
;;	...
;;	5368
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	768	+1	...
;;	8	...
;;	...
;;	760
;;	1536
;;	...

;; Do 768 seven_complex_fft operations
;;	distance between fft data elements is 768
;;	do 96 macros each processing 112 data values

	start_timer 4
	mov	rdi, xsincos_complex	;; The sin/cos data
	zloop_init 96			;; 96 iterations
	zloop_set_rw_prefetch ZDOUBLE	;; Prefetch 192 cache lines
	zr7f_seven_complex_djbfft_preload
b1b:	zr7f_seven_complex_djbfft rsi, 128, 3*dist32, rdi, ZMM_SCD3, 32, L1PREFETCH_ALL, 128
	zloop	32, b1b, -32*128+dist32	;; Test loop counter, next source pointer
	zloop	3, b1b, -3*dist32	;; Test loop counter, restore source pointer
	end_timer 4

;; Do FFT levels 4,5,6
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	768	+1	...
;;	8	...
;;	...
;;	760
;;	1536
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	64	+1	...
;;	8	...
;;	...
;;	56
;;	128
;;	...

;; Do 448 twelve_complex_fft operations
;;	distance between fft data elements is 64
;;	do 56 macros each processing 192 data values

	start_timer 5
	zloop_init 56			;; 7 iterations of 8
	zloop_set_rw_prefetch ZTRIPLE	;; Prefetch 168 cache lines
	zr12_twelve_complex_djbfft_preload
b2b:	zr12_twelve_complex_djbfft rsi, 128, 8*128, 16*128, dist32, rdi, ZMM_SCD6, 8, L1PREFETCH_ALL, 128
	zloop	8, b2b, -8*128+3*dist32, rdi, -8*ZMM_SCD6 ;; Test loop counter, restore source pointer, restore sin/cos pointer
	zloop	7, b2b, -7*3*dist32, rdi, 8*ZMM_SCD6 ;; Test loop counter, restore source pointer, next sin/cos pointer
	end_timer 5

;; Do FFT levels 7,8,9,10,11,12 as well as inverse FFT levels 7,8,9,10,11,12
;; On input and output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	64	+1	...
;;	8	...
;;	...
;;	56
;;	128
;;	...

;; Do 84 sixtyfour_complex_fft operations
;;	distance between fft data elements is 1 (8 for swizzling)
;;	do 84 macros each processing 128 data values

;; Execute the right middle step

pfing	bump	rcx, -28*64		;; We will prefetch 28 cache lines too many
	cmp	ffttype, 2
	jl	zpass2_5376_complex_1
	jg	zpass2_5376_complex_3

zpass2_5376_complex_2:
	start_timer 9
	zloop_init 84			;; 84 iterations
	zloop_set_rw_prefetch ZOCT	;; Prefetch 672 cache lines
	mov	r8, DEST2ARG
	zr64_sixtyfour_complex_with_square_preload
b3a:	zr64_sixtyfour_complex_with_square rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 4, L1PREFETCH_ALL, 8*128
	zloop	4, b3a, -4*8*128+dist32	;; Test loop counter
	zloop	21, b3a			;; Test loop counter
	add	rsi, r8
	end_timer 9
	jmp	zpass2_5376_complex_unfft

zpass2_5376_complex_1:
	start_timer 9
	zloop_init 84			;; 84 iterations
	zloop_set_rw_prefetch ZOCT	;; Prefetch 672 cache lines
	zr64_sixtyfour_complex_fft_final_preload
b3b:	zr64_sixtyfour_complex_fft_final rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 4, L1PREFETCH_ALL, 8*128
	zloop	4, b3b, -4*8*128+dist32	;; Test loop counter
	zloop	21, b3b			;; Test loop counter
	end_timer 9
	jmp	zpass2_5376_complex_done

zpass2_5376_complex_3:
	start_timer 9
	zloop_init 84			;; 84 iterations
	zloop_set_rw_prefetch ZOCT	;; Prefetch 672 cache lines
	mov	r8, DEST2ARG
	mov	rbp, DIST_TO_MULSRCARG
	zr64_sixtyfour_complex_with_mult_preload
b3c:	zr64_sixtyfour_complex_with_mult rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 4, L1PREFETCH_ALL, 8*128
	zloop	4, b3c, -4*8*128+dist32	;; Test loop counter
	zloop	21, b3c			;; Test loop counter
	add	rsi, r8
	end_timer 9
	jmp	zpass2_5376_complex_unfft

zpass2_5376_complex_4:
	start_timer 9
	zloop_init 84			;; 84 iterations
	zloop_set_rw_prefetch ZOCT	;; Prefetch 672 cache lines
	mov	rbp, DIST_TO_MULSRCARG
	mov	rdi, xsincos_complex	;; The sin/cos data
	bump	rdi, 96*ZMM_SCD3+8*ZMM_SCD6
	zr64f_sixtyfour_complex_with_mulf_preload
b3d:	zr64f_sixtyfour_complex_with_mulf rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 4, L1PREFETCH_ALL, 8*128
	zloop	4, b3d, -4*8*128+dist32	;; Test loop counter
	zloop	21, b3d			;; Test loop counter
	end_timer 9
zpass2_5376_complex_unfft:
	bump	rsi, -21*dist32		;; Restore source pointer

;; Do inverse FFT levels 4,5,6
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	64	+1	...
;;	8	...
;;	...
;;	56
;;	128
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	768	+1	...
;;	8	...
;;	...
;;	760
;;	1536
;;	...

;; Do 448 twelve_complex_unfft operations
;;	distance between fft data elements is 64
;;	do 56 macros each processing 192 data values

	start_timer 13
	bump	rdi, -8*ZMM_SCD6
	zloop_init 56			;; 7 iterations of 8
	zloop_set_rw_prefetch ZTRIPLE	;; Prefetch 168 cache lines
	zr12_twelve_complex_djbunfft_preload
c2b:	zr12_twelve_complex_djbunfft rsi, 128, 8*128, 16*128, dist32, rdi, ZMM_SCD6, 8, L1PREFETCH_ALL, 128
	zloop	8, c2b, -8*128+3*dist32, rdi, -8*ZMM_SCD6 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	7, c2b, -7*3*dist32	;; Test loop counter, restore source pointer
	end_timer 13

;; Do inverse FFT levels 1,2,3
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	768	+1	...
;;	8	...
;;	...
;;	760
;;	1536
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	5376	+1	...
;;	8	...
;;	...
;;	5368

;; Do 768 seven_complex_unfft operations
;;	distance between fft data elements is 768
;;	do 96 macros each processing 112 data values

	start_timer 14
	bump	rdi, -96*ZMM_SCD3
	zloop_init 96			;; 96 iterations
	zloop_set_rw_prefetch ZDOUBLE	;; Prefetch 192 cache lines
	zr7_seven_complex_djbunfft_preload
c1b:	zr7_seven_complex_djbunfft rsi, 128, 3*dist32, rdi, ZMM_SCD3, 32, L1PREFETCH_ALL, 128
	zloop	32, c1b, -32*128+dist32	;; Test loop counter, next source pointer
	zloop	3, c1b			;; Test loop counter
	end_timer 14

zpass2_5376_complex_done:
	ENDM


; *************** radix8-pass2-6144-real macro ******************
; This macro takes 12288 real values and performs the final 13.585 levels of the FFT.
; The results are squared, and the first 13.585 levels of the inverse FFT are done.

zr4dwpn_pass2_6144_real MACRO
	LOCAL	b1b, b2a, b2b, b3a, b3b, b3c, b3d, c1b, c2a, c2b
	LOCAL	zpass2_6144_real_1, zpass2_6144_real_2
	LOCAL	zpass2_6144_real_3, zpass2_6144_real_4
	LOCAL	zpass2_6144_real_unfft, zpass2_6144_real_done

;; Type 4 FFTs skip the forward FFT process

	start_timer 2
	mov	rsi, DATA_ADDR		;; Load source address
	mov	rbx, DIST_TO_FFTSRCARG
	mov	rcx, DATA_PREFETCH	;; Prefetch pointer (we need to prefetch 1536 cache lines + 23 pad cache lines)
	cmp	ffttype, 4
	je	zpass2_6144_real_4

;; Do FFT levels 1,2,3,4
;;	0	+1	+1	+1	+1	+1	+1	+1	6144	+1	...
;;	8	...
;;	...
;;	6136
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	512	+1	...
;;	8	...
;;	...
;;	504
;;	1K
;;	...

;; Do 512 twentyfour_real_fft operations
;;	distance between fft data elements is 512
;;	do 64 macros each processing 192 data values

	mov	rdx, sincos3		;; Real sin/cos data
	mov	rdi, xsincos_complex	;; Complex sin/cos data
	zloop_init 64			;; 64 iterations
	zloop_set_rw_prefetch ZTRIPLE	;; Prefetch 192 cache lines
	zr12f_2sc_twentyfour_reals_fft_preload
b1b:	zr12f_2sc_twentyfour_reals_fft rsi, 128, 2*dist32, 4*dist32, 8*dist32, rdx, ZMM_SCD6, rdi, ZMM_SCD6, 32
	zloop	32, b1b, -32*128+dist32	;; Test loop counter, next source pointer
	zloop	2, b1b, -2*dist32	;; Test loop counter, restore source pointer

;; Do FFT levels 5,6,7
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	512	+1	...
;;	8	...
;;	...
;;	504
;;	1K
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	64	+1	...
;;	8	...
;;	...
;;	56
;;	128
;;	...

;; Do 64 sixteen_reals_fft operations
;; Do 704 eight_complex_fft operations
;;	distance between fft data elements is 64
;;	do 96 macros each processing 128 data values

	zloop_init 96			;; 12 iterations of 8
	zloop_set_rw_prefetch ZDOUBLE	;; Prefetch 192 cache lines
	zr8_2sc_sixteen_reals_fft_preload
b2a:	zr8_2sc_sixteen_reals_fft rsi, 128, 8*128, 16*128, dist32, rdx, ZMM_SCD4, rdi, ZMM_SCD4, 8
	zloop	8, b2a, -8*128+2*dist32, rdi, -8*ZMM_SCD4 ;; Test loop counter, next source pointer, restore sin/cos pointer
	loops_reset			;; 11 more iters of 8
	zr8_eight_complex_djbfft_preload
b2b:	zr8_eight_complex_djbfft rsi, 128, 8*128, 16*128, dist32, rdi, ZMM_SCD4, 8
	zloop	8, b2b, -8*128+2*dist32, rdi, -8*ZMM_SCD4 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	12, b2b, -12*2*dist32, rdi, 8*ZMM_SCD4 ;; Test loop counter, restore source pointer, next sin/cos pointer

;; Do FFT levels 8,9,10,11,12,13 as well as inverse FFT levels 8,9,10,11,12,13
;; On input and output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	64	+1	...
;;	8	...
;;	...
;;	56
;;	128
;;	...

;; Do 1 hundredtwentyeight_reals_fft operation
;; Do 95 sixtyfour_complex_fft operations
;;	distance between fft data elements is 1 (8 for swizzling)
;;	do 96 macros each processing 128 data values

;; Execute the proper middle step

pfing	bump	rcx, -33*64		;; We will prefetch 33 cache lines too many
	cmp	ffttype, 2
	jl	zpass2_6144_real_1
	jg	zpass2_6144_real_3

zpass2_6144_real_2:
	mov	r8, DEST2ARG
;;	zr64_hundredtwentyeight_real_with_square_preload
;;	zr64_hundredtwentyeight_real_with_square rsi, 8*128, 1*128, 2*128, 4*128, rdx, 0*ZMM_SCD4, rdi, 0*ZMM_SCD4, 1
	CALLP	zreal_square
	zloop_init 96, 1, 4		;; 95 more iterations
	zloop_set_rw_prefetch ZOCT	;; Prefetch 760 cache lines
	zr64_sixtyfour_complex_with_square_preload
b3a:	zr64_sixtyfour_complex_with_square rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 1, L1PREFETCH_ALL, 8*128
	zloop	4, b3a, -4*8*128+dist32	;; Test loop counter, next source pointer
	zloop	24, b3a			;; Test loop counter
	add	rsi, r8
	jmp	zpass2_6144_real_unfft

zpass2_6144_real_1:
;;	zr64_hundredtwentyeight_real_fft_final_preload
;;	zr64_hundredtwentyeight_real_fft_final rsi, 8*128, 1*128, 2*128, 4*128, rdx, 0*ZMM_SCD4, rdi, 0*ZMM_SCD4, 1
	CALLP	zreal_fft_final
	zloop_init 96, 1, 4		;; 95 more iterations
	zloop_set_rw_prefetch ZOCT	;; Prefetch 760 cache lines
	zr64_sixtyfour_complex_fft_final_preload
b3b:	zr64_sixtyfour_complex_fft_final rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 1, L1PREFETCH_ALL, 8*128
	zloop	4, b3b, -4*8*128+dist32	;; Test loop counter, next source pointer
	zloop	24, b3b			;; Test loop counter
	jmp	zpass2_6144_real_done

zpass2_6144_real_3:
	mov	r8, DEST2ARG
	mov	rbp, DIST_TO_MULSRCARG
;;	zr64_hundredtwentyeight_real_with_mult_preload
;;	zr64_hundredtwentyeight_real_with_mult rsi, 8*128, 1*128, 2*128, 4*128, rdx, 0*ZMM_SCD4, rdi, 0*ZMM_SCD4, 1
	CALLP	zreal_mult
	zloop_init 96, 1, 4		;; 95 more iterations
	zloop_set_rw_prefetch ZOCT	;; Prefetch 760 cache lines
	zr64_sixtyfour_complex_with_mult_preload
b3c:	zr64_sixtyfour_complex_with_mult rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 1, L1PREFETCH_ALL, 8*128
	zloop	4, b3c, -4*8*128+dist32	;; Test loop counter, next source pointer
	zloop	24, b3c			;; Test loop counter
	add	rsi, r8
	jmp	zpass2_6144_real_unfft

zpass2_6144_real_4:
	mov	rbp, DIST_TO_MULSRCARG
	mov	rdx, sincos3		;; Real sin/cos data
	mov	rdi, xsincos_complex	;; Complex sin/cos data
	bump	rdx, 64*ZMM_SCD6+8*ZMM_SCD4
	bump	rdi, 64*ZMM_SCD6+8*ZMM_SCD4
;;	zr64f_hundredtwentyeight_real_with_mulf_preload
;;	zr64f_hundredtwentyeight_real_with_mulf rsi, 8*128, 1*128, 2*128, 4*128, rdx, 0*ZMM_SCD4, rdi, 0*ZMM_SCD4, 1
	CALLP	zreal_mulf
	zloop_init 96, 1, 4		;; 95 more iterations
	zloop_set_rw_prefetch ZOCT	;; Prefetch 760 cache lines
	zr64f_sixtyfour_complex_with_mulf_preload
b3d:	zr64f_sixtyfour_complex_with_mulf rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 1, L1PREFETCH_ALL, 8*128
	zloop	4, b3d, -4*8*128+dist32	;; Test loop counter, next source pointer
	zloop	24, b3d			;; Test loop counter
zpass2_6144_real_unfft:
	bump	rsi, -24*dist32		;; Restore source pointer

;; Do inverse FFT levels 5,6,7
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	64	+1	...
;;	8	...
;;	...
;;	56
;;	128
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	512	+1	...
;;	8	...
;;	...
;;	504
;;	1K
;;	...

;; Do 64 sixteen_reals_unfft operations
;; Do 704 eight_complex_unfft operations
;;	distance between fft data elements is 64
;;	do 96 macros each processing 128 data values

	zloop_init 96			;; 12 iterations of 8
	zloop_set_rw_prefetch ZDOUBLE	;; Prefetch 192 cache lines
	bump	rdx, -8*ZMM_SCD4
	bump	rdi, -8*ZMM_SCD4
	zr8_2sc_sixteen_reals_unfft_preload
c2a:	zr8_2sc_sixteen_reals_unfft rsi, 128, 8*128, 16*128, dist32, rdx, ZMM_SCD4, rdi, ZMM_SCD4, 8
	zloop	8, c2a, -8*128+2*dist32, rdi, -8*ZMM_SCD4 ;; Test loop counter, next source pointer, restore sin/cos pointer
	loops_reset			;; 11 more iters of 8
	zr8_eight_complex_djbunfft_preload
c2b:	zr8_eight_complex_djbunfft rsi, 128, 8*128, 16*128, dist32, rdi, ZMM_SCD4, 8
	zloop	8, c2b, -8*128+2*dist32, rdi, -8*ZMM_SCD4 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	12, c2b, -12*2*dist32	;; Test loop counter, restore source pointer

;; Do inverse FFT levels 1,2,3,4
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	512	+1	...
;;	8	...
;;	...
;;	504
;;	1K
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	6144	+1	...
;;	8	...
;;	...
;;	6136

;; Do 512 twentyfour_reals_last_unfft operations
;;	distance between fft data elements is 512
;;	do 64 macros each processing 192 data values

	bump	rdx, -8*ZMM_SCD4-64*ZMM_SCD6
	bump	rdi, -64*ZMM_SCD6
	zloop_init 64			;; 64 iterations
	zloop_set_rw_prefetch ZQUAD	;; Prefetch 256 cache lines
	zr12_2sc_twentyfour_reals_unfft_preload
c1b:	zr12_2sc_twentyfour_reals_unfft rsi, 128, 2*dist32, 4*dist32, 8*dist32, rdx, ZMM_SCD6, rdi, ZMM_SCD6, 32
	zloop	32, c1b, -32*128+dist32	;; Test loop counter, next source pointer
	zloop	2, c1b			;; Test loop counter

zpass2_6144_real_done:
	end_timer 2
	ENDM

; *************** radix8-pass2-6144-complex macro ******************
; This macro takes 6144 complex values and performs the final 12.585 levels of a standard
; radix-8 FFT, squares the results, and does the first 12.585 levels of the inverse FFT.

zr4dwpn_pass2_6144_complex MACRO
	LOCAL	b1b, b2b, b3a, b3b, b3c, b3d, c1b, c2b
	LOCAL	zpass2_6144_complex_1, zpass2_6144_complex_2
	LOCAL	zpass2_6144_complex_3, zpass2_6144_complex_4
	LOCAL	zpass2_6144_complex_unfft, zpass2_6144_complex_done

;; Type 4 FFTs skip the forward FFT process

	mov	rsi, DATA_ADDR		;; Load address of FFT data
	mov	rbx, DIST_TO_FFTSRCARG
	mov	rcx, DATA_PREFETCH	;; Prefetch pointer (we need to prefetch 1536 cache lines + 23 pad cache lines)
	cmp	ffttype, 4
	je	zpass2_6144_complex_4

;; Do FFT levels 1,2,3
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	6144	+1	...
;;	8	...
;;	...
;;	6136
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	512	+1	...
;;	8	...
;;	...
;;	504
;;	1K
;;	...

;; Do 512 twelve_complex_fft operations
;;	distance between fft data elements is 512
;;	do 64 macros each processing 192 data values

	start_timer 4
	mov	rdi, xsincos_complex	;; The sin/cos data
	zloop_init 64			;; 64 iterations
	zloop_set_rw_prefetch ZTRIPLE	;; Prefetch 192 cache lines
	zr12f_twelve_complex_djbfft_preload
b1b:	zr12f_twelve_complex_djbfft rsi, 128, 2*dist32, 4*dist32, 8*dist32, rdi, ZMM_SCD6, 32, L1PREFETCH_ALL, 128
	zloop	32, b1b, -32*128+dist32	;; Test loop counter, next source pointer
	zloop	2, b1b, -2*dist32	;; Test loop counter, restore source pointer
	end_timer 4

;; Do FFT levels 4,5,6
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	512	+1	...
;;	8	...
;;	...
;;	504
;;	1K
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	64	+1	...
;;	8	...
;;	...
;;	56
;;	128
;;	...

;; Do 768 eight_complex_fft operations
;;	distance between fft data elements is 64
;;	do 96 macros each processing 128 data values

	start_timer 5
	zloop_init 96			;; 12 iterations of 8
	zloop_set_rw_prefetch ZDOUBLE	;; Prefetch 192 cache lines
	zr8_eight_complex_djbfft_preload
b2b:	zr8_eight_complex_djbfft rsi, 128, 8*128, 16*128, dist32, rdi, ZMM_SCD4, 8, L1PREFETCH_ALL, 128
	zloop	8, b2b, -8*128+2*dist32, rdi, -8*ZMM_SCD4 ;; Test loop counter, restore source pointer, restore sin/cos pointer
	zloop	12, b2b, -12*2*dist32, rdi, 8*ZMM_SCD4 ;; Test loop counter, restore source pointer, next sin/cos pointer
	end_timer 5

;; Do FFT levels 7,8,9,10,11,12 as well as inverse FFT levels 7,8,9,10,11,12
;; On input and output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	64	+1	...
;;	8	...
;;	...
;;	56
;;	128
;;	...

;; Do 96 sixtyfour_complex_fft operations
;;	distance between fft data elements is 1 (8 for swizzling)
;;	do 96 macros each processing 128 data values

;; Execute the right middle step

pfing	bump	rcx, -41*64		;; We will prefetch 41 cache lines too many
	cmp	ffttype, 2
	jl	zpass2_6144_complex_1
	jg	zpass2_6144_complex_3

zpass2_6144_complex_2:
	start_timer 9
	zloop_init 96			;; 96 iterations
	zloop_set_rw_prefetch ZOCT	;; Prefetch 768 cache lines
	mov	r8, DEST2ARG
	zr64_sixtyfour_complex_with_square_preload
b3a:	zr64_sixtyfour_complex_with_square rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 4, L1PREFETCH_ALL, 8*128
	zloop	4, b3a, -4*8*128+dist32	;; Test loop counter, next source pointer
	zloop	24, b3a			;; Test loop counter
	add	rsi, r8
	end_timer 9
	jmp	zpass2_6144_complex_unfft

zpass2_6144_complex_1:
	start_timer 9
	zloop_init 96			;; 96 iterations
	zloop_set_rw_prefetch ZOCT	;; Prefetch 768 cache lines
	zr64_sixtyfour_complex_fft_final_preload
b3b:	zr64_sixtyfour_complex_fft_final rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 4, L1PREFETCH_ALL, 8*128
	zloop	4, b3b, -4*8*128+dist32	;; Test loop counter, next source pointer
	zloop	24, b3b			;; Test loop counter
	end_timer 9
	jmp	zpass2_6144_complex_done

zpass2_6144_complex_3:
	start_timer 9
	zloop_init 96			;; 96 iterations
	zloop_set_rw_prefetch ZOCT	;; Prefetch 768 cache lines
	mov	r8, DEST2ARG
	mov	rbp, DIST_TO_MULSRCARG
	zr64_sixtyfour_complex_with_mult_preload
b3c:	zr64_sixtyfour_complex_with_mult rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 4, L1PREFETCH_ALL, 8*128
	zloop	4, b3c, -4*8*128+dist32	;; Test loop counter, next source pointer
	zloop	24, b3c			;; Test loop counter
	add	rsi, r8
	end_timer 9
	jmp	zpass2_6144_complex_unfft

zpass2_6144_complex_4:
	start_timer 9
	zloop_init 96			;; 96 iterations
	zloop_set_rw_prefetch ZOCT	;; Prefetch 768 cache lines
	mov	rbp, DIST_TO_MULSRCARG
	mov	rdi, xsincos_complex	;; The sin/cos data
	bump	rdi, 64*ZMM_SCD6+8*ZMM_SCD4
	zr64f_sixtyfour_complex_with_mulf_preload
b3d:	zr64f_sixtyfour_complex_with_mulf rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 4, L1PREFETCH_ALL, 8*128
	zloop	4, b3d, -4*8*128+dist32	;; Test loop counter, next source pointer
	zloop	24, b3d			;; Test loop counter
	end_timer 9
zpass2_6144_complex_unfft:
	bump	rsi, -24*dist32		;; Restore source pointer

;; Do inverse FFT levels 4,5,6
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	64	+1	...
;;	8	...
;;	...
;;	56
;;	128
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	512	+1	...
;;	8	...
;;	...
;;	504
;;	1K
;;	...

;; Do 768 eight_complex_unfft operations
;;	distance between fft data elements is 64
;;	do 96 macros each processing 128 data values

	start_timer 13
	bump	rdi, -8*ZMM_SCD4
	zloop_init 96			;; 12 iterations of 8
	zloop_set_rw_prefetch ZDOUBLE	;; Prefetch 192 cache lines
	zr8_eight_complex_djbunfft_preload
c2b:	zr8_eight_complex_djbunfft rsi, 128, 8*128, 16*128, dist32, rdi, ZMM_SCD4, 8, L1PREFETCH_ALL, 128
	zloop	8, c2b, -8*128+2*dist32, rdi, -8*ZMM_SCD4 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	12, c2b, -12*2*dist32	;; Test loop counter, restore source pointer
	end_timer 13

;; Do inverse FFT levels 1,2,3
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	512	+1	...
;;	8	...
;;	...
;;	504
;;	1K
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	6144	+1	...
;;	8	...
;;	...
;;	6136

;; Do 512 twelve_complex_unfft operations
;;	distance between fft data elements is 512
;;	do 64 macros each processing 192 data values

	start_timer 14
	bump	rdi, -64*ZMM_SCD6
	zloop_init 64			;; 64 iterations
	zloop_set_rw_prefetch ZQUAD	;; Prefetch 256 cache lines
	zr12_twelve_complex_djbunfft_preload
c1b:	zr12_twelve_complex_djbunfft rsi, 128, 2*dist32, 4*dist32, 8*dist32, rdi, ZMM_SCD6, 32, L1PREFETCH_ALL, 128
	zloop	32, c1b, -32*128+dist32	;; Test loop counter, next source pointer
	zloop	2, c1b			;; Test loop counter
	end_timer 14

zpass2_6144_complex_done:
	ENDM


; *************** radix8-pass2-6400-real macro ******************
; This macro takes 12800 real values and performs the final 13.644 levels of the FFT.
; The results are squared, and the first 13.644 levels of the inverse FFT are done.

zr4dwpn_pass2_6400_real MACRO
	LOCAL	b1b, b2a, b2b, b3a, b3b, b3c, b3d, c1b, c2a, c2b
	LOCAL	zpass2_6400_real_1, zpass2_6400_real_2
	LOCAL	zpass2_6400_real_3, zpass2_6400_real_4
	LOCAL	zpass2_6400_real_unfft, zpass2_6400_real_done

;; Type 4 FFTs skip the forward FFT process

	start_timer 2
	mov	rsi, DATA_ADDR		;; Load source address
	mov	rbx, DIST_TO_FFTSRCARG
	mov	rcx, DATA_PREFETCH	;; Prefetch pointer (we need to prefetch 1600 cache lines + ?? pad cache lines)
	cmp	ffttype, 4
	je	zpass2_6400_real_4

;; Do FFT levels 1,2,3,4
;;	0	+1	+1	+1	+1	+1	+1	+1	6400	+1	...
;;	8	...
;;	...
;;	6392
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	640	+1	...
;;	8	...
;;	...
;;	632
;;	1280
;;	...

;; Do 640 twenty_real_fft operations
;;	distance between fft data elements is 640
;;	do 80 macros each processing 160 data values

	mov	rdx, sincos3		;; Real sin/cos data
	mov	rdi, xsincos_complex	;; Complex sin/cos data
	zloop_init 80			;; 80 iterations
	zloop_set_rw_prefetch ZDOUBLE	;; Prefetch 160 cache lines
	zr10f_2sc_twenty_reals_fft_preload
b1b:	zr10f_2sc_twenty_reals_fft rsi, 128, 80*128, rdx, ZMM_SCD5, rdi, ZMM_SCD5, 80
	zloop	80, b1b, -80*128	;; Test loop counter, restore source pointer

;; Do FFT levels 5,6,7
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	640	+1	...
;;	8	...
;;	...
;;	632
;;	1280
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	64	+1	...
;;	8	...
;;	...
;;	56
;;	128
;;	...

;; Do 64 twenty_reals_fft operations
;; Do 576 ten_complex_fft operations
;;	distance between fft data elements is 64
;;	do 80 macros each processing 160 data values

	zloop_init 80			;; 10 iterations of 8
	zloop_set_rw_prefetch ZTRIPLE	;; Prefetch 240 cache lines
	zr10_2sc_twenty_reals_fft_preload
b2a:	zr10_2sc_twenty_reals_fft rsi, 128, 8*128, rdx, ZMM_SCD5, rdi, ZMM_SCD5, 8
	zloop	8, b2a, -8*128+80*128, rdi, -8*ZMM_SCD5 ;; Test loop counter, next source pointer, restore sin/cos pointer
	loops_reset			;; 9 more iters of 8
	zr10_ten_complex_djbfft_preload
b2b:	zr10_ten_complex_djbfft rsi, 128, 8*128, rdi, ZMM_SCD5, 8
	zloop	8, b2b, -8*128+80*128, rdi, -8*ZMM_SCD5 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	10, b2b, -10*80*128, rdi, 8*ZMM_SCD5 ;; Test loop counter, restore source pointer, next sin/cos pointer

;; Do FFT levels 8,9,10,11,12,13 as well as inverse FFT levels 8,9,10,11,12,13
;; On input and output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	64	+1	...
;;	8	...
;;	...
;;	56
;;	128
;;	...

;; Do 1 hundredtwentyeight_reals_fft operation
;; Do 99 sixtyfour_complex_fft operations
;;	distance between fft data elements is 1 (8 for swizzling)
;;	do 100 macros each processing 128 data values

;; Execute the proper middle step

	cmp	ffttype, 2
	jl	zpass2_6400_real_1
	jg	zpass2_6400_real_3

zpass2_6400_real_2:
	mov	r8, DEST2ARG
;;	zr64_hundredtwentyeight_real_with_square_preload
;;	zr64_hundredtwentyeight_real_with_square rsi, 8*128, 1*128, 2*128, 4*128, rdx, 0*ZMM_SCD4, rdi, 0*ZMM_SCD4, 1
	CALLP	zreal_square
	zloop_init 99			;; 99 more iterations
	zloop_set_rw_prefetch ZOCT	;; Prefetch 792 cache lines
	zr64_sixtyfour_complex_with_square_preload
b3a:	zr64_sixtyfour_complex_with_square rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 99, L1PREFETCH_ALL, 8*128
	zloop	99, b3a			;; Test loop counter
	add	rsi, r8
	jmp	zpass2_6400_real_unfft

zpass2_6400_real_1:
;;	zr64_hundredtwentyeight_real_fft_final_preload
;;	zr64_hundredtwentyeight_real_fft_final rsi, 8*128, 1*128, 2*128, 4*128, rdx, 0*ZMM_SCD4, rdi, 0*ZMM_SCD4, 1
	CALLP	zreal_fft_final
	zloop_init 99			;; 99 more iterations
	zloop_set_rw_prefetch ZOCT	;; Prefetch 792 cache lines
	zr64_sixtyfour_complex_fft_final_preload
b3b:	zr64_sixtyfour_complex_fft_final rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 99, L1PREFETCH_ALL, 8*128
	zloop	99, b3b			;; Test loop counter
	jmp	zpass2_6400_real_done

zpass2_6400_real_3:
	mov	r8, DEST2ARG
	mov	rbp, DIST_TO_MULSRCARG
;;	zr64_hundredtwentyeight_real_with_mult_preload
;;	zr64_hundredtwentyeight_real_with_mult rsi, 8*128, 1*128, 2*128, 4*128, rdx, 0*ZMM_SCD4, rdi, 0*ZMM_SCD4, 1
	CALLP	zreal_mult
	zloop_init 99			;; 99 more iterations
	zloop_set_rw_prefetch ZOCT	;; Prefetch 792 cache lines
	zr64_sixtyfour_complex_with_mult_preload
b3c:	zr64_sixtyfour_complex_with_mult rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 99, L1PREFETCH_ALL, 8*128
	zloop	99, b3c			;; Test loop counter
	add	rsi, r8
	jmp	zpass2_6400_real_unfft

zpass2_6400_real_4:
	mov	rbp, DIST_TO_MULSRCARG
	mov	rdx, sincos3		;; Real sin/cos data
	mov	rdi, xsincos_complex	;; Complex sin/cos data
	bump	rdx, 80*ZMM_SCD5+8*ZMM_SCD5
	bump	rdi, 80*ZMM_SCD5+8*ZMM_SCD5
;;	zr64f_hundredtwentyeight_real_with_mulf_preload
;;	zr64f_hundredtwentyeight_real_with_mulf rsi, 8*128, 1*128, 2*128, 4*128, rdx, 0*ZMM_SCD4, rdi, 0*ZMM_SCD4, 1
	CALLP	zreal_mulf
	zloop_init 99			;; 99 more iterations
	zloop_set_rw_prefetch ZOCT	;; Prefetch 792 cache lines
	zr64f_sixtyfour_complex_with_mulf_preload
b3d:	zr64f_sixtyfour_complex_with_mulf rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 99, L1PREFETCH_ALL, 8*128
	zloop	99, b3d			;; Test loop counter
zpass2_6400_real_unfft:
	bump	rsi, -100*8*128		;; Restore source pointer

;; Do inverse FFT levels 5,6,7
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	64	+1	...
;;	8	...
;;	...
;;	56
;;	128
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	640	+1	...
;;	8	...
;;	...
;;	632
;;	1280
;;	...

;; Do 64 twenty_reals_unfft operations
;; Do 576 ten_complex_unfft operations
;;	distance between fft data elements is 64
;;	do 80 macros each processing 160 data values

	zloop_init 80			;; 10 iterations of 8
	zloop_set_rw_prefetch ZDOUBLE	;; Prefetch 160 cache lines
	bump	rdx, -8*ZMM_SCD5
	bump	rdi, -8*ZMM_SCD5
	zr10_2sc_twenty_reals_unfft_preload
c2a:	zr10_2sc_twenty_reals_unfft rsi, 128, 8*128, rdx, ZMM_SCD5, rdi, ZMM_SCD5, 8
	zloop	8, c2a, -8*128+80*128, rdi, -8*ZMM_SCD5 ;; Test loop counter, next source pointer, restore sin/cos pointer
	loops_reset			;; 6 more iters of 8
	zr10_ten_complex_djbunfft_preload
c2b:	zr10_ten_complex_djbunfft rsi, 128, 8*128, rdi, ZMM_SCD5, 8
	zloop	8, c2b, -8*128+80*128, rdi, -8*ZMM_SCD5 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	10, c2b, -10*80*128	;; Test loop counter, restore source pointer

;; Do inverse FFT levels 1,2,3,4
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	640	+1	...
;;	8	...
;;	...
;;	632
;;	1280
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	6400	+1	...
;;	8	...
;;	...
;;	6392

;; Do 640 twenty_reals_last_unfft operations
;;	distance between fft data elements is 640
;;	do 80 macros each processing 160 data values

	bump	rdx, -8*ZMM_SCD5-80*ZMM_SCD5
	bump	rdi, -80*ZMM_SCD5
	zloop_init 80			;; 80 iterations
	zloop_set_rw_prefetch ZTRIPLE	;; Prefetch 240 cache lines
	zr10_2sc_twenty_reals_unfft_preload
c1b:	zr10_2sc_twenty_reals_unfft rsi, 128, 80*128, rdx, ZMM_SCD5, rdi, ZMM_SCD5, 80
	zloop	80, c1b			;; Test loop counter

zpass2_6400_real_done:
	end_timer 2
	ENDM

; *************** radix8-pass2-6400-complex macro ******************
; This macro takes 6400 complex values and performs the final 12.644 levels of a standard
; radix-8 FFT, squares the results, and does the first 12.644 levels of the inverse FFT.

zr4dwpn_pass2_6400_complex MACRO
	LOCAL	b1b, b2b, b3a, b3b, b3c, b3d, c1b, c2b
	LOCAL	zpass2_6400_complex_1, zpass2_6400_complex_2
	LOCAL	zpass2_6400_complex_3, zpass2_6400_complex_4
	LOCAL	zpass2_6400_complex_unfft, zpass2_6400_complex_done

;; Type 4 FFTs skip the forward FFT process

	mov	rsi, DATA_ADDR		;; Load address of FFT data
	mov	rbx, DIST_TO_FFTSRCARG
	mov	rcx, DATA_PREFETCH	;; Prefetch pointer (we need to prefetch 1600 cache lines + ?? pad cache lines)
	cmp	ffttype, 4
	je	zpass2_6400_complex_4

;; Do FFT levels 1,2,3
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	6400	+1	...
;;	8	...
;;	...
;;	6392
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	640	+1	...
;;	8	...
;;	...
;;	632
;;	1280
;;	...

;; Do 640 ten_complex_fft operations
;;	distance between fft data elements is 640
;;	do 80 macros each processing 160 data values

	start_timer 4
	mov	rdi, xsincos_complex	;; The sin/cos data
	zloop_init 80			;; 80 iterations
	zloop_set_rw_prefetch ZDOUBLE	;; Prefetch 160 cache lines
	zr10f_ten_complex_djbfft_preload
b1b:	zr10f_ten_complex_djbfft rsi, 128, 80*128, rdi, ZMM_SCD5, 80, L1PREFETCH_ALL, 128
	zloop	80, b1b, -80*128	;; Test loop counter, restore source pointer
	end_timer 4

;; Do FFT levels 4,5,6
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	640	+1	...
;;	8	...
;;	...
;;	632
;;	1280
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	64	+1	...
;;	8	...
;;	...
;;	56
;;	128
;;	...

;; Do 640 ten_complex_fft operations
;;	distance between fft data elements is 64
;;	do 80 macros each processing 160 data values

	start_timer 5
	zloop_init 80			;; 10 iterations of 8
	zloop_set_rw_prefetch ZTRIPLE	;; Prefetch 240 cache lines
	zr10_ten_complex_djbfft_preload
b2b:	zr10_ten_complex_djbfft rsi, 128, 8*128, rdi, ZMM_SCD5, 8, L1PREFETCH_ALL, 128
	zloop	8, b2b, -8*128+80*128, rdi, -8*ZMM_SCD5	;; Test loop counter, restore source pointer, restore sin/cos pointer
	zloop	10, b2b, -10*80*128, rdi, 8*ZMM_SCD5 ;; Test loop counter, restore source pointer, next sin/cos pointer
	end_timer 5

;; Do FFT levels 7,8,9,10,11,12 as well as inverse FFT levels 7,8,9,10,11,12
;; On input and output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	64	+1	...
;;	8	...
;;	...
;;	56
;;	128
;;	...

;; Do 100 sixtyfour_complex_fft operations
;;	distance between fft data elements is 1 (8 for swizzling)
;;	do 100 macros each processing 128 data values

;; Execute the right middle step

	cmp	ffttype, 2
	jl	zpass2_6400_complex_1
	jg	zpass2_6400_complex_3

zpass2_6400_complex_2:
	start_timer 9
	zloop_init 100			;; 100 iterations
	zloop_set_rw_prefetch ZOCT	;; Prefetch 800 cache lines
	mov	r8, DEST2ARG
	zr64_sixtyfour_complex_with_square_preload
b3a:	zr64_sixtyfour_complex_with_square rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 100, L1PREFETCH_ALL, 8*128
	zloop	100, b3a		;; Test loop counter
	add	rsi, r8
	end_timer 9
	jmp	zpass2_6400_complex_unfft

zpass2_6400_complex_1:
	start_timer 9
	zloop_init 100			;; 100 iterations
	zloop_set_rw_prefetch ZOCT	;; Prefetch 800 cache lines
	zr64_sixtyfour_complex_fft_final_preload
b3b:	zr64_sixtyfour_complex_fft_final rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 100, L1PREFETCH_ALL, 8*128
	zloop	100, b3b		;; Test loop counter
	end_timer 9
	jmp	zpass2_6400_complex_done

zpass2_6400_complex_3:
	start_timer 9
	zloop_init 100			;; 100 iterations
	zloop_set_rw_prefetch ZOCT	;; Prefetch 800 cache lines
	mov	r8, DEST2ARG
	mov	rbp, DIST_TO_MULSRCARG
	zr64_sixtyfour_complex_with_mult_preload
b3c:	zr64_sixtyfour_complex_with_mult rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 100, L1PREFETCH_ALL, 8*128
	zloop	100, b3c		;; Test loop counter
	add	rsi, r8
	end_timer 9
	jmp	zpass2_6400_complex_unfft

zpass2_6400_complex_4:
	start_timer 9
	zloop_init 100			;; 100 iterations
	zloop_set_rw_prefetch ZOCT	;; Prefetch 800 cache lines
	mov	rbp, DIST_TO_MULSRCARG
	mov	rdi, xsincos_complex	;; The sin/cos data
	bump	rdi, 80*ZMM_SCD5+8*ZMM_SCD5
	zr64f_sixtyfour_complex_with_mulf_preload
b3d:	zr64f_sixtyfour_complex_with_mulf rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 100, L1PREFETCH_ALL, 8*128
	zloop	100, b3d		;; Test loop counter
	end_timer 9
zpass2_6400_complex_unfft:
	bump	rsi, -100*8*128		;; Restore source pointer

;; Do inverse FFT levels 4,5,6
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	64	+1	...
;;	8	...
;;	...
;;	56
;;	128
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	640	+1	...
;;	8	...
;;	...
;;	632
;;	1280
;;	...

;; Do 640 ten_complex_unfft operations
;;	distance between fft data elements is 64
;;	do 80 macros each processing 160 data values

	start_timer 13
	bump	rdi, -8*ZMM_SCD5
	zloop_init 80			;; 10 iterations of 8
	zloop_set_rw_prefetch ZDOUBLE	;; Prefetch 160 cache lines
	zr10_ten_complex_djbunfft_preload
c2b:	zr10_ten_complex_djbunfft rsi, 128, 8*128, rdi, ZMM_SCD5, 8, L1PREFETCH_ALL, 128
	zloop	8, c2b, -8*128+80*128, rdi, -8*ZMM_SCD5 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	10, c2b, -10*80*128	;; Test loop counter, restore source pointer
	end_timer 13

;; Do inverse FFT levels 1,2,3
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	640	+1	...
;;	8	...
;;	...
;;	632
;;	1280
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	6400	+1	...
;;	8	...
;;	...
;;	6392

;; Do 640 ten_complex_unfft operations
;;	distance between fft data elements is 640
;;	do 80 macros each processing 160 data values

	start_timer 14
	bump	rdi, -80*ZMM_SCD5
	zloop_init 80			;; 80 iterations
	zloop_set_rw_prefetch ZTRIPLE	;; Prefetch 240 cache lines
	zr10_ten_complex_djbunfft_preload
c1b:	zr10_ten_complex_djbunfft rsi, 128, 80*128, rdi, ZMM_SCD5, 80, L1PREFETCH_ALL, 128
	zloop	80, c1b			;; Test loop counter
	end_timer 14

zpass2_6400_complex_done:
	ENDM


; *************** radix8-pass2-7168-levels-real macro ******************
; This macro takes 14336 real values and performs the final 13.807 levels of the FFT.
; The results are squared, and the first 13.807 levels of the inverse FFT are done.

zr4dwpn_pass2_7168_real MACRO
	LOCAL	b1b, b2a, b2b, b3a, b3b, b3c, b3d, c1b, c2a, c2b
	LOCAL	zpass2_7168_levels_real_1, zpass2_7168_levels_real_2
	LOCAL	zpass2_7168_levels_real_3, zpass2_7168_levels_real_4
	LOCAL	zpass2_7168_levels_real_unfft, zpass2_7168_levels_real_done

;; Type 4 FFTs skip the forward FFT process

	start_timer 2
	mov	rsi, DATA_ADDR		;; Load source address
	mov	rbx, DIST_TO_FFTSRCARG
	mov	rcx, DATA_PREFETCH	;; Prefetch pointer (we need to prefetch 1792 cache lines + 27 pad cache lines)
	cmp	ffttype, 4
	je	zpass2_7168_levels_real_4

;; Do FFT levels 1,2,3,4
;;	0	+1	+1	+1	+1	+1	+1	+1	7K	+1	...
;;	8	...
;;	...
;;	7K-8
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	1K	+1	...
;;	8	...
;;	...
;;	1K-8
;;	2K
;;	...

;; Do 1024 fourteen_real_fft operations
;;	distance between fft data elements is 1024
;;	do 128 macros each processing 112 data values

	mov	rdx, sincos3		;; Real sin/cos data
	mov	rdi, xsincos_complex	;; Complex sin/cos data
	zloop_init 128			;; 128 iterations
	zloop_set_rw_prefetch ZDOUBLE	;; Prefetch 256 cache lines
	zr7f_2sc_fourteen_reals_fft_preload
b1b:	zr7f_2sc_fourteen_reals_fft rsi, 128, 4*dist32, rdx, ZMM_SCD3, rdi, ZMM_SCD3, 32
	zloop	32, b1b, -32*128+dist32	;; Test loop counter, next source pointer
	zloop	4, b1b, -4*dist32	;; Test loop counter, restore source pointer

;; Do FFT levels 5,6,7
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	1K	+1	...
;;	8	...
;;	...
;;	1K-8
;;	2K
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	64	+1	...
;;	8	...
;;	...
;;	56
;;	128
;;	...

;; Do 64 thirtytwo_reals_fft operations
;; Do 384 sixteen_complex_fft operations
;;	distance between fft data elements is 64
;;	do 56 macros each processing 256 data values

	zloop_init 56			;; 7 iterations of 8
	zloop_set_rw_prefetch ZQUAD	;; Prefetch 224 cache lines
	zr16_2sc_thirtytwo_reals_fft_preload
b2a:	zr16_2sc_thirtytwo_reals_fft rsi, 128, 8*128, 16*128, dist32, 2*dist32, rdx, ZMM_SCD8, rdi, ZMM_SCD8, 8
	zloop	8, b2a, -8*128+4*dist32, rdi, -8*ZMM_SCD8 ;; Test loop counter, next source pointer, restore sin/cos pointer
	loops_reset			;; 6 more iters of 8
	zr16_sixteen_complex_djbfft_preload
b2b:	zr16_sixteen_complex_djbfft rsi, 128, 8*128, 16*128, dist32, 2*dist32, rdi, ZMM_SCD8, 8
	zloop	8, b2b, -8*128+4*dist32, rdi, -8*ZMM_SCD8 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	7, b2b, -7*4*dist32, rdi, 8*ZMM_SCD8 ;; Test loop counter, restore source pointer, next sin/cos pointer

;; Do FFT levels 8,9,10,11,12,13 as well as inverse FFT levels 8,9,10,11,12,13
;; On input and output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	64	+1	...
;;	8	...
;;	...
;;	56
;;	128
;;	...

;; Do 1 hundredtwentyeight_reals_fft operation
;; Do 111 sixtyfour_complex_fft operations
;;	distance between fft data elements is 1 (8 for swizzling)
;;	do 112 macros each processing 128 data values

;; Execute the proper middle step

pfing	bump	rcx, -29*64		;; We will prefetch 29 cache lines too many
	cmp	ffttype, 2
	jl	zpass2_7168_levels_real_1
	jg	zpass2_7168_levels_real_3

zpass2_7168_levels_real_2:
	mov	r8, DEST2ARG
;;	zr64_hundredtwentyeight_real_with_square_preload
;;	zr64_hundredtwentyeight_real_with_square rsi, 8*128, 1*128, 2*128, 4*128, rdx, 0*ZMM_SCD4, rdi, 0*ZMM_SCD4, 1
	CALLP	zreal_square
	zloop_init 112, 1, 4		;; 111 more iterations
	zloop_set_rw_prefetch ZOCT	;; Prefetch 888 cache lines
	zr64_sixtyfour_complex_with_square_preload
b3a:	zr64_sixtyfour_complex_with_square rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 1, L1PREFETCH_ALL, 8*128
	zloop	4, b3a, -4*8*128+dist32	;; Test loop counter, next source pointer
	zloop	28, b3a			;; Test loop counter
	add	rsi, r8
	jmp	zpass2_7168_levels_real_unfft

zpass2_7168_levels_real_1:
;;	zr64_hundredtwentyeight_real_fft_final_preload
;;	zr64_hundredtwentyeight_real_fft_final rsi, 8*128, 1*128, 2*128, 4*128, rdx, 0*ZMM_SCD4, rdi, 0*ZMM_SCD4, 1
	CALLP	zreal_fft_final
	zloop_init 112, 1, 4		;; 111 more iterations
	zloop_set_rw_prefetch ZOCT	;; Prefetch 888 cache lines
	zr64_sixtyfour_complex_fft_final_preload
b3b:	zr64_sixtyfour_complex_fft_final rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 1, L1PREFETCH_ALL, 8*128
	zloop	4, b3b, -4*8*128+dist32	;; Test loop counter, next source pointer
	zloop	28, b3b			;; Test loop counter
	jmp	zpass2_7168_levels_real_done

zpass2_7168_levels_real_3:
	mov	r8, DEST2ARG
	mov	rbp, DIST_TO_MULSRCARG
;;	zr64_hundredtwentyeight_real_with_mult_preload
;;	zr64_hundredtwentyeight_real_with_mult rsi, 8*128, 1*128, 2*128, 4*128, rdx, 0*ZMM_SCD4, rdi, 0*ZMM_SCD4, 1
	CALLP	zreal_mult
	zloop_init 112, 1, 4		;; 111 more iterations
	zloop_set_rw_prefetch ZOCT	;; Prefetch 888 cache lines
	zr64_sixtyfour_complex_with_mult_preload
b3c:	zr64_sixtyfour_complex_with_mult rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 1, L1PREFETCH_ALL, 8*128
	zloop	4, b3c, -4*8*128+dist32	;; Test loop counter, next source pointer
	zloop	28, b3c			;; Test loop counter
	add	rsi, r8
	jmp	zpass2_7168_levels_real_unfft

zpass2_7168_levels_real_4:
	mov	rbp, DIST_TO_MULSRCARG
	mov	rdx, sincos3		;; Real sin/cos data
	mov	rdi, xsincos_complex	;; Complex sin/cos data
	bump	rdx, 128*ZMM_SCD3+8*ZMM_SCD8
	bump	rdi, 128*ZMM_SCD3+8*ZMM_SCD8
;;	zr64f_hundredtwentyeight_real_with_mulf_preload
;;	zr64f_hundredtwentyeight_real_with_mulf rsi, 8*128, 1*128, 2*128, 4*128, rdx, 0*ZMM_SCD4, rdi, 0*ZMM_SCD4, 1
	CALLP	zreal_mulf
	zloop_init 112, 1, 4		;; 111 more iterations
	zloop_set_rw_prefetch ZOCT	;; Prefetch 888 cache lines
	zr64f_sixtyfour_complex_with_mulf_preload
b3d:	zr64f_sixtyfour_complex_with_mulf rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 1, L1PREFETCH_ALL, 8*128
	zloop	4, b3d, -4*8*128+dist32	;; Test loop counter, next source pointer
	zloop	28, b3d			;; Test loop counter
zpass2_7168_levels_real_unfft:
	bump	rsi, -28*dist32		;; Restore source pointer

;; Do inverse FFT levels 5,6,7
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	64	+1	...
;;	8	...
;;	...
;;	56
;;	128
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	1K	+1	...
;;	8	...
;;	...
;;	1K-8
;;	2K
;;	...

;; Do 64 thirtytwo_reals_unfft operations
;; Do 384 sixteen_complex_unfft operations
;;	distance between fft data elements is 64
;;	do 56 macros each processing 256 data values

	zloop_init 56			;; 7 iterations of 8
	zloop_set_rw_prefetch ZQUAD	;; Prefetch 224 cache lines
	bump	rdx, -8*ZMM_SCD8
	bump	rdi, -8*ZMM_SCD8
	zr16_2sc_thirtytwo_reals_unfft_preload
c2a:	zr16_2sc_thirtytwo_reals_unfft rsi, 128, 8*128, 16*128, dist32, 2*dist32, rdx, ZMM_SCD8, rdi, ZMM_SCD8, 8
	zloop	8, c2a, -8*128+4*dist32, rdi, -8*ZMM_SCD8 ;; Test loop counter, next source pointer, restore sin/cos pointer
	loops_reset			;; 6 more iters of 8
	zr16_sixteen_complex_djbunfft_preload
c2b:	zr16_sixteen_complex_djbunfft rsi, 128, 8*128, 16*128, dist32, 2*dist32, rdi, ZMM_SCD8, 8
	zloop	8, c2b, -8*128+4*dist32, rdi, -8*ZMM_SCD8 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	7, c2b, -7*4*dist32	;; Test loop counter, restore source pointer

;; Do inverse FFT levels 1,2,3,4
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	1K	+1	...
;;	8	...
;;	...
;;	1K-8
;;	2K
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	7K	+1	...
;;	8	...
;;	...
;;	7K-8

;; Do 1024 fourteen_reals_last_unfft operations
;;	distance between fft data elements is 1024
;;	do 128 macros each processing 112 data values

	bump	rdx, -8*ZMM_SCD8-128*ZMM_SCD3
	bump	rdi, -128*ZMM_SCD3
	zloop_init 128			;; 128 iterations
	zloop_set_rw_prefetch ZDOUBLE	;; Prefetch 256 cache lines
	zr7_2sc_fourteen_reals_unfft_preload
c1b:	zr7_2sc_fourteen_reals_unfft rsi, 128, 4*dist32, rdx, ZMM_SCD3, rdi, ZMM_SCD3, 32
	zloop	32, c1b, -32*128+dist32		;; Test loop counter, next source pointer
	zloop	4, c1b				;; Test loop counter

zpass2_7168_levels_real_done:
	end_timer 2
	ENDM

; *************** radix8-pass2-7168-levels-complex macro ******************
; This macro takes 7168 complex values and performs the final 12.807 levels of a standard
; radix-8 FFT, squares the results, and does the first 12.807 levels of the inverse FFT.

zr4dwpn_pass2_7168_complex MACRO
	LOCAL	b1b, b2b, b3a, b3b, b3c, b3d, c1b, c2b
	LOCAL	zpass2_7168_levels_complex_1, zpass2_7168_levels_complex_2
	LOCAL	zpass2_7168_levels_complex_3, zpass2_7168_levels_complex_4
	LOCAL	zpass2_7168_levels_complex_unfft, zpass2_7168_levels_complex_done

;; Type 4 FFTs skip the forward FFT process

	mov	rsi, DATA_ADDR		;; Load address of FFT data
	mov	rbx, DIST_TO_FFTSRCARG
	mov	rcx, DATA_PREFETCH	;; Prefetch pointer (we need to prefetch 1792 cache lines + 27 pad cache lines)
	cmp	ffttype, 4
	je	zpass2_7168_levels_complex_4

;; Do FFT levels 1,2,3
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	7K	+1	...
;;	8	...
;;	...
;;	7K-8
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	1K	+1	...
;;	8	...
;;	...
;;	1K-8
;;	2K
;;	...

;; Do 1024 seven_complex_fft operations
;;	distance between fft data elements is 1024
;;	do 128 macros each processing 112 data values

	start_timer 4
	mov	rdi, xsincos_complex	;; The sin/cos data
	zloop_init 128			;; 128 iterations
	zloop_set_rw_prefetch ZDOUBLE	;; Prefetch 256 cache lines
	zr7f_seven_complex_djbfft_preload
b1b:	zr7f_seven_complex_djbfft rsi, 128, 4*dist32, rdi, ZMM_SCD3, 32, L1PREFETCH_ALL, 128
	zloop	32, b1b, -32*128+dist32	;; Test loop counter, next source pointer
	zloop	4, b1b, -4*dist32	;; Test loop counter, restore source pointer
	end_timer 4

;; Do FFT levels 4,5,6
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	1K	+1	...
;;	8	...
;;	...
;;	1K-8
;;	2K
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	64	+1	...
;;	8	...
;;	...
;;	56
;;	128
;;	...

;; Do 448 sixteen_complex_fft operations
;;	distance between fft data elements is 64
;;	do 56 macros each processing 256 data values

	start_timer 5
	zloop_init 56			;; 7 iterations of 8
	zloop_set_rw_prefetch ZQUAD	;; Prefetch 224 cache lines
	zr16_sixteen_complex_djbfft_preload
b2b:	zr16_sixteen_complex_djbfft rsi, 128, 8*128, 16*128, dist32, 2*dist32, rdi, ZMM_SCD8, 8, L1PREFETCH_ALL, 128
	zloop	8, b2b, -8*128+4*dist32, rdi, -8*ZMM_SCD8 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	7, b2b, -7*4*dist32, rdi, 8*ZMM_SCD8 ;; Test loop counter, restore source pointer, next sin/cos pointer
	end_timer 5

;; Do FFT levels 7,8,9,10,11,12 as well as inverse FFT levels 7,8,9,10,11,12
;; On input and output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	64	+1	...
;;	8	...
;;	...
;;	56
;;	128
;;	...

;; Do 112 sixtyfour_complex_fft operations
;;	distance between fft data elements is 1 (8 for swizzling)
;;	do 112 macros each processing 128 data values

;; Execute the right middle step

pfing	bump	rcx, -37*64		;; We will prefetch 37 cache lines too many
	cmp	ffttype, 2
	jl	zpass2_7168_levels_complex_1
	jg	zpass2_7168_levels_complex_3

zpass2_7168_levels_complex_2:
	start_timer 9
	zloop_init 112			;; 112 iterations
	zloop_set_rw_prefetch ZOCT	;; Prefetch 896 cache lines
	mov	r8, DEST2ARG
	zr64_sixtyfour_complex_with_square_preload
b3a:	zr64_sixtyfour_complex_with_square rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 112, L1PREFETCH_ALL, 8*128
	zloop	4, b3a, -4*8*128+dist32	;; Test loop counter, next source pointer
	zloop	28, b3a			;; Test loop counter
	add	rsi, r8
	end_timer 9
	jmp	zpass2_7168_levels_complex_unfft

zpass2_7168_levels_complex_1:
	start_timer 9
	zloop_init 112			;; 112 iterations
	zloop_set_rw_prefetch ZOCT	;; Prefetch 896 cache lines
	zr64_sixtyfour_complex_fft_final_preload
b3b:	zr64_sixtyfour_complex_fft_final rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 4, L1PREFETCH_ALL, 8*128
	zloop	4, b3b, -4*8*128+dist32	;; Test loop counter, next source pointer
	zloop	28, b3b			;; Test loop counter
	end_timer 9
	jmp	zpass2_7168_levels_complex_done

zpass2_7168_levels_complex_3:
	start_timer 9
	zloop_init 112			;; 112 iterations
	zloop_set_rw_prefetch ZOCT	;; Prefetch 896 cache lines
	mov	r8, DEST2ARG
	mov	rbp, DIST_TO_MULSRCARG
	zr64_sixtyfour_complex_with_mult_preload
b3c:	zr64_sixtyfour_complex_with_mult rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 4, L1PREFETCH_ALL, 8*128
	zloop	4, b3c, -4*8*128+dist32	;; Test loop counter, next source pointer
	zloop	28, b3c			;; Test loop counter
	add	rsi, r8
	end_timer 9
	jmp	zpass2_7168_levels_complex_unfft

zpass2_7168_levels_complex_4:
	start_timer 9
	zloop_init 112			;; 112 iterations
	zloop_set_rw_prefetch ZOCT	;; Prefetch 896 cache lines
	mov	rbp, DIST_TO_MULSRCARG
	mov	rdi, xsincos_complex	;; The sin/cos data
	bump	rdi, 128*ZMM_SCD3+8*ZMM_SCD8
	zr64f_sixtyfour_complex_with_mulf_preload
b3d:	zr64f_sixtyfour_complex_with_mulf rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 4, L1PREFETCH_ALL, 8*128
	zloop	4, b3d, -4*8*128+dist32	;; Test loop counter, next source pointer
	zloop	28, b3d			;; Test loop counter
	end_timer 9
zpass2_7168_levels_complex_unfft:
	bump	rsi, -28*dist32		;; Restore source pointer

;; Do inverse FFT levels 4,5,6
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	64	+1	...
;;	8	...
;;	...
;;	56
;;	128
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	1K	+1	...
;;	8	...
;;	...
;;	1K-8
;;	2K
;;	...

;; Do 448 sixteen_complex_unfft operations
;;	distance between fft data elements is 64
;;	do 56 macros each processing 256 data values

	start_timer 13
	bump	rdi, -8*ZMM_SCD8
	zloop_init 56			;; 7 iterations of 8
	zloop_set_rw_prefetch ZQUAD	;; Prefetch 224 cache lines
	zr16_sixteen_complex_djbunfft_preload
c2b:	zr16_sixteen_complex_djbunfft rsi, 128, 8*128, 16*128, dist32, 2*dist32, rdi, ZMM_SCD8, 8, L1PREFETCH_ALL, 128
	zloop	8, c2b, -8*128+4*dist32, rdi, -8*ZMM_SCD8 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	7, c2b, -7*4*dist32	;; Test loop counter, restore source pointer
	end_timer 13

;; Do inverse FFT levels 1,2,3
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	1K	+1	...
;;	8	...
;;	...
;;	1K-8
;;	2K
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	7K	+1	...
;;	8	...
;;	...
;;	7K-8

;; Do 1024 seven_complex_unfft operations
;;	distance between fft data elements is 1024
;;	do 128 macros each processing 112 data values

	start_timer 14
	bump	rdi, -128*ZMM_SCD3
	zloop_init 128			;; 128 iterations
	zloop_set_rw_prefetch ZDOUBLE	;; Prefetch 256 cache lines
	zr7_seven_complex_djbunfft_preload
c1b:	zr7_seven_complex_djbunfft rsi, 128, 4*dist32, rdi, ZMM_SCD3, 32, L1PREFETCH_ALL, 128
	zloop	32, c1b, -32*128+dist32	;; Test loop counter, next source pointer
	zloop	4, c1b			;; Test loop counter
	end_timer 14

zpass2_7168_levels_complex_done:
	ENDM


; *************** radix8-pass2-7680-real macro ******************
; This macro takes 15360 real values and performs the final 13.907 levels of the FFT.
; The results are squared, and the first 13.907 levels of the inverse FFT are done.

zr4dwpn_pass2_7680_real MACRO
	LOCAL	b1b, b2a, b2b, b3a, b3b, b3c, b3d, c1b, c2a, c2b
	LOCAL	zpass2_7680_real_1, zpass2_7680_real_2
	LOCAL	zpass2_7680_real_3, zpass2_7680_real_4
	LOCAL	zpass2_7680_real_unfft, zpass2_7680_real_done

;; Type 4 FFTs skip the forward FFT process

	start_timer 2
	mov	rsi, DATA_ADDR		;; Load source address
	mov	rbx, DIST_TO_FFTSRCARG
	mov	rcx, DATA_PREFETCH	;; Prefetch pointer (we need to prefetch 1920 cache lines + 29 pad cache lines)
	cmp	ffttype, 4
	je	zpass2_7680_real_4

;; Do FFT levels 1,2,3,4
;;	0	+1	+1	+1	+1	+1	+1	+1	7680	+1	...
;;	8	...
;;	...
;;	7672
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	768	+1	...
;;	8	...
;;	...
;;	760
;;	1536
;;	...

;; Do 768 twenty_real_fft operations
;;	distance between fft data elements is 768
;;	do 96 macros each processing 160 data values

	mov	rdx, sincos3		;; Real sin/cos data
	mov	rdi, xsincos_complex	;; Complex sin/cos data
	zloop_init 96			;; 96 iterations
	zloop_set_rw_prefetch ZTRIPLE	;; Prefetch 288 cache lines
	zr10f_2sc_twenty_reals_fft_preload
b1b:	zr10f_2sc_twenty_reals_fft rsi, 128, 3*dist32, rdx, ZMM_SCD5, rdi, ZMM_SCD5, 32
	zloop	32, b1b, -32*128+dist32	;; Test loop counter, next source pointer
	zloop	3, b1b, -3*dist32	;; Test loop counter, restore source pointer

;; Do FFT levels 5,6,7
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	768	+1	...
;;	8	...
;;	...
;;	760
;;	1536
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	64	+1	...
;;	8	...
;;	...
;;	56
;;	128
;;	...

;; Do 64 twentyfour_reals_fft operations
;; Do 576 twelve_complex_fft operations
;;	distance between fft data elements is 64
;;	do 80 macros each processing 192 data values

	zloop_init 80			;; 10 iterations of 8
	zloop_set_rw_prefetch ZTRIPLE	;; Prefetch 240 cache lines
	zr12_2sc_twentyfour_reals_fft_preload
b2a:	zr12_2sc_twentyfour_reals_fft rsi, 128, 8*128, 16*128, dist32, rdx, ZMM_SCD6, rdi, ZMM_SCD6, 8
	zloop	8, b2a, -8*128+3*dist32, rdi, -8*ZMM_SCD6 ;; Test loop counter, next source pointer, restore sin/cos pointer
	loops_reset			;; 6 more iters of 8
	zr12_twelve_complex_djbfft_preload
b2b:	zr12_twelve_complex_djbfft rsi, 128, 8*128, 16*128, dist32, rdi, ZMM_SCD6, 8
	zloop	8, b2b, -8*128+3*dist32, rdi, -8*ZMM_SCD6 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	10, b2b, -10*3*dist32, rdi, 8*ZMM_SCD6 ;; Test loop counter, restore source pointer, next sin/cos pointer

;; Do FFT levels 8,9,10,11,12,13 as well as inverse FFT levels 8,9,10,11,12,13
;; On input and output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	64	+1	...
;;	8	...
;;	...
;;	56
;;	128
;;	...

;; Do 1 hundredtwentyeight_reals_fft operation
;; Do 119 sixtyfour_complex_fft operations
;;	distance between fft data elements is 1 (8 for swizzling)
;;	do 120 macros each processing 128 data values

;; Execute the proper middle step

;pfing	bump	rcx, -3*64		;; We will prefetch 4 cache lines too few
	cmp	ffttype, 2
	jl	zpass2_7680_real_1
	jg	zpass2_7680_real_3

zpass2_7680_real_2:
	mov	r8, DEST2ARG
;;	zr64_hundredtwentyeight_real_with_square_preload
;;	zr64_hundredtwentyeight_real_with_square rsi, 8*128, 1*128, 2*128, 4*128, rdx, 0*ZMM_SCD4, rdi, 0*ZMM_SCD4, 1
	CALLP	zreal_square
	zloop_init 120, 1, 4		;; 119 more iterations
	zloop_set_rw_prefetch ZSEXT	;; Prefetch 714 cache lines
	zr64_sixtyfour_complex_with_square_preload
b3a:	zr64_sixtyfour_complex_with_square rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 1, L1PREFETCH_ALL, 8*128
	zloop	4, b3a, -4*8*128+dist32	;; Test loop counter, next source pointer
	zloop	30, b3a			;; Test loop counter
	add	rsi, r8
	jmp	zpass2_7680_real_unfft

zpass2_7680_real_1:
;;	zr64_hundredtwentyeight_real_fft_final_preload
;;	zr64_hundredtwentyeight_real_fft_final rsi, 8*128, 1*128, 2*128, 4*128, rdx, 0*ZMM_SCD4, rdi, 0*ZMM_SCD4, 1
	CALLP	zreal_fft_final
	zloop_init 120, 1, 4		;; 119 more iterations
	zloop_set_rw_prefetch ZSEXT	;; Prefetch 714 cache lines
	zr64_sixtyfour_complex_fft_final_preload
b3b:	zr64_sixtyfour_complex_fft_final rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 1, L1PREFETCH_ALL, 8*128
	zloop	4, b3b, -4*8*128+dist32	;; Test loop counter, next source pointer
	zloop	30, b3b			;; Test loop counter
	jmp	zpass2_7680_real_done

zpass2_7680_real_3:
	mov	r8, DEST2ARG
	mov	rbp, DIST_TO_MULSRCARG
;;	zr64_hundredtwentyeight_real_with_mult_preload
;;	zr64_hundredtwentyeight_real_with_mult rsi, 8*128, 1*128, 2*128, 4*128, rdx, 0*ZMM_SCD4, rdi, 0*ZMM_SCD4, 1
	CALLP	zreal_mult
	zloop_init 120, 1, 4		;; 119 more iterations
	zloop_set_rw_prefetch ZSEXT	;; Prefetch 714 cache lines
	zr64_sixtyfour_complex_with_mult_preload
b3c:	zr64_sixtyfour_complex_with_mult rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 1, L1PREFETCH_ALL, 8*128
	zloop	4, b3c, -4*8*128+dist32	;; Test loop counter, next source pointer
	zloop	30, b3c			;; Test loop counter
	add	rsi, r8
	jmp	zpass2_7680_real_unfft

zpass2_7680_real_4:
	mov	rbp, DIST_TO_MULSRCARG
	mov	rdx, sincos3		;; Real sin/cos data
	mov	rdi, xsincos_complex	;; Complex sin/cos data
	bump	rdx, 96*ZMM_SCD5+8*ZMM_SCD6
	bump	rdi, 96*ZMM_SCD5+8*ZMM_SCD6
;;	zr64f_hundredtwentyeight_real_with_mulf_preload
;;	zr64f_hundredtwentyeight_real_with_mulf rsi, 8*128, 1*128, 2*128, 4*128, rdx, 0*ZMM_SCD4, rdi, 0*ZMM_SCD4, 1
	CALLP	zreal_mulf
	zloop_init 120, 1, 4		;; 119 more iterations
	zloop_set_rw_prefetch ZSEXT	;; Prefetch 714 cache lines
	zr64f_sixtyfour_complex_with_mulf_preload
b3d:	zr64f_sixtyfour_complex_with_mulf rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 1, L1PREFETCH_ALL, 8*128
	zloop	4, b3d, -4*8*128+dist32	;; Test loop counter, next source pointer
	zloop	30, b3d			;; Test loop counter
zpass2_7680_real_unfft:
	bump	rsi, -30*dist32		;; Restore source pointer

;; Do inverse FFT levels 5,6,7
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	64	+1	...
;;	8	...
;;	...
;;	56
;;	128
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	768	+1	...
;;	8	...
;;	...
;;	760
;;	1536
;;	...

;; Do 64 twentyfour_reals_unfft operations
;; Do 576 twelve_complex_unfft operations
;;	distance between fft data elements is 64
;;	do 80 macros each processing 192 data values

	zloop_init 80			;; 10 iterations of 8
	zloop_set_rw_prefetch ZQUAD	;; Prefetch 320 cache lines
	bump	rdx, -8*ZMM_SCD6
	bump	rdi, -8*ZMM_SCD6
	zr12_2sc_twentyfour_reals_unfft_preload
c2a:	zr12_2sc_twentyfour_reals_unfft rsi, 128, 8*128, 16*128, dist32, rdx, ZMM_SCD6, rdi, ZMM_SCD6, 8
	zloop	8, c2a, -8*128+3*dist32, rdi, -8*ZMM_SCD6 ;; Test loop counter, next source pointer, restore sin/cos pointer
	loops_reset			;; 6 more iters of 8
	zr12_twelve_complex_djbunfft_preload
c2b:	zr12_twelve_complex_djbunfft rsi, 128, 8*128, 16*128, dist32, rdi, ZMM_SCD6, 8
	zloop	8, c2b, -8*128+3*dist32, rdi, -8*ZMM_SCD6 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	10, c2b, -10*3*dist32	;; Test loop counter, restore source pointer

;; Do inverse FFT levels 1,2,3,4
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	768	+1	...
;;	8	...
;;	...
;;	760
;;	1536
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	7680	+1	...
;;	8	...
;;	...
;;	7672

;; Do 768 twenty_reals_last_unfft operations
;;	distance between fft data elements is 768
;;	do 96 macros each processing 160 data values

	bump	rdx, -8*ZMM_SCD6-96*ZMM_SCD5
	bump	rdi, -96*ZMM_SCD5
	zloop_init 96			;; 96 iterations
	zloop_set_rw_prefetch ZQUAD	;; Prefetch 384 cache lines
	zr10_2sc_twenty_reals_unfft_preload
c1b:	zr10_2sc_twenty_reals_unfft rsi, 128, 3*dist32, rdx, ZMM_SCD5, rdi, ZMM_SCD5, 32
	zloop	32, c1b, -32*128+dist32	;; Test loop counter, next source pointer
	zloop	3, c1b			;; Test loop counter

zpass2_7680_real_done:
	end_timer 2
	ENDM

; *************** radix8-pass2-7680-complex macro ******************
; This macro takes 7680 complex values and performs the final 12.907 levels of a standard
; radix-8 FFT, squares the results, and does the first 12.907 levels of the inverse FFT.

zr4dwpn_pass2_7680_complex MACRO
	LOCAL	b1b, b2b, b3a, b3b, b3c, b3d, c1b, c2b
	LOCAL	zpass2_7680_complex_1, zpass2_7680_complex_2
	LOCAL	zpass2_7680_complex_3, zpass2_7680_complex_4
	LOCAL	zpass2_7680_complex_unfft, zpass2_7680_complex_done

;; Type 4 FFTs skip the forward FFT process

	mov	rsi, DATA_ADDR		;; Load address of FFT data
	mov	rbx, DIST_TO_FFTSRCARG
	mov	rcx, DATA_PREFETCH	;; Prefetch pointer (we need to prefetch 1920 cache lines + 29 pad cache lines)
	cmp	ffttype, 4
	je	zpass2_7680_complex_4

;; Do FFT levels 1,2,3
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	7680	+1	...
;;	8	...
;;	...
;;	7672
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	768	+1	...
;;	8	...
;;	...
;;	760
;;	1536
;;	...

;; Do 768 ten_complex_fft operations
;;	distance between fft data elements is 768
;;	do 96 macros each processing 160 data values

	start_timer 4
	mov	rdi, xsincos_complex	;; The sin/cos data
	zloop_init 96			;; 96 iterations
	zloop_set_rw_prefetch ZTRIPLE	;; Prefetch 288 cache lines
	zr10f_ten_complex_djbfft_preload
b1b:	zr10f_ten_complex_djbfft rsi, 128, 3*dist32, rdi, ZMM_SCD5, 32, L1PREFETCH_ALL, 128
	zloop	32, b1b, -32*128+dist32	;; Test loop counter, next source pointer
	zloop	3, b1b, -3*dist32	;; Test loop counter, restore source pointer
	end_timer 4

;; Do FFT levels 4,5,6
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	768	+1	...
;;	8	...
;;	...
;;	760
;;	1536
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	64	+1	...
;;	8	...
;;	...
;;	56
;;	128
;;	...

;; Do 640 twelve_complex_fft operations
;;	distance between fft data elements is 64
;;	do 80 macros each processing 192 data values

	start_timer 5
	zloop_init 80			;; 10 iterations of 8
	zloop_set_rw_prefetch ZTRIPLE	;; Prefetch 240 cache lines
	zr12_twelve_complex_djbfft_preload
b2b:	zr12_twelve_complex_djbfft rsi, 128, 8*128, 16*128, dist32, rdi, ZMM_SCD6, 8, L1PREFETCH_ALL, 128
	zloop	8, b2b, -8*128+3*dist32, rdi, -8*ZMM_SCD6 ;; Test loop counter, restore source pointer, restore sin/cos pointer
	zloop	10, b2b, -10*3*dist32, rdi, 8*ZMM_SCD6 ;; Test loop counter, restore source pointer, next sin/cos pointer
	end_timer 5

;; Do FFT levels 7,8,9,10,11,12 as well as inverse FFT levels 7,8,9,10,11,12
;; On input and output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	64	+1	...
;;	8	...
;;	...
;;	56
;;	128
;;	...

;; Do 120 sixtyfour_complex_fft operations
;;	distance between fft data elements is 1 (8 for swizzling)
;;	do 120 macros each processing 128 data values

;; Execute the right middle step

pfing	bump	rcx, -3*64		;; We will prefetch 3 cache lines too many
	cmp	ffttype, 2
	jl	zpass2_7680_complex_1
	jg	zpass2_7680_complex_3

zpass2_7680_complex_2:
	start_timer 9
	zloop_init 120			;; 120 iterations
	zloop_set_rw_prefetch ZSEXT	;; Prefetch 720 cache lines
	mov	r8, DEST2ARG
	zr64_sixtyfour_complex_with_square_preload
b3a:	zr64_sixtyfour_complex_with_square rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 120, L1PREFETCH_ALL, 8*128
	zloop	4, b3a, -4*8*128+dist32	;; Test loop counter, next source pointer
	zloop	30, b3a			;; Test loop counter
	add	rsi, r8
	end_timer 9
	jmp	zpass2_7680_complex_unfft

zpass2_7680_complex_1:
	start_timer 9
	zloop_init 120			;; 120 iterations
	zloop_set_rw_prefetch ZSEXT	;; Prefetch 720 cache lines
	zr64_sixtyfour_complex_fft_final_preload
b3b:	zr64_sixtyfour_complex_fft_final rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 120, L1PREFETCH_ALL, 8*128
	zloop	4, b3b, -4*8*128+dist32	;; Test loop counter, next source pointer
	zloop	30, b3b			;; Test loop counter
	end_timer 9
	jmp	zpass2_7680_complex_done

zpass2_7680_complex_3:
	start_timer 9
	zloop_init 120			;; 120 iterations
	zloop_set_rw_prefetch ZSEXT	;; Prefetch 720 cache lines
	mov	r8, DEST2ARG
	mov	rbp, DIST_TO_MULSRCARG
	zr64_sixtyfour_complex_with_mult_preload
b3c:	zr64_sixtyfour_complex_with_mult rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 120, L1PREFETCH_ALL, 8*128
	zloop	4, b3c, -4*8*128+dist32	;; Test loop counter, next source pointer
	zloop	30, b3c			;; Test loop counter
	add	rsi, r8
	end_timer 9
	jmp	zpass2_7680_complex_unfft

zpass2_7680_complex_4:
	start_timer 9
	zloop_init 120			;; 120 iterations
	zloop_set_rw_prefetch ZSEXT	;; Prefetch 720 cache lines
	mov	rbp, DIST_TO_MULSRCARG
	mov	rdi, xsincos_complex	;; The sin/cos data
	bump	rdi, 96*ZMM_SCD5+8*ZMM_SCD6
	zr64f_sixtyfour_complex_with_mulf_preload
b3d:	zr64f_sixtyfour_complex_with_mulf rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 120, L1PREFETCH_ALL, 8*128
	zloop	4, b3d, -4*8*128+dist32	;; Test loop counter, next source pointer
	zloop	30, b3d			;; Test loop counter
	end_timer 9
zpass2_7680_complex_unfft:
	bump	rsi, -30*dist32		;; Restore source pointer

;; Do inverse FFT levels 4,5,6
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	64	+1	...
;;	8	...
;;	...
;;	56
;;	128
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	768	+1	...
;;	8	...
;;	...
;;	760
;;	1536
;;	...

;; Do 640 twelve_complex_unfft operations
;;	distance between fft data elements is 64
;;	do 80 macros each processing 192 data values

	start_timer 13
	bump	rdi, -8*ZMM_SCD6
	zloop_init 80			;; 10 iterations of 8
	zloop_set_rw_prefetch ZQUAD	;; Prefetch 320 cache lines
	zr12_twelve_complex_djbunfft_preload
c2b:	zr12_twelve_complex_djbunfft rsi, 128, 8*128, 16*128, dist32, rdi, ZMM_SCD6, 8, L1PREFETCH_ALL, 128
	zloop	8, c2b, -8*128+3*dist32, rdi, -8*ZMM_SCD6 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	10, c2b, -10*3*dist32	;; Test loop counter, restore source pointer
	end_timer 13

;; Do inverse FFT levels 1,2,3
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	768	+1	...
;;	8	...
;;	...
;;	760
;;	1536
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	7680	+1	...
;;	8	...
;;	...
;;	7672

;; Do 768 ten_complex_unfft operations
;;	distance between fft data elements is 768
;;	do 96 macros each processing 160 data values

	start_timer 14
	bump	rdi, -96*ZMM_SCD5
	zloop_init 96			;; 96 iterations
	zloop_set_rw_prefetch ZQUAD	;; Prefetch 384 cache lines
	zr10_ten_complex_djbunfft_preload
c1b:	zr10_ten_complex_djbunfft rsi, 128, 3*dist32, rdi, ZMM_SCD5, 32, L1PREFETCH_ALL, 128
	zloop	32, c1b, -32*128+dist32	;; Test loop counter
	zloop	3, c1b			;; Test loop counter
	end_timer 14

zpass2_7680_complex_done:
	ENDM


; *************** radix8-pass2-8192-levels-real macro ******************
; This macro takes 16384 real values and performs the final 14 levels of the FFT.
; The results are squared, and the first 14 levels of the inverse FFT are done.

zr4dwpn_pass2_8192_real MACRO
	LOCAL	b1b, b2a, b2b, b3a, b3b, b3c, b3d, c1b, c2a, c2b
	LOCAL	zpass2_8192_levels_real_1, zpass2_8192_levels_real_2
	LOCAL	zpass2_8192_levels_real_3, zpass2_8192_levels_real_4
	LOCAL	zpass2_8192_levels_real_unfft, zpass2_8192_levels_real_done

;; Type 4 FFTs skip the forward FFT process

	start_timer 2
	mov	rsi, DATA_ADDR		;; Load source address
	mov	rbx, DIST_TO_FFTSRCARG
	mov	rcx, DATA_PREFETCH	;; Prefetch pointer (we need to prefetch 2048 cache lines + 31 pad cache lines)
	cmp	ffttype, 4
	je	zpass2_8192_levels_real_4

;; Do FFT levels 1,2,3,4
;;	0	+1	+1	+1	+1	+1	+1	+1	8192	+1	...
;;	8	...
;;	...
;;	8184
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	512	+1	...
;;	8	...
;;	...
;;	504
;;	1K
;;	...

;; Do 512 thirtytwo_real_fft operations
;;	distance between fft data elements is 512
;;	do 64 macros each processing 256 data values

	mov	rdx, sincos3		;; Real sin/cos data
	mov	rdi, xsincos_complex	;; Complex sin/cos data
	zloop_init 64			;; 64 iterations
	zloop_set_rw_prefetch ZQUAD	;; Prefetch 256 cache lines
	zr16f_2sc_thirtytwo_reals_fft_preload
b1b:	zr16f_2sc_thirtytwo_reals_fft rsi, 128, 2*dist32, 4*dist32, 8*dist32, 16*dist32, rdx, ZMM_SCD8, rdi, ZMM_SCD8, 32
	zloop	32, b1b, -32*128+dist32	;; Test loop counter, next source pointer
	zloop	2, b1b, -2*dist32	;; Test loop counter, restore source pointer

;; Do FFT levels 5,6,7
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	512	+1	...
;;	8	...
;;	...
;;	504
;;	1K
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	64	+1	...
;;	8	...
;;	...
;;	56
;;	128
;;	...

;; Do 64 sixteen_reals_fft operations
;; Do 960 eight_complex_fft operations
;;	distance between fft data elements is 64
;;	do 128 macros each processing 128 data values

	zloop_init 128			;; 16 iterations of 8
	zloop_set_rw_prefetch ZDOUBLE	;; Prefetch 256 cache lines
	zr8_2sc_sixteen_reals_fft_preload
b2a:	zr8_2sc_sixteen_reals_fft rsi, 128, 8*128, 16*128, dist32, rdx, ZMM_SCD4, rdi, ZMM_SCD4, 8
	zloop	8, b2a, -8*128+2*dist32, rdi, -8*ZMM_SCD4 ;; Test loop counter, next source pointer, restore sin/cos pointer
	loops_reset			;; 15 more iters of 8
	zr8_eight_complex_djbfft_preload
b2b:	zr8_eight_complex_djbfft rsi, 128, 8*128, 16*128, dist32, rdi, ZMM_SCD4, 8
	zloop	8, b2b, -8*128+2*dist32, rdi, -8*ZMM_SCD4 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	16, b2b, -16*2*dist32, rdi, 8*ZMM_SCD4 ;; Test loop counter, restore source pointer, next sin/cos pointer

;; Do FFT levels 8,9,10,11,12,13 as well as inverse FFT levels 8,9,10,11,12,13
;; On input and output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	64	+1	...
;;	8	...
;;	...
;;	56
;;	128
;;	...

;; Do 1 hundredtwentyeight_reals_fft operation
;; Do 127 sixtyfour_complex_fft operations
;;	distance between fft data elements is 1 (8 for swizzling)
;;	do 128 macros each processing 128 data values

;; Execute the proper middle step

pfing	bump	rcx, -25*64		;; We will prefetch 25 cache lines too many
	cmp	ffttype, 2
	jl	zpass2_8192_levels_real_1
	jg	zpass2_8192_levels_real_3

zpass2_8192_levels_real_2:
	mov	r8, DEST2ARG
;;	zr64_hundredtwentyeight_real_with_square_preload
;;	zr64_hundredtwentyeight_real_with_square rsi, 8*128, 1*128, 2*128, 4*128, rdx, 0*ZMM_SCD4, rdi, 0*ZMM_SCD4, 1
	CALLP	zreal_square
	zloop_init 128, 1, 4		;; 127 more iterations
	zloop_set_rw_prefetch ZOCT	;; Prefetch 1016 cache lines
	zr64_sixtyfour_complex_with_square_preload
b3a:	zr64_sixtyfour_complex_with_square rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 1, L1PREFETCH_ALL, 8*128
	zloop	4, b3a, -4*8*128+dist32	;; Test loop counter, next source pointer
	zloop	32, b3a			;; Test loop counter
	add	rsi, r8
	jmp	zpass2_8192_levels_real_unfft

zpass2_8192_levels_real_1:
;;	zr64_hundredtwentyeight_real_fft_final_preload
;;	zr64_hundredtwentyeight_real_fft_final rsi, 8*128, 1*128, 2*128, 4*128, rdx, 0*ZMM_SCD4, rdi, 0*ZMM_SCD4, 1
	CALLP	zreal_fft_final
	zloop_init 128, 1, 4		;; 127 more iterations
	zloop_set_rw_prefetch ZOCT	;; Prefetch 1016 cache lines
	zr64_sixtyfour_complex_fft_final_preload
b3b:	zr64_sixtyfour_complex_fft_final rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 1, L1PREFETCH_ALL, 8*128
	zloop	4, b3b, -4*8*128+dist32	;; Test loop counter, next source pointer
	zloop	32, b3b			;; Test loop counter
	jmp	zpass2_8192_levels_real_done

zpass2_8192_levels_real_3:
	mov	r8, DEST2ARG
	mov	rbp, DIST_TO_MULSRCARG
;;	zr64_hundredtwentyeight_real_with_mult_preload
;;	zr64_hundredtwentyeight_real_with_mult rsi, 8*128, 1*128, 2*128, 4*128, rdx, 0*ZMM_SCD4, rdi, 0*ZMM_SCD4, 1
	CALLP	zreal_mult
	zloop_init 128, 1, 4		;; 127 more iterations
	zloop_set_rw_prefetch ZOCT	;; Prefetch 1016 cache lines
	zr64_sixtyfour_complex_with_mult_preload
b3c:	zr64_sixtyfour_complex_with_mult rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 1, L1PREFETCH_ALL, 8*128
	zloop	4, b3c, -4*8*128+dist32	;; Test loop counter, next source pointer
	zloop	32, b3c			;; Test loop counter
	add	rsi, r8
	jmp	zpass2_8192_levels_real_unfft

zpass2_8192_levels_real_4:
	mov	rbp, DIST_TO_MULSRCARG
	mov	rdx, sincos3		;; Real sin/cos data
	mov	rdi, xsincos_complex	;; Complex sin/cos data
	bump	rdx, 64*ZMM_SCD8+8*ZMM_SCD4
	bump	rdi, 64*ZMM_SCD8+8*ZMM_SCD4
;;	zr64f_hundredtwentyeight_real_with_mulf_preload
;;	zr64f_hundredtwentyeight_real_with_mulf rsi, 8*128, 1*128, 2*128, 4*128, rdx, 0*ZMM_SCD4, rdi, 0*ZMM_SCD4, 1
	CALLP	zreal_mulf
	zloop_init 128, 1, 4		;; 127 more iterations
	zloop_set_rw_prefetch ZOCT	;; Prefetch 1016 cache lines
	zr64f_sixtyfour_complex_with_mulf_preload
b3d:	zr64f_sixtyfour_complex_with_mulf rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 1, L1PREFETCH_ALL, 8*128
	zloop	4, b3d, -4*8*128+dist32	;; Test loop counter, next source pointer
	zloop	32, b3d			;; Test loop counter
zpass2_8192_levels_real_unfft:
	bump	rsi, -32*dist32		;; Restore source pointer

;; Do inverse FFT levels 5,6,7
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	64	+1	...
;;	8	...
;;	...
;;	56
;;	128
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	512	+1	...
;;	8	...
;;	...
;;	504
;;	1K
;;	...

;; Do 64 sixteen_reals_unfft operations
;; Do 960 eight_complex_unfft operations
;;	distance between fft data elements is 64
;;	do 128 macros each processing 128 data values

	zloop_init 128			;; 16 iterations of 8
	zloop_set_rw_prefetch ZDOUBLE	;; Prefetch 256 cache lines
	bump	rdx, -8*ZMM_SCD4
	bump	rdi, -8*ZMM_SCD4
	zr8_2sc_sixteen_reals_unfft_preload
c2a:	zr8_2sc_sixteen_reals_unfft rsi, 128, 8*128, 16*128, dist32, rdx, ZMM_SCD4, rdi, ZMM_SCD4, 8
	zloop	8, c2a, -8*128+2*dist32, rdi, -8*ZMM_SCD4 ;; Test loop counter, next source pointer, restore sin/cos pointer
	loops_reset			;; 15 more iters of 8
	zr8_eight_complex_djbunfft_preload
c2b:	zr8_eight_complex_djbunfft rsi, 128, 8*128, 16*128, dist32, rdi, ZMM_SCD4, 8
	zloop	8, c2b, -8*128+2*dist32, rdi, -8*ZMM_SCD4 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	16, c2b, -16*2*dist32	;; Test loop counter, restore source pointer

;; Do inverse FFT levels 1,2,3,4
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	512	+1	...
;;	8	...
;;	...
;;	504
;;	1K
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	8192	+1	...
;;	8	...
;;	...
;;	8184

;; Do 512 thirtytwo_reals_last_unfft operations
;;	distance between fft data elements is 512
;;	do 64 macros each processing 256 data values

	bump	rdx, -8*ZMM_SCD4-64*ZMM_SCD8
	bump	rdi, -64*ZMM_SCD8
	zloop_init 64			;; 64 iterations
	zloop_set_rw_prefetch ZQUINT	;; Prefetch 320 cache lines
	zr16_2sc_thirtytwo_reals_unfft_preload
c1b:	zr16_2sc_thirtytwo_reals_unfft rsi, 128, 2*dist32, 4*dist32, 8*dist32, 16*dist32, rdx, ZMM_SCD8, rdi, ZMM_SCD8, 32
	zloop	32, c1b, -32*128+dist32	;; Test loop counter, next source pointer
	zloop	2, c1b			;; Test loop counter

zpass2_8192_levels_real_done:
	end_timer 2
	ENDM

; *************** radix8-pass2-8192-levels-complex macro ******************
; This macro takes 8192 complex values and performs the final 13 levels of a standard
; radix-8 FFT, squares the results, and does the first 13 levels of the inverse FFT.

zr4dwpn_pass2_8192_complex MACRO
	LOCAL	b1b, b2b, b3a, b3b, b3c, b3d, c1b, c2b
	LOCAL	zpass2_8192_levels_complex_1, zpass2_8192_levels_complex_2
	LOCAL	zpass2_8192_levels_complex_3, zpass2_8192_levels_complex_4
	LOCAL	zpass2_8192_levels_complex_unfft, zpass2_8192_levels_complex_done

;; Type 4 FFTs skip the forward FFT process

	mov	rsi, DATA_ADDR		;; Load address of FFT data
	mov	rbx, DIST_TO_FFTSRCARG
	mov	rcx, DATA_PREFETCH	;; Prefetch pointer (we need to prefetch 2048 cache lines + 31 pad cache lines)
	cmp	ffttype, 4
	je	zpass2_8192_levels_complex_4

;; Do FFT levels 1,2,3
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	8192	+1	...
;;	8	...
;;	...
;;	8184
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	512	+1	...
;;	8	...
;;	...
;;	504
;;	1K
;;	...

;; Do 512 sixteen_complex_fft operations
;;	distance between fft data elements is 512
;;	do 64 macros each processing 256 data values

	start_timer 4
	mov	rdi, xsincos_complex	;; The sin/cos data
	zloop_init 64			;; 64 iterations
	zloop_set_rw_prefetch ZQUAD	;; Prefetch 256 cache lines
	zr16f_sixteen_complex_djbfft_preload
b1b:	zr16f_sixteen_complex_djbfft rsi, 128, 2*dist32, 4*dist32, 8*dist32, 16*dist32, rdi, ZMM_SCD8, 32, L1PREFETCH_ALL, 128
	zloop	32, b1b, -32*128+dist32	;; Test loop counter, next source pointer
	zloop	2, b1b, -2*dist32	;; Test loop counter, restore source pointer
	end_timer 4

;; Do FFT levels 4,5,6
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	512	+1	...
;;	8	...
;;	...
;;	504
;;	1K
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	64	+1	...
;;	8	...
;;	...
;;	56
;;	128
;;	...

;; Do 1024 eight_complex_fft operations
;;	distance between fft data elements is 64
;;	do 128 macros each processing 128 data values

	start_timer 5
	zloop_init 128			;; 16 iterations of 8
	zloop_set_rw_prefetch ZDOUBLE	;; Prefetch 256 cache lines
	zr8_eight_complex_djbfft_preload
b2b:	zr8_eight_complex_djbfft rsi, 128, 8*128, 16*128, dist32, rdi, ZMM_SCD4, 8, L1PREFETCH_ALL, 128
	zloop	8, b2b, -8*128+2*dist32, rdi, -8*ZMM_SCD4	;; Test loop counter, restore source pointer, restore sin/cos pointer
	zloop	16, b2b, -16*2*dist32, rdi, 8*ZMM_SCD4 ;; Test loop counter, restore source pointer, next sin/cos pointer
	end_timer 5

;; Do FFT levels 7,8,9,10,11,12 as well as inverse FFT levels 7,8,9,10,11,12
;; On input and output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	64	+1	...
;;	8	...
;;	...
;;	56
;;	128
;;	...

;; Do 128 sixtyfour_complex_fft operations
;;	distance between fft data elements is 1 (8 for swizzling)
;;	do 128 macros each processing 128 data values

;; Execute the right middle step

pfing	bump	rcx, -33*64		;; We will prefetch 33 cache lines too many
	cmp	ffttype, 2
	jl	zpass2_8192_levels_complex_1
	jg	zpass2_8192_levels_complex_3

zpass2_8192_levels_complex_2:
	start_timer 9
	zloop_init 128			;; 128 iterations
	zloop_set_rw_prefetch ZOCT	;; Prefetch 1024 cache lines
	mov	r8, DEST2ARG
	zr64_sixtyfour_complex_with_square_preload
b3a:	zr64_sixtyfour_complex_with_square rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 4, L1PREFETCH_ALL, 8*128
	zloop	4, b3a, -4*8*128+dist32	;; Test loop counter, next source pointer
	zloop	32, b3a			;; Test loop counter
	add	rsi, r8
	end_timer 9
	jmp	zpass2_8192_levels_complex_unfft

zpass2_8192_levels_complex_1:
	start_timer 9
	zloop_init 128			;; 128 iterations
	zloop_set_rw_prefetch ZOCT	;; Prefetch 1024 cache lines
	zr64_sixtyfour_complex_fft_final_preload
b3b:	zr64_sixtyfour_complex_fft_final rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 4, L1PREFETCH_ALL, 8*128
	zloop	4, b3b, -4*8*128+dist32	;; Test loop counter, next source pointer
	zloop	32, b3b			;; Test loop counter
	end_timer 9
	jmp	zpass2_8192_levels_complex_done

zpass2_8192_levels_complex_3:
	start_timer 9
	zloop_init 128			;; 128 iterations
	zloop_set_rw_prefetch ZOCT	;; Prefetch 1024 cache lines
	mov	r8, DEST2ARG
	mov	rbp, DIST_TO_MULSRCARG
	zr64_sixtyfour_complex_with_mult_preload
b3c:	zr64_sixtyfour_complex_with_mult rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 4, L1PREFETCH_ALL, 8*128
	zloop	4, b3c, -4*8*128+dist32	;; Test loop counter, next source pointer
	zloop	32, b3c			;; Test loop counter
	add	rsi, r8
	end_timer 9
	jmp	zpass2_8192_levels_complex_unfft

zpass2_8192_levels_complex_4:
	start_timer 9
	zloop_init 128			;; 128 iterations
	zloop_set_rw_prefetch ZOCT	;; Prefetch 1024 cache lines
	mov	rbp, DIST_TO_MULSRCARG
	mov	rdi, xsincos_complex	;; The sin/cos data
	bump	rdi, 64*ZMM_SCD8+8*ZMM_SCD4
	zr64f_sixtyfour_complex_with_mulf_preload
b3d:	zr64f_sixtyfour_complex_with_mulf rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 4, L1PREFETCH_ALL, 8*128
	zloop	4, b3d, -4*8*128+dist32	;; Test loop counter, next source pointer
	zloop	32, b3d			;; Test loop counter
	end_timer 9
zpass2_8192_levels_complex_unfft:
	bump	rsi, -32*dist32		;; Restore source pointer

;; Do inverse FFT levels 4,5,6
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	64	+1	...
;;	8	...
;;	...
;;	56
;;	128
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	512	+1	...
;;	8	...
;;	...
;;	504
;;	1K
;;	...

;; Do 1024 eight_complex_unfft operations
;;	distance between fft data elements is 64
;;	do 128 macros each processing 128 data values

	start_timer 13
	bump	rdi, -8*ZMM_SCD4
	zloop_init 128			;; 16 iterations of 8
	zloop_set_rw_prefetch ZDOUBLE	;; Prefetch 256 cache lines
	zr8_eight_complex_djbunfft_preload
c2b:	zr8_eight_complex_djbunfft rsi, 128, 8*128, 16*128, dist32, rdi, ZMM_SCD4, 8, L1PREFETCH_ALL, 128
	zloop	8, c2b, -8*128+2*dist32, rdi, -8*ZMM_SCD4 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	16, c2b, -16*2*dist32	;; Test loop counter, restore source pointer
	end_timer 13

;; Do inverse FFT levels 1,2,3
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	512	+1	...
;;	8	...
;;	...
;;	504
;;	1K
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	8192	+1	...
;;	8	...
;;	...
;;	8184

;; Do 512 sixteen_complex_unfft operations
;;	distance between fft data elements is 512
;;	do 64 macros each processing 256 data values

	start_timer 14
	bump	rdi, -64*ZMM_SCD8
	zloop_init 64			;; 64 iterations
	zloop_set_rw_prefetch ZQUINT	;; Prefetch 320 cache lines
	zr16_sixteen_complex_djbunfft_preload
c1b:	zr16_sixteen_complex_djbunfft rsi, 128, 2*dist32, 4*dist32, 8*dist32, 16*dist32, rdi, ZMM_SCD8, 32, L1PREFETCH_ALL, 128
	zloop	32, c1b, -32*128+dist32	;; Test loop counter, next source pointer
	zloop	2, c1b			;; Test loop counter
	end_timer 14

zpass2_8192_levels_complex_done:
	ENDM


; *************** radix8-pass2-9216-real macro ******************
; This macro takes 18432 real values and performs the final 14.17 levels of the FFT.
; The results are squared, and the first 14.17 levels of the inverse FFT are done.

zr4dwpn_pass2_9216_real MACRO
	LOCAL	b1b, b2a, b2b, b3a, b3b, b3c, b3d, c1b, c2a, c2b
	LOCAL	zpass2_9216_real_1, zpass2_9216_real_2
	LOCAL	zpass2_9216_real_3, zpass2_9216_real_4
	LOCAL	zpass2_9216_real_unfft, zpass2_9216_real_done

;; Type 4 FFTs skip the forward FFT process

	start_timer 2
	mov	rsi, DATA_ADDR		;; Load source address
	mov	rbx, DIST_TO_FFTSRCARG
	mov	rcx, DATA_PREFETCH	;; Prefetch pointer (we need to prefetch 2304 cache lines + 35 pad cache lines)
	cmp	ffttype, 4
	je	zpass2_9216_real_4

;; Do FFT levels 1,2,3,4
;;	0	+1	+1	+1	+1	+1	+1	+1	9216	+1	...
;;	8	...
;;	...
;;	9208
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	768	+1	...
;;	8	...
;;	...
;;	760
;;	1536
;;	...

;; Do 768 twentyfour_real_fft operations
;;	distance between fft data elements is 768
;;	do 96 macros each processing 192 data values

	mov	rdx, sincos3		;; Real sin/cos data
	mov	rdi, xsincos_complex	;; Complex sin/cos data
	zloop_init 96			;; 96 iterations
	zloop_set_rw_prefetch ZDOUBLE	;; Prefetch 192 cache lines
	zr12f_2sc_twentyfour_reals_fft_preload
b1b:	zr12f_2sc_twentyfour_reals_fft rsi, 128, 3*dist32, 6*dist32, 12*dist32, rdx, ZMM_SCD6, rdi, ZMM_SCD6, 32
	zloop	32, b1b, -32*128+dist32	;; Test loop counter, next source pointer
	zloop	3, b1b, -3*dist32	;; Test loop counter, restore source pointer

;; Do FFT levels 5,6,7
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	768	+1	...
;;	8	...
;;	...
;;	760
;;	1536
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	64	+1	...
;;	8	...
;;	...
;;	56
;;	128
;;	...

;; Do 64 twentyfour_reals_fft operations
;; Do 704 twelve_complex_fft operations
;;	distance between fft data elements is 64
;;	do 96 macros each processing 192 data values

	zloop_init 96			;; 12 iterations of 8
	zloop_set_rw_prefetch ZTRIPLE	;; Prefetch 288 cache lines
	zr12_2sc_twentyfour_reals_fft_preload
b2a:	zr12_2sc_twentyfour_reals_fft rsi, 128, 8*128, 16*128, dist32, rdx, ZMM_SCD6, rdi, ZMM_SCD6, 8
	zloop	8, b2a, -8*128+3*dist32, rdi, -8*ZMM_SCD6 ;; Test loop counter, next source pointer, restore sin/cos pointer
	loops_reset			;; 11 more iters of 8
	zr12_twelve_complex_djbfft_preload
b2b:	zr12_twelve_complex_djbfft rsi, 128, 8*128, 16*128, dist32, rdi, ZMM_SCD6, 8
	zloop	8, b2b, -8*128+3*dist32, rdi, -8*ZMM_SCD6 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	12, b2b, -12*3*dist32, rdi, 8*ZMM_SCD6 ;; Test loop counter, restore source pointer, next sin/cos pointer

;; Do FFT levels 8,9,10,11,12,13 as well as inverse FFT levels 8,9,10,11,12,13
;; On input and output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	64	+1	...
;;	8	...
;;	...
;;	56
;;	128
;;	...

;; Do 1 hundredtwentyeight_reals_fft operation
;; Do 143 sixtyfour_complex_fft operations
;;	distance between fft data elements is 1 (8 for swizzling)
;;	do 144 macros each processing 128 data values

;; Execute the proper middle step

pfing	bump	rcx, -8*64		;; We will prefetch 8 cache lines too many
	cmp	ffttype, 2
	jl	zpass2_9216_real_1
	jg	zpass2_9216_real_3

zpass2_9216_real_2:
	mov	r8, DEST2ARG
;;	zr64_hundredtwentyeight_real_with_square_preload
;;	zr64_hundredtwentyeight_real_with_square rsi, 8*128, 1*128, 2*128, 4*128, rdx, 0*ZMM_SCD4, rdi, 0*ZMM_SCD4, 1
	CALLP	zreal_square
	zloop_init 144, 1, 4		;; 143 more iterations
	zloop_set_rw_prefetch ZQUINT	;; Prefetch 715 cache lines
	zr64_sixtyfour_complex_with_square_preload
b3a:	zr64_sixtyfour_complex_with_square rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 1, L1PREFETCH_ALL, 8*128
	zloop	4, b3a, -4*8*128+dist32	;; Test loop counter, next source pointer
	zloop	36, b3a			;; Test loop counter
	add	rsi, r8
	jmp	zpass2_9216_real_unfft

zpass2_9216_real_1:
;;	zr64_hundredtwentyeight_real_fft_final_preload
;;	zr64_hundredtwentyeight_real_fft_final rsi, 8*128, 1*128, 2*128, 4*128, rdx, 0*ZMM_SCD4, rdi, 0*ZMM_SCD4, 1
	CALLP	zreal_fft_final
	zloop_init 144, 1, 4		;; 143 more iterations
	zloop_set_rw_prefetch ZQUINT	;; Prefetch 715 cache lines
	zr64_sixtyfour_complex_fft_final_preload
b3b:	zr64_sixtyfour_complex_fft_final rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 1, L1PREFETCH_ALL, 8*128
	zloop	4, b3b, -4*8*128+dist32	;; Test loop counter, next source pointer
	zloop	36, b3b			;; Test loop counter
	jmp	zpass2_9216_real_done

zpass2_9216_real_3:
	mov	r8, DEST2ARG
	mov	rbp, DIST_TO_MULSRCARG
;;	zr64_hundredtwentyeight_real_with_mult_preload
;;	zr64_hundredtwentyeight_real_with_mult rsi, 8*128, 1*128, 2*128, 4*128, rdx, 0*ZMM_SCD4, rdi, 0*ZMM_SCD4, 1
	CALLP	zreal_mult
	zloop_init 144, 1, 4		;; 143 more iterations
	zloop_set_rw_prefetch ZQUINT	;; Prefetch 715 cache lines
	zr64_sixtyfour_complex_with_mult_preload
b3c:	zr64_sixtyfour_complex_with_mult rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 1, L1PREFETCH_ALL, 8*128
	zloop	4, b3c, -4*8*128+dist32	;; Test loop counter, next source pointer
	zloop	36, b3c			;; Test loop counter
	add	rsi, r8
	jmp	zpass2_9216_real_unfft

zpass2_9216_real_4:
	mov	rbp, DIST_TO_MULSRCARG
	mov	rdx, sincos3		;; Real sin/cos data
	mov	rdi, xsincos_complex	;; Complex sin/cos data
	bump	rdx, 96*ZMM_SCD6+8*ZMM_SCD6
	bump	rdi, 96*ZMM_SCD6+8*ZMM_SCD6
;;	zr64f_hundredtwentyeight_real_with_mulf_preload
;;	zr64f_hundredtwentyeight_real_with_mulf rsi, 8*128, 1*128, 2*128, 4*128, rdx, 0*ZMM_SCD4, rdi, 0*ZMM_SCD4, 1
	CALLP	zreal_mulf
	zloop_init 144, 1, 4		;; 143 more iterations
	zloop_set_rw_prefetch ZQUINT	;; Prefetch 715 cache lines
	zr64f_sixtyfour_complex_with_mulf_preload
b3d:	zr64f_sixtyfour_complex_with_mulf rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 1, L1PREFETCH_ALL, 8*128
	zloop	4, b3d, -4*8*128+dist32	;; Test loop counter, next source pointer
	zloop	36, b3d			;; Test loop counter
zpass2_9216_real_unfft:
	bump	rsi, -36*dist32		;; Restore source pointer

;; Do inverse FFT levels 5,6,7
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	64	+1	...
;;	8	...
;;	...
;;	56
;;	128
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	768	+1	...
;;	8	...
;;	...
;;	760
;;	1536
;;	...

;; Do 64 twentyfour_reals_unfft operations
;; Do 704 twelve_complex_unfft operations
;;	distance between fft data elements is 64
;;	do 96 macros each processing 192 data values

	zloop_init 96			;; 12 iterations of 8
	zloop_set_rw_prefetch ZTRIPLE	;; Prefetch 288 cache lines
	bump	rdx, -8*ZMM_SCD6
	bump	rdi, -8*ZMM_SCD6
	zr12_2sc_twentyfour_reals_unfft_preload
c2a:	zr12_2sc_twentyfour_reals_unfft rsi, 128, 8*128, 16*128, dist32, rdx, ZMM_SCD6, rdi, ZMM_SCD6, 8
	zloop	8, c2a, -8*128+3*dist32, rdi, -8*ZMM_SCD6 ;; Test loop counter, next source pointer, restore sin/cos pointer
	loops_reset			;; 11 more iters of 8
	zr12_twelve_complex_djbunfft_preload
c2b:	zr12_twelve_complex_djbunfft rsi, 128, 8*128, 16*128, dist32, rdi, ZMM_SCD6, 8
	zloop	8, c2b, -8*128+3*dist32, rdi, -8*ZMM_SCD6 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	12, c2b, -12*3*dist32	;; Test loop counter, restore source pointer

;; Do inverse FFT levels 1,2,3,4
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	768	+1	...
;;	8	...
;;	...
;;	760
;;	1536
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	9216	+1	...
;;	8	...
;;	...
;;	9208

;; Do 768 twentyfour_reals_last_unfft operations
;;	distance between fft data elements is 768
;;	do 96 macros each processing 192 data values

	bump	rdx, -8*ZMM_SCD6-96*ZMM_SCD6
	bump	rdi, -96*ZMM_SCD6
	zloop_init 96			;; 96 iterations
	zloop_set_rw_prefetch ZTRIPLE	;; Prefetch 288 cache lines
	zr12_2sc_twentyfour_reals_unfft_preload
c1b:	zr12_2sc_twentyfour_reals_unfft rsi, 128, 3*dist32, 6*dist32, 12*dist32, rdx, ZMM_SCD6, rdi, ZMM_SCD6, 32
	zloop	32, c1b, -32*128+dist32	;; Test loop counter, next source pointer
	zloop	3, c1b			;; Test loop counter

zpass2_9216_real_done:
	end_timer 2
	ENDM

; *************** radix8-pass2-9216-complex macro ******************
; This macro takes 9216 complex values and performs the final 13.17 levels of a standard
; radix-8 FFT, squares the results, and does the first 13.17 levels of the inverse FFT.

zr4dwpn_pass2_9216_complex MACRO
	LOCAL	b1b, b2b, b3a, b3b, b3c, b3d, c1b, c2b
	LOCAL	zpass2_9216_complex_1, zpass2_9216_complex_2
	LOCAL	zpass2_9216_complex_3, zpass2_9216_complex_4
	LOCAL	zpass2_9216_complex_unfft, zpass2_9216_complex_done

;; Type 4 FFTs skip the forward FFT process

	mov	rsi, DATA_ADDR		;; Load address of FFT data
	mov	rbx, DIST_TO_FFTSRCARG
	mov	rcx, DATA_PREFETCH	;; Prefetch pointer (we need to prefetch 2304 cache lines + 35 pad cache lines)
	cmp	ffttype, 4
	je	zpass2_9216_complex_4

;; Do FFT levels 1,2,3
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	9216	+1	...
;;	8	...
;;	...
;;	9208
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	768	+1	...
;;	8	...
;;	...
;;	760
;;	1536
;;	...

;; Do 768 twelve_complex_fft operations
;;	distance between fft data elements is 768
;;	do 96 macros each processing 192 data values

	start_timer 4
	mov	rdi, xsincos_complex	;; The sin/cos data
	zloop_init 96			;; 96 iterations
	zloop_set_rw_prefetch ZDOUBLE	;; Prefetch 192 cache lines
	zr12f_twelve_complex_djbfft_preload
b1b:	zr12f_twelve_complex_djbfft rsi, 128, 3*dist32, 6*dist32, 12*dist32, rdi, ZMM_SCD6, 32, L1PREFETCH_ALL, 128
	zloop	32, b1b, -32*128+dist32	;; Test loop counter, next source pointer
	zloop	3, b1b, -3*dist32	;; Test loop counter, restore source pointer
	end_timer 4

;; Do FFT levels 4,5,6
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	768	+1	...
;;	8	...
;;	...
;;	760
;;	1536
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	64	+1	...
;;	8	...
;;	...
;;	56
;;	128
;;	...

;; Do 768 twelve_complex_fft operations
;;	distance between fft data elements is 64
;;	do 96 macros each processing 192 data values

	start_timer 5
	zloop_init 96			;; 12 iterations of 8
	zloop_set_rw_prefetch ZTRIPLE	;; Prefetch 288 cache lines
	zr12_twelve_complex_djbfft_preload
b2b:	zr12_twelve_complex_djbfft rsi, 128, 8*128, 16*128, dist32, rdi, ZMM_SCD6, 8, L1PREFETCH_ALL, 128
	zloop	8, b2b, -8*128+3*dist32, rdi, -8*ZMM_SCD6 ;; Test loop counter, restore source pointer, restore sin/cos pointer
	zloop	12, b2b, -12*3*dist32, rdi, 8*ZMM_SCD6 ;; Test loop counter, restore source pointer, next sin/cos pointer
	end_timer 5

;; Do FFT levels 7,8,9,10,11,12 as well as inverse FFT levels 7,8,9,10,11,12
;; On input and output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	64	+1	...
;;	8	...
;;	...
;;	56
;;	128
;;	...

;; Do 144 sixtyfour_complex_fft operations
;;	distance between fft data elements is 1 (8 for swizzling)
;;	do 144 macros each processing 128 data values

;; Execute the right middle step

pfing	bump	rcx, -13*64		;; We will prefetch 13 cache lines too many
	cmp	ffttype, 2
	jl	zpass2_9216_complex_1
	jg	zpass2_9216_complex_3

zpass2_9216_complex_2:
	start_timer 9
	zloop_init 144			;; 144 iterations
	zloop_set_rw_prefetch ZQUINT	;; Prefetch 720 cache lines
	mov	r8, DEST2ARG
	zr64_sixtyfour_complex_with_square_preload
b3a:	zr64_sixtyfour_complex_with_square rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 4, L1PREFETCH_ALL, 8*128
	zloop	4, b3a, -4*8*128+dist32	;; Test loop counter, next source pointer
	zloop	36, b3a			;; Test loop counter
	add	rsi, r8
	end_timer 9
	jmp	zpass2_9216_complex_unfft

zpass2_9216_complex_1:
	start_timer 9
	zloop_init 144			;; 144 iterations
	zloop_set_rw_prefetch ZQUINT	;; Prefetch 720 cache lines
	zr64_sixtyfour_complex_fft_final_preload
b3b:	zr64_sixtyfour_complex_fft_final rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 4, L1PREFETCH_ALL, 8*128
	zloop	4, b3b, -4*8*128+dist32	;; Test loop counter, next source pointer
	zloop	36, b3b			;; Test loop counter
	end_timer 9
	jmp	zpass2_9216_complex_done

zpass2_9216_complex_3:
	start_timer 9
	zloop_init 144			;; 144 iterations
	zloop_set_rw_prefetch ZQUINT	;; Prefetch 720 cache lines
	mov	r8, DEST2ARG
	mov	rbp, DIST_TO_MULSRCARG
	zr64_sixtyfour_complex_with_mult_preload
b3c:	zr64_sixtyfour_complex_with_mult rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 4, L1PREFETCH_ALL, 8*128
	zloop	4, b3c, -4*8*128+dist32	;; Test loop counter, next source pointer
	zloop	36, b3c			;; Test loop counter
	add	rsi, r8
	end_timer 9
	jmp	zpass2_9216_complex_unfft

zpass2_9216_complex_4:
	start_timer 9
	zloop_init 144			;; 144 iterations
	zloop_set_rw_prefetch ZQUINT	;; Prefetch 720 cache lines
	mov	rbp, DIST_TO_MULSRCARG
	mov	rdi, xsincos_complex	;; The sin/cos data
	bump	rdi, 96*ZMM_SCD6+8*ZMM_SCD6
	zr64f_sixtyfour_complex_with_mulf_preload
b3d:	zr64f_sixtyfour_complex_with_mulf rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 4, L1PREFETCH_ALL, 8*128
	zloop	4, b3d, -4*8*128+dist32	;; Test loop counter, next source pointer
	zloop	36, b3d			;; Test loop counter
	end_timer 9
zpass2_9216_complex_unfft:
	bump	rsi, -36*dist32		;; Restore source pointer

;; Do inverse FFT levels 4,5,6
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	64	+1	...
;;	8	...
;;	...
;;	56
;;	128
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	768	+1	...
;;	8	...
;;	...
;;	760
;;	1536
;;	...

;; Do 768 twelve_complex_unfft operations
;;	distance between fft data elements is 64
;;	do 96 macros each processing 192 data values

	start_timer 13
	bump	rdi, -8*ZMM_SCD6
	zloop_init 96			;; 12 iterations of 8
	zloop_set_rw_prefetch ZTRIPLE	;; Prefetch 288 cache lines
	zr12_twelve_complex_djbunfft_preload
c2b:	zr12_twelve_complex_djbunfft rsi, 128, 8*128, 16*128, dist32, rdi, ZMM_SCD6, 8, L1PREFETCH_ALL, 128
	zloop	8, c2b, -8*128+3*dist32, rdi, -8*ZMM_SCD6 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	12, c2b, -12*3*dist32	;; Test loop counter, restore source pointer
	end_timer 13

;; Do inverse FFT levels 1,2,3
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	768	+1	...
;;	8	...
;;	...
;;	760
;;	1536
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	9216	+1	...
;;	8	...
;;	...
;;	9208

;; Do 768 twelve_complex_unfft operations
;;	distance between fft data elements is 768
;;	do 96 macros each processing 192 data values

	start_timer 14
	bump	rdi, -96*ZMM_SCD6
	zloop_init 96			;; 96 iterations
	zloop_set_rw_prefetch ZTRIPLE	;; Prefetch 288 cache lines
	zr12_twelve_complex_djbunfft_preload
c1b:	zr12_twelve_complex_djbunfft rsi, 128, 3*dist32, 6*dist32, 12*dist32, rdi, ZMM_SCD6, 32, L1PREFETCH_ALL, 128
	zloop	32, c1b, -32*128+dist32	;; Test loop counter
	zloop	3, c1b			;; Test loop counter
	end_timer 14

zpass2_9216_complex_done:
	ENDM


; *************** radix8-pass2-10240-levels-real macro ******************
; This macro takes 20480 real values and performs the final 14.322 levels of the FFT.
; The results are squared, and the first 14.322 levels of the inverse FFT are done.

zr4dwpn_pass2_10240_real MACRO
	LOCAL	b1b, b2a, b2b, b3a, b3b, b3c, b3d, c1b, c2a, c2b
	LOCAL	zpass2_10240_levels_real_1, zpass2_10240_levels_real_2
	LOCAL	zpass2_10240_levels_real_3, zpass2_10240_levels_real_4
	LOCAL	zpass2_10240_levels_real_unfft, zpass2_10240_levels_real_done

;; Type 4 FFTs skip the forward FFT process

	start_timer 2
	mov	rsi, DATA_ADDR		;; Load source address
	mov	rbx, DIST_TO_FFTSRCARG
	mov	rcx, DATA_PREFETCH	;; Prefetch pointer (we need to prefetch 2560 cache lines + 39 pad cache lines)
	cmp	ffttype, 4
	je	zpass2_10240_levels_real_4

;; Do FFT levels 1,2,3,4
;;	0	+1	+1	+1	+1	+1	+1	+1	10K	+1	...
;;	8	...
;;	...
;;	10K-8
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	1K	+1	...
;;	8	...
;;	...
;;	1K-8
;;	2K
;;	...

;; Do 1024 twenty_real_fft operations
;;	distance between fft data elements is 1024
;;	do 128 macros each processing 160 data values

	mov	rdx, sincos3		;; Real sin/cos data
	mov	rdi, xsincos_complex	;; Complex sin/cos data
	zloop_init 128			;; 128 iterations
	zloop_set_rw_prefetch ZTRIPLE	;; Prefetch 384 cache lines
	zr10f_2sc_twenty_reals_fft_preload
b1b:	zr10f_2sc_twenty_reals_fft rsi, 128, 4*dist32, rdx, ZMM_SCD5, rdi, ZMM_SCD5, 32
	zloop	32, b1b, -32*128+dist32	;; Test loop counter, next source pointer
	zloop	4, b1b, -4*dist32	;; Test loop counter, restore source pointer

;; Do FFT levels 5,6,7
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	1K	+1	...
;;	8	...
;;	...
;;	1K-8
;;	2K
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	64	+1	...
;;	8	...
;;	...
;;	56
;;	128
;;	...

;; Do 64 thirtytwo_reals_fft operations
;; Do 576 sixteen_complex_fft operations
;;	distance between fft data elements is 64
;;	do 80 macros each processing 256 data values

	zloop_init 80			;; 10 iterations of 8
	zloop_set_rw_prefetch ZTRIPLE	;; Prefetch 240 cache lines
	zr16_2sc_thirtytwo_reals_fft_preload
b2a:	zr16_2sc_thirtytwo_reals_fft rsi, 128, 8*128, 16*128, dist32, 2*dist32, rdx, ZMM_SCD8, rdi, ZMM_SCD8, 8
	zloop	8, b2a, -8*128+4*dist32, rdi, -8*ZMM_SCD8 ;; Test loop counter, next source pointer, restore sin/cos pointer
	loops_reset			;; 6 more iters of 8
	zr16_sixteen_complex_djbfft_preload
b2b:	zr16_sixteen_complex_djbfft rsi, 128, 8*128, 16*128, dist32, 2*dist32, rdi, ZMM_SCD8, 8
	zloop	8, b2b, -8*128+4*dist32, rdi, -8*ZMM_SCD8 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	10, b2b, -10*4*dist32, rdi, 8*ZMM_SCD8 ;; Test loop counter, restore source pointer, next sin/cos pointer

;; Do FFT levels 8,9,10,11,12,13 as well as inverse FFT levels 8,9,10,11,12,13
;; On input and output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	64	+1	...
;;	8	...
;;	...
;;	56
;;	128
;;	...

;; Do 1 hundredtwentyeight_reals_fft operation
;; Do 159 sixtyfour_complex_fft operations
;;	distance between fft data elements is 1 (8 for swizzling)
;;	do 160 macros each processing 128 data values

;; Execute the proper middle step

pfing	bump	rcx, -1*64		;; We will prefetch 1 cache lines too many
	cmp	ffttype, 2
	jl	zpass2_10240_levels_real_1
	jg	zpass2_10240_levels_real_3

zpass2_10240_levels_real_2:
	mov	r8, DEST2ARG
;;	zr64_hundredtwentyeight_real_with_square_preload
;;	zr64_hundredtwentyeight_real_with_square rsi, 8*128, 1*128, 2*128, 4*128, rdx, 0*ZMM_SCD4, rdi, 0*ZMM_SCD4, 1
	CALLP	zreal_square
	zloop_init 160, 1, 4		;; 159 more iterations
	zloop_set_rw_prefetch ZOCT	;; Prefetch 1272 cache lines
	zr64_sixtyfour_complex_with_square_preload
b3a:	zr64_sixtyfour_complex_with_square rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 1, L1PREFETCH_ALL, 8*128
	zloop	4, b3a, -4*8*128+dist32	;; Test loop counter, next source pointer
	zloop	40, b3a			;; Test loop counter
	add	rsi, r8
	jmp	zpass2_10240_levels_real_unfft

zpass2_10240_levels_real_1:
;;	zr64_hundredtwentyeight_real_fft_final_preload
;;	zr64_hundredtwentyeight_real_fft_final rsi, 8*128, 1*128, 2*128, 4*128, rdx, 0*ZMM_SCD4, rdi, 0*ZMM_SCD4, 1
	CALLP	zreal_fft_final
	zloop_init 160, 1, 4		;; 159 more iterations
	zloop_set_rw_prefetch ZOCT	;; Prefetch 1272 cache lines
	zr64_sixtyfour_complex_fft_final_preload
b3b:	zr64_sixtyfour_complex_fft_final rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 1, L1PREFETCH_ALL, 8*128
	zloop	4, b3b, -4*8*128+dist32	;; Test loop counter, next source pointer
	zloop	40, b3b			;; Test loop counter
	jmp	zpass2_10240_levels_real_done

zpass2_10240_levels_real_3:
	mov	r8, DEST2ARG
	mov	rbp, DIST_TO_MULSRCARG
;;	zr64_hundredtwentyeight_real_with_mult_preload
;;	zr64_hundredtwentyeight_real_with_mult rsi, 8*128, 1*128, 2*128, 4*128, rdx, 0*ZMM_SCD4, rdi, 0*ZMM_SCD4, 1
	CALLP	zreal_mult
	zloop_init 160, 1, 4		;; 159 more iterations
	zloop_set_rw_prefetch ZOCT	;; Prefetch 1272 cache lines
	zr64_sixtyfour_complex_with_mult_preload
b3c:	zr64_sixtyfour_complex_with_mult rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 1, L1PREFETCH_ALL, 8*128
	zloop	4, b3c, -4*8*128+dist32	;; Test loop counter, next source pointer
	zloop	40, b3c			;; Test loop counter
	add	rsi, r8
	jmp	zpass2_10240_levels_real_unfft

zpass2_10240_levels_real_4:
	mov	rbp, DIST_TO_MULSRCARG
	mov	rdx, sincos3		;; Real sin/cos data
	mov	rdi, xsincos_complex	;; Complex sin/cos data
	bump	rdx, 128*ZMM_SCD5+8*ZMM_SCD8
	bump	rdi, 128*ZMM_SCD5+8*ZMM_SCD8
;;	zr64f_hundredtwentyeight_real_with_mulf_preload
;;	zr64f_hundredtwentyeight_real_with_mulf rsi, 8*128, 1*128, 2*128, 4*128, rdx, 0*ZMM_SCD4, rdi, 0*ZMM_SCD4, 1
	CALLP	zreal_mulf
	zloop_init 160, 1, 4		;; 159 more iterations
	zloop_set_rw_prefetch ZOCT	;; Prefetch 1272 cache lines
	zr64f_sixtyfour_complex_with_mulf_preload
b3d:	zr64f_sixtyfour_complex_with_mulf rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 1, L1PREFETCH_ALL, 8*128
	zloop	4, b3d, -4*8*128+dist32	;; Test loop counter, next source pointer
	zloop	40, b3d			;; Test loop counter
zpass2_10240_levels_real_unfft:
	bump	rsi, -40*dist32		;; Restore source pointer

;; Do inverse FFT levels 5,6,7
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	64	+1	...
;;	8	...
;;	...
;;	56
;;	128
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	1K	+1	...
;;	8	...
;;	...
;;	1K-8
;;	2K
;;	...

;; Do 64 thirtytwo_reals_unfft operations
;; Do 576 sixteen_complex_unfft operations
;;	distance between fft data elements is 64
;;	do 80 macros each processing 256 data values

	zloop_init 80			;; 10 iterations of 8
	zloop_set_rw_prefetch ZQUAD	;; Prefetch 320 cache lines
	bump	rdx, -8*ZMM_SCD8
	bump	rdi, -8*ZMM_SCD8
	zr16_2sc_thirtytwo_reals_unfft_preload
c2a:	zr16_2sc_thirtytwo_reals_unfft rsi, 128, 8*128, 16*128, dist32, 2*dist32, rdx, ZMM_SCD8, rdi, ZMM_SCD8, 8
	zloop	8, c2a, -8*128+4*dist32, rdi, -8*ZMM_SCD8 ;; Test loop counter, next source pointer, restore sin/cos pointer
	loops_reset			;; 6 more iters of 8
	zr16_sixteen_complex_djbunfft_preload
c2b:	zr16_sixteen_complex_djbunfft rsi, 128, 8*128, 16*128, dist32, 2*dist32, rdi, ZMM_SCD8, 8
	zloop	8, c2b, -8*128+4*dist32, rdi, -8*ZMM_SCD8 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	10, c2b, -10*4*dist32	;; Test loop counter, restore source pointer

;; Do inverse FFT levels 1,2,3,4
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	1K	+1	...
;;	8	...
;;	...
;;	1K-8
;;	2K
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	10K	+1	...
;;	8	...
;;	...
;;	10K-8

;; Do 1024 twenty_reals_last_unfft operations
;;	distance between fft data elements is 1024
;;	do 128 macros each processing 160 data values

	bump	rdx, -8*ZMM_SCD8-128*ZMM_SCD5
	bump	rdi, -128*ZMM_SCD5
	zloop_init 128			;; 128 iterations
	zloop_set_rw_prefetch ZTRIPLE	;; Prefetch 384 cache lines
	zr10_2sc_twenty_reals_unfft_preload
c1b:	zr10_2sc_twenty_reals_unfft rsi, 128, 4*dist32, rdx, ZMM_SCD5, rdi, ZMM_SCD5, 32
	zloop	32, c1b, -32*128+dist32	;; Test loop counter, next source pointer
	zloop	4, c1b			;; Test loop counter

zpass2_10240_levels_real_done:
	end_timer 2
	ENDM

; *************** radix8-pass2-10240-levels-complex macro ******************
; This macro takes 10240 complex values and performs the final 13.322 levels of a standard
; radix-8 FFT, squares the results, and does the first 13.322 levels of the inverse FFT.

zr4dwpn_pass2_10240_complex MACRO
	LOCAL	b1b, b2b, b3a, b3b, b3c, b3d, c1b, c2b
	LOCAL	zpass2_10240_levels_complex_1, zpass2_10240_levels_complex_2
	LOCAL	zpass2_10240_levels_complex_3, zpass2_10240_levels_complex_4
	LOCAL	zpass2_10240_levels_complex_unfft, zpass2_10240_levels_complex_done

;; Type 4 FFTs skip the forward FFT process

	mov	rsi, DATA_ADDR		;; Load address of FFT data
	mov	rbx, DIST_TO_FFTSRCARG
	mov	rcx, DATA_PREFETCH	;; Prefetch pointer (we need to prefetch 2560 cache lines + 39 pad cache lines)
	cmp	ffttype, 4
	je	zpass2_10240_levels_complex_4

;; Do FFT levels 1,2,3
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	10K	+1	...
;;	8	...
;;	...
;;	10K-8
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	1K	+1	...
;;	8	...
;;	...
;;	1K-8
;;	2K
;;	...

;; Do 1024 ten_complex_fft operations
;;	distance between fft data elements is 1024
;;	do 128 macros each processing 160 data values

	start_timer 4
	mov	rdi, xsincos_complex	;; The sin/cos data
	zloop_init 128			;; 128 iterations
	zloop_set_rw_prefetch ZTRIPLE	;; Prefetch 384 cache lines
	zr10f_ten_complex_djbfft_preload
b1b:	zr10f_ten_complex_djbfft rsi, 128, 4*dist32, rdi, ZMM_SCD5, 32, L1PREFETCH_ALL, 128
	zloop	32, b1b, -32*128+dist32	;; Test loop counter, next source pointer
	zloop	4, b1b, -4*dist32	;; Test loop counter, restore source pointer
	end_timer 4

;; Do FFT levels 4,5,6
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	1K	+1	...
;;	8	...
;;	...
;;	1K-8
;;	2K
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	64	+1	...
;;	8	...
;;	...
;;	56
;;	128
;;	...

;; Do 640 sixteen_complex_fft operations
;;	distance between fft data elements is 64
;;	do 80 macros each processing 256 data values

	start_timer 5
	zloop_init 80			;; 10 iterations of 8
	zloop_set_rw_prefetch ZTRIPLE	;; Prefetch 240 cache lines
	zr16_sixteen_complex_djbfft_preload
b2b:	zr16_sixteen_complex_djbfft rsi, 128, 8*128, 16*128, dist32, 2*dist32, rdi, ZMM_SCD8, 8, L1PREFETCH_ALL, 128
	zloop	8, b2b, -8*128+4*dist32, rdi, -8*ZMM_SCD8 ;; Test loop counter, restore source pointer, restore sin/cos pointer
	zloop	10, b2b, -10*4*dist32, rdi, 8*ZMM_SCD8 ;; Test loop counter, restore source pointer, next sin/cos pointer
	end_timer 5

;; Do FFT levels 7,8,9,10,11,12 as well as inverse FFT levels 7,8,9,10,11,12
;; On input and output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	64	+1	...
;;	8	...
;;	...
;;	56
;;	128
;;	...

;; Do 160 sixtyfour_complex_fft operations
;;	distance between fft data elements is 1 (8 for swizzling)
;;	do 160 macros each processing 128 data values

;; Execute the right middle step

pfing	bump	rcx, -9*64		;; We will prefetch 9 cache lines too many
	cmp	ffttype, 2
	jl	zpass2_10240_levels_complex_1
	jg	zpass2_10240_levels_complex_3

zpass2_10240_levels_complex_2:
	start_timer 9
	zloop_init 160			;; 160 iterations
	zloop_set_rw_prefetch ZOCT	;; Prefetch 1280 cache lines
	mov	r8, DEST2ARG
	zr64_sixtyfour_complex_with_square_preload
b3a:	zr64_sixtyfour_complex_with_square rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 4, L1PREFETCH_ALL, 8*128
	zloop	4, b3a, -4*8*128+dist32	;; Test loop counter, next source pointer
	zloop	40, b3a			;; Test loop counter
	add	rsi, r8
	end_timer 9
	jmp	zpass2_10240_levels_complex_unfft

zpass2_10240_levels_complex_1:
	start_timer 9
	zloop_init 160			;; 160 iterations
	zloop_set_rw_prefetch ZOCT	;; Prefetch 1280 cache lines
	zr64_sixtyfour_complex_fft_final_preload
b3b:	zr64_sixtyfour_complex_fft_final rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 4, L1PREFETCH_ALL, 8*128
	zloop	4, b3b, -4*8*128+dist32	;; Test loop counter, next source pointer
	zloop	40, b3b			;; Test loop counter
	end_timer 9
	jmp	zpass2_10240_levels_complex_done

zpass2_10240_levels_complex_3:
	start_timer 9
	zloop_init 160			;; 160 iterations
	zloop_set_rw_prefetch ZOCT	;; Prefetch 1280 cache lines
	mov	r8, DEST2ARG
	mov	rbp, DIST_TO_MULSRCARG
	zr64_sixtyfour_complex_with_mult_preload
b3c:	zr64_sixtyfour_complex_with_mult rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 4, L1PREFETCH_ALL, 8*128
	zloop	4, b3c, -4*8*128+dist32	;; Test loop counter, next source pointer
	zloop	40, b3c			;; Test loop counter
	add	rsi, r8
	end_timer 9
	jmp	zpass2_10240_levels_complex_unfft

zpass2_10240_levels_complex_4:
	start_timer 9
	zloop_init 160			;; 160 iterations
	zloop_set_rw_prefetch ZOCT	;; Prefetch 1280 cache lines
	mov	rbp, DIST_TO_MULSRCARG
	mov	rdi, xsincos_complex	;; The sin/cos data
	bump	rdi, 128*ZMM_SCD5+8*ZMM_SCD8
	zr64f_sixtyfour_complex_with_mulf_preload
b3d:	zr64f_sixtyfour_complex_with_mulf rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 4, L1PREFETCH_ALL, 8*128
	zloop	4, b3d, -4*8*128+dist32	;; Test loop counter, next source pointer
	zloop	40, b3d			;; Test loop counter
	end_timer 9
zpass2_10240_levels_complex_unfft:
	bump	rsi, -40*dist32		;; Restore source pointer

;; Do inverse FFT levels 4,5,6
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	64	+1	...
;;	8	...
;;	...
;;	56
;;	128
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	1K	+1	...
;;	8	...
;;	...
;;	1K-8
;;	2K
;;	...

;; Do 640 sixteen_complex_unfft operations
;;	distance between fft data elements is 64
;;	do 80 macros each processing 256 data values

	start_timer 13
	bump	rdi, -8*ZMM_SCD8
	zloop_init 80			;; 10 iterations of 8
	zloop_set_rw_prefetch ZQUAD	;; Prefetch 320 cache lines
	zr16_sixteen_complex_djbunfft_preload
c2b:	zr16_sixteen_complex_djbunfft rsi, 128, 8*128, 16*128, dist32, 2*dist32, rdi, ZMM_SCD8, 8, L1PREFETCH_ALL, 128
	zloop	8, c2b, -8*128+4*dist32, rdi, -8*ZMM_SCD8 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	10, c2b, -10*4*dist32	;; Test loop counter, restore source pointer
	end_timer 13

;; Do inverse FFT levels 1,2,3
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	1K	+1	...
;;	8	...
;;	...
;;	1K-8
;;	2K
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	10K	+1	...
;;	8	...
;;	...
;;	10K-8

;; Do 1024 ten_complex_unfft operations
;;	distance between fft data elements is 1024
;;	do 128 macros each processing 160 data values

	start_timer 14
	bump	rdi, -128*ZMM_SCD5
	zloop_init 128			;; 128 iterations
	zloop_set_rw_prefetch ZTRIPLE	;; Prefetch 384 cache lines
	zr10_ten_complex_djbunfft_preload
c1b:	zr10_ten_complex_djbunfft rsi, 128, 4*dist32, rdi, ZMM_SCD5, 32, L1PREFETCH_ALL, 128
	zloop	32, c1b, -32*128+dist32	;; Test loop counter, next source pointer
	zloop	4, c1b			;; Test loop counter
	end_timer 14

zpass2_10240_levels_complex_done:
	ENDM


; *************** radix8-pass2-12288-levels-real macro ******************
; This macro takes 24576 real values and performs the final 14.585 levels of the FFT.
; The results are squared, and the first 14.585 levels of the inverse FFT are done.

zr4dwpn_pass2_12288_real MACRO
	LOCAL	b1b, b2a, b2b, b3a, b3b, b3c, b3d, c1b, c2a, c2b
	LOCAL	zpass2_12288_levels_real_1, zpass2_12288_levels_real_2
	LOCAL	zpass2_12288_levels_real_3, zpass2_12288_levels_real_4
	LOCAL	zpass2_12288_levels_real_unfft, zpass2_12288_levels_real_done

;; Type 4 FFTs skip the forward FFT process

	start_timer 2
	mov	rsi, DATA_ADDR		;; Load source address
	mov	rbx, DIST_TO_FFTSRCARG
	mov	rcx, DATA_PREFETCH	;; Prefetch pointer (we need to prefetch 3072 cache lines + 47 pad cache lines)
	cmp	ffttype, 4
	je	zpass2_12288_levels_real_4

;; Do FFT levels 1,2,3,4
;;	0	+1	+1	+1	+1	+1	+1	+1	12K	+1	...
;;	8	...
;;	...
;;	12K-8
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	1K	+1	...
;;	8	...
;;	...
;;	1K-8
;;	2K
;;	...

;; Do 1024 twentyfour_real_fft operations
;;	distance between fft data elements is 1024
;;	do 128 macros each processing 192 data values

	mov	rdx, sincos3		;; Real sin/cos data
	mov	rdi, xsincos_complex	;; Complex sin/cos data
	zloop_init 128			;; 128 iterations
	zloop_set_rw_prefetch ZTRIPLE	;; Prefetch 384 cache lines
	zr12f_2sc_twentyfour_reals_fft_preload
b1b:	zr12f_2sc_twentyfour_reals_fft rsi, 128, 4*dist32, 8*dist32, 16*dist32, rdx, ZMM_SCD6, rdi, ZMM_SCD6, 32
	zloop	32, b1b, -32*128+dist32	;; Test loop counter, next source pointer
	zloop	4, b1b, -4*dist32	;; Test loop counter, restore source pointer

;; Do FFT levels 5,6,7
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	1K	+1	...
;;	8	...
;;	...
;;	1K-8
;;	2K
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	64	+1	...
;;	8	...
;;	...
;;	56
;;	128
;;	...

;; Do 64 thirtytwo_reals_fft operations
;; Do 704 sixteen_complex_fft operations
;;	distance between fft data elements is 64
;;	do 96 macros each processing 256 data values

	zloop_init 96			;; 12 iterations of 8
	zloop_set_rw_prefetch ZQUINT	;; Prefetch 480 cache lines
	zr16_2sc_thirtytwo_reals_fft_preload
b2a:	zr16_2sc_thirtytwo_reals_fft rsi, 128, 8*128, 16*128, dist32, 2*dist32, rdx, ZMM_SCD8, rdi, ZMM_SCD8, 8
	zloop	8, b2a, -8*128+4*dist32, rdi, -8*ZMM_SCD8 ;; Test loop counter, next source pointer, restore sin/cos pointer
	loops_reset			;; 11 more iters of 8
	zr16_sixteen_complex_djbfft_preload
b2b:	zr16_sixteen_complex_djbfft rsi, 128, 8*128, 16*128, dist32, 2*dist32, rdi, ZMM_SCD8, 8
	zloop	8, b2b, -8*128+4*dist32, rdi, -8*ZMM_SCD8 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	12, b2b, -12*4*dist32, rdi, 8*ZMM_SCD8 ;; Test loop counter, restore source pointer, next sin/cos pointer

;; Do FFT levels 8,9,10,11,12,13 as well as inverse FFT levels 8,9,10,11,12,13
;; On input and output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	64	+1	...
;;	8	...
;;	...
;;	56
;;	128
;;	...

;; Do 1 hundredtwentyeight_reals_fft operation
;; Do 191 sixtyfour_complex_fft operations
;;	distance between fft data elements is 1 (8 for swizzling)
;;	do 192 macros each processing 128 data values

;; Execute the proper middle step

pfing	bump	rcx, -41*64		;; We will prefetch 41 cache lines too many
	cmp	ffttype, 2
	jl	zpass2_12288_levels_real_1
	jg	zpass2_12288_levels_real_3

zpass2_12288_levels_real_2:
	mov	r8, DEST2ARG
;;	zr64_hundredtwentyeight_real_with_square_preload
;;	zr64_hundredtwentyeight_real_with_square rsi, 8*128, 1*128, 2*128, 4*128, rdx, 0*ZMM_SCD4, rdi, 0*ZMM_SCD4, 1
	CALLP	zreal_square
	zloop_init 192, 1, 4		;; 191 more iterations
	zloop_set_rw_prefetch ZOCT	;; Prefetch 1528 cache lines
	zr64_sixtyfour_complex_with_square_preload
b3a:	zr64_sixtyfour_complex_with_square rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 1, L1PREFETCH_ALL, 8*128
	zloop	4, b3a, -4*8*128+dist32	;; Test loop counter, next source pointer
	zloop	48, b3a			;; Test loop counter
	add	rsi, r8
	jmp	zpass2_12288_levels_real_unfft

zpass2_12288_levels_real_1:
;;	zr64_hundredtwentyeight_real_fft_final_preload
;;	zr64_hundredtwentyeight_real_fft_final rsi, 8*128, 1*128, 2*128, 4*128, rdx, 0*ZMM_SCD4, rdi, 0*ZMM_SCD4, 1
	CALLP	zreal_fft_final
	zloop_init 192, 1, 4		;; 191 more iterations
	zloop_set_rw_prefetch ZOCT	;; Prefetch 1528 cache lines
	zr64_sixtyfour_complex_fft_final_preload
b3b:	zr64_sixtyfour_complex_fft_final rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 1, L1PREFETCH_ALL, 8*128
	zloop	4, b3b, -4*8*128+dist32	;; Test loop counter, next source pointer
	zloop	48, b3b			;; Test loop counter
	jmp	zpass2_12288_levels_real_done

zpass2_12288_levels_real_3:
	mov	r8, DEST2ARG
	mov	rbp, DIST_TO_MULSRCARG
;;	zr64_hundredtwentyeight_real_with_mult_preload
;;	zr64_hundredtwentyeight_real_with_mult rsi, 8*128, 1*128, 2*128, 4*128, rdx, 0*ZMM_SCD4, rdi, 0*ZMM_SCD4, 1
	CALLP	zreal_mult
	zloop_init 192, 1, 4		;; 191 more iterations
	zloop_set_rw_prefetch ZOCT	;; Prefetch 1528 cache lines
	zr64_sixtyfour_complex_with_mult_preload
b3c:	zr64_sixtyfour_complex_with_mult rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 1, L1PREFETCH_ALL, 8*128
	zloop	4, b3c, -4*8*128+dist32	;; Test loop counter, next source pointer
	zloop	48, b3c			;; Test loop counter
	add	rsi, r8
	jmp	zpass2_12288_levels_real_unfft

zpass2_12288_levels_real_4:
	mov	rbp, DIST_TO_MULSRCARG
	mov	rdx, sincos3		;; Real sin/cos data
	mov	rdi, xsincos_complex	;; Complex sin/cos data
	bump	rdx, 128*ZMM_SCD6+8*ZMM_SCD8
	bump	rdi, 128*ZMM_SCD6+8*ZMM_SCD8
;;	zr64f_hundredtwentyeight_real_with_mulf_preload
;;	zr64f_hundredtwentyeight_real_with_mulf rsi, 8*128, 1*128, 2*128, 4*128, rdx, 0*ZMM_SCD4, rdi, 0*ZMM_SCD4, 1
	CALLP	zreal_mulf
	zloop_init 192, 1, 4		;; 191 more iterations
	zloop_set_rw_prefetch ZOCT	;; Prefetch 1528 cache lines
	zr64f_sixtyfour_complex_with_mulf_preload
b3d:	zr64f_sixtyfour_complex_with_mulf rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 1, L1PREFETCH_ALL, 8*128
	zloop	4, b3d, -4*8*128+dist32	;; Test loop counter, next source pointer
	zloop	48, b3d			;; Test loop counter
zpass2_12288_levels_real_unfft:
	bump	rsi, -48*dist32		;; Restore source pointer

;; Do inverse FFT levels 5,6,7
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	64	+1	...
;;	8	...
;;	...
;;	56
;;	128
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	1K	+1	...
;;	8	...
;;	...
;;	1K-8
;;	2K
;;	...

;; Do 64 thirtytwo_reals_unfft operations
;; Do 704 sixteen_complex_unfft operations
;;	distance between fft data elements is 64
;;	do 96 macros each processing 256 data values

	zloop_init 96			;; 12 iterations of 8
	zloop_set_rw_prefetch ZQUAD	;; Prefetch 384 cache lines
	bump	rdx, -8*ZMM_SCD8
	bump	rdi, -8*ZMM_SCD8
	zr16_2sc_thirtytwo_reals_unfft_preload
c2a:	zr16_2sc_thirtytwo_reals_unfft rsi, 128, 8*128, 16*128, dist32, 2*dist32, rdx, ZMM_SCD8, rdi, ZMM_SCD8, 8
	zloop	8, c2a, -8*128+4*dist32, rdi, -8*ZMM_SCD8 ;; Test loop counter, next source pointer, restore sin/cos pointer
	loops_reset			;; 11 more iters of 8
	zr16_sixteen_complex_djbunfft_preload
c2b:	zr16_sixteen_complex_djbunfft rsi, 128, 8*128, 16*128, dist32, 2*dist32, rdi, ZMM_SCD8, 8
	zloop	8, c2b, -8*128+4*dist32, rdi, -8*ZMM_SCD8 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	12, c2b, -12*4*dist32	;; Test loop counter, restore source pointer

;; Do inverse FFT levels 1,2,3,4
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	1K	+1	...
;;	8	...
;;	...
;;	1K-8
;;	2K
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	12K	+1	...
;;	8	...
;;	...
;;	12K-8

;; Do 1024 twentyfour_reals_last_unfft operations
;;	distance between fft data elements is 1024
;;	do 128 macros each processing 192 data values

	bump	rdx, -8*ZMM_SCD8-128*ZMM_SCD6
	bump	rdi, -128*ZMM_SCD6
	zloop_init 128			;; 128 iterations
	zloop_set_rw_prefetch ZTRIPLE	;; Prefetch 384 cache lines
	zr12_2sc_twentyfour_reals_unfft_preload
c1b:	zr12_2sc_twentyfour_reals_unfft rsi, 128, 4*dist32, 8*dist32, 16*dist32, rdx, ZMM_SCD6, rdi, ZMM_SCD6, 32
	zloop	32, c1b, -32*128+dist32	;; Test loop counter, next source pointer
	zloop	4, c1b			;; Test loop counter

zpass2_12288_levels_real_done:
	end_timer 2
	ENDM

; *************** radix8-pass2-12288-levels-complex macro ******************
; This macro takes 12288 complex values and performs the final 13.585 levels of a standard
; radix-8 FFT, squares the results, and does the first 13.585 levels of the inverse FFT.

zr4dwpn_pass2_12288_complex MACRO
	LOCAL	b1b, b2b, b3a, b3b, b3c, b3d, c1b, c2b
	LOCAL	zpass2_12288_levels_complex_1, zpass2_12288_levels_complex_2
	LOCAL	zpass2_12288_levels_complex_3, zpass2_12288_levels_complex_4
	LOCAL	zpass2_12288_levels_complex_unfft, zpass2_12288_levels_complex_done

;; Type 4 FFTs skip the forward FFT process

	mov	rsi, DATA_ADDR		;; Load address of FFT data
	mov	rbx, DIST_TO_FFTSRCARG
	mov	rcx, DATA_PREFETCH	;; Prefetch pointer (we need to prefetch 3072 cache lines + 47 pad cache lines)
	cmp	ffttype, 4
	je	zpass2_12288_levels_complex_4

;; Do FFT levels 1,2,3
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	12K	+1	...
;;	8	...
;;	...
;;	12K-8
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	1K	+1	...
;;	8	...
;;	...
;;	1K-8
;;	2K
;;	...

;; Do 1024 twelve_complex_fft operations
;;	distance between fft data elements is 1024
;;	do 128 macros each processing 192 data values

	start_timer 4
	mov	rdi, xsincos_complex	;; The sin/cos data
	zloop_init 128			;; 128 iterations
	zloop_set_rw_prefetch ZTRIPLE	;; Prefetch 384 cache lines
	zr12f_twelve_complex_djbfft_preload
b1b:	zr12f_twelve_complex_djbfft rsi, 128, 4*dist32, 8*dist32, 16*dist32, rdi, ZMM_SCD6, 32, L1PREFETCH_ALL, 128
	zloop	32, b1b, -32*128+dist32	;; Test loop counter, next source pointer
	zloop	4, b1b, -4*dist32	;; Test loop counter, restore source pointer
	end_timer 4

;; Do FFT levels 4,5,6
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	1K	+1	...
;;	8	...
;;	...
;;	1K-8
;;	2K
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	64	+1	...
;;	8	...
;;	...
;;	56
;;	128
;;	...

;; Do 768 sixteen_complex_fft operations
;;	distance between fft data elements is 64
;;	do 96 macros each processing 256 data values

	start_timer 5
	zloop_init 96			;; 12 iterations of 8
	zloop_set_rw_prefetch ZQUINT	;; Prefetch 480 cache lines
	zr16_sixteen_complex_djbfft_preload
b2b:	zr16_sixteen_complex_djbfft rsi, 128, 8*128, 16*128, dist32, 2*dist32, rdi, ZMM_SCD8, 8, L1PREFETCH_ALL, 128
	zloop	8, b2b, -8*128+4*dist32, rdi, -8*ZMM_SCD8 ;; Test loop counter, restore source pointer, restore sin/cos pointer
	zloop	12, b2b, -12*4*dist32, rdi, 8*ZMM_SCD8 ;; Test loop counter, restore source pointer, next sin/cos pointer
	end_timer 5

;; Do FFT levels 7,8,9,10,11,12 as well as inverse FFT levels 7,8,9,10,11,12
;; On input and output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	64	+1	...
;;	8	...
;;	...
;;	56
;;	128
;;	...

;; Do 192 sixtyfour_complex_fft operations
;;	distance between fft data elements is 1 (8 for swizzling)
;;	do 192 macros each processing 128 data values

;; Execute the right middle step

pfing	bump	rcx, -49*64		;; We will prefetch 49 cache lines too many
	cmp	ffttype, 2
	jl	zpass2_12288_levels_complex_1
	jg	zpass2_12288_levels_complex_3

zpass2_12288_levels_complex_2:
	start_timer 9
	zloop_init 192			;; 192 iterations
	zloop_set_rw_prefetch ZOCT	;; Prefetch 1536 cache lines
	mov	r8, DEST2ARG
	zr64_sixtyfour_complex_with_square_preload
b3a:	zr64_sixtyfour_complex_with_square rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 4, L1PREFETCH_ALL, 8*128
	zloop	4, b3a, -4*8*128+dist32	;; Test loop counter, next source pointer
	zloop	48, b3a			;; Test loop counter
	add	rsi, r8
	end_timer 9
	jmp	zpass2_12288_levels_complex_unfft

zpass2_12288_levels_complex_1:
	start_timer 9
	zloop_init 192			;; 192 iterations
	zloop_set_rw_prefetch ZOCT	;; Prefetch 1536 cache lines
	zr64_sixtyfour_complex_fft_final_preload
b3b:	zr64_sixtyfour_complex_fft_final rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 4, L1PREFETCH_ALL, 8*128
	zloop	4, b3b, -4*8*128+dist32	;; Test loop counter, next source pointer
	zloop	48, b3b			;; Test loop counter
	end_timer 9
	jmp	zpass2_12288_levels_complex_done

zpass2_12288_levels_complex_3:
	start_timer 9
	zloop_init 192			;; 192 iterations
	zloop_set_rw_prefetch ZOCT	;; Prefetch 1536 cache lines
	mov	r8, DEST2ARG
	mov	rbp, DIST_TO_MULSRCARG
	zr64_sixtyfour_complex_with_mult_preload
b3c:	zr64_sixtyfour_complex_with_mult rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 4, L1PREFETCH_ALL, 8*128
	zloop	4, b3c, -4*8*128+dist32	;; Test loop counter, next source pointer
	zloop	48, b3c			;; Test loop counter
	add	rsi, r8
	end_timer 9
	jmp	zpass2_12288_levels_complex_unfft

zpass2_12288_levels_complex_4:
	start_timer 9
	zloop_init 192			;; 192 iterations
	zloop_set_rw_prefetch ZOCT	;; Prefetch 1536 cache lines
	mov	rbp, DIST_TO_MULSRCARG
	mov	rdi, xsincos_complex	;; The sin/cos data
	bump	rdi, 128*ZMM_SCD6+8*ZMM_SCD8
	zr64f_sixtyfour_complex_with_mulf_preload
b3d:	zr64f_sixtyfour_complex_with_mulf rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 4, L1PREFETCH_ALL, 8*128
	zloop	4, b3d, -4*8*128+dist32	;; Test loop counter, next source pointer
	zloop	48, b3d			;; Test loop counter
	end_timer 9
zpass2_12288_levels_complex_unfft:
	bump	rsi, -48*dist32		;; Restore source pointer

;; Do inverse FFT levels 4,5,6
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	64	+1	...
;;	8	...
;;	...
;;	56
;;	128
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	1K	+1	...
;;	8	...
;;	...
;;	1K-8
;;	2K
;;	...

;; Do 768 sixteen_complex_unfft operations
;;	distance between fft data elements is 64
;;	do 96 macros each processing 256 data values

	start_timer 13
	bump	rdi, -8*ZMM_SCD8
	zloop_init 96			;; 12 iterations of 8
	zloop_set_rw_prefetch ZQUAD	;; Prefetch 384 cache lines
	zr16_sixteen_complex_djbunfft_preload
c2b:	zr16_sixteen_complex_djbunfft rsi, 128, 8*128, 16*128, dist32, 2*dist32, rdi, ZMM_SCD8, 8, L1PREFETCH_ALL, 128
	zloop	8, c2b, -8*128+4*dist32, rdi, -8*ZMM_SCD8 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	12, c2b, -12*4*dist32	;; Test loop counter, restore source pointer
	end_timer 13

;; Do inverse FFT levels 1,2,3
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	1K	+1	...
;;	8	...
;;	...
;;	1K-8
;;	2K
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	12K	+1	...
;;	8	...
;;	...
;;	12K-8

;; Do 1024 twelve_complex_unfft operations
;;	distance between fft data elements is 1024
;;	do 128 macros each processing 192 data values

	start_timer 14
	bump	rdi, -128*ZMM_SCD6
	zloop_init 128			;; 128 iterations
	zloop_set_rw_prefetch ZTRIPLE	;; Prefetch 384 cache lines
	zr12_twelve_complex_djbunfft_preload
c1b:	zr12_twelve_complex_djbunfft rsi, 128, 4*dist32, 8*dist32, 16*dist32, rdi, ZMM_SCD6, 32, L1PREFETCH_ALL, 128
	zloop	32, c1b, -32*128+dist32	;; Test loop counter
	zloop	4, c1b			;; Test loop counter
	end_timer 14

zpass2_12288_levels_complex_done:
	ENDM


; *************** radix8-pass2-16384-levels-real macro ******************
; This macro takes 32768 real values and performs the final 15 levels of the FFT.
; The results are squared, and the first 15 levels of the inverse FFT are done.

zr4dwpn_pass2_16384_real MACRO
	LOCAL	b1b, b2a, b2b, b3a, b3b, b3c, b3d, c1b, c2a, c2b
	LOCAL	zpass2_16384_levels_real_1, zpass2_16384_levels_real_2
	LOCAL	zpass2_16384_levels_real_3, zpass2_16384_levels_real_4
	LOCAL	zpass2_16384_levels_real_unfft, zpass2_16384_levels_real_done

;; Type 4 FFTs skip the forward FFT process

	start_timer 2
	mov	rsi, DATA_ADDR		;; Load source address
	mov	rbx, DIST_TO_FFTSRCARG
	mov	rcx, DATA_PREFETCH	;; Prefetch pointer (we need to prefetch 4096 cache lines + 63 pad cache lines)
	cmp	ffttype, 4
	je	zpass2_16384_levels_real_4

;; Do FFT levels 1,2,3,4
;;	0	+1	+1	+1	+1	+1	+1	+1	16K	+1	...
;;	8	...
;;	...
;;	16K-8
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	1K	+1	...
;;	8	...
;;	...
;;	1K-8
;;	2K
;;	...

;; Do 1024 thirtytwo_real_fft operations
;;	distance between fft data elements is 1024
;;	do 128 macros each processing 256 data values

	start_timer 5
	mov	rdx, sincos3		;; Real sin/cos data
	mov	rdi, xsincos_complex	;; Complex sin/cos data
	zloop_init 128			;; 128 iterations
	zloop_set_rw_prefetch ZQUAD	;; Prefetch 512 cache lines
	zr16f_2sc_thirtytwo_reals_fft_preload
b1b:	zr16f_2sc_thirtytwo_reals_fft rsi, 128, 4*dist32, 8*dist32, 16*dist32, 32*dist32, rdx, ZMM_SCD8, rdi, ZMM_SCD8, 32, L1PREFETCH_ALL, 128
	zloop	32, b1b, -32*128+dist32	;; Test loop counter, next source pointer
	zloop	4, b1b, -4*dist32	;; Test loop counter, restore source pointer
	end_timer 5

;; Do FFT levels 5,6,7
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	1K	+1	...
;;	8	...
;;	...
;;	1K-8
;;	2K
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	64	+1	...
;;	8	...
;;	...
;;	56
;;	128
;;	...

;; Do 64 thirtytwo_reals_fft operations
;; Do 960 sixteen_complex_fft operations
;;	distance between fft data elements is 64
;;	do 128 macros each processing 256 data values

	start_timer 6
	zloop_init 128			;; 16 iterations of 8
	zloop_set_rw_prefetch ZQUAD	;; Prefetch 512 cache lines
	zr16_2sc_thirtytwo_reals_fft_preload
b2a:	zr16_2sc_thirtytwo_reals_fft rsi, 128, 8*128, 16*128, dist32, 2*dist32, rdx, ZMM_SCD8, rdi, ZMM_SCD8, 8
	zloop	8, b2a, -8*128+4*dist32, rdi, -8*ZMM_SCD8 ;; Test loop counter, next source pointer, restore sin/cos pointer
	loops_reset			;; 15 more iters of 8
	zr16_sixteen_complex_djbfft_preload
b2b:	zr16_sixteen_complex_djbfft rsi, 128, 8*128, 16*128, dist32, 2*dist32, rdi, ZMM_SCD8, 8, L1PREFETCH_ALL, 128
	zloop	8, b2b, -8*128+4*dist32, rdi, -8*ZMM_SCD8 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	16, b2b, -16*4*dist32, rdi, 8*ZMM_SCD8 ;; Test loop counter, restore source pointer, next sin/cos pointer
	end_timer 6

;; Do FFT levels 8,9,10,11,12,13 as well as inverse FFT levels 8,9,10,11,12,13
;; On input and output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	64	+1	...
;;	8	...
;;	...
;;	56
;;	128
;;	...

;; Do 1 hundredtwentyeight_reals_fft operation
;; Do 255 sixtyfour_complex_fft operations
;;	distance between fft data elements is 1 (8 for swizzling)
;;	do 256 macros each processing 128 data values

;; Execute the proper middle step

	start_timer 7
pfing	bump	rcx, -57*64		;; We will prefetch 57 cache lines too many
	cmp	ffttype, 2
	jl	zpass2_16384_levels_real_1
	jg	zpass2_16384_levels_real_3

zpass2_16384_levels_real_2:
	mov	r8, DEST2ARG
;;	zr64_hundredtwentyeight_real_with_square_preload
;;	zr64_hundredtwentyeight_real_with_square rsi, 8*128, 1*128, 2*128, 4*128, rdx, 0*ZMM_SCD4, rdi, 0*ZMM_SCD4, 1
	CALLP	zreal_square
	zloop_init 256, 1, 4		;; 255 more iterations
	zloop_set_rw_prefetch ZOCT	;; Prefetch 2040 cache lines
	zr64_sixtyfour_complex_with_square_preload
b3a:	zr64_sixtyfour_complex_with_square rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 1;;, L1PREFETCH_ALL, 8*128
	zloop	4, b3a, -4*8*128+dist32	;; Test loop counter, next source pointer
	zloop	64, b3a			;; Test loop counter
	add	rsi, r8
	jmp	zpass2_16384_levels_real_unfft

zpass2_16384_levels_real_1:
;;	zr64_hundredtwentyeight_real_fft_final_preload
;;	zr64_hundredtwentyeight_real_fft_final rsi, 8*128, 1*128, 2*128, 4*128, rdx, 0*ZMM_SCD4, rdi, 0*ZMM_SCD4, 1
	CALLP	zreal_fft_final
	zloop_init 256, 1, 4		;; 255 more iterations
	zloop_set_rw_prefetch ZOCT	;; Prefetch 2040 cache lines
	zr64_sixtyfour_complex_fft_final_preload
b3b:	zr64_sixtyfour_complex_fft_final rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 1, L1PREFETCH_ALL, 8*128
	zloop	4, b3b, -4*8*128+dist32	;; Test loop counter, next source pointer
	zloop	64, b3b			;; Test loop counter
	jmp	zpass2_16384_levels_real_done

zpass2_16384_levels_real_3:
	mov	r8, DEST2ARG
	mov	rbp, DIST_TO_MULSRCARG
;;	zr64_hundredtwentyeight_real_with_mult_preload
;;	zr64_hundredtwentyeight_real_with_mult rsi, 8*128, 1*128, 2*128, 4*128, rdx, 0*ZMM_SCD4, rdi, 0*ZMM_SCD4, 1
	CALLP	zreal_mult
	zloop_init 256, 1, 4		;; 255 more iterations
	zloop_set_rw_prefetch ZOCT	;; Prefetch 2040 cache lines
	zr64_sixtyfour_complex_with_mult_preload
b3c:	zr64_sixtyfour_complex_with_mult rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 1, L1PREFETCH_ALL, 8*128
	zloop	4, b3c, -4*8*128+dist32	;; Test loop counter, next source pointer
	zloop	64, b3c			;; Test loop counter
	add	rsi, r8
	jmp	zpass2_16384_levels_real_unfft

zpass2_16384_levels_real_4:
	mov	rbp, DIST_TO_MULSRCARG
	mov	rdx, sincos3		;; Real sin/cos data
	mov	rdi, xsincos_complex	;; Complex sin/cos data
	bump	rdx, 128*ZMM_SCD8+8*ZMM_SCD8
	bump	rdi, 128*ZMM_SCD8+8*ZMM_SCD8
;;	zr64f_hundredtwentyeight_real_with_mulf_preload
;;	zr64f_hundredtwentyeight_real_with_mulf rsi, 8*128, 1*128, 2*128, 4*128, rdx, 0*ZMM_SCD4, rdi, 0*ZMM_SCD4, 1
	CALLP	zreal_mulf
	zloop_init 256, 1, 4		;; 255 more iterations
	zloop_set_rw_prefetch ZOCT	;; Prefetch 2040 cache lines
	zr64f_sixtyfour_complex_with_mulf_preload
b3d:	zr64f_sixtyfour_complex_with_mulf rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 1, L1PREFETCH_ALL, 8*128
	zloop	4, b3d, -4*8*128+dist32	;; Test loop counter, next source pointer
	zloop	64, b3d			;; Test loop counter
zpass2_16384_levels_real_unfft:
	bump	rsi, -64*dist32		;; Restore source pointer
	end_timer 7

;; Do inverse FFT levels 5,6,7
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	64	+1	...
;;	8	...
;;	...
;;	56
;;	128
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	1K	+1	...
;;	8	...
;;	...
;;	1K-8
;;	2K
;;	...

;; Do 64 thirtytwo_reals_unfft operations
;; Do 960 sixteen_complex_unfft operations
;;	distance between fft data elements is 64
;;	do 128 macros each processing 256 data values

	start_timer 8
	zloop_init 128			;; 16 iterations of 8
	zloop_set_rw_prefetch ZQUINT	;; Prefetch 640 cache lines
	bump	rdx, -8*ZMM_SCD8
	bump	rdi, -8*ZMM_SCD8
	zr16_2sc_thirtytwo_reals_unfft_preload
c2a:	zr16_2sc_thirtytwo_reals_unfft rsi, 128, 8*128, 16*128, dist32, 2*dist32, rdx, ZMM_SCD8, rdi, ZMM_SCD8, 8
	zloop	8, c2a, -8*128+4*dist32, rdi, -8*ZMM_SCD8 ;; Test loop counter, next source pointer, restore sin/cos pointer
	loops_reset			;; 15 more iters of 8
	zr16_sixteen_complex_djbunfft_preload
c2b:	zr16_sixteen_complex_djbunfft rsi, 128, 8*128, 16*128, dist32, 2*dist32, rdi, ZMM_SCD8, 8, L1PREFETCH_ALL, 128
	zloop	8, c2b, -8*128+4*dist32, rdi, -8*ZMM_SCD8 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	16, c2b, -16*4*dist32	;; Test loop counter, restore source pointer
	end_timer 8

;; Do inverse FFT levels 1,2,3,4
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	1K	+1	...
;;	8	...
;;	...
;;	1K-8
;;	2K
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	16K	+1	...
;;	8	...
;;	...
;;	16K-8

;; Do 1024 thirtytwo_reals_last_unfft operations
;;	distance between fft data elements is 1024
;;	do 128 macros each processing 256 data values

	start_timer 9
	bump	rdx, -8*ZMM_SCD8-128*ZMM_SCD8
	bump	rdi, -128*ZMM_SCD8
	zloop_init 128			;; 128 iterations
	zloop_set_rw_prefetch ZQUAD	;; Prefetch 512 cache lines
	zr16_2sc_thirtytwo_reals_unfft_preload
c1b:	zr16_2sc_thirtytwo_reals_unfft rsi, 128, 4*dist32, 8*dist32, 16*dist32, 32*dist32, rdx, ZMM_SCD8, rdi, ZMM_SCD8, 32, L1PREFETCH_ALL, 128
	zloop	32, c1b, -32*128+dist32	;; Test loop counter, next source pointer
	zloop	4, c1b			;; Test loop counter
	end_timer 9

zpass2_16384_levels_real_done:
	end_timer 2
	ENDM

; *************** radix8-pass2-16384-levels-complex macro ******************
; This macro takes 16384 complex values and performs the final 14 levels of a standard
; radix-8 FFT, squares the results, and does the first 14 levels of the inverse FFT.

zr4dwpn_pass2_16384_complex MACRO
	LOCAL	b1b, b2b, b3a, b3b, b3c, b3d, c1b, c2b
	LOCAL	zpass2_16384_levels_complex_1, zpass2_16384_levels_complex_2
	LOCAL	zpass2_16384_levels_complex_3, zpass2_16384_levels_complex_4
	LOCAL	zpass2_16384_levels_complex_unfft, zpass2_16384_levels_complex_done

;; Type 4 FFTs skip the forward FFT process

	mov	rsi, DATA_ADDR		;; Load address of FFT data
	mov	rbx, DIST_TO_FFTSRCARG
	mov	rcx, DATA_PREFETCH	;; Prefetch pointer (we need to prefetch 4096 cache lines + 63 pad cache lines)
	cmp	ffttype, 4
	je	zpass2_16384_levels_complex_4

;; Do FFT levels 1,2,3
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	16K	+1	...
;;	8	...
;;	...
;;	16K-8
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	1K	+1	...
;;	8	...
;;	...
;;	1K-8
;;	2K
;;	...

;; Do 1024 sixteen_complex_fft operations
;;	distance between fft data elements is 1024
;;	do 128 macros each processing 256 data values

	start_timer 4
	mov	rdi, xsincos_complex	;; The sin/cos data
	zloop_init 128			;; 128 iterations
	zloop_set_rw_prefetch ZQUAD	;; Prefetch 512 cache lines
	zr16f_sixteen_complex_djbfft_preload
b1b:	zr16f_sixteen_complex_djbfft rsi, 128, 4*dist32, 8*dist32, 16*dist32, 32*dist32, rdi, ZMM_SCD8, 32, L1PREFETCH_ALL, 128
	zloop	32, b1b, -32*128+dist32	;; Test loop counter, next source pointer
	zloop	4, b1b, -4*dist32	;; Test loop counter, restore source pointer
	end_timer 4

;; Do FFT levels 4,5,6
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	1K	+1	...
;;	8	...
;;	...
;;	1K-8
;;	2K
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	64	+1	...
;;	8	...
;;	...
;;	56
;;	128
;;	...

;; Do 1024 sixteen_complex_fft operations
;;	distance between fft data elements is 64
;;	do 128 macros each processing 256 data values

	start_timer 5
	zloop_init 128			;; 16 iterations of 8
	zloop_set_rw_prefetch ZQUAD	;; Prefetch 512 cache lines
	zr16_sixteen_complex_djbfft_preload
b2b:	zr16_sixteen_complex_djbfft rsi, 128, 8*128, 16*128, dist32, 2*dist32, rdi, ZMM_SCD8, 8, L1PREFETCH_ALL, 128
	zloop	8, b2b, -8*128+4*dist32, rdi, -8*ZMM_SCD8 ;; Test loop counter, restore source pointer, restore sin/cos pointer
	zloop	16, b2b, -16*4*dist32, rdi, 8*ZMM_SCD8 ;; Test loop counter, restore source pointer, next sin/cos pointer
	end_timer 5

;; Do FFT levels 7,8,9,10,11,12 as well as inverse FFT levels 7,8,9,10,11,12
;; On input and output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	64	+1	...
;;	8	...
;;	...
;;	56
;;	128
;;	...

;; Do 256 sixtyfour_complex_fft operations
;;	distance between fft data elements is 1 (8 for swizzling)
;;	do 256 macros each processing 128 data values

;; Execute the right middle step

pfing	bump	rcx, -65*64		;; We will prefetch 65 cache lines too many
	cmp	ffttype, 2
	jl	zpass2_16384_levels_complex_1
	jg	zpass2_16384_levels_complex_3

zpass2_16384_levels_complex_2:
	start_timer 9
	zloop_init 256			;; 256 iterations
	zloop_set_rw_prefetch ZOCT	;; Prefetch 2048 cache lines
	mov	r8, DEST2ARG
	zr64_sixtyfour_complex_with_square_preload
b3a:	zr64_sixtyfour_complex_with_square rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 4, L1PREFETCH_ALL, 8*128
	zloop	4, b3a, -4*8*128+dist32	;; Test loop counter, next source pointer
	zloop	64, b3a			;; Test loop counter
	add	rsi, r8
	end_timer 9
	jmp	zpass2_16384_levels_complex_unfft

zpass2_16384_levels_complex_1:
	start_timer 9
	zloop_init 256			;; 256 iterations
	zloop_set_rw_prefetch ZOCT	;; Prefetch 2048 cache lines
	zr64_sixtyfour_complex_fft_final_preload
b3b:	zr64_sixtyfour_complex_fft_final rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 256, L1PREFETCH_ALL, 8*128
	zloop	4, b3b, -4*8*128+dist32	;; Test loop counter, next source pointer
	zloop	64, b3b			;; Test loop counter
	end_timer 9
	jmp	zpass2_16384_levels_complex_done

zpass2_16384_levels_complex_3:
	start_timer 9
	zloop_init 256			;; 256 iterations
	zloop_set_rw_prefetch ZOCT	;; Prefetch 2048 cache lines
	mov	r8, DEST2ARG
	mov	rbp, DIST_TO_MULSRCARG
	zr64_sixtyfour_complex_with_mult_preload
b3c:	zr64_sixtyfour_complex_with_mult rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 256, L1PREFETCH_ALL, 8*128
	zloop	4, b3c, -4*8*128+dist32	;; Test loop counter, next source pointer
	zloop	64, b3c			;; Test loop counter
	add	rsi, r8
	end_timer 9
	jmp	zpass2_16384_levels_complex_unfft

zpass2_16384_levels_complex_4:
	start_timer 9
	zloop_init 256			;; 256 iterations
	zloop_set_rw_prefetch ZOCT	;; Prefetch 2048 cache lines
	mov	rbp, DIST_TO_MULSRCARG
	mov	rdi, xsincos_complex	;; The sin/cos data
	bump	rdi, 128*ZMM_SCD8+8*ZMM_SCD8
	zr64f_sixtyfour_complex_with_mulf_preload
b3d:	zr64f_sixtyfour_complex_with_mulf rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 256, L1PREFETCH_ALL, 8*128
	zloop	4, b3d, -4*8*128+dist32	;; Test loop counter, next source pointer
	zloop	64, b3d			;; Test loop counter
	end_timer 9
zpass2_16384_levels_complex_unfft:
	bump	rsi, -64*dist32		;; Restore source pointer

;; Do inverse FFT levels 4,5,6
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	64	+1	...
;;	8	...
;;	...
;;	56
;;	128
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	1K	+1	...
;;	8	...
;;	...
;;	1K-8
;;	2K
;;	...

;; Do 1024 sixteen_complex_unfft operations
;;	distance between fft data elements is 64
;;	do 128 macros each processing 256 data values

	start_timer 13
	bump	rdi, -8*ZMM_SCD8
	zloop_init 128			;; 16 iterations of 8
	zloop_set_rw_prefetch ZQUINT	;; Prefetch 640 cache lines
	zr16_sixteen_complex_djbunfft_preload
c2b:	zr16_sixteen_complex_djbunfft rsi, 128, 8*128, 16*128, dist32, 2*dist32, rdi, ZMM_SCD8, 8, L1PREFETCH_ALL, 128
	zloop	8, c2b, -8*128+4*dist32, rdi, -8*ZMM_SCD8 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	16, c2b, -16*4*dist32	;; Test loop counter, restore source pointer
	end_timer 13

;; Do inverse FFT levels 1,2,3
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	1K	+1	...
;;	8	...
;;	...
;;	1K-8
;;	2K
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	16K	+1	...
;;	8	...
;;	...
;;	16K-8

;; Do 1024 sixteen_complex_unfft operations
;;	distance between fft data elements is 1024
;;	do 128 macros each processing 256 data values

	start_timer 14
	bump	rdi, -128*ZMM_SCD8
	zloop_init 128			;; 128 iterations
	zloop_set_rw_prefetch ZQUAD	;; Prefetch 512 cache lines
	zr16_sixteen_complex_djbunfft_preload
c1b:	zr16_sixteen_complex_djbunfft rsi, 128, 4*dist32, 8*dist32, 16*dist32, 32*dist32, rdi, ZMM_SCD8, 32, L1PREFETCH_ALL, 128
	zloop	32, c1b, -32*128+dist32	;; Test loop counter, next source pointer
	zloop	4, c1b			;; Test loop counter
	end_timer 14

zpass2_16384_levels_complex_done:
	ENDM



;*********************************************************************************************
;			Macros derived from pass2-15-levels
;*********************************************************************************************

; *************** radix8-pass2-12800-real macro ******************
; This macro takes 25600 real values and performs the final 14.644 levels of the FFT.
; The results are squared, and the first 14.644 levels of the inverse FFT are done.

zr4dwpn_pass2_12800_real MACRO
	LOCAL	b1b, b2a, b2b, b3a, b3b, b4a, b4b, b4c, b4d, c1b, c2a, c2b, c3a, c3b
	LOCAL	zpass2_12800_real_1, zpass2_12800_real_2
	LOCAL	zpass2_12800_real_3, zpass2_12800_real_4
	LOCAL	zpass2_12800_real_unfft, zpass2_12800_real_done

;; Type 4 FFTs skip the forward FFT process

	start_timer 2
	mov	rsi, DATA_ADDR		;; Load source address
	mov	rbx, DIST_TO_FFTSRCARG
	mov	rcx, DATA_PREFETCH	;; Prefetch pointer (we need to prefetch 3200 cache lines + 49 pad cache lines)
	cmp	ffttype, 4
	je	zpass2_12800_real_4

;; Do FFT levels 1,2,3,4
;;	0	+1	+1	+1	+1	+1	+1	+1	12800	+1	...
;;	8	...
;;	...
;;	12792
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	3K	+1	...
;;	8	...
;;	...
;;	3K-8
;;	6K
;;	...

;; Do 2560 ten_real_fft operations
;;	distance between fft data elements is 3K
;;	do 320 macros each processing 80 data values

	mov	rdx, sincos3		;; Real sin/cos data
	mov	rdi, xsincos_complex	;; Complex sin/cos data
	zloop_init 320			;; 320 iterations
	zloop_set_rw_prefetch ZSINGLE	;; Prefetch 320 cache lines
	zr5f_2sc_ten_reals_fft_preload
b1b:	zr5f_2sc_ten_reals_fft rsi, 128, 10*dist32, rdx, ZMM_SCD2, rdi, ZMM_SCD2, 32
	zloop	32, b1b, -32*128+dist32	;; Test loop counter, next source pointer
	zloop	10, b1b, -10*dist32	;; Test loop counter, restore source pointer

;; Do FFT levels 5,6,7
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	3K	+1	...
;;	8	...
;;	...
;;	3K-8
;;	6K
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	512	+1	...
;;	8	...
;;	...
;;	504
;;	1K
;;	...

;; Do 512 ten_reals_fft operations
;; Do 2048 five_complex_fft operations
;;	distance between fft data elements is 512
;;	do 320 macros each processing 80 data values

	zloop_init 320			;; 5 iterations of 64
	zloop_set_rw_prefetch ZSINGLE	;; Prefetch 320 cache lines
	zr5_2sc_ten_reals_fft_preload
b2a:	zr5_2sc_ten_reals_fft rsi, 128, 2*dist32, rdx, ZMM_SCD2, rdi, ZMM_SCD2, 32
	zloop	32, b2a, -32*128+dist32	;; Test loop counter, next source pointer
	zloop	2, b2a, -2*dist32+10*dist32, rdi, -64*ZMM_SCD2 ;; Test loop counter, next source pointer, restore sin/cos pointer
	loops_reset			;; 5 more iters of 64
	zr5_five_complex_djbfft_preload
b2b:	zr5_five_complex_djbfft rsi, 128, 2*dist32, rdi, ZMM_SCD2, 32
	zloop	32, b2b, -32*128+dist32	;; Test loop counter, next source pointer
	zloop	2, b2b, -2*dist32+10*dist32, rdi, -64*ZMM_SCD2 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	5, b2b, -5*10*dist32, rdi, 64*ZMM_SCD2 ;; Test loop counter, restore source pointer, next sin/cos pointer

;; Do FFT levels 8,9,10
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	512	+1	...
;;	8	...
;;	...
;;	504
;;	1K
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	64	+1	...
;;	8	...
;;	...
;;	56
;;	128
;;	...

;; Do 64 sixteen_reals_fft operations
;; Do 1536 eight_complex_fft operations
;;	distance between fft data elements is 64
;;	do 200 macros each processing 128 data values

	zloop_init 200			;; 25 iterations of 8
	zloop_set_rw_prefetch ZDOUBLE	;; Prefetch 400 cache lines
	zr8_2sc_sixteen_reals_fft_preload
b3a:	zr8_2sc_sixteen_reals_fft rsi, 128, 8*128, 16*128, dist32, rdx, ZMM_SCD4, rdi, ZMM_SCD4, 8
	zloop	8, b3a, -8*128+2*dist32, rdi, -8*ZMM_SCD4 ;; Test loop counter, next source pointer, restore sin/cos pointer
	loops_reset			;; 24 more iters of 8
	zr8_eight_complex_djbfft_preload
b3b:	zr8_eight_complex_djbfft rsi, 128, 8*128, 16*128, dist32, rdi, ZMM_SCD4, 8
	zloop	8, b3b, -8*128+2*dist32, rdi, -8*ZMM_SCD4 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	25, b3b, -25*2*dist32, rdi, 8*ZMM_SCD4 ;; Test loop counter, restore source pointer, next sin/cos pointer

;; Do FFT levels 11,12,13,14,15,16 as well as inverse FFT levels 11,12,13,14,15,16
;; On input and output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	64	+1	...
;;	8	...
;;	...
;;	56
;;	128
;;	...

;; Do 1 hundredtwentyeight_reals_fft operation
;; Do 199 sixtyfour_complex_fft operations
;;	distance between fft data elements is 1 (8 for swizzling)
;;	do 200 macros each processing 128 data values

;; Execute the proper middle step

pfing	bump	rcx, -25*64		;; We will prefetch 25 cache lines too many
	cmp	ffttype, 2
	jl	zpass2_12800_real_1
	jg	zpass2_12800_real_3

zpass2_12800_real_2:
	mov	r8, DEST2ARG
;;	zr64_hundredtwentyeight_real_with_square_preload
;;	zr64_hundredtwentyeight_real_with_square rsi, 8*128, 1*128, 2*128, 4*128, rdx, 0*ZMM_SCD4, rdi, 0*ZMM_SCD4, 1
	CALLP	zreal_square
	zloop_init 200, 1, 4		;; 199 more iterations
	zloop_set_rw_prefetch ZSEXT	;; Prefetch 1194 cache lines
	zr64_sixtyfour_complex_with_square_preload
b4a:	zr64_sixtyfour_complex_with_square rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 1, L1PREFETCH_ALL, 8*128
	zloop	4, b4a, -4*8*128+dist32	;; Test loop counter, next source pointer
	zloop	50, b4a			;; Test loop counter
	add	rsi, r8
	jmp	zpass2_12800_real_unfft

zpass2_12800_real_1:
;;	zr64_hundredtwentyeight_real_fft_final_preload
;;	zr64_hundredtwentyeight_real_fft_final rsi, 8*128, 1*128, 2*128, 4*128, rdx, 0*ZMM_SCD4, rdi, 0*ZMM_SCD4, 1
	CALLP	zreal_fft_final
	zloop_init 200, 1, 4		;; 199 more iterations
	zloop_set_rw_prefetch ZSEXT	;; Prefetch 1194 cache lines
	zr64_sixtyfour_complex_fft_final_preload
b4b:	zr64_sixtyfour_complex_fft_final rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 1, L1PREFETCH_ALL, 8*128
	zloop	4, b4b, -4*8*128+dist32	;; Test loop counter, next source pointer
	zloop	50, b4b			;; Test loop counter
	jmp	zpass2_12800_real_done

zpass2_12800_real_3:
	mov	r8, DEST2ARG
	mov	rbp, DIST_TO_MULSRCARG
;;	zr64_hundredtwentyeight_real_with_mult_preload
;;	zr64_hundredtwentyeight_real_with_mult rsi, 8*128, 1*128, 2*128, 4*128, rdx, 0*ZMM_SCD4, rdi, 0*ZMM_SCD4, 1
	CALLP	zreal_mult
	zloop_init 200, 1, 4		;; 199 more iterations
	zloop_set_rw_prefetch ZSEXT	;; Prefetch 1194 cache lines
	zr64_sixtyfour_complex_with_mult_preload
b4c:	zr64_sixtyfour_complex_with_mult rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 1, L1PREFETCH_ALL, 8*128
	zloop	4, b4c, -4*8*128+dist32	;; Test loop counter, next source pointer
	zloop	50, b4c			;; Test loop counter
	add	rsi, r8
	jmp	zpass2_12800_real_unfft

zpass2_12800_real_4:
	mov	rbp, DIST_TO_MULSRCARG
	mov	rdx, sincos3		;; Real sin/cos data
	mov	rdi, xsincos_complex	;; Complex sin/cos data
	bump	rdx, 320*ZMM_SCD2+64*ZMM_SCD2+8*ZMM_SCD4
	bump	rdi, 320*ZMM_SCD2+64*ZMM_SCD2+8*ZMM_SCD4
;;	zr64f_hundredtwentyeight_real_with_mulf_preload
;;	zr64f_hundredtwentyeight_real_with_mulf rsi, 8*128, 1*128, 2*128, 4*128, rdx, 0*ZMM_SCD4, rdi, 0*ZMM_SCD4, 1
	CALLP	zreal_mulf
	zloop_init 200, 1, 4		;; 199 more iterations
	zloop_set_rw_prefetch ZSEXT	;; Prefetch 1194 cache lines
	zr64f_sixtyfour_complex_with_mulf_preload
b4d:	zr64f_sixtyfour_complex_with_mulf rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 1, L1PREFETCH_ALL, 8*128
	zloop	4, b4d, -4*8*128+dist32	;; Test loop counter, next source pointer
	zloop	50, b4d			;; Test loop counter
zpass2_12800_real_unfft:
	bump	rsi, -50*dist32		;; Restore source pointer

;; Do inverse FFT levels 8,9,10
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	64	+1	...
;;	8	...
;;	...
;;	56
;;	128
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	512	+1	...
;;	8	...
;;	...
;;	504
;;	1K
;;	...

;; Do 64 sixteen_reals_unfft operations
;; Do 1536 eight_complex_unfft operations
;;	distance between fft data elements is 64
;;	do 200 macros each processing 128 data values

	zloop_init 200			;; 25 iterations of 8
	zloop_set_rw_prefetch ZDOUBLE	;; Prefetch 400 cache lines
	bump	rdx, -8*ZMM_SCD4
	bump	rdi, -8*ZMM_SCD4
	zr8_2sc_sixteen_reals_unfft_preload
c3a:	zr8_2sc_sixteen_reals_unfft rsi, 128, 8*128, 16*128, dist32, rdx, ZMM_SCD4, rdi, ZMM_SCD4, 8
	zloop	8, c3a, -8*128+2*dist32, rdi, -8*ZMM_SCD4 ;; Test loop counter, next source pointer, restore sin/cos pointer
	loops_reset			;; 24 more iters of 8
	zr8_eight_complex_djbunfft_preload
c3b:	zr8_eight_complex_djbunfft rsi, 128, 8*128, 16*128, dist32, rdi, ZMM_SCD4, 8
	zloop	8, c3b, -8*128+2*dist32, rdi, -8*ZMM_SCD4 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	25, c3b, -25*2*dist32	;; Test loop counter, restore source pointer

;; Do inverse FFT levels 5,6,7
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	512	+1	...
;;	8	...
;;	...
;;	504
;;	1K
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	3K	+1	...
;;	8	...
;;	...
;;	3K-8
;;	6K
;;	...

;; Do 512 ten_reals_unfft operations
;; Do 2048 five_complex_unfft operations
;;	distance between fft data elements is 512
;;	do 320 macros each processing 80 data values

	zloop_init 320			;; 5 iterations of 64
	zloop_set_rw_prefetch ZSINGLE	;; Prefetch 320 cache lines
	bump	rdx, -8*ZMM_SCD4-64*ZMM_SCD2
	bump	rdi, -64*ZMM_SCD2
	zr5_2sc_ten_reals_unfft_preload
c2a:	zr5_2sc_ten_reals_unfft rsi, 128, 2*dist32, rdx, ZMM_SCD2, rdi, ZMM_SCD2, 32
	zloop	32, c2a, -32*128+dist32	;; Test loop counter, next source pointer
	zloop	2, c2a, -2*dist32+10*dist32, rdi, -64*ZMM_SCD2 ;; Test loop counter, next source pointer, restore sin/cos pointer
	loops_reset			;; 4 more iters of 64
	zr5_five_complex_djbunfft_preload
c2b:	zr5_five_complex_djbunfft rsi, 128, 2*dist32, rdi, ZMM_SCD2, 32
	zloop	32, c2b, -32*128+dist32	;; Test loop counter, next source pointer
	zloop	2, c2b, -2*dist32+10*dist32, rdi, -64*ZMM_SCD2 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	5, c2b, -5*10*dist32	;; Test loop counter, restore source pointer

;; Do inverse FFT levels 1,2,3,4
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	3K	+1	...
;;	8	...
;;	...
;;	3K-8
;;	6K
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	12800	+1	...
;;	8	...
;;	...
;;	12792

;; Do 3072 ten_reals_last_unfft operations
;;	distance between fft data elements is 3K
;;	do 320 macros each processing 80 data values

	bump	rdx, -64*ZMM_SCD2-320*ZMM_SCD2
	bump	rdi, -320*ZMM_SCD2
	zloop_init 320			;; 320 iterations
	zloop_set_rw_prefetch ZSINGLE	;; Prefetch 320 cache lines
	zr5_2sc_ten_reals_unfft_preload
c1b:	zr5_2sc_ten_reals_unfft rsi, 128, 10*dist32, rdx, ZMM_SCD2, rdi, ZMM_SCD2, 32
	zloop	32, c1b, -32*128+dist32	;; Test loop counter, next source pointer
	zloop	10, c1b			;; Test loop counter

zpass2_12800_real_done:
	end_timer 2
	ENDM

; *************** radix8-pass2-12800-complex macro ******************
; This macro takes 12800 complex values and performs the final 13.644 levels of a standard
; radix-8 FFT, squares the results, and does the first 13.644 levels of the inverse FFT.

zr4dwpn_pass2_12800_complex MACRO
	LOCAL	b1b, b2b, b3b, b4a, b4b, b4c, b4d, c1b, c2b, c3b
	LOCAL	zpass2_12800_complex_1, zpass2_12800_complex_2
	LOCAL	zpass2_12800_complex_3, zpass2_12800_complex_4
	LOCAL	zpass2_12800_complex_unfft, zpass2_12800_complex_done

;; Type 4 FFTs skip the forward FFT process

	mov	rsi, DATA_ADDR		;; Load address of FFT data
	mov	rbx, DIST_TO_FFTSRCARG
	mov	rcx, DATA_PREFETCH	;; Prefetch pointer (we need to prefetch 3200 cache lines + 49 pad cache lines)
	cmp	ffttype, 4
	je	zpass2_12800_complex_4

;; Do FFT levels 1,2,3
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	12800	+1	...
;;	8	...
;;	...
;;	12792
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	3K	+1	...
;;	8	...
;;	...
;;	3K-8
;;	6K
;;	...

;; Do 2560 five_complex_fft operations
;;	distance between fft data elements is 3K
;;	do 320 macros each processing 80 data values

	start_timer 4
	mov	rdi, xsincos_complex	;; The sin/cos data
	zloop_init 320			;; 320 iterations
	zloop_set_rw_prefetch ZSINGLE	;; Prefetch 320 cache lines
	zr5f_five_complex_djbfft_preload
b1b:	zr5f_five_complex_djbfft rsi, 128, 10*dist32, rdi, ZMM_SCD2, 32, L1PREFETCH_ALL, 128
	zloop	32, b1b, -32*128+dist32	;; Test loop counter, next source pointer
	zloop	10, b1b, -10*dist32	;; Test loop counter, restore source pointer
	end_timer 4

;; Do FFT levels 4,5,6
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	3K	+1	...
;;	8	...
;;	...
;;	3K-8
;;	6K
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	512	+1	...
;;	8	...
;;	...
;;	504
;;	1K
;;	...

;; Do 2560 five_complex_fft operations
;;	distance between fft data elements is 512
;;	do 320 macros each processing 80 data values

	start_timer 5
	zloop_init 320			;; 5 iterations of 64
	zloop_set_rw_prefetch ZSINGLE	;; Prefetch 320 cache lines
	zr5_five_complex_djbfft_preload
b2b:	zr5_five_complex_djbfft rsi, 128, 2*dist32, rdi, ZMM_SCD2, 32, L1PREFETCH_ALL, 128
	zloop	32, b2b, -32*128+dist32	;; Test loop counter, next source pointer
	zloop	2, b2b, -2*dist32+10*dist32, rdi, -64*ZMM_SCD2 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	5, b2b, -5*10*dist32, rdi, 64*ZMM_SCD2 ;; Test loop counter, restore source pointer, next sin/cos pointer
	end_timer 5

;; Do FFT levels 7,8,9
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	512	+1	...
;;	8	...
;;	...
;;	504
;;	1K
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	64	+1	...
;;	8	...
;;	...
;;	56
;;	128
;;	...

;; Do 1600 eight_complex_fft operations
;;	distance between fft data elements is 64
;;	do 200 macros each processing 128 data values

	start_timer 6
	zloop_init 200			;; 25 iterations of 8
	zloop_set_rw_prefetch ZDOUBLE	;; Prefetch 400 cache lines
	zr8_eight_complex_djbfft_preload
b3b:	zr8_eight_complex_djbfft rsi, 128, 8*128, 16*128, dist32, rdi, ZMM_SCD4, 8, L1PREFETCH_ALL, 128
	zloop	8, b3b, -8*128+2*dist32, rdi, -8*ZMM_SCD4 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	25, b3b, -25*2*dist32, rdi, 8*ZMM_SCD4 ;; Test loop counter, restore source pointer, next sin/cos pointer
	end_timer 6

;; Do FFT levels 10,11,12,13,14,15 as well as inverse FFT levels 10,11,12,13,14,15
;; On input and output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	64	+1	...
;;	8	...
;;	...
;;	56
;;	128
;;	...

;; Do 200 sixtyfour_complex_fft operations
;;	distance between fft data elements is 1 (8 for swizzling)
;;	do 200 macros each processing 128 data values

;; Execute the right middle step

pfing	bump	rcx, -31*64		;; We will prefetch 31 cache lines too many
	cmp	ffttype, 2
	jl	zpass2_12800_complex_1
	jg	zpass2_12800_complex_3

zpass2_12800_complex_2:
	start_timer 9
	zloop_init 200			;; 200 iterations
	zloop_set_rw_prefetch ZSEXT	;; Prefetch 1200 cache lines
	mov	r8, DEST2ARG
	zr64_sixtyfour_complex_with_square_preload
b4a:	zr64_sixtyfour_complex_with_square rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 4, L1PREFETCH_ALL, 8*128
	zloop	4, b4a, -4*8*128+dist32	;; Test loop counter, next source pointer
	zloop	50, b4a			;; Test loop counter
	add	rsi, r8
	end_timer 9
	jmp	zpass2_12800_complex_unfft

zpass2_12800_complex_1:
	start_timer 9
	zloop_init 200			;; 200 iterations
	zloop_set_rw_prefetch ZSEXT	;; Prefetch 1200 cache lines
	zr64_sixtyfour_complex_fft_final_preload
b4b:	zr64_sixtyfour_complex_fft_final rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 4, L1PREFETCH_ALL, 8*128
	zloop	4, b4b, -4*8*128+dist32	;; Test loop counter, next source pointer
	zloop	50, b4b			;; Test loop counter
	end_timer 9
	jmp	zpass2_12800_complex_done

zpass2_12800_complex_3:
	start_timer 9
	zloop_init 200			;; 200 iterations
	zloop_set_rw_prefetch ZSEXT	;; Prefetch 1200 cache lines
	mov	r8, DEST2ARG
	mov	rbp, DIST_TO_MULSRCARG
	zr64_sixtyfour_complex_with_mult_preload
b4c:	zr64_sixtyfour_complex_with_mult rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 4, L1PREFETCH_ALL, 8*128
	zloop	4, b4c, -4*8*128+dist32	;; Test loop counter, next source pointer
	zloop	50, b4c			;; Test loop counter
	add	rsi, r8
	end_timer 9
	jmp	zpass2_12800_complex_unfft

zpass2_12800_complex_4:
	start_timer 9
	zloop_init 200			;; 200 iterations
	zloop_set_rw_prefetch ZSEXT	;; Prefetch 1200 cache lines
	mov	rbp, DIST_TO_MULSRCARG
	mov	rdi, xsincos_complex	;; The sin/cos data
	bump	rdi, 320*ZMM_SCD2+64*ZMM_SCD2+8*ZMM_SCD4
	zr64f_sixtyfour_complex_with_mulf_preload
b4d:	zr64f_sixtyfour_complex_with_mulf rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 4, L1PREFETCH_ALL, 8*128
	zloop	4, b4d, -4*8*128+dist32	;; Test loop counter, next source pointer
	zloop	50, b4d			;; Test loop counter
	end_timer 9
zpass2_12800_complex_unfft:
	bump	rsi, -50*dist32		;; Restore source pointer

;; Do inverse FFT levels 7,8,9
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	64	+1	...
;;	8	...
;;	...
;;	56
;;	128
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	512	+1	...
;;	8	...
;;	...
;;	504
;;	1K
;;	...

;; Do 1600 eight_complex_unfft operations
;;	distance between fft data elements is 64
;;	do 200 macros each processing 128 data values

	start_timer 12
	bump	rdi, -8*ZMM_SCD4
	zloop_init 200			;; 25 iterations of 8
	zloop_set_rw_prefetch ZDOUBLE	;; Prefetch 400 cache lines
	zr8_eight_complex_djbunfft_preload
c3b:	zr8_eight_complex_djbunfft rsi, 128, 8*128, 16*128, dist32, rdi, ZMM_SCD4, 8, L1PREFETCH_ALL, 128
	zloop	8, c3b, -8*128+2*dist32, rdi, -8*ZMM_SCD4 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	25, c3b, -25*2*dist32	;; Test loop counter, restore source pointer
	end_timer 12

;; Do inverse FFT levels 4,5,6
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	512	+1	...
;;	8	...
;;	...
;;	504
;;	1K
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	3K	+1	...
;;	8	...
;;	...
;;	3K-8
;;	6K
;;	...

;; Do 2560 five_complex_unfft operations
;;	distance between fft data elements is 512
;;	do 320 macros each processing 80 data values

	start_timer 13
	bump	rdi, -64*ZMM_SCD2
	zloop_init 320			;; 5 iterations of 64
	zloop_set_rw_prefetch ZSINGLE	;; Prefetch 320 cache lines
	zr5_five_complex_djbunfft_preload
c2b:	zr5_five_complex_djbunfft rsi, 128, 2*dist32, rdi, ZMM_SCD2, 32, L1PREFETCH_ALL, 128
	zloop	32, c2b, -32*128+dist32	;; Test loop counter, next source pointer
	zloop	2, c2b, -2*dist32+10*dist32, rdi, -64*ZMM_SCD2 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	5, c2b, -5*10*dist32	;; Test loop counter, restore source pointer
	end_timer 13

;; Do inverse FFT levels 1,2,3
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	3K	+1	...
;;	8	...
;;	...
;;	3K-8
;;	6K
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	12800	+1	...
;;	8	...
;;	...
;;	12792

;; Do 3072 five_complex_unfft operations
;;	distance between fft data elements is 3K
;;	do 320 macros each processing 80 data values

	start_timer 14
	bump	rdi, -320*ZMM_SCD2
	zloop_init 320			;; 320 iterations
	zloop_set_rw_prefetch ZSINGLE	;; Prefetch 320 cache lines
	zr5_five_complex_djbunfft_preload
c1b:	zr5_five_complex_djbunfft rsi, 128, 10*dist32, rdi, ZMM_SCD2, 32, L1PREFETCH_ALL, 128
	zloop	32, c1b, -32*128+dist32	;; Test loop counter, next source pointer
	zloop	10, c1b			;; Test loop counter
	end_timer 14

zpass2_12800_complex_done:
	ENDM


; *************** radix8-pass2-15360-real macro ******************
; This macro takes 30720 real values and performs the final 14.907 levels of the FFT.
; The results are squared, and the first 14.907 levels of the inverse FFT are done.

zr4dwpn_pass2_15360_real MACRO
	LOCAL	b1b, b2a, b2b, b3a, b3b, b4a, b4b, b4c, b4d, c1b, c2a, c2b, c3a, c3b
	LOCAL	zpass2_15360_real_1, zpass2_15360_real_2
	LOCAL	zpass2_15360_real_3, zpass2_15360_real_4
	LOCAL	zpass2_15360_real_unfft, zpass2_15360_real_done

;; Type 4 FFTs skip the forward FFT process

	start_timer 2
	mov	rsi, DATA_ADDR		;; Load source address
	mov	rbx, DIST_TO_FFTSRCARG
	mov	rcx, DATA_PREFETCH	;; Prefetch pointer (we need to prefetch 3840 cache lines + ??? pad cache lines)
	cmp	ffttype, 4
	je	zpass2_15360_real_4

;; Do FFT levels 1,2,3,4
;;	0	+1	+1	+1	+1	+1	+1	+1	15K	+1	...
;;	8	...
;;	...
;;	15K-8
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	3K	+1	...
;;	8	...
;;	...
;;	3K-8
;;	6K
;;	...

;; Do 3072 ten_real_fft operations
;;	distance between fft data elements is 3K
;;	do 384 macros each processing 80 data values

	mov	rdx, sincos3		;; Real sin/cos data
	mov	rdi, xsincos_complex	;; Complex sin/cos data
	zloop_init 384			;; 384 iterations
	zloop_set_rw_prefetch ZSINGLE	;; Prefetch 384 cache lines
	zr5f_2sc_ten_reals_fft_preload
b1b:	zr5f_2sc_ten_reals_fft rsi, 128, 12*dist32, rdx, ZMM_SCD2, rdi, ZMM_SCD2, 32
	zloop	32, b1b, -32*128+dist32	;; Test loop counter, next source pointer
	zloop	12, b1b, -12*dist32	;; Test loop counter, restore source pointer

;; Do FFT levels 5,6,7
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	3K	+1	...
;;	8	...
;;	...
;;	3K-8
;;	6K
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	512	+1	...
;;	8	...
;;	...
;;	504
;;	1K
;;	...

;; Do 512 twelve_reals_fft operations
;; Do 2048 six_complex_fft operations
;;	distance between fft data elements is 512
;;	do 320 macros each processing 96 data values

	zloop_init 320			;; 5 iterations of 64
	zloop_set_rw_prefetch ZSINGLE	;; Prefetch 320 cache lines
	zr6_2sc_twelve_reals_fft_preload
b2a:	zr6_2sc_twelve_reals_fft rsi, 128, 2*dist32, rdx, ZMM_SCD3, rdi, ZMM_SCD3, 32
	zloop	32, b2a, -32*128+dist32	;; Test loop counter, next source pointer
	zloop	2, b2a, -2*dist32+12*dist32, rdi, -64*ZMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer
	loops_reset			;; 4 more iters of 64
	zr6_six_complex_djbfft_preload
b2b:	zr6_six_complex_djbfft rsi, 128, 2*dist32, rdi, ZMM_SCD3, 32
	zloop	32, b2b, -32*128+dist32	;; Test loop counter, next source pointer
	zloop	2, b2b, -2*dist32+12*dist32, rdi, -64*ZMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	5, b2b, -5*12*dist32, rdi, 64*ZMM_SCD3 ;; Test loop counter, restore source pointer, next sin/cos pointer

;; Do FFT levels 8,9,10
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	512	+1	...
;;	8	...
;;	...
;;	504
;;	1K
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	64	+1	...
;;	8	...
;;	...
;;	56
;;	128
;;	...

;; Do 64 sixteen_reals_fft operations
;; Do 1856 eight_complex_fft operations
;;	distance between fft data elements is 64
;;	do 240 macros each processing 128 data values

	zloop_init 240			;; 30 iterations of 8
	zloop_set_rw_prefetch ZDOUBLE	;; Prefetch 480 cache lines
	zr8_2sc_sixteen_reals_fft_preload
b3a:	zr8_2sc_sixteen_reals_fft rsi, 128, 8*128, 16*128, dist32, rdx, ZMM_SCD4, rdi, ZMM_SCD4, 8
	zloop	8, b3a, -8*128+2*dist32, rdi, -8*ZMM_SCD4 ;; Test loop counter, next source pointer, restore sin/cos pointer
	loops_reset			;; 29 more iters of 8
	zr8_eight_complex_djbfft_preload
b3b:	zr8_eight_complex_djbfft rsi, 128, 8*128, 16*128, dist32, rdi, ZMM_SCD4, 8
	zloop	8, b3b, -8*128+2*dist32, rdi, -8*ZMM_SCD4 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	30, b3b, -30*2*dist32, rdi, 8*ZMM_SCD4 ;; Test loop counter, restore source pointer, next sin/cos pointer

;; Do FFT levels 11,12,13,14,15,16 as well as inverse FFT levels 11,12,13,14,15,16
;; On input and output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	64	+1	...
;;	8	...
;;	...
;;	56
;;	128
;;	...

;; Do 1 hundredtwentyeight_reals_fft operation
;; Do 239 sixtyfour_complex_fft operations
;;	distance between fft data elements is 1 (8 for swizzling)
;;	do 240 macros each processing 128 data values

;; Execute the proper middle step

pfing	bump	rcx, -142*64		;; We will prefetch 142 cache lines too many
	cmp	ffttype, 2
	jl	zpass2_15360_real_1
	jg	zpass2_15360_real_3

zpass2_15360_real_2:
	mov	r8, DEST2ARG
;;	zr64_hundredtwentyeight_real_with_square_preload
;;	zr64_hundredtwentyeight_real_with_square rsi, 8*128, 1*128, 2*128, 4*128, rdx, 0*ZMM_SCD4, rdi, 0*ZMM_SCD4, 1
	CALLP	zreal_square
	zloop_init 240, 1, 4		;; 239 more iterations
	zloop_set_rw_prefetch ZSEPT	;; Prefetch 1673 cache lines
	zr64_sixtyfour_complex_with_square_preload
b4a:	zr64_sixtyfour_complex_with_square rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 1, L1PREFETCH_ALL, 8*128
	zloop	4, b4a, -4*8*128+dist32	;; Test loop counter, next source pointer
	zloop	60, b4a			;; Test loop counter
	add	rsi, r8
	jmp	zpass2_15360_real_unfft

zpass2_15360_real_1:
;;	zr64_hundredtwentyeight_real_fft_final_preload
;;	zr64_hundredtwentyeight_real_fft_final rsi, 8*128, 1*128, 2*128, 4*128, rdx, 0*ZMM_SCD4, rdi, 0*ZMM_SCD4, 1
	CALLP	zreal_fft_final
	zloop_init 240, 1, 4		;; 239 more iterations
	zloop_set_rw_prefetch ZSEPT	;; Prefetch 1673 cache lines
	zr64_sixtyfour_complex_fft_final_preload
b4b:	zr64_sixtyfour_complex_fft_final rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 1, L1PREFETCH_ALL, 8*128
	zloop	4, b4b, -4*8*128+dist32	;; Test loop counter, next source pointer
	zloop	60, b4b			;; Test loop counter
	jmp	zpass2_15360_real_done

zpass2_15360_real_3:
	mov	r8, DEST2ARG
	mov	rbp, DIST_TO_MULSRCARG
;;	zr64_hundredtwentyeight_real_with_mult_preload
;;	zr64_hundredtwentyeight_real_with_mult rsi, 8*128, 1*128, 2*128, 4*128, rdx, 0*ZMM_SCD4, rdi, 0*ZMM_SCD4, 1
	CALLP	zreal_mult
	zloop_init 240, 1, 4		;; 239 more iterations
	zloop_set_rw_prefetch ZSEPT	;; Prefetch 1673 cache lines
	zr64_sixtyfour_complex_with_mult_preload
b4c:	zr64_sixtyfour_complex_with_mult rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 1, L1PREFETCH_ALL, 8*128
	zloop	4, b4c, -4*8*128+dist32	;; Test loop counter, next source pointer
	zloop	60, b4c			;; Test loop counter
	add	rsi, r8
	jmp	zpass2_15360_real_unfft

zpass2_15360_real_4:
	mov	rbp, DIST_TO_MULSRCARG
	mov	rdx, sincos3		;; Real sin/cos data
	mov	rdi, xsincos_complex	;; Complex sin/cos data
	bump	rdx, 384*ZMM_SCD2+64*ZMM_SCD3+8*ZMM_SCD4
	bump	rdi, 384*ZMM_SCD2+64*ZMM_SCD3+8*ZMM_SCD4
;;	zr64f_hundredtwentyeight_real_with_mulf_preload
;;	zr64f_hundredtwentyeight_real_with_mulf rsi, 8*128, 1*128, 2*128, 4*128, rdx, 0*ZMM_SCD4, rdi, 0*ZMM_SCD4, 1
	CALLP	zreal_mulf
	zloop_init 240, 1, 4		;; 239 more iterations
	zloop_set_rw_prefetch ZSEPT	;; Prefetch 1673 cache lines
	zr64f_sixtyfour_complex_with_mulf_preload
b4d:	zr64f_sixtyfour_complex_with_mulf rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 1, L1PREFETCH_ALL, 8*128
	zloop	4, b4d, -4*8*128+dist32	;; Test loop counter, next source pointer
	zloop	60, b4d			;; Test loop counter
zpass2_15360_real_unfft:
	bump	rsi, -60*dist32		;; Restore source pointer

;; Do inverse FFT levels 8,9,10
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	64	+1	...
;;	8	...
;;	...
;;	56
;;	128
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	512	+1	...
;;	8	...
;;	...
;;	504
;;	1K
;;	...

;; Do 64 sixteen_reals_unfft operations
;; Do 1856 eight_complex_unfft operations
;;	distance between fft data elements is 64
;;	do 240 macros each processing 128 data values

	zloop_init 240			;; 30 iterations of 8
	zloop_set_rw_prefetch ZDOUBLE	;; Prefetch 480 cache lines
	bump	rdx, -8*ZMM_SCD4
	bump	rdi, -8*ZMM_SCD4
	zr8_2sc_sixteen_reals_unfft_preload
c3a:	zr8_2sc_sixteen_reals_unfft rsi, 128, 8*128, 16*128, dist32, rdx, ZMM_SCD4, rdi, ZMM_SCD4, 8
	zloop	8, c3a, -8*128+2*dist32, rdi, -8*ZMM_SCD4 ;; Test loop counter, next source pointer, restore sin/cos pointer
	loops_reset			;; 29 more iters of 8
	zr8_eight_complex_djbunfft_preload
c3b:	zr8_eight_complex_djbunfft rsi, 128, 8*128, 16*128, dist32, rdi, ZMM_SCD4, 8
	zloop	8, c3b, -8*128+2*dist32, rdi, -8*ZMM_SCD4 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	30, c3b, -30*2*dist32	;; Test loop counter, restore source pointer

;; Do inverse FFT levels 5,6,7
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	512	+1	...
;;	8	...
;;	...
;;	504
;;	1K
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	3K	+1	...
;;	8	...
;;	...
;;	3K-8
;;	6K
;;	...

;; Do 512 twelve_reals_unfft operations
;; Do 2048 six_complex_unfft operations
;;	distance between fft data elements is 512
;;	do 320 macros each processing 96 data values

	zloop_init 320			;; 5 iterations of 64
	zloop_set_rw_prefetch ZSINGLE	;; Prefetch 320 cache lines
	bump	rdx, -8*ZMM_SCD4-64*ZMM_SCD3
	bump	rdi, -64*ZMM_SCD3
	zr6_2sc_twelve_reals_unfft_preload
c2a:	zr6_2sc_twelve_reals_unfft rsi, 128, 2*dist32, rdx, ZMM_SCD3, rdi, ZMM_SCD3, 32
	zloop	32, c2a, -32*128+dist32	;; Test loop counter, next source pointer
	zloop	2, c2a, -2*dist32+12*dist32, rdi, -64*ZMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer
	loops_reset			;; 4 more iters of 64
	zr6_six_complex_djbunfft_preload
c2b:	zr6_six_complex_djbunfft rsi, 128, 2*dist32, rdi, ZMM_SCD3, 32
	zloop	32, c2b, -32*128+dist32	;; Test loop counter, next source pointer
	zloop	2, c2b, -2*dist32+12*dist32, rdi, -64*ZMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	5, c2b, -5*12*dist32	;; Test loop counter, restore source pointer

;; Do inverse FFT levels 1,2,3,4
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	3K	+1	...
;;	8	...
;;	...
;;	3K-8
;;	6K
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	15K	+1	...
;;	8	...
;;	...
;;	15K-8

;; Do 3072 ten_reals_last_unfft operations
;;	distance between fft data elements is 3K
;;	do 384 macros each processing 80 data values

	bump	rdx, -64*ZMM_SCD3-384*ZMM_SCD2
	bump	rdi, -384*ZMM_SCD2
	zloop_init 384			;; 384 iterations
	zloop_set_rw_prefetch ZSINGLE	;; Prefetch 384 cache lines
	zr5_2sc_ten_reals_unfft_preload
c1b:	zr5_2sc_ten_reals_unfft rsi, 128, 12*dist32, rdx, ZMM_SCD2, rdi, ZMM_SCD2, 32
	zloop	32, c1b, -32*128+dist32	;; Test loop counter, next source pointer
	zloop	12, c1b			;; Test loop counter

zpass2_15360_real_done:
	end_timer 2
	ENDM

; *************** radix8-pass2-15360-complex macro ******************
; This macro takes 15360 complex values and performs the final 13.907 levels of a standard
; radix-8 FFT, squares the results, and does the first 13.907 levels of the inverse FFT.

zr4dwpn_pass2_15360_complex MACRO
	LOCAL	b1b, b2b, b3b, b4a, b4b, b4c, b4d, c1b, c2b, c3b
	LOCAL	zpass2_15360_complex_1, zpass2_15360_complex_2
	LOCAL	zpass2_15360_complex_3, zpass2_15360_complex_4
	LOCAL	zpass2_15360_complex_unfft, zpass2_15360_complex_done

;; Type 4 FFTs skip the forward FFT process

	mov	rsi, DATA_ADDR		;; Load address of FFT data
	mov	rbx, DIST_TO_FFTSRCARG
	mov	rcx, DATA_PREFETCH	;; Prefetch pointer (we need to prefetch 3840 cache lines + 59 pad cache lines)
	cmp	ffttype, 4
	je	zpass2_15360_complex_4

;; Do FFT levels 1,2,3
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	15K	+1	...
;;	8	...
;;	...
;;	15K-8
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	3K	+1	...
;;	8	...
;;	...
;;	3K-8
;;	6K
;;	...

;; Do 3072 five_complex_fft operations
;;	distance between fft data elements is 3K
;;	do 384 macros each processing 80 data values

	start_timer 4
	mov	rdi, xsincos_complex	;; The sin/cos data
	zloop_init 384			;; 384 iterations
	zloop_set_rw_prefetch ZSINGLE	;; Prefetch 384 cache lines
	zr5f_five_complex_djbfft_preload
b1b:	zr5f_five_complex_djbfft rsi, 128, 12*dist32, rdi, ZMM_SCD2, 32, L1PREFETCH_ALL, 128
	zloop	32, b1b, -32*128+dist32	;; Test loop counter, next source pointer
	zloop	12, b1b, -12*dist32	;; Test loop counter, restore source pointer
	end_timer 4

;; Do FFT levels 4,5,6
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	3K	+1	...
;;	8	...
;;	...
;;	3K-8
;;	6K
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	512	+1	...
;;	8	...
;;	...
;;	504
;;	1K
;;	...

;; Do 2560 six_complex_fft operations
;;	distance between fft data elements is 512
;;	do 320 macros each processing 96 data values

	start_timer 5
	zloop_init 320			;; 5 iterations of 64
	zloop_set_rw_prefetch ZSINGLE	;; Prefetch 320 cache lines
	zr6_six_complex_djbfft_preload
b2b:	zr6_six_complex_djbfft rsi, 128, 2*dist32, rdi, ZMM_SCD3, 32, L1PREFETCH_ALL, 128
	zloop	32, b2b, -32*128+dist32	;; Test loop counter, next source pointer
	zloop	2, b2b, -2*dist32+12*dist32, rdi, -64*ZMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	5, b2b, -5*12*dist32, rdi, 64*ZMM_SCD3 ;; Test loop counter, restore source pointer, next sin/cos pointer
	end_timer 5

;; Do FFT levels 7,8,9
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	512	+1	...
;;	8	...
;;	...
;;	504
;;	1K
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	64	+1	...
;;	8	...
;;	...
;;	56
;;	128
;;	...

;; Do 1920 eight_complex_fft operations
;;	distance between fft data elements is 64
;;	do 240 macros each processing 128 data values

	start_timer 6
	zloop_init 240			;; 30 iterations of 8
	zloop_set_rw_prefetch ZDOUBLE	;; Prefetch 480 cache lines
	zr8_eight_complex_djbfft_preload
b3b:	zr8_eight_complex_djbfft rsi, 128, 8*128, 16*128, dist32, rdi, ZMM_SCD4, 8, L1PREFETCH_ALL, 128
	zloop	8, b3b, -8*128+2*dist32, rdi, -8*ZMM_SCD4 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	30, b3b, -30*2*dist32, rdi, 8*ZMM_SCD4 ;; Test loop counter, restore source pointer, next sin/cos pointer
	end_timer 6

;; Do FFT levels 10,11,12,13,14,15 as well as inverse FFT levels 10,11,12,13,14,15
;; On input and output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	64	+1	...
;;	8	...
;;	...
;;	56
;;	128
;;	...

;; Do 240 sixtyfour_complex_fft operations
;;	distance between fft data elements is 1 (8 for swizzling)
;;	do 240 macros each processing 128 data values

;; Execute the right middle step

pfing	bump	rcx, -149*64		;; We will prefetch 149 cache lines too many
	cmp	ffttype, 2
	jl	zpass2_15360_complex_1
	jg	zpass2_15360_complex_3

zpass2_15360_complex_2:
	start_timer 9
	zloop_init 240			;; 240 iterations
	zloop_set_rw_prefetch ZSEPT	;; Prefetch 1680 cache lines
	mov	r8, DEST2ARG
	zr64_sixtyfour_complex_with_square_preload
b4a:	zr64_sixtyfour_complex_with_square rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 4, L1PREFETCH_ALL, 8*128
	zloop	4, b4a, -4*8*128+dist32	;; Test loop counter, next source pointer
	zloop	60, b4a			;; Test loop counter
	add	rsi, r8
	end_timer 9
	jmp	zpass2_15360_complex_unfft

zpass2_15360_complex_1:
	start_timer 9
	zloop_init 240			;; 240 iterations
	zloop_set_rw_prefetch ZSEPT	;; Prefetch 1680 cache lines
	zr64_sixtyfour_complex_fft_final_preload
b4b:	zr64_sixtyfour_complex_fft_final rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 4, L1PREFETCH_ALL, 8*128
	zloop	4, b4b, -4*8*128+dist32	;; Test loop counter, next source pointer
	zloop	60, b4b			;; Test loop counter
	end_timer 9
	jmp	zpass2_15360_complex_done

zpass2_15360_complex_3:
	start_timer 9
	zloop_init 240			;; 240 iterations
	zloop_set_rw_prefetch ZSEPT	;; Prefetch 1680 cache lines
	mov	r8, DEST2ARG
	mov	rbp, DIST_TO_MULSRCARG
	zr64_sixtyfour_complex_with_mult_preload
b4c:	zr64_sixtyfour_complex_with_mult rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 4, L1PREFETCH_ALL, 8*128
	zloop	4, b4c, -4*8*128+dist32	;; Test loop counter, next source pointer
	zloop	60, b4c			;; Test loop counter
	add	rsi, r8
	end_timer 9
	jmp	zpass2_15360_complex_unfft

zpass2_15360_complex_4:
	start_timer 9
	zloop_init 240			;; 240 iterations
	zloop_set_rw_prefetch ZSEPT	;; Prefetch 1680 cache lines
	mov	rbp, DIST_TO_MULSRCARG
	mov	rdi, xsincos_complex	;; The sin/cos data
	bump	rdi, 384*ZMM_SCD2+64*ZMM_SCD3+8*ZMM_SCD4
	zr64f_sixtyfour_complex_with_mulf_preload
b4d:	zr64f_sixtyfour_complex_with_mulf rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 4, L1PREFETCH_ALL, 8*128
	zloop	4, b4d, -4*8*128+dist32	;; Test loop counter, next source pointer
	zloop	60, b4d			;; Test loop counter
	end_timer 9
zpass2_15360_complex_unfft:
	bump	rsi, -60*dist32		;; Restore source pointer

;; Do inverse FFT levels 7,8,9
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	64	+1	...
;;	8	...
;;	...
;;	56
;;	128
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	512	+1	...
;;	8	...
;;	...
;;	504
;;	1K
;;	...

;; Do 1920 eight_complex_unfft operations
;;	distance between fft data elements is 64
;;	do 240 macros each processing 128 data values

	start_timer 12
	bump	rdi, -8*ZMM_SCD4
	zloop_init 240			;; 30 iterations of 8
	zloop_set_rw_prefetch ZDOUBLE	;; Prefetch 480 cache lines
	zr8_eight_complex_djbunfft_preload
c3b:	zr8_eight_complex_djbunfft rsi, 128, 8*128, 16*128, dist32, rdi, ZMM_SCD4, 8, L1PREFETCH_ALL, 128
	zloop	8, c3b, -8*128+2*dist32, rdi, -8*ZMM_SCD4 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	30, c3b, -30*2*dist32	;; Test loop counter, restore source pointer
	end_timer 12

;; Do inverse FFT levels 4,5,6
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	512	+1	...
;;	8	...
;;	...
;;	504
;;	1K
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	3K	+1	...
;;	8	...
;;	...
;;	3K-8
;;	6K
;;	...

;; Do 2560 six_complex_unfft operations
;;	distance between fft data elements is 512
;;	do 320 macros each processing 96 data values

	start_timer 13
	bump	rdi, -64*ZMM_SCD3
	zloop_init 320			;; 5 iterations of 64
	zloop_set_rw_prefetch ZSINGLE	;; Prefetch 320 cache lines
	zr6_six_complex_djbunfft_preload
c2b:	zr6_six_complex_djbunfft rsi, 128, 2*dist32, rdi, ZMM_SCD3, 32, L1PREFETCH_ALL, 128
	zloop	32, c2b, -32*128+dist32	;; Test loop counter, next source pointer
	zloop	2, c2b, -2*dist32+12*dist32, rdi, -64*ZMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	5, c2b, -5*12*dist32	;; Test loop counter, restore source pointer
	end_timer 13

;; Do inverse FFT levels 1,2,3
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	3K	+1	...
;;	8	...
;;	...
;;	3K-8
;;	6K
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	15K	+1	...
;;	8	...
;;	...
;;	15K-8

;; Do 3072 five_complex_unfft operations
;;	distance between fft data elements is 3K
;;	do 384 macros each processing 80 data values

	start_timer 14
	bump	rdi, -384*ZMM_SCD2
	zloop_init 384			;; 384 iterations
	zloop_set_rw_prefetch ZSINGLE	;; Prefetch 384 cache lines
	zr5_five_complex_djbunfft_preload
c1b:	zr5_five_complex_djbunfft rsi, 128, 12*dist32, rdi, ZMM_SCD2, 32, L1PREFETCH_ALL, 128
	zloop	32, c1b, -32*128+dist32	;; Test loop counter, next source pointer
	zloop	12, c1b			;; Test loop counter
	end_timer 14

zpass2_15360_complex_done:
	ENDM


; *************** radix8-pass2-17920-real macro ******************
; This macro takes 35840 real values and performs the final 15.129 levels of the FFT.
; The results are squared, and the first 15.129 levels of the inverse FFT are done.

zr4dwpn_pass2_17920_real MACRO
	LOCAL	b1b, b2a, b2b, b3a, b3b, b4a, b4b, b4c, b4d, c1b, c2a, c2b, c3a, c3b
	LOCAL	zpass2_17920_real_1, zpass2_17920_real_2
	LOCAL	zpass2_17920_real_3, zpass2_17920_real_4
	LOCAL	zpass2_17920_real_unfft, zpass2_17920_real_done

;; Type 4 FFTs skip the forward FFT process

	start_timer 2
	mov	rsi, DATA_ADDR		;; Load source address
	mov	rbx, DIST_TO_FFTSRCARG
	mov	rcx, DATA_PREFETCH	;; Prefetch pointer (we need to prefetch 4480 cache lines + ??? pad cache lines)
	cmp	ffttype, 4
	je	zpass2_17920_real_4

;; Do FFT levels 1,2,3,4
;;	0	+1	+1	+1	+1	+1	+1	+1	17920	+1	...
;;	8	...
;;	...
;;	17912
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	3584	+1	...
;;	8	...
;;	...
;;	3576
;;	7168
;;	...

;; Do 3584 ten_real_fft operations
;;	distance between fft data elements is 3584
;;	do 448 macros each processing 80 data values

	mov	rdx, sincos3		;; Real sin/cos data
	mov	rdi, xsincos_complex	;; Complex sin/cos data
	zloop_init 448			;; 448 iterations
	zloop_set_rw_prefetch ZSINGLE	;; Prefetch 448 cache lines
	zr5f_2sc_ten_reals_fft_preload
b1b:	zr5f_2sc_ten_reals_fft rsi, 128, 14*dist32, rdx, ZMM_SCD2, rdi, ZMM_SCD2, 32
	zloop	32, b1b, -32*128+dist32	;; Test loop counter, next source pointer
	zloop	14, b1b, -14*dist32	;; Test loop counter, restore source pointer

;; Do FFT levels 5,6,7
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	3584	+1	...
;;	8	...
;;	...
;;	3576
;;	7168
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	512	+1	...
;;	8	...
;;	...
;;	504
;;	1K
;;	...

;; Do 512 fourteen_reals_fft operations
;; Do 2048 seven_complex_fft operations
;;	distance between fft data elements is 512
;;	do 320 macros each processing 112 data values

	zloop_init 320			;; 5 iterations of 64
	zloop_set_rw_prefetch ZSINGLE	;; Prefetch 320 cache lines
	zr7_2sc_fourteen_reals_fft_preload
b2a:	zr7_2sc_fourteen_reals_fft rsi, 128, 2*dist32, rdx, ZMM_SCD3, rdi, ZMM_SCD3, 32
	zloop	32, b2a, -32*128+dist32	;; Test loop counter, next source pointer
	zloop	2, b2a, -2*dist32+14*dist32, rdi, -64*ZMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer
	loops_reset			;; 4 more iters of 64
	zr7_seven_complex_djbfft_preload
b2b:	zr7_seven_complex_djbfft rsi, 128, 2*dist32, rdi, ZMM_SCD3, 32
	zloop	32, b2b, -32*128+dist32	;; Test loop counter, next source pointer
	zloop	2, b2b, -2*dist32+14*dist32, rdi, -64*ZMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	5, b2b, -5*14*dist32, rdi, 64*ZMM_SCD3 ;; Test loop counter, restore source pointer, next sin/cos pointer

;; Do FFT levels 8,9,10
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	512	+1	...
;;	8	...
;;	...
;;	504
;;	1K
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	64	+1	...
;;	8	...
;;	...
;;	56
;;	128
;;	...

;; Do 64 sixteen_reals_fft operations
;; Do 2176 eight_complex_fft operations
;;	distance between fft data elements is 64
;;	do 280 macros each processing 128 data values

	zloop_init 280			;; 35 iterations of 8
	zloop_set_rw_prefetch ZDOUBLE	;; Prefetch 560 cache lines
	zr8_2sc_sixteen_reals_fft_preload
b3a:	zr8_2sc_sixteen_reals_fft rsi, 128, 8*128, 16*128, dist32, rdx, ZMM_SCD4, rdi, ZMM_SCD4, 8
	zloop	8, b3a, -8*128+2*dist32, rdi, -8*ZMM_SCD4 ;; Test loop counter, next source pointer, restore sin/cos pointer
	loops_reset			;; 34 more iters of 8
	zr8_eight_complex_djbfft_preload
b3b:	zr8_eight_complex_djbfft rsi, 128, 8*128, 16*128, dist32, rdi, ZMM_SCD4, 8
	zloop	8, b3b, -8*128+2*dist32, rdi, -8*ZMM_SCD4 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	35, b3b, -35*2*dist32, rdi, 8*ZMM_SCD4 ;; Test loop counter, restore source pointer, next sin/cos pointer

;; Do FFT levels 11,12,13,14,15,16 as well as inverse FFT levels 11,12,13,14,15,16
;; On input and output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	64	+1	...
;;	8	...
;;	...
;;	56
;;	128
;;	...

;; Do 1 hundredtwentyeight_reals_fft operation
;; Do 279 sixtyfour_complex_fft operations
;;	distance between fft data elements is 1 (8 for swizzling)
;;	do 280 macros each processing 128 data values

;; Execute the proper middle step

pfing	bump	rcx, -60*64		;; We will prefetch 60 cache lines too many
	cmp	ffttype, 2
	jl	zpass2_17920_real_1
	jg	zpass2_17920_real_3

zpass2_17920_real_2:
	mov	r8, DEST2ARG
;;	zr64_hundredtwentyeight_real_with_square_preload
;;	zr64_hundredtwentyeight_real_with_square rsi, 8*128, 1*128, 2*128, 4*128, rdx, 0*ZMM_SCD4, rdi, 0*ZMM_SCD4, 1
	CALLP	zreal_square
	zloop_init 280, 1, 4		;; 279 more iterations
	zloop_set_rw_prefetch ZSEPT	;; Prefetch 1953 cache lines
	zr64_sixtyfour_complex_with_square_preload
b4a:	zr64_sixtyfour_complex_with_square rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 1, L1PREFETCH_ALL, 8*128
	zloop	4, b4a, -4*8*128+dist32	;; Test loop counter, next source pointer
	zloop	70, b4a			;; Test loop counter
	add	rsi, r8
	jmp	zpass2_17920_real_unfft

zpass2_17920_real_1:
;;	zr64_hundredtwentyeight_real_fft_final_preload
;;	zr64_hundredtwentyeight_real_fft_final rsi, 8*128, 1*128, 2*128, 4*128, rdx, 0*ZMM_SCD4, rdi, 0*ZMM_SCD4, 1
	CALLP	zreal_fft_final
	zloop_init 280, 1, 4		;; 279 more iterations
	zloop_set_rw_prefetch ZSEPT	;; Prefetch 1953 cache lines
	zr64_sixtyfour_complex_fft_final_preload
b4b:	zr64_sixtyfour_complex_fft_final rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 1, L1PREFETCH_ALL, 8*128
	zloop	4, b4b, -4*8*128+dist32	;; Test loop counter, next source pointer
	zloop	70, b4b			;; Test loop counter
	jmp	zpass2_17920_real_done

zpass2_17920_real_3:
	mov	r8, DEST2ARG
	mov	rbp, DIST_TO_MULSRCARG
;;	zr64_hundredtwentyeight_real_with_mult_preload
;;	zr64_hundredtwentyeight_real_with_mult rsi, 8*128, 1*128, 2*128, 4*128, rdx, 0*ZMM_SCD4, rdi, 0*ZMM_SCD4, 1
	CALLP	zreal_mult
	zloop_init 280, 1, 4		;; 279 more iterations
	zloop_set_rw_prefetch ZSEPT	;; Prefetch 1953 cache lines
	zr64_sixtyfour_complex_with_mult_preload
b4c:	zr64_sixtyfour_complex_with_mult rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 1, L1PREFETCH_ALL, 8*128
	zloop	4, b4c, -4*8*128+dist32	;; Test loop counter, next source pointer
	zloop	70, b4c			;; Test loop counter
	add	rsi, r8
	jmp	zpass2_17920_real_unfft

zpass2_17920_real_4:
	mov	rbp, DIST_TO_MULSRCARG
	mov	rdx, sincos3		;; Real sin/cos data
	mov	rdi, xsincos_complex	;; Complex sin/cos data
	bump	rdx, 448*ZMM_SCD2+64*ZMM_SCD3+8*ZMM_SCD4
	bump	rdi, 448*ZMM_SCD2+64*ZMM_SCD3+8*ZMM_SCD4
;;	zr64f_hundredtwentyeight_real_with_mulf_preload
;;	zr64f_hundredtwentyeight_real_with_mulf rsi, 8*128, 1*128, 2*128, 4*128, rdx, 0*ZMM_SCD4, rdi, 0*ZMM_SCD4, 1
	CALLP	zreal_mulf
	zloop_init 280, 1, 4		;; 279 more iterations
	zloop_set_rw_prefetch ZSEPT	;; Prefetch 1953 cache lines
	zr64f_sixtyfour_complex_with_mulf_preload
b4d:	zr64f_sixtyfour_complex_with_mulf rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 1, L1PREFETCH_ALL, 8*128
	zloop	4, b4d, -4*8*128+dist32	;; Test loop counter, next source pointer
	zloop	70, b4d			;; Test loop counter
zpass2_17920_real_unfft:
	bump	rsi, -70*dist32		;; Restore source pointer

;; Do inverse FFT levels 8,9,10
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	64	+1	...
;;	8	...
;;	...
;;	56
;;	128
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	512	+1	...
;;	8	...
;;	...
;;	504
;;	1K
;;	...

;; Do 64 sixteen_reals_unfft operations
;; Do 2176 eight_complex_unfft operations
;;	distance between fft data elements is 64
;;	do 280 macros each processing 128 data values

	zloop_init 280			;; 35 iterations of 8
	zloop_set_rw_prefetch ZDOUBLE	;; Prefetch 560 cache lines
	bump	rdx, -8*ZMM_SCD4
	bump	rdi, -8*ZMM_SCD4
	zr8_2sc_sixteen_reals_unfft_preload
c3a:	zr8_2sc_sixteen_reals_unfft rsi, 128, 8*128, 16*128, dist32, rdx, ZMM_SCD4, rdi, ZMM_SCD4, 8
	zloop	8, c3a, -8*128+2*dist32, rdi, -8*ZMM_SCD4 ;; Test loop counter, next source pointer, restore sin/cos pointer
	loops_reset			;; 34 more iters of 8
	zr8_eight_complex_djbunfft_preload
c3b:	zr8_eight_complex_djbunfft rsi, 128, 8*128, 16*128, dist32, rdi, ZMM_SCD4, 8
	zloop	8, c3b, -8*128+2*dist32, rdi, -8*ZMM_SCD4 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	35, c3b, -35*2*dist32	;; Test loop counter, restore source pointer

;; Do inverse FFT levels 5,6,7
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	512	+1	...
;;	8	...
;;	...
;;	504
;;	1K
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	3584	+1	...
;;	8	...
;;	...
;;	3576
;;	7168
;;	...

;; Do 512 fourteen_reals_unfft operations
;; Do 2048 seven_complex_unfft operations
;;	distance between fft data elements is 512
;;	do 320 macros each processing 112 data values

	zloop_init 320			;; 5 iterations of 64
	zloop_set_rw_prefetch ZSINGLE	;; Prefetch 320 cache lines
	bump	rdx, -8*ZMM_SCD4-64*ZMM_SCD3
	bump	rdi, -64*ZMM_SCD3
	zr7_2sc_fourteen_reals_unfft_preload
c2a:	zr7_2sc_fourteen_reals_unfft rsi, 128, 2*dist32, rdx, ZMM_SCD3, rdi, ZMM_SCD3, 32
	zloop	32, c2a, -32*128+dist32	;; Test loop counter, next source pointer
	zloop	2, c2a, -2*dist32+14*dist32, rdi, -64*ZMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer
	loops_reset			;; 4 more iters of 64
	zr7_seven_complex_djbunfft_preload
c2b:	zr7_seven_complex_djbunfft rsi, 128, 2*dist32, rdi, ZMM_SCD3, 32
	zloop	32, c2b, -32*128+dist32	;; Test loop counter, next source pointer
	zloop	2, c2b, -2*dist32+14*dist32, rdi, -64*ZMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	5, c2b, -5*14*dist32	;; Test loop counter, restore source pointer

;; Do inverse FFT levels 1,2,3,4
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	3584	+1	...
;;	8	...
;;	...
;;	3576
;;	7168
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	17920	+1	...
;;	8	...
;;	...
;;	17912

;; Do 3584 ten_reals_last_unfft operations
;;	distance between fft data elements is 3584
;;	do 448 macros each processing 80 data values

	bump	rdx, -64*ZMM_SCD3-448*ZMM_SCD2
	bump	rdi, -448*ZMM_SCD2
	zloop_init 448			;; 448 iterations
	zloop_set_rw_prefetch ZSINGLE	;; Prefetch 448 cache lines
	zr5_2sc_ten_reals_unfft_preload
c1b:	zr5_2sc_ten_reals_unfft rsi, 128, 14*dist32, rdx, ZMM_SCD2, rdi, ZMM_SCD2, 32
	zloop	32, c1b, -32*128+dist32	;; Test loop counter, next source pointer
	zloop	14, c1b			;; Test loop counter

zpass2_17920_real_done:
	end_timer 2
	ENDM

; *************** radix8-pass2-17920-complex macro ******************
; This macro takes 17920 complex values and performs the final 14.129 levels of a standard
; radix-8 FFT, squares the results, and does the first 14.129 levels of the inverse FFT.

zr4dwpn_pass2_17920_complex MACRO
	LOCAL	b1b, b2b, b3b, b4a, b4b, b4c, b4d, c1b, c2b, c3b
	LOCAL	zpass2_17920_complex_1, zpass2_17920_complex_2
	LOCAL	zpass2_17920_complex_3, zpass2_17920_complex_4
	LOCAL	zpass2_17920_complex_unfft, zpass2_17920_complex_done

;; Type 4 FFTs skip the forward FFT process

	mov	rsi, DATA_ADDR		;; Load address of FFT data
	mov	rbx, DIST_TO_FFTSRCARG
	mov	rcx, DATA_PREFETCH	;; Prefetch pointer (we need to prefetch 4480 cache lines + 69 pad cache lines)
	cmp	ffttype, 4
	je	zpass2_17920_complex_4

;; Do FFT levels 1,2,3
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	17920	+1	...
;;	8	...
;;	...
;;	17912
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	3584	+1	...
;;	8	...
;;	...
;;	3576
;;	7168
;;	...

;; Do 3584 five_complex_fft operations
;;	distance between fft data elements is 3584
;;	do 448 macros each processing 80 data values

	start_timer 4
	mov	rdi, xsincos_complex	;; The sin/cos data
	zloop_init 448			;; 448 iterations
	zloop_set_rw_prefetch ZSINGLE	;; Prefetch 448 cache lines
	zr5f_five_complex_djbfft_preload
b1b:	zr5f_five_complex_djbfft rsi, 128, 14*dist32, rdi, ZMM_SCD2, 32, L1PREFETCH_ALL, 128
	zloop	32, b1b, -32*128+dist32	;; Test loop counter, next source pointer
	zloop	14, b1b, -14*dist32	;; Test loop counter, restore source pointer
	end_timer 4

;; Do FFT levels 4,5,6
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	3584	+1	...
;;	8	...
;;	...
;;	3576
;;	7168
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	512	+1	...
;;	8	...
;;	...
;;	504
;;	1K
;;	...

;; Do 2560 seven_complex_fft operations
;;	distance between fft data elements is 512
;;	do 320 macros each processing 112 data values

	start_timer 5
	zloop_init 320			;; 5 iterations of 64
	zloop_set_rw_prefetch ZSINGLE	;; Prefetch 320 cache lines
	zr7_seven_complex_djbfft_preload
b2b:	zr7_seven_complex_djbfft rsi, 128, 2*dist32, rdi, ZMM_SCD3, 32, L1PREFETCH_ALL, 128
	zloop	32, b2b, -32*128+dist32	;; Test loop counter, next source pointer
	zloop	2, b2b, -2*dist32+14*dist32, rdi, -64*ZMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	5, b2b, -5*14*dist32, rdi, 64*ZMM_SCD3 ;; Test loop counter, restore source pointer, next sin/cos pointer
	end_timer 5

;; Do FFT levels 7,8,9
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	512	+1	...
;;	8	...
;;	...
;;	504
;;	1K
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	64	+1	...
;;	8	...
;;	...
;;	56
;;	128
;;	...

;; Do 2240 eight_complex_fft operations
;;	distance between fft data elements is 64
;;	do 280 macros each processing 128 data values

	start_timer 6
	zloop_init 280			;; 35 iterations of 8
	zloop_set_rw_prefetch ZDOUBLE	;; Prefetch 560 cache lines
	zr8_eight_complex_djbfft_preload
b3b:	zr8_eight_complex_djbfft rsi, 128, 8*128, 16*128, dist32, rdi, ZMM_SCD4, 8, L1PREFETCH_ALL, 128
	zloop	8, b3b, -8*128+2*dist32, rdi, -8*ZMM_SCD4 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	35, b3b, -35*2*dist32, rdi, 8*ZMM_SCD4 ;; Test loop counter, restore source pointer, next sin/cos pointer
	end_timer 6

;; Do FFT levels 10,11,12,13,14,15 as well as inverse FFT levels 10,11,12,13,14,15
;; On input and output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	64	+1	...
;;	8	...
;;	...
;;	56
;;	128
;;	...

;; Do 280 sixtyfour_complex_fft operations
;;	distance between fft data elements is 1 (8 for swizzling)
;;	do 280 macros each processing 128 data values

;; Execute the right middle step

pfing	bump	rcx, -67*64		;; We will prefetch 67 cache lines too many
	cmp	ffttype, 2
	jl	zpass2_17920_complex_1
	jg	zpass2_17920_complex_3

zpass2_17920_complex_2:
	start_timer 9
	zloop_init 280			;; 280 iterations
	zloop_set_rw_prefetch ZSEPT	;; Prefetch 1960 cache lines
	mov	r8, DEST2ARG
	zr64_sixtyfour_complex_with_square_preload
b4a:	zr64_sixtyfour_complex_with_square rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 4, L1PREFETCH_ALL, 8*128
	zloop	4, b4a, -4*8*128+dist32	;; Test loop counter, next source pointer
	zloop	70, b4a			;; Test loop counter
	add	rsi, r8
	end_timer 9
	jmp	zpass2_17920_complex_unfft

zpass2_17920_complex_1:
	start_timer 9
	zloop_init 280			;; 280 iterations
	zloop_set_rw_prefetch ZSEPT	;; Prefetch 1960 cache lines
	zr64_sixtyfour_complex_fft_final_preload
b4b:	zr64_sixtyfour_complex_fft_final rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 4, L1PREFETCH_ALL, 8*128
	zloop	4, b4b, -4*8*128+dist32	;; Test loop counter, next source pointer
	zloop	70, b4b			;; Test loop counter
	end_timer 9
	jmp	zpass2_17920_complex_done

zpass2_17920_complex_3:
	start_timer 9
	zloop_init 280			;; 280 iterations
	zloop_set_rw_prefetch ZSEPT	;; Prefetch 1960 cache lines
	mov	r8, DEST2ARG
	mov	rbp, DIST_TO_MULSRCARG
	zr64_sixtyfour_complex_with_mult_preload
b4c:	zr64_sixtyfour_complex_with_mult rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 4, L1PREFETCH_ALL, 8*128
	zloop	4, b4c, -4*8*128+dist32	;; Test loop counter, next source pointer
	zloop	70, b4c			;; Test loop counter
	add	rsi, r8
	end_timer 9
	jmp	zpass2_17920_complex_unfft

zpass2_17920_complex_4:
	start_timer 9
	zloop_init 280			;; 280 iterations
	zloop_set_rw_prefetch ZSEPT	;; Prefetch 1960 cache lines
	mov	rbp, DIST_TO_MULSRCARG
	mov	rdi, xsincos_complex	;; The sin/cos data
	bump	rdi, 448*ZMM_SCD2+64*ZMM_SCD3+8*ZMM_SCD4
	zr64f_sixtyfour_complex_with_mulf_preload
b4d:	zr64f_sixtyfour_complex_with_mulf rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 4, L1PREFETCH_ALL, 8*128
	zloop	4, b4d, -4*8*128+dist32	;; Test loop counter, next source pointer
	zloop	70, b4d			;; Test loop counter
	end_timer 9
zpass2_17920_complex_unfft:
	bump	rsi, -70*dist32		;; Restore source pointer

;; Do inverse FFT levels 7,8,9
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	64	+1	...
;;	8	...
;;	...
;;	56
;;	128
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	512	+1	...
;;	8	...
;;	...
;;	504
;;	1K
;;	...

;; Do 2240 eight_complex_unfft operations
;;	distance between fft data elements is 64
;;	do 280 macros each processing 128 data values

	start_timer 12
	bump	rdi, -8*ZMM_SCD4
	zloop_init 280			;; 35 iterations of 8
	zloop_set_rw_prefetch ZDOUBLE	;; Prefetch 560 cache lines
	zr8_eight_complex_djbunfft_preload
c3b:	zr8_eight_complex_djbunfft rsi, 128, 8*128, 16*128, dist32, rdi, ZMM_SCD4, 8, L1PREFETCH_ALL, 128
	zloop	8, c3b, -8*128+2*dist32, rdi, -8*ZMM_SCD4 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	35, c3b, -35*2*dist32	;; Test loop counter, restore source pointer
	end_timer 12

;; Do inverse FFT levels 4,5,6
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	512	+1	...
;;	8	...
;;	...
;;	504
;;	1K
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	3584	+1	...
;;	8	...
;;	...
;;	3576
;;	7168
;;	...

;; Do 2560 seven_complex_unfft operations
;;	distance between fft data elements is 512
;;	do 320 macros each processing 112 data values

	start_timer 13
	bump	rdi, -64*ZMM_SCD3
	zloop_init 320			;; 5 iterations of 64
	zloop_set_rw_prefetch ZSINGLE	;; Prefetch 320 cache lines
	zr7_seven_complex_djbunfft_preload
c2b:	zr7_seven_complex_djbunfft rsi, 128, 2*dist32, rdi, ZMM_SCD3, 32, L1PREFETCH_ALL, 128
	zloop	32, c2b, -32*128+dist32	;; Test loop counter, next source pointer
	zloop	2, c2b, -2*dist32+14*dist32, rdi, -64*ZMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	5, c2b, -5*14*dist32	;; Test loop counter, restore source pointer
	end_timer 13

;; Do inverse FFT levels 1,2,3
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	3584	+1	...
;;	8	...
;;	...
;;	3576
;;	7168
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	17920	+1	...
;;	8	...
;;	...
;;	17912

;; Do 3584 five_complex_unfft operations
;;	distance between fft data elements is 3584
;;	do 448 macros each processing 80 data values

	start_timer 14
	bump	rdi, -448*ZMM_SCD2
	zloop_init 448			;; 448 iterations
	zloop_set_rw_prefetch ZSINGLE	;; Prefetch 448 cache lines
	zr5_five_complex_djbunfft_preload
c1b:	zr5_five_complex_djbunfft rsi, 128, 14*dist32, rdi, ZMM_SCD2, 32, L1PREFETCH_ALL, 128
	zloop	32, c1b, -32*128+dist32	;; Test loop counter, next source pointer
	zloop	14, c1b			;; Test loop counter
	end_timer 14

zpass2_17920_complex_done:
	ENDM


; *************** radix8-pass2-18432-real macro ******************
; This macro takes 36864 real values and performs the final 15.17 levels of the FFT.
; The results are squared, and the first 15.17 levels of the inverse FFT are done.

zr4dwpn_pass2_18432_real MACRO
	LOCAL	b1b, b2a, b2b, b3a, b3b, b4a, b4b, b4c, b4d, c1b, c2a, c2b, c3a, c3b
	LOCAL	zpass2_18432_real_1, zpass2_18432_real_2
	LOCAL	zpass2_18432_real_3, zpass2_18432_real_4
	LOCAL	zpass2_18432_real_unfft, zpass2_18432_real_done

;; Type 4 FFTs skip the forward FFT process

	start_timer 2
	mov	rsi, DATA_ADDR		;; Load source address
	mov	rbx, DIST_TO_FFTSRCARG
	mov	rcx, DATA_PREFETCH	;; Prefetch pointer (we need to prefetch 4608 cache lines + ??? pad cache lines)
	cmp	ffttype, 4
	je	zpass2_18432_real_4

;; Do FFT levels 1,2,3,4
;;	0	+1	+1	+1	+1	+1	+1	+1	18K	+1	...
;;	8	...
;;	...
;;	18K-8
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	3K	+1	...
;;	8	...
;;	...
;;	3K-8
;;	6K
;;	...

;; Do 3072 twelve_real_fft operations
;;	distance between fft data elements is 3K
;;	do 384 macros each processing 96 data values

	mov	rdx, sincos3		;; Real sin/cos data
	mov	rdi, xsincos_complex	;; Complex sin/cos data
	zloop_init 384			;; 384 iterations
	zloop_set_rw_prefetch ZSINGLE	;; Prefetch 384 cache lines
	zr6f_2sc_twelve_reals_fft_preload
b1b:	zr6f_2sc_twelve_reals_fft rsi, 128, 12*dist32, rdx, ZMM_SCD3, rdi, ZMM_SCD3, 32
	zloop	32, b1b, -32*128+dist32	;; Test loop counter, next source pointer
	zloop	12, b1b, -12*dist32	;; Test loop counter, restore source pointer

;; Do FFT levels 5,6,7
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	3K	+1	...
;;	8	...
;;	...
;;	3K-8
;;	6K
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	512	+1	...
;;	8	...
;;	...
;;	504
;;	1K
;;	...

;; Do 512 twelve_reals_fft operations
;; Do 2560 six_complex_fft operations
;;	distance between fft data elements is 512
;;	do 384 macros each processing 96 data values

	zloop_init 384			;; 6 iterations of 64
	zloop_set_rw_prefetch ZSINGLE	;; Prefetch 384 cache lines
	zr6_2sc_twelve_reals_fft_preload
b2a:	zr6_2sc_twelve_reals_fft rsi, 128, 2*dist32, rdx, ZMM_SCD3, rdi, ZMM_SCD3, 32
	zloop	32, b2a, -32*128+dist32	;; Test loop counter, next source pointer
	zloop	2, b2a, -2*dist32+12*dist32, rdi, -64*ZMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer
	loops_reset			;; 5 more iters of 64
	zr6_six_complex_djbfft_preload
b2b:	zr6_six_complex_djbfft rsi, 128, 2*dist32, rdi, ZMM_SCD3, 32
	zloop	32, b2b, -32*128+dist32	;; Test loop counter, next source pointer
	zloop	2, b2b, -2*dist32+12*dist32, rdi, -64*ZMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	6, b2b, -6*12*dist32, rdi, 64*ZMM_SCD3 ;; Test loop counter, restore source pointer, next sin/cos pointer

;; Do FFT levels 8,9,10
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	512	+1	...
;;	8	...
;;	...
;;	504
;;	1K
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	64	+1	...
;;	8	...
;;	...
;;	56
;;	128
;;	...

;; Do 64 sixteen_reals_fft operations
;; Do 2240 eight_complex_fft operations
;;	distance between fft data elements is 64
;;	do 288 macros each processing 128 data values

	zloop_init 288			;; 36 iterations of 8
	zloop_set_rw_prefetch ZDOUBLE	;; Prefetch 576 cache lines
	zr8_2sc_sixteen_reals_fft_preload
b3a:	zr8_2sc_sixteen_reals_fft rsi, 128, 8*128, 16*128, dist32, rdx, ZMM_SCD4, rdi, ZMM_SCD4, 8
	zloop	8, b3a, -8*128+2*dist32, rdi, -8*ZMM_SCD4 ;; Test loop counter, next source pointer, restore sin/cos pointer
	loops_reset			;; 35 more iters of 8
	zr8_eight_complex_djbfft_preload
b3b:	zr8_eight_complex_djbfft rsi, 128, 8*128, 16*128, dist32, rdi, ZMM_SCD4, 8
	zloop	8, b3b, -8*128+2*dist32, rdi, -8*ZMM_SCD4 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	36, b3b, -36*2*dist32, rdi, 8*ZMM_SCD4 ;; Test loop counter, restore source pointer, next sin/cos pointer

;; Do FFT levels 11,12,13,14,15,16 as well as inverse FFT levels 11,12,13,14,15,16
;; On input and output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	64	+1	...
;;	8	...
;;	...
;;	56
;;	128
;;	...

;; Do 1 hundredtwentyeight_reals_fft operation
;; Do 287 sixtyfour_complex_fft operations
;;	distance between fft data elements is 1 (8 for swizzling)
;;	do 288 macros each processing 128 data values

;; Execute the proper middle step

pfing	bump	rcx, -18*64		;; We will prefetch 18 cache lines too many
	cmp	ffttype, 2
	jl	zpass2_18432_real_1
	jg	zpass2_18432_real_3

zpass2_18432_real_2:
	mov	r8, DEST2ARG
;;	zr64_hundredtwentyeight_real_with_square_preload
;;	zr64_hundredtwentyeight_real_with_square rsi, 8*128, 1*128, 2*128, 4*128, rdx, 0*ZMM_SCD4, rdi, 0*ZMM_SCD4, 1
	CALLP	zreal_square
	zloop_init 288, 1, 4		;; 287 more iterations
	zloop_set_rw_prefetch ZSEPT	;; Prefetch 2009 cache lines
	zr64_sixtyfour_complex_with_square_preload
b4a:	zr64_sixtyfour_complex_with_square rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 1, L1PREFETCH_ALL, 8*128
	zloop	4, b4a, -4*8*128+dist32	;; Test loop counter, next source pointer
	zloop	72, b4a			;; Test loop counter
	add	rsi, r8
	jmp	zpass2_18432_real_unfft

zpass2_18432_real_1:
;;	zr64_hundredtwentyeight_real_fft_final_preload
;;	zr64_hundredtwentyeight_real_fft_final rsi, 8*128, 1*128, 2*128, 4*128, rdx, 0*ZMM_SCD4, rdi, 0*ZMM_SCD4, 1
	CALLP	zreal_fft_final
	zloop_init 288, 1, 4		;; 287 more iterations
	zloop_set_rw_prefetch ZSEPT	;; Prefetch 2009 cache lines
	zr64_sixtyfour_complex_fft_final_preload
b4b:	zr64_sixtyfour_complex_fft_final rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 1, L1PREFETCH_ALL, 8*128
	zloop	4, b4b, -4*8*128+dist32	;; Test loop counter, next source pointer
	zloop	72, b4b			;; Test loop counter
	jmp	zpass2_18432_real_done

zpass2_18432_real_3:
	mov	r8, DEST2ARG
	mov	rbp, DIST_TO_MULSRCARG
;;	zr64_hundredtwentyeight_real_with_mult_preload
;;	zr64_hundredtwentyeight_real_with_mult rsi, 8*128, 1*128, 2*128, 4*128, rdx, 0*ZMM_SCD4, rdi, 0*ZMM_SCD4, 1
	CALLP	zreal_mult
	zloop_init 288, 1, 4		;; 287 more iterations
	zloop_set_rw_prefetch ZSEPT	;; Prefetch 2009 cache lines
	zr64_sixtyfour_complex_with_mult_preload
b4c:	zr64_sixtyfour_complex_with_mult rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 1, L1PREFETCH_ALL, 8*128
	zloop	4, b4c, -4*8*128+dist32	;; Test loop counter, next source pointer
	zloop	72, b4c			;; Test loop counter
	add	rsi, r8
	jmp	zpass2_18432_real_unfft

zpass2_18432_real_4:
	mov	rbp, DIST_TO_MULSRCARG
	mov	rdx, sincos3		;; Real sin/cos data
	mov	rdi, xsincos_complex	;; Complex sin/cos data
	bump	rdx, 384*ZMM_SCD3+64*ZMM_SCD3+8*ZMM_SCD4
	bump	rdi, 384*ZMM_SCD3+64*ZMM_SCD3+8*ZMM_SCD4
;;	zr64f_hundredtwentyeight_real_with_mulf_preload
;;	zr64f_hundredtwentyeight_real_with_mulf rsi, 8*128, 1*128, 2*128, 4*128, rdx, 0*ZMM_SCD4, rdi, 0*ZMM_SCD4, 1
	CALLP	zreal_mulf
	zloop_init 288, 1, 4		;; 287 more iterations
	zloop_set_rw_prefetch ZSEPT	;; Prefetch 2009 cache lines
	zr64f_sixtyfour_complex_with_mulf_preload
b4d:	zr64f_sixtyfour_complex_with_mulf rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 1, L1PREFETCH_ALL, 8*128
	zloop	4, b4d, -4*8*128+dist32	;; Test loop counter, next source pointer
	zloop	72, b4d			;; Test loop counter
zpass2_18432_real_unfft:
	bump	rsi, -72*dist32		;; Restore source pointer

;; Do inverse FFT levels 8,9,10
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	64	+1	...
;;	8	...
;;	...
;;	56
;;	128
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	512	+1	...
;;	8	...
;;	...
;;	504
;;	1K
;;	...

;; Do 64 sixteen_reals_unfft operations
;; Do 2240 eight_complex_unfft operations
;;	distance between fft data elements is 64
;;	do 288 macros each processing 128 data values

	zloop_init 288			;; 36 iterations of 8
	zloop_set_rw_prefetch ZDOUBLE	;; Prefetch 576 cache lines
	bump	rdx, -8*ZMM_SCD4
	bump	rdi, -8*ZMM_SCD4
	zr8_2sc_sixteen_reals_unfft_preload
c3a:	zr8_2sc_sixteen_reals_unfft rsi, 128, 8*128, 16*128, dist32, rdx, ZMM_SCD4, rdi, ZMM_SCD4, 8
	zloop	8, c3a, -8*128+2*dist32, rdi, -8*ZMM_SCD4 ;; Test loop counter, next source pointer, restore sin/cos pointer
	loops_reset			;; 35 more iters of 8
	zr8_eight_complex_djbunfft_preload
c3b:	zr8_eight_complex_djbunfft rsi, 128, 8*128, 16*128, dist32, rdi, ZMM_SCD4, 8
	zloop	8, c3b, -8*128+2*dist32, rdi, -8*ZMM_SCD4 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	36, c3b, -36*2*dist32	;; Test loop counter, restore source pointer

;; Do inverse FFT levels 5,6,7
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	512	+1	...
;;	8	...
;;	...
;;	504
;;	1K
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	3K	+1	...
;;	8	...
;;	...
;;	3K-8
;;	6K
;;	...

;; Do 512 twelve_reals_unfft operations
;; Do 2560 six_complex_unfft operations
;;	distance between fft data elements is 512
;;	do 384 macros each processing 96 data values

	zloop_init 384			;; 6 iterations of 64
	zloop_set_rw_prefetch ZSINGLE	;; Prefetch 384 cache lines
	bump	rdx, -8*ZMM_SCD4-64*ZMM_SCD3
	bump	rdi, -64*ZMM_SCD3
	zr6_2sc_twelve_reals_unfft_preload
c2a:	zr6_2sc_twelve_reals_unfft rsi, 128, 2*dist32, rdx, ZMM_SCD3, rdi, ZMM_SCD3, 32
	zloop	32, c2a, -32*128+dist32	;; Test loop counter, next source pointer
	zloop	2, c2a, -2*dist32+12*dist32, rdi, -64*ZMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer
	loops_reset			;; 5 more iters of 64
	zr6_six_complex_djbunfft_preload
c2b:	zr6_six_complex_djbunfft rsi, 128, 2*dist32, rdi, ZMM_SCD3, 32
	zloop	32, c2b, -32*128+dist32	;; Test loop counter, next source pointer
	zloop	2, c2b, -2*dist32+12*dist32, rdi, -64*ZMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	6, c2b, -6*12*dist32	;; Test loop counter, restore source pointer

;; Do inverse FFT levels 1,2,3,4
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	3K	+1	...
;;	8	...
;;	...
;;	3K-8
;;	6K
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	18K	+1	...
;;	8	...
;;	...
;;	18K-8

;; Do 3072 twelve_reals_last_unfft operations
;;	distance between fft data elements is 3K
;;	do 384 macros each processing 96 data values

	bump	rdx, -64*ZMM_SCD3-384*ZMM_SCD3
	bump	rdi, -384*ZMM_SCD3
	zloop_init 384			;; 384 iterations
	zloop_set_rw_prefetch ZSINGLE	;; Prefetch 384 cache lines
	zr6_2sc_twelve_reals_unfft_preload
c1b:	zr6_2sc_twelve_reals_unfft rsi, 128, 12*dist32, rdx, ZMM_SCD3, rdi, ZMM_SCD3, 32
	zloop	32, c1b, -32*128+dist32	;; Test loop counter, next source pointer
	zloop	12, c1b			;; Test loop counter

zpass2_18432_real_done:
	end_timer 2
	ENDM

; *************** radix8-pass2-18432-complex macro ******************
; This macro takes 18432 complex values and performs the final 14.17 levels of a standard
; radix-8 FFT, squares the results, and does the first 14.17 levels of the inverse FFT.

zr4dwpn_pass2_18432_complex MACRO
	LOCAL	b1b, b2b, b3b, b4a, b4b, b4c, b4d, c1b, c2b, c3b
	LOCAL	zpass2_18432_complex_1, zpass2_18432_complex_2
	LOCAL	zpass2_18432_complex_3, zpass2_18432_complex_4
	LOCAL	zpass2_18432_complex_unfft, zpass2_18432_complex_done

;; Type 4 FFTs skip the forward FFT process

	mov	rsi, DATA_ADDR		;; Load address of FFT data
	mov	rbx, DIST_TO_FFTSRCARG
	mov	rcx, DATA_PREFETCH	;; Prefetch pointer (we need to prefetch 4608 cache lines + 71 pad cache lines)
	cmp	ffttype, 4
	je	zpass2_18432_complex_4

;; Do FFT levels 1,2,3
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	18K	+1	...
;;	8	...
;;	...
;;	18K-8
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	3K	+1	...
;;	8	...
;;	...
;;	3K-8
;;	6K
;;	...

;; Do 3072 six_complex_fft operations
;;	distance between fft data elements is 3K
;;	do 384 macros each processing 96 data values

	start_timer 4
	mov	rdi, xsincos_complex	;; The sin/cos data
	zloop_init 384			;; 384 iterations
	zloop_set_rw_prefetch ZSINGLE	;; Prefetch 384 cache lines
	zr6f_six_complex_djbfft_preload
b1b:	zr6f_six_complex_djbfft rsi, 128, 12*dist32, rdi, ZMM_SCD3, 32, L1PREFETCH_ALL, 128
	zloop	32, b1b, -32*128+dist32	;; Test loop counter, next source pointer
	zloop	12, b1b, -12*dist32	;; Test loop counter, restore source pointer
	end_timer 4

;; Do FFT levels 4,5,6
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	3K	+1	...
;;	8	...
;;	...
;;	3K-8
;;	6K
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	512	+1	...
;;	8	...
;;	...
;;	504
;;	1K
;;	...

;; Do 3072 six_complex_fft operations
;;	distance between fft data elements is 512
;;	do 384 macros each processing 96 data values

	start_timer 5
	zloop_init 384			;; 6 iterations of 64
	zloop_set_rw_prefetch ZSINGLE	;; Prefetch 384 cache lines
	zr6_six_complex_djbfft_preload
b2b:	zr6_six_complex_djbfft rsi, 128, 2*dist32, rdi, ZMM_SCD3, 32, L1PREFETCH_ALL, 128
	zloop	32, b2b, -32*128+dist32	;; Test loop counter, next source pointer
	zloop	2, b2b, -2*dist32+12*dist32, rdi, -64*ZMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	6, b2b, -6*12*dist32, rdi, 64*ZMM_SCD3 ;; Test loop counter, restore source pointer, next sin/cos pointer
	end_timer 5

;; Do FFT levels 7,8,9
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	512	+1	...
;;	8	...
;;	...
;;	504
;;	1K
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	64	+1	...
;;	8	...
;;	...
;;	56
;;	128
;;	...

;; Do 2304 eight_complex_fft operations
;;	distance between fft data elements is 64
;;	do 288 macros each processing 128 data values

	start_timer 6
	zloop_init 288			;; 36 iterations of 8
	zloop_set_rw_prefetch ZDOUBLE	;; Prefetch 576 cache lines
	zr8_eight_complex_djbfft_preload
b3b:	zr8_eight_complex_djbfft rsi, 128, 8*128, 16*128, dist32, rdi, ZMM_SCD4, 8, L1PREFETCH_ALL, 128
	zloop	8, b3b, -8*128+2*dist32, rdi, -8*ZMM_SCD4 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	36, b3b, -36*2*dist32, rdi, 8*ZMM_SCD4 ;; Test loop counter, restore source pointer, next sin/cos pointer
	end_timer 6

;; Do FFT levels 10,11,12,13,14,15 as well as inverse FFT levels 10,11,12,13,14,15
;; On input and output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	64	+1	...
;;	8	...
;;	...
;;	56
;;	128
;;	...

;; Do 288 sixtyfour_complex_fft operations
;;	distance between fft data elements is 1 (8 for swizzling)
;;	do 288 macros each processing 128 data values

;; Execute the right middle step

pfing	bump	rcx, -25*64		;; We will prefetch 25 cache lines too many
	cmp	ffttype, 2
	jl	zpass2_18432_complex_1
	jg	zpass2_18432_complex_3

zpass2_18432_complex_2:
	start_timer 9
	zloop_init 288			;; 288 iterations
	zloop_set_rw_prefetch ZSEPT	;; Prefetch 2016 cache lines
	mov	r8, DEST2ARG
	zr64_sixtyfour_complex_with_square_preload
b4a:	zr64_sixtyfour_complex_with_square rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 4, L1PREFETCH_ALL, 8*128
	zloop	4, b4a, -4*8*128+dist32	;; Test loop counter, next source pointer
	zloop	72, b4a			;; Test loop counter
	add	rsi, r8
	end_timer 9
	jmp	zpass2_18432_complex_unfft

zpass2_18432_complex_1:
	start_timer 9
	zloop_init 288			;; 288 iterations
	zloop_set_rw_prefetch ZSEPT	;; Prefetch 2016 cache lines
	zr64_sixtyfour_complex_fft_final_preload
b4b:	zr64_sixtyfour_complex_fft_final rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 4, L1PREFETCH_ALL, 8*128
	zloop	4, b4b, -4*8*128+dist32	;; Test loop counter, next source pointer
	zloop	72, b4b			;; Test loop counter
	end_timer 9
	jmp	zpass2_18432_complex_done

zpass2_18432_complex_3:
	start_timer 9
	zloop_init 288			;; 288 iterations
	zloop_set_rw_prefetch ZSEPT	;; Prefetch 2016 cache lines
	mov	r8, DEST2ARG
	mov	rbp, DIST_TO_MULSRCARG
	zr64_sixtyfour_complex_with_mult_preload
b4c:	zr64_sixtyfour_complex_with_mult rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 4, L1PREFETCH_ALL, 8*128
	zloop	4, b4c, -4*8*128+dist32	;; Test loop counter, next source pointer
	zloop	72, b4c			;; Test loop counter
	add	rsi, r8
	end_timer 9
	jmp	zpass2_18432_complex_unfft

zpass2_18432_complex_4:
	start_timer 9
	zloop_init 288			;; 288 iterations
	zloop_set_rw_prefetch ZSEPT	;; Prefetch 2016 cache lines
	mov	rbp, DIST_TO_MULSRCARG
	mov	rdi, xsincos_complex	;; The sin/cos data
	bump	rdi, 384*ZMM_SCD3+64*ZMM_SCD3+8*ZMM_SCD4
	zr64f_sixtyfour_complex_with_mulf_preload
b4d:	zr64f_sixtyfour_complex_with_mulf rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 4, L1PREFETCH_ALL, 8*128
	zloop	4, b4d, -4*8*128+dist32	;; Test loop counter, next source pointer
	zloop	72, b4d			;; Test loop counter
	end_timer 9
zpass2_18432_complex_unfft:
	bump	rsi, -72*dist32		;; Restore source pointer

;; Do inverse FFT levels 7,8,9
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	64	+1	...
;;	8	...
;;	...
;;	56
;;	128
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	512	+1	...
;;	8	...
;;	...
;;	504
;;	1K
;;	...

;; Do 2304 eight_complex_unfft operations
;;	distance between fft data elements is 64
;;	do 288 macros each processing 128 data values

	start_timer 12
	bump	rdi, -8*ZMM_SCD4
	zloop_init 288			;; 36 iterations of 8
	zloop_set_rw_prefetch ZDOUBLE	;; Prefetch 576 cache lines
	zr8_eight_complex_djbunfft_preload
c3b:	zr8_eight_complex_djbunfft rsi, 128, 8*128, 16*128, dist32, rdi, ZMM_SCD4, 8, L1PREFETCH_ALL, 128
	zloop	8, c3b, -8*128+2*dist32, rdi, -8*ZMM_SCD4 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	36, c3b, -36*2*dist32	;; Test loop counter, restore source pointer
	end_timer 12

;; Do inverse FFT levels 4,5,6
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	512	+1	...
;;	8	...
;;	...
;;	504
;;	1K
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	3K	+1	...
;;	8	...
;;	...
;;	3K-8
;;	6K
;;	...

;; Do 3072 six_complex_unfft operations
;;	distance between fft data elements is 512
;;	do 384 macros each processing 96 data values

	start_timer 13
	bump	rdi, -64*ZMM_SCD3
	zloop_init 384			;; 6 iterations of 64
	zloop_set_rw_prefetch ZSINGLE	;; Prefetch 384 cache lines
	zr6_six_complex_djbunfft_preload
c2b:	zr6_six_complex_djbunfft rsi, 128, 2*dist32, rdi, ZMM_SCD3, 32, L1PREFETCH_ALL, 128
	zloop	32, c2b, -32*128+dist32	;; Test loop counter, next source pointer
	zloop	2, c2b, -2*dist32+12*dist32, rdi, -64*ZMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	6, c2b, -6*12*dist32	;; Test loop counter, restore source pointer
	end_timer 13

;; Do inverse FFT levels 1,2,3
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	3K	+1	...
;;	8	...
;;	...
;;	3K-8
;;	6K
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	18K	+1	...
;;	8	...
;;	...
;;	18K-8

;; Do 3072 six_complex_unfft operations
;;	distance between fft data elements is 3K
;;	do 384 macros each processing 96 data values

	start_timer 14
	bump	rdi, -384*ZMM_SCD3
	zloop_init 384			;; 384 iterations
	zloop_set_rw_prefetch ZSINGLE	;; Prefetch 384 cache lines
	zr6_six_complex_djbunfft_preload
c1b:	zr6_six_complex_djbunfft rsi, 128, 12*dist32, rdi, ZMM_SCD3, 32, L1PREFETCH_ALL, 128
	zloop	32, c1b, -32*128+dist32	;; Test loop counter, next source pointer
	zloop	12, c1b			;; Test loop counter
	end_timer 14

zpass2_18432_complex_done:
	ENDM


; *************** radix8-pass2-20480-real macro ******************
; This macro takes 40960 real values and performs the final 15.322 levels of the FFT.
; The results are squared, and the first 15.322 levels of the inverse FFT are done.

zr4dwpn_pass2_20480_real MACRO
	LOCAL	b1b, b2a, b2b, b3a, b3b, b4a, b4b, b4c, b4d, c1b, c2a, c2b, c3a, c3b
	LOCAL	zpass2_20480_real_1, zpass2_20480_real_2
	LOCAL	zpass2_20480_real_3, zpass2_20480_real_4
	LOCAL	zpass2_20480_real_unfft, zpass2_20480_real_done

;; Type 4 FFTs skip the forward FFT process

	start_timer 2
	mov	rsi, DATA_ADDR		;; Load source address
	mov	rbx, DIST_TO_FFTSRCARG
	mov	rcx, DATA_PREFETCH	;; Prefetch pointer (we need to prefetch 5120 cache lines + 79 pad cache lines)
	cmp	ffttype, 4
	je	zpass2_20480_real_4

;; Do FFT levels 1,2,3,4
;;	0	+1	+1	+1	+1	+1	+1	+1	20K	+1	...
;;	8	...
;;	...
;;	20K-8
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	4K	+1	...
;;	8	...
;;	...
;;	4K-8
;;	8K
;;	...

;; Do 4096 ten_real_fft operations
;;	distance between fft data elements is 4K
;;	do 512 macros each processing 80 data values

	mov	rdx, sincos3		;; Real sin/cos data
	mov	rdi, xsincos_complex	;; Complex sin/cos data
	zloop_init 512			;; 512 iterations
	zloop_set_rw_prefetch ZSINGLE	;; Prefetch 512 cache lines
	zr5f_2sc_ten_reals_fft_preload
b1b:	zr5f_2sc_ten_reals_fft rsi, 128, 16*dist32, rdx, ZMM_SCD2, rdi, ZMM_SCD2, 32
	zloop	32, b1b, -32*128+dist32	;; Test loop counter, next source pointer
	zloop	16, b1b, -16*dist32	;; Test loop counter, restore source pointer

;; Do FFT levels 5,6,7
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	4K	+1	...
;;	8	...
;;	...
;;	4K-8
;;	8K
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	512	+1	...
;;	8	...
;;	...
;;	504
;;	1K
;;	...

;; Do 512 sixteen_reals_fft operations
;; Do 2048 eight_complex_fft operations
;;	distance between fft data elements is 512
;;	do 320 macros each processing 128 data values

	zloop_init 320			;; 5 iterations of 64
	zloop_set_rw_prefetch ZDOUBLE	;; Prefetch 640 cache lines
	zr8_2sc_sixteen_reals_fft_preload
b2a:	zr8_2sc_sixteen_reals_fft rsi, 128, 2*dist32, 4*dist32, 8*dist32, rdx, ZMM_SCD4, rdi, ZMM_SCD4, 32
	zloop	32, b2a, -32*128+dist32	;; Test loop counter, next source pointer
	zloop	2, b2a, -2*dist32+16*dist32, rdi, -64*ZMM_SCD4 ;; Test loop counter, next source pointer, restore sin/cos pointer
	loops_reset			;; 4 more iters of 64
	zr8_eight_complex_djbfft_preload
b2b:	zr8_eight_complex_djbfft rsi, 128, 2*dist32, 4*dist32, 8*dist32, rdi, ZMM_SCD4, 32
	zloop	32, b2b, -32*128+dist32	;; Test loop counter, next source pointer
	zloop	2, b2b, -2*dist32+16*dist32, rdi, -64*ZMM_SCD4 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	5, b2b, -5*16*dist32, rdi, 64*ZMM_SCD4 ;; Test loop counter, restore source pointer, next sin/cos pointer

;; Do FFT levels 8,9,10
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	512	+1	...
;;	8	...
;;	...
;;	504
;;	1K
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	64	+1	...
;;	8	...
;;	...
;;	56
;;	128
;;	...

;; Do 64 sixteen_reals_fft operations
;; Do 2496 eight_complex_fft operations
;;	distance between fft data elements is 64
;;	do 320 macros each processing 128 data values

	zloop_init 320			;; 40 iterations of 8
	zloop_set_rw_prefetch ZDOUBLE	;; Prefetch 640 cache lines
	zr8_2sc_sixteen_reals_fft_preload
b3a:	zr8_2sc_sixteen_reals_fft rsi, 128, 8*128, 16*128, dist32, rdx, ZMM_SCD4, rdi, ZMM_SCD4, 8
	zloop	8, b3a, -8*128+2*dist32, rdi, -8*ZMM_SCD4 ;; Test loop counter, next source pointer, restore sin/cos pointer
	loops_reset			;; 47 more iters of 8
	zr8_eight_complex_djbfft_preload
b3b:	zr8_eight_complex_djbfft rsi, 128, 8*128, 16*128, dist32, rdi, ZMM_SCD4, 8
	zloop	8, b3b, -8*128+2*dist32, rdi, -8*ZMM_SCD4 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	40, b3b, -40*2*dist32, rdi, 8*ZMM_SCD4 ;; Test loop counter, restore source pointer, next sin/cos pointer

;; Do FFT levels 11,12,13,14,15,16 as well as inverse FFT levels 11,12,13,14,15,16
;; On input and output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	64	+1	...
;;	8	...
;;	...
;;	56
;;	128
;;	...

;; Do 1 hundredtwentyeight_reals_fft operation
;; Do 319 sixtyfour_complex_fft operations
;;	distance between fft data elements is 1 (8 for swizzling)
;;	do 320 macros each processing 128 data values

;; Execute the proper middle step

pfing	bump	rcx, -299*64		;; We will prefetch 299 cache lines too many
	cmp	ffttype, 2
	jl	zpass2_20480_real_1
	jg	zpass2_20480_real_3

zpass2_20480_real_2:
	mov	r8, DEST2ARG
;;	zr64_hundredtwentyeight_real_with_square_preload
;;	zr64_hundredtwentyeight_real_with_square rsi, 8*128, 1*128, 2*128, 4*128, rdx, 0*ZMM_SCD4, rdi, 0*ZMM_SCD4, 1
	CALLP	zreal_square
	zloop_init 320, 1, 4		;; 319 more iterations
	zloop_set_rw_prefetch ZSEXT	;; Prefetch 1914 cache lines
	zr64_sixtyfour_complex_with_square_preload
b4a:	zr64_sixtyfour_complex_with_square rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 1, L1PREFETCH_ALL, 8*128
	zloop	4, b4a, -4*8*128+dist32	;; Test loop counter, next source pointer
	zloop	80, b4a			;; Test loop counter
	add	rsi, r8
	jmp	zpass2_20480_real_unfft

zpass2_20480_real_1:
;;	zr64_hundredtwentyeight_real_fft_final_preload
;;	zr64_hundredtwentyeight_real_fft_final rsi, 8*128, 1*128, 2*128, 4*128, rdx, 0*ZMM_SCD4, rdi, 0*ZMM_SCD4, 1
	CALLP	zreal_fft_final
	zloop_init 320, 1, 4		;; 319 more iterations
	zloop_set_rw_prefetch ZSEXT	;; Prefetch 1914 cache lines
	zr64_sixtyfour_complex_fft_final_preload
b4b:	zr64_sixtyfour_complex_fft_final rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 1, L1PREFETCH_ALL, 8*128
	zloop	4, b4b, -4*8*128+dist32	;; Test loop counter, next source pointer
	zloop	80, b4b			;; Test loop counter
	jmp	zpass2_20480_real_done

zpass2_20480_real_3:
	mov	r8, DEST2ARG
	mov	rbp, DIST_TO_MULSRCARG
;;	zr64_hundredtwentyeight_real_with_mult_preload
;;	zr64_hundredtwentyeight_real_with_mult rsi, 8*128, 1*128, 2*128, 4*128, rdx, 0*ZMM_SCD4, rdi, 0*ZMM_SCD4, 1
	CALLP	zreal_mult
	zloop_init 320, 1, 4		;; 319 more iterations
	zloop_set_rw_prefetch ZSEXT	;; Prefetch 1914 cache lines
	zr64_sixtyfour_complex_with_mult_preload
b4c:	zr64_sixtyfour_complex_with_mult rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 1, L1PREFETCH_ALL, 8*128
	zloop	4, b4c, -4*8*128+dist32	;; Test loop counter, next source pointer
	zloop	80, b4c			;; Test loop counter
	add	rsi, r8
	jmp	zpass2_20480_real_unfft

zpass2_20480_real_4:
	mov	rbp, DIST_TO_MULSRCARG
	mov	rdx, sincos3		;; Real sin/cos data
	mov	rdi, xsincos_complex	;; Complex sin/cos data
	bump	rdx, 512*ZMM_SCD2+64*ZMM_SCD4+8*ZMM_SCD4
	bump	rdi, 512*ZMM_SCD2+64*ZMM_SCD4+8*ZMM_SCD4
;;	zr64f_hundredtwentyeight_real_with_mulf_preload
;;	zr64f_hundredtwentyeight_real_with_mulf rsi, 8*128, 1*128, 2*128, 4*128, rdx, 0*ZMM_SCD4, rdi, 0*ZMM_SCD4, 1
	CALLP	zreal_mulf
	zloop_init 320, 1, 4		;; 319 more iterations
	zloop_set_rw_prefetch ZSEXT	;; Prefetch 1914 cache lines
	zr64f_sixtyfour_complex_with_mulf_preload
b4d:	zr64f_sixtyfour_complex_with_mulf rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 1, L1PREFETCH_ALL, 8*128
	zloop	4, b4d, -4*8*128+dist32	;; Test loop counter, next source pointer
	zloop	80, b4d			;; Test loop counter
zpass2_20480_real_unfft:
	bump	rsi, -80*dist32		;; Restore source pointer

;; Do inverse FFT levels 8,9,10
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	64	+1	...
;;	8	...
;;	...
;;	56
;;	128
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	512	+1	...
;;	8	...
;;	...
;;	504
;;	1K
;;	...

;; Do 64 sixteen_reals_unfft operations
;; Do 2496 eight_complex_unfft operations
;;	distance between fft data elements is 64
;;	do 320 macros each processing 128 data values

	zloop_init 320			;; 40 iterations of 8
	zloop_set_rw_prefetch ZDOUBLE	;; Prefetch 640 cache lines
	bump	rdx, -8*ZMM_SCD4
	bump	rdi, -8*ZMM_SCD4
	zr8_2sc_sixteen_reals_unfft_preload
c3a:	zr8_2sc_sixteen_reals_unfft rsi, 128, 8*128, 16*128, dist32, rdx, ZMM_SCD4, rdi, ZMM_SCD4, 8
	zloop	8, c3a, -8*128+2*dist32, rdi, -8*ZMM_SCD4 ;; Test loop counter, next source pointer, restore sin/cos pointer
	loops_reset			;; 47 more iters of 8
	zr8_eight_complex_djbunfft_preload
c3b:	zr8_eight_complex_djbunfft rsi, 128, 8*128, 16*128, dist32, rdi, ZMM_SCD4, 8
	zloop	8, c3b, -8*128+2*dist32, rdi, -8*ZMM_SCD4 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	40, c3b, -40*2*dist32	;; Test loop counter, restore source pointer

;; Do inverse FFT levels 5,6,7
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	512	+1	...
;;	8	...
;;	...
;;	504
;;	1K
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	4K	+1	...
;;	8	...
;;	...
;;	4K-8
;;	8K
;;	...

;; Do 512 sixteen_reals_unfft operations
;; Do 2048 eight_complex_unfft operations
;;	distance between fft data elements is 512
;;	do 320 macros each processing 128 data values

	zloop_init 320			;; 5 iterations of 64
	zloop_set_rw_prefetch ZDOUBLE	;; Prefetch 640 cache lines
	bump	rdx, -8*ZMM_SCD4-64*ZMM_SCD4
	bump	rdi, -64*ZMM_SCD4
	zr8_2sc_sixteen_reals_unfft_preload
c2a:	zr8_2sc_sixteen_reals_unfft rsi, 128, 2*dist32, 4*dist32, 8*dist32, rdx, ZMM_SCD4, rdi, ZMM_SCD4, 32
	zloop	32, c2a, -32*128+dist32	;; Test loop counter, next source pointer
	zloop	2, c2a, -2*dist32+16*dist32, rdi, -64*ZMM_SCD4 ;; Test loop counter, next source pointer, restore sin/cos pointer
	loops_reset			;; 5 more iters of 64
	zr8_eight_complex_djbunfft_preload
c2b:	zr8_eight_complex_djbunfft rsi, 128, 2*dist32, 4*dist32, 8*dist32, rdi, ZMM_SCD4, 32
	zloop	32, c2b, -32*128+dist32	;; Test loop counter, next source pointer
	zloop	2, c2b, -2*dist32+16*dist32, rdi, -64*ZMM_SCD4 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	5, c2b, -5*16*dist32	;; Test loop counter, restore source pointer

;; Do inverse FFT levels 1,2,3,4
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	4K	+1	...
;;	8	...
;;	...
;;	4K-8
;;	8K
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	20K	+1	...
;;	8	...
;;	...
;;	20K-8

;; Do 4096 ten_reals_last_unfft operations
;;	distance between fft data elements is 4K
;;	do 512 macros each processing 80 data values

	bump	rdx, -64*ZMM_SCD4-512*ZMM_SCD2
	bump	rdi, -512*ZMM_SCD2
	zloop_init 512			;; 512 iterations
	zloop_set_rw_prefetch ZSINGLE	;; Prefetch 512 cache lines
	zr5_2sc_ten_reals_unfft_preload
c1b:	zr5_2sc_ten_reals_unfft rsi, 128, 16*dist32, rdx, ZMM_SCD2, rdi, ZMM_SCD2, 32
	zloop	32, c1b, -32*128+dist32	;; Test loop counter, next source pointer
	zloop	16, c1b			;; Test loop counter

zpass2_20480_real_done:
	end_timer 2
	ENDM

; *************** radix8-pass2-20480-complex macro ******************
; This macro takes 20480 complex values and performs the final 14.322 levels of a standard
; radix-8 FFT, squares the results, and does the first 14.322 levels of the inverse FFT.

zr4dwpn_pass2_20480_complex MACRO
	LOCAL	b1b, b2b, b3b, b4a, b4b, b4c, b4d, c1b, c2b, c3b
	LOCAL	zpass2_20480_complex_1, zpass2_20480_complex_2
	LOCAL	zpass2_20480_complex_3, zpass2_20480_complex_4
	LOCAL	zpass2_20480_complex_unfft, zpass2_20480_complex_done

;; Type 4 FFTs skip the forward FFT process

	mov	rsi, DATA_ADDR		;; Load address of FFT data
	mov	rbx, DIST_TO_FFTSRCARG
	mov	rcx, DATA_PREFETCH	;; Prefetch pointer (we need to prefetch 5120 cache lines + 79 pad cache lines)
	cmp	ffttype, 4
	je	zpass2_20480_complex_4

;; Do FFT levels 1,2,3
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	20K	+1	...
;;	8	...
;;	...
;;	20K-8
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	4K	+1	...
;;	8	...
;;	...
;;	4K-8
;;	8K
;;	...

;; Do 4096 five_complex_fft operations
;;	distance between fft data elements is 4K
;;	do 512 macros each processing 80 data values

	start_timer 4
	mov	rdi, xsincos_complex	;; The sin/cos data
	zloop_init 512			;; 512 iterations
	zloop_set_rw_prefetch ZSINGLE	;; Prefetch 512 cache lines
	zr5f_five_complex_djbfft_preload
b1b:	zr5f_five_complex_djbfft rsi, 128, 16*dist32, rdi, ZMM_SCD2, 32, L1PREFETCH_ALL, 128
	zloop	32, b1b, -32*128+dist32	;; Test loop counter, next source pointer
	zloop	16, b1b, -16*dist32	;; Test loop counter, restore source pointer
	end_timer 4

;; Do FFT levels 4,5,6
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	4K	+1	...
;;	8	...
;;	...
;;	4K-8
;;	8K
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	512	+1	...
;;	8	...
;;	...
;;	504
;;	1K
;;	...

;; Do 2560 eight_complex_fft operations
;;	distance between fft data elements is 512
;;	do 320 macros each processing 128 data values

	start_timer 5
	zloop_init 320			;; 5 iterations of 64
	zloop_set_rw_prefetch ZDOUBLE	;; Prefetch 640 cache lines
	zr8_eight_complex_djbfft_preload
b2b:	zr8_eight_complex_djbfft rsi, 128, 2*dist32, 4*dist32, 8*dist32, rdi, ZMM_SCD4, 32, L1PREFETCH_ALL, 128
	zloop	32, b2b, -32*128+dist32	;; Test loop counter, next source pointer
	zloop	2, b2b, -2*dist32+16*dist32, rdi, -64*ZMM_SCD4 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	5, b2b, -5*16*dist32, rdi, 64*ZMM_SCD4 ;; Test loop counter, restore source pointer, next sin/cos pointer
	end_timer 5

;; Do FFT levels 7,8,9
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	512	+1	...
;;	8	...
;;	...
;;	504
;;	1K
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	64	+1	...
;;	8	...
;;	...
;;	56
;;	128
;;	...

;; Do 2560 eight_complex_fft operations
;;	distance between fft data elements is 64
;;	do 320 macros each processing 128 data values

	start_timer 6
	zloop_init 320			;; 40 iterations of 8
	zloop_set_rw_prefetch ZDOUBLE	;; Prefetch 640 cache lines
	zr8_eight_complex_djbfft_preload
b3b:	zr8_eight_complex_djbfft rsi, 128, 8*128, 16*128, dist32, rdi, ZMM_SCD4, 8, L1PREFETCH_ALL, 128
	zloop	8, b3b, -8*128+2*dist32, rdi, -8*ZMM_SCD4 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	40, b3b, -40*2*dist32, rdi, 8*ZMM_SCD4 ;; Test loop counter, restore source pointer, next sin/cos pointer
	end_timer 6

;; Do FFT levels 10,11,12,13,14,15 as well as inverse FFT levels 10,11,12,13,14,15
;; On input and output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	64	+1	...
;;	8	...
;;	...
;;	56
;;	128
;;	...

;; Do 320 sixtyfour_complex_fft operations
;;	distance between fft data elements is 1 (8 for swizzling)
;;	do 320 macros each processing 128 data values

;; Execute the right middle step

pfing	bump	rcx, -305*64		;; We will prefetch 305 cache lines too many
	cmp	ffttype, 2
	jl	zpass2_20480_complex_1
	jg	zpass2_20480_complex_3

zpass2_20480_complex_2:
	start_timer 9
	zloop_init 320			;; 320 iterations
	zloop_set_rw_prefetch ZSEXT	;; Prefetch 1920 cache lines
	mov	r8, DEST2ARG
	zr64_sixtyfour_complex_with_square_preload
b4a:	zr64_sixtyfour_complex_with_square rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 4, L1PREFETCH_ALL, 8*128
	zloop	4, b4a, -4*8*128+dist32	;; Test loop counter, next source pointer
	zloop	80, b4a			;; Test loop counter
	add	rsi, r8
	end_timer 9
	jmp	zpass2_20480_complex_unfft

zpass2_20480_complex_1:
	start_timer 9
	zloop_init 320			;; 320 iterations
	zloop_set_rw_prefetch ZSEXT	;; Prefetch 1920 cache lines
	zr64_sixtyfour_complex_fft_final_preload
b4b:	zr64_sixtyfour_complex_fft_final rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 4, L1PREFETCH_ALL, 8*128
	zloop	4, b4b, -4*8*128+dist32	;; Test loop counter, next source pointer
	zloop	80, b4b			;; Test loop counter
	end_timer 9
	jmp	zpass2_20480_complex_done

zpass2_20480_complex_3:
	start_timer 9
	zloop_init 320			;; 320 iterations
	zloop_set_rw_prefetch ZSEXT	;; Prefetch 1920 cache lines
	mov	r8, DEST2ARG
	mov	rbp, DIST_TO_MULSRCARG
	zr64_sixtyfour_complex_with_mult_preload
b4c:	zr64_sixtyfour_complex_with_mult rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 4, L1PREFETCH_ALL, 8*128
	zloop	4, b4c, -4*8*128+dist32	;; Test loop counter, next source pointer
	zloop	80, b4c			;; Test loop counter
	add	rsi, r8
	end_timer 9
	jmp	zpass2_20480_complex_unfft

zpass2_20480_complex_4:
	start_timer 9
	zloop_init 320			;; 320 iterations
	zloop_set_rw_prefetch ZSEXT	;; Prefetch 1920 cache lines
	mov	rbp, DIST_TO_MULSRCARG
	mov	rdi, xsincos_complex	;; The sin/cos data
	bump	rdi, 512*ZMM_SCD2+64*ZMM_SCD4+8*ZMM_SCD4
	zr64f_sixtyfour_complex_with_mulf_preload
b4d:	zr64f_sixtyfour_complex_with_mulf rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 4, L1PREFETCH_ALL, 8*128
	zloop	4, b4d, -4*8*128+dist32	;; Test loop counter, next source pointer
	zloop	80, b4d			;; Test loop counter
	end_timer 9
zpass2_20480_complex_unfft:
	bump	rsi, -80*dist32		;; Restore source pointer

;; Do inverse FFT levels 7,8,9
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	64	+1	...
;;	8	...
;;	...
;;	56
;;	128
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	512	+1	...
;;	8	...
;;	...
;;	504
;;	1K
;;	...

;; Do 2560 eight_complex_unfft operations
;;	distance between fft data elements is 64
;;	do 320 macros each processing 128 data values

	start_timer 12
	bump	rdi, -8*ZMM_SCD4
	zloop_init 320			;; 40 iterations of 8
	zloop_set_rw_prefetch ZDOUBLE	;; Prefetch 640 cache lines
	zr8_eight_complex_djbunfft_preload
c3b:	zr8_eight_complex_djbunfft rsi, 128, 8*128, 16*128, dist32, rdi, ZMM_SCD4, 8, L1PREFETCH_ALL, 128
	zloop	8, c3b, -8*128+2*dist32, rdi, -8*ZMM_SCD4 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	40, c3b, -40*2*dist32	;; Test loop counter, restore source pointer
	end_timer 12

;; Do inverse FFT levels 4,5,6
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	512	+1	...
;;	8	...
;;	...
;;	504
;;	1K
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	4K	+1	...
;;	8	...
;;	...
;;	4K-8
;;	8K
;;	...

;; Do 2560 eight_complex_unfft operations
;;	distance between fft data elements is 512
;;	do 320 macros each processing 128 data values

	start_timer 13
	bump	rdi, -64*ZMM_SCD4
	zloop_init 320			;; 5 iterations of 64
	zloop_set_rw_prefetch ZDOUBLE	;; Prefetch 640 cache lines
	zr8_eight_complex_djbunfft_preload
c2b:	zr8_eight_complex_djbunfft rsi, 128, 2*dist32, 4*dist32, 8*dist32, rdi, ZMM_SCD4, 32, L1PREFETCH_ALL, 128
	zloop	32, c2b, -32*128+dist32	;; Test loop counter, next source pointer
	zloop	2, c2b, -2*dist32+16*dist32, rdi, -64*ZMM_SCD4 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	5, c2b, -5*16*dist32	;; Test loop counter, restore source pointer
	end_timer 13

;; Do inverse FFT levels 1,2,3
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	4K	+1	...
;;	8	...
;;	...
;;	4K-8
;;	8K
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	24K	+1	...
;;	8	...
;;	...
;;	24K-8

;; Do 4096 five_complex_unfft operations
;;	distance between fft data elements is 4K
;;	do 512 macros each processing 80 data values

	start_timer 14
	bump	rdi, -512*ZMM_SCD2
	zloop_init 512			;; 512 iterations
	zloop_set_rw_prefetch ZSINGLE	;; Prefetch 512 cache lines
	zr5_five_complex_djbunfft_preload
c1b:	zr5_five_complex_djbunfft rsi, 128, 16*dist32, rdi, ZMM_SCD2, 32, L1PREFETCH_ALL, 128
	zloop	32, c1b, -32*128+dist32	;; Test loop counter, next source pointer
	zloop	16, c1b			;; Test loop counter
	end_timer 14

zpass2_20480_complex_done:
	ENDM


; *************** radix8-pass2-21504-real macro ******************
; This macro takes 43008 real values and performs the final 15.392 levels of the FFT.
; The results are squared, and the first 15.392 levels of the inverse FFT are done.

zr4dwpn_pass2_21504_real MACRO
	LOCAL	b1b, b2a, b2b, b3a, b3b, b4a, b4b, b4c, b4d, c1b, c2a, c2b, c3a, c3b
	LOCAL	zpass2_21504_real_1, zpass2_21504_real_2
	LOCAL	zpass2_21504_real_3, zpass2_21504_real_4
	LOCAL	zpass2_21504_real_unfft, zpass2_21504_real_done

;; Type 4 FFTs skip the forward FFT process

	start_timer 2
	mov	rsi, DATA_ADDR		;; Load source address
	mov	rbx, DIST_TO_FFTSRCARG
	mov	rcx, DATA_PREFETCH	;; Prefetch pointer (we need to prefetch 5376 cache lines + ??? pad cache lines)
	cmp	ffttype, 4
	je	zpass2_21504_real_4

;; Do FFT levels 1,2,3,4
;;	0	+1	+1	+1	+1	+1	+1	+1	21K	+1	...
;;	8	...
;;	...
;;	21K-8
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	3K	+1	...
;;	8	...
;;	...
;;	3K-8
;;	6K
;;	...

;; Do 3072 fourteen_real_fft operations
;;	distance between fft data elements is 3K
;;	do 384 macros each processing 112 data values

	mov	rdx, sincos3		;; Real sin/cos data
	mov	rdi, xsincos_complex	;; Complex sin/cos data
	zloop_init 384			;; 384 iterations
	zloop_set_rw_prefetch ZSINGLE	;; Prefetch 768 cache lines
	zr7f_2sc_fourteen_reals_fft_preload
b1b:	zr7f_2sc_fourteen_reals_fft rsi, 128, 12*dist32, rdx, ZMM_SCD3, rdi, ZMM_SCD3, 32
	zloop	32, b1b, -32*128+dist32	;; Test loop counter, next source pointer
	zloop	12, b1b, -12*dist32	;; Test loop counter, restore source pointer

;; Do FFT levels 5,6,7
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	3K	+1	...
;;	8	...
;;	...
;;	3K-8
;;	6K
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	512	+1	...
;;	8	...
;;	...
;;	504
;;	1K
;;	...

;; Do 512 twelve_reals_fft operations
;; Do 3072 six_complex_fft operations
;;	distance between fft data elements is 512
;;	do 448 macros each processing 96 data values

	zloop_init 448			;; 7 iterations of 64
	zloop_set_rw_prefetch ZSINGLE	;; Prefetch 448 cache lines
	zr6_2sc_twelve_reals_fft_preload
b2a:	zr6_2sc_twelve_reals_fft rsi, 128, 2*dist32, rdx, ZMM_SCD3, rdi, ZMM_SCD3, 32
	zloop	32, b2a, -32*128+dist32	;; Test loop counter, next source pointer
	zloop	2, b2a, -2*dist32+12*dist32, rdi, -64*ZMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer
	loops_reset			;; 6 more iters of 64
	zr6_six_complex_djbfft_preload
b2b:	zr6_six_complex_djbfft rsi, 128, 2*dist32, rdi, ZMM_SCD3, 32
	zloop	32, b2b, -32*128+dist32	;; Test loop counter, next source pointer
	zloop	2, b2b, -2*dist32+12*dist32, rdi, -64*ZMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	7, b2b, -7*12*dist32, rdi, 64*ZMM_SCD3 ;; Test loop counter, restore source pointer, next sin/cos pointer

;; Do FFT levels 8,9,10
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	512	+1	...
;;	8	...
;;	...
;;	504
;;	1K
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	64	+1	...
;;	8	...
;;	...
;;	56
;;	128
;;	...

;; Do 64 twelve_reals_fft operations
;; Do 2624 eight_complex_fft operations
;;	distance between fft data elements is 64
;;	do 336 macros each processing 128 data values

	zloop_init 336			;; 42 iterations of 8
	zloop_set_rw_prefetch ZDOUBLE	;; Prefetch 672 cache lines
	zr8_2sc_sixteen_reals_fft_preload
b3a:	zr8_2sc_sixteen_reals_fft rsi, 128, 8*128, 16*128, dist32, rdx, ZMM_SCD4, rdi, ZMM_SCD4, 8
	zloop	8, b3a, -8*128+2*dist32, rdi, -8*ZMM_SCD4 ;; Test loop counter, next source pointer, restore sin/cos pointer
	loops_reset			;; 41 more iters of 8
	zr8_eight_complex_djbfft_preload
b3b:	zr8_eight_complex_djbfft rsi, 128, 8*128, 16*128, dist32, rdi, ZMM_SCD4, 8
	zloop	8, b3b, -8*128+2*dist32, rdi, -8*ZMM_SCD4 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	42, b3b, -42*2*dist32, rdi, 8*ZMM_SCD4 ;; Test loop counter, restore source pointer, next sin/cos pointer

;; Do FFT levels 11,12,13,14,15,16 as well as inverse FFT levels 11,12,13,14,15,16
;; On input and output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	64	+1	...
;;	8	...
;;	...
;;	56
;;	128
;;	...

;; Do 1 hundredtwentyeight_reals_fft operation
;; Do 335 sixtyfour_complex_fft operations
;;	distance between fft data elements is 1 (8 for swizzling)
;;	do 336 macros each processing 128 data values

;; Execute the proper middle step

pfing	bump	rcx, -7*64		;; We will prefetch 7 cache lines too many
	cmp	ffttype, 2
	jl	zpass2_21504_real_1
	jg	zpass2_21504_real_3

zpass2_21504_real_2:
	mov	r8, DEST2ARG
;;	zr64_hundredtwentyeight_real_with_square_preload
;;	zr64_hundredtwentyeight_real_with_square rsi, 8*128, 1*128, 2*128, 4*128, rdx, 0*ZMM_SCD4, rdi, 0*ZMM_SCD4, 1
	CALLP	zreal_square
	zloop_init 336, 1, 4		;; 335 more iterations
	zloop_set_rw_prefetch ZSEXT	;; Prefetch 2010 cache lines
	zr64_sixtyfour_complex_with_square_preload
b4a:	zr64_sixtyfour_complex_with_square rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 1, L1PREFETCH_ALL, 8*128
	zloop	4, b4a, -4*8*128+dist32	;; Test loop counter, next source pointer
	zloop	84, b4a			;; Test loop counter
	add	rsi, r8
	jmp	zpass2_21504_real_unfft

zpass2_21504_real_1:
;;	zr64_hundredtwentyeight_real_fft_final_preload
;;	zr64_hundredtwentyeight_real_fft_final rsi, 8*128, 1*128, 2*128, 4*128, rdx, 0*ZMM_SCD4, rdi, 0*ZMM_SCD4, 1
	CALLP	zreal_fft_final
	zloop_init 336, 1, 4		;; 335 more iterations
	zloop_set_rw_prefetch ZSEXT	;; Prefetch 2010 cache lines
	zr64_sixtyfour_complex_fft_final_preload
b4b:	zr64_sixtyfour_complex_fft_final rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 1, L1PREFETCH_ALL, 8*128
	zloop	4, b4b, -4*8*128+dist32	;; Test loop counter, next source pointer
	zloop	84, b4b			;; Test loop counter
	jmp	zpass2_21504_real_done

zpass2_21504_real_3:
	mov	r8, DEST2ARG
	mov	rbp, DIST_TO_MULSRCARG
;;	zr64_hundredtwentyeight_real_with_mult_preload
;;	zr64_hundredtwentyeight_real_with_mult rsi, 8*128, 1*128, 2*128, 4*128, rdx, 0*ZMM_SCD4, rdi, 0*ZMM_SCD4, 1
	CALLP	zreal_mult
	zloop_init 336, 1, 4		;; 335 more iterations
	zloop_set_rw_prefetch ZSEXT	;; Prefetch 2010 cache lines
	zr64_sixtyfour_complex_with_mult_preload
b4c:	zr64_sixtyfour_complex_with_mult rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 1, L1PREFETCH_ALL, 8*128
	zloop	4, b4c, -4*8*128+dist32	;; Test loop counter, next source pointer
	zloop	84, b4c			;; Test loop counter
	add	rsi, r8
	jmp	zpass2_21504_real_unfft

zpass2_21504_real_4:
	mov	rbp, DIST_TO_MULSRCARG
	mov	rdx, sincos3		;; Real sin/cos data
	mov	rdi, xsincos_complex	;; Complex sin/cos data
	bump	rdx, 384*ZMM_SCD3+64*ZMM_SCD3+8*ZMM_SCD4
	bump	rdi, 384*ZMM_SCD3+64*ZMM_SCD3+8*ZMM_SCD4
;;	zr64f_hundredtwentyeight_real_with_mulf_preload
;;	zr64f_hundredtwentyeight_real_with_mulf rsi, 8*128, 1*128, 2*128, 4*128, rdx, 0*ZMM_SCD4, rdi, 0*ZMM_SCD4, 1
	CALLP	zreal_mulf
	zloop_init 336, 1, 4		;; 335 more iterations
	zloop_set_rw_prefetch ZSEXT	;; Prefetch 2010 cache lines
	zr64f_sixtyfour_complex_with_mulf_preload
b4d:	zr64f_sixtyfour_complex_with_mulf rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 1, L1PREFETCH_ALL, 8*128
	zloop	4, b4d, -4*8*128+dist32	;; Test loop counter, next source pointer
	zloop	84, b4d			;; Test loop counter
zpass2_21504_real_unfft:
	bump	rsi, -84*dist32		;; Restore source pointer

;; Do inverse FFT levels 8,9,10
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	64	+1	...
;;	8	...
;;	...
;;	56
;;	128
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	512	+1	...
;;	8	...
;;	...
;;	504
;;	1K
;;	...

;; Do 64 twelve_reals_unfft operations
;; Do 2624 eight_complex_unfft operations
;;	distance between fft data elements is 64
;;	do 336 macros each processing 128 data values

	zloop_init 336			;; 42 iterations of 8
	zloop_set_rw_prefetch ZDOUBLE	;; Prefetch 672 cache lines
	bump	rdx, -8*ZMM_SCD4
	bump	rdi, -8*ZMM_SCD4
	zr8_2sc_sixteen_reals_unfft_preload
c3a:	zr8_2sc_sixteen_reals_unfft rsi, 128, 8*128, 16*128, dist32, rdx, ZMM_SCD4, rdi, ZMM_SCD4, 8
	zloop	8, c3a, -8*128+2*dist32, rdi, -8*ZMM_SCD4 ;; Test loop counter, next source pointer, restore sin/cos pointer
	loops_reset			;; 41 more iters of 8
	zr8_eight_complex_djbunfft_preload
c3b:	zr8_eight_complex_djbunfft rsi, 128, 8*128, 16*128, dist32, rdi, ZMM_SCD4, 8
	zloop	8, c3b, -8*128+2*dist32, rdi, -8*ZMM_SCD4 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	42, c3b, -42*2*dist32	;; Test loop counter, restore source pointer

;; Do inverse FFT levels 5,6,7
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	512	+1	...
;;	8	...
;;	...
;;	504
;;	1K
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	3K	+1	...
;;	8	...
;;	...
;;	3K-8
;;	6K
;;	...

;; Do 512 twelve_reals_unfft operations
;; Do 3072 six_complex_unfft operations
;;	distance between fft data elements is 512
;;	do 448 macros each processing 96 data values

	zloop_init 448			;; 7 iterations of 64
	zloop_set_rw_prefetch ZDOUBLE	;; Prefetch 896 cache lines
	bump	rdx, -8*ZMM_SCD4-64*ZMM_SCD3
	bump	rdi, -64*ZMM_SCD3
	zr6_2sc_twelve_reals_unfft_preload
c2a:	zr6_2sc_twelve_reals_unfft rsi, 128, 2*dist32, rdx, ZMM_SCD3, rdi, ZMM_SCD3, 32
	zloop	32, c2a, -32*128+dist32	;; Test loop counter, next source pointer
	zloop	2, c2a, -2*dist32+12*dist32, rdi, -64*ZMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer
	loops_reset			;; 6 more iters of 64
	zr6_six_complex_djbunfft_preload
c2b:	zr6_six_complex_djbunfft rsi, 128, 2*dist32, rdi, ZMM_SCD3, 32
	zloop	32, c2b, -32*128+dist32	;; Test loop counter, next source pointer
	zloop	2, c2b, -2*dist32+12*dist32, rdi, -64*ZMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	7, c2b, -7*12*dist32	;; Test loop counter, restore source pointer

;; Do inverse FFT levels 1,2,3,4
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	3K	+1	...
;;	8	...
;;	...
;;	3K-8
;;	4K
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	21K	+1	...
;;	8	...
;;	...
;;	21K-8

;; Do 3072 fourteen_reals_last_unfft operations
;;	distance between fft data elements is 3K
;;	do 384 macros each processing 112 data values

	bump	rdx, -64*ZMM_SCD3-384*ZMM_SCD3
	bump	rdi, -384*ZMM_SCD3
	zloop_init 384			;; 384 iterations
	zloop_set_rw_prefetch ZSINGLE	;; Prefetch 384 cache lines
	zr7_2sc_fourteen_reals_unfft_preload
c1b:	zr7_2sc_fourteen_reals_unfft rsi, 128, 12*dist32, rdx, ZMM_SCD3, rdi, ZMM_SCD3, 32
	zloop	32, c1b, -32*128+dist32	;; Test loop counter, next source pointer
	zloop	12, c1b			;; Test loop counter

zpass2_21504_real_done:
	end_timer 2
	ENDM

; *************** radix8-pass2-21504-complex macro ******************
; This macro takes 21504 complex values and performs the final 14.392 levels of a standard
; radix-8 FFT, squares the results, and does the first 14.392 levels of the inverse FFT.

zr4dwpn_pass2_21504_complex MACRO
	LOCAL	b1b, b2b, b3b, b4a, b4b, b4c, b4d, c1b, c2b, c3b
	LOCAL	zpass2_21504_complex_1, zpass2_21504_complex_2
	LOCAL	zpass2_21504_complex_3, zpass2_21504_complex_4
	LOCAL	zpass2_21504_complex_unfft, zpass2_21504_complex_done

;; Type 4 FFTs skip the forward FFT process

	mov	rsi, DATA_ADDR		;; Load address of FFT data
	mov	rbx, DIST_TO_FFTSRCARG
	mov	rcx, DATA_PREFETCH	;; Prefetch pointer (we need to prefetch 5376 cache lines + 83 pad cache lines)
	cmp	ffttype, 4
	je	zpass2_21504_complex_4

;; Do FFT levels 1,2,3
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	21K	+1	...
;;	8	...
;;	...
;;	21K-8
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	3K	+1	...
;;	8	...
;;	...
;;	3K-8
;;	6K
;;	...

;; Do 3072 seven_complex_fft operations
;;	distance between fft data elements is 3K
;;	do 384 macros each processing 112 data values

	start_timer 4
	mov	rdi, xsincos_complex	;; The sin/cos data
	zloop_init 384			;; 384 iterations
	zloop_set_rw_prefetch ZSINGLE	;; Prefetch 768 cache lines
	zr7f_seven_complex_djbfft_preload
b1b:	zr7f_seven_complex_djbfft rsi, 128, 12*dist32, rdi, ZMM_SCD3, 32, L1PREFETCH_ALL, 128
	zloop	32, b1b, -32*128+dist32	;; Test loop counter, next source pointer
	zloop	12, b1b, -12*dist32	;; Test loop counter, restore source pointer
	end_timer 4

;; Do FFT levels 4,5,6
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	3K	+1	...
;;	8	...
;;	...
;;	3K-8
;;	6K
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	512	+1	...
;;	8	...
;;	...
;;	504
;;	1K
;;	...

;; Do 3584 six_complex_fft operations
;;	distance between fft data elements is 512
;;	do 448 macros each processing 96 data values

	start_timer 5
	zloop_init 448			;; 7 iterations of 64
	zloop_set_rw_prefetch ZSINGLE	;; Prefetch 448 cache lines
	zr6_six_complex_djbfft_preload
b2b:	zr6_six_complex_djbfft rsi, 128, 2*dist32, rdi, ZMM_SCD3, 32, L1PREFETCH_ALL, 128
	zloop	32, b2b, -32*128+dist32	;; Test loop counter, next source pointer
	zloop	2, b2b, -2*dist32+12*dist32, rdi, -64*ZMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	7, b2b, -7*12*dist32, rdi, 64*ZMM_SCD3 ;; Test loop counter, restore source pointer, next sin/cos pointer
	end_timer 5

;; Do FFT levels 7,8,9
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	512	+1	...
;;	8	...
;;	...
;;	504
;;	1K
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	64	+1	...
;;	8	...
;;	...
;;	56
;;	128
;;	...

;; Do 2688 eight_complex_fft operations
;;	distance between fft data elements is 64
;;	do 336 macros each processing 128 data values

	start_timer 6
	zloop_init 336			;; 42 iterations of 8
	zloop_set_rw_prefetch ZDOUBLE	;; Prefetch 672 cache lines
	zr8_eight_complex_djbfft_preload
b3b:	zr8_eight_complex_djbfft rsi, 128, 8*128, 16*128, dist32, rdi, ZMM_SCD4, 8, L1PREFETCH_ALL, 128
	zloop	8, b3b, -8*128+2*dist32, rdi, -8*ZMM_SCD4 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	42, b3b, -42*2*dist32, rdi, 8*ZMM_SCD4 ;; Test loop counter, restore source pointer, next sin/cos pointer
	end_timer 6

;; Do FFT levels 10,11,12,13,14,15 as well as inverse FFT levels 10,11,12,13,14,15
;; On input and output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	64	+1	...
;;	8	...
;;	...
;;	56
;;	128
;;	...

;; Do 336 sixtyfour_complex_fft operations
;;	distance between fft data elements is 1 (8 for swizzling)
;;	do 336 macros each processing 128 data values

;; Execute the right middle step

pfing	bump	rcx, -13*64		;; We will prefetch 13 cache lines too many
	cmp	ffttype, 2
	jl	zpass2_21504_complex_1
	jg	zpass2_21504_complex_3

zpass2_21504_complex_2:
	start_timer 9
	zloop_init 336			;; 336 iterations
	zloop_set_rw_prefetch ZSEXT	;; Prefetch 2016 cache lines
	mov	r8, DEST2ARG
	zr64_sixtyfour_complex_with_square_preload
b4a:	zr64_sixtyfour_complex_with_square rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 4, L1PREFETCH_ALL, 8*128
	zloop	4, b4a, -4*8*128+dist32	;; Test loop counter, next source pointer
	zloop	84, b4a			;; Test loop counter
	add	rsi, r8
	end_timer 9
	jmp	zpass2_21504_complex_unfft

zpass2_21504_complex_1:
	start_timer 9
	zloop_init 336			;; 336 iterations
	zloop_set_rw_prefetch ZSEXT	;; Prefetch 2016 cache lines
	zr64_sixtyfour_complex_fft_final_preload
b4b:	zr64_sixtyfour_complex_fft_final rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 4, L1PREFETCH_ALL, 8*128
	zloop	4, b4b, -4*8*128+dist32	;; Test loop counter, next source pointer
	zloop	84, b4b			;; Test loop counter
	end_timer 9
	jmp	zpass2_21504_complex_done

zpass2_21504_complex_3:
	start_timer 9
	zloop_init 336			;; 336 iterations
	zloop_set_rw_prefetch ZSEXT	;; Prefetch 2016 cache lines
	mov	r8, DEST2ARG
	mov	rbp, DIST_TO_MULSRCARG
	zr64_sixtyfour_complex_with_mult_preload
b4c:	zr64_sixtyfour_complex_with_mult rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 4, L1PREFETCH_ALL, 8*128
	zloop	4, b4c, -4*8*128+dist32	;; Test loop counter, next source pointer
	zloop	84, b4c			;; Test loop counter
	add	rsi, r8
	end_timer 9
	jmp	zpass2_21504_complex_unfft

zpass2_21504_complex_4:
	start_timer 9
	zloop_init 336			;; 336 iterations
	zloop_set_rw_prefetch ZSEXT	;; Prefetch 2016 cache lines
	mov	rbp, DIST_TO_MULSRCARG
	mov	rdi, xsincos_complex	;; The sin/cos data
	bump	rdi, 384*ZMM_SCD3+64*ZMM_SCD3+8*ZMM_SCD4
	zr64f_sixtyfour_complex_with_mulf_preload
b4d:	zr64f_sixtyfour_complex_with_mulf rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 4, L1PREFETCH_ALL, 8*128
	zloop	4, b4d, -4*8*128+dist32	;; Test loop counter, next source pointer
	zloop	84, b4d			;; Test loop counter
	end_timer 9
zpass2_21504_complex_unfft:
	bump	rsi, -84*dist32		;; Restore source pointer

;; Do inverse FFT levels 7,8,9
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	64	+1	...
;;	8	...
;;	...
;;	56
;;	128
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	512	+1	...
;;	8	...
;;	...
;;	504
;;	1K
;;	...

;; Do 2688 eight_complex_unfft operations
;;	distance between fft data elements is 64
;;	do 336 macros each processing 128 data values

	start_timer 12
	bump	rdi, -8*ZMM_SCD4
	zloop_init 336			;; 42 iterations of 8
	zloop_set_rw_prefetch ZDOUBLE	;; Prefetch 672 cache lines
	zr8_eight_complex_djbunfft_preload
c3b:	zr8_eight_complex_djbunfft rsi, 128, 8*128, 16*128, dist32, rdi, ZMM_SCD4, 8, L1PREFETCH_ALL, 128
	zloop	8, c3b, -8*128+2*dist32, rdi, -8*ZMM_SCD4 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	42, c3b, -42*2*dist32	;; Test loop counter, restore source pointer
	end_timer 12

;; Do inverse FFT levels 4,5,6
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	512	+1	...
;;	8	...
;;	...
;;	504
;;	1K
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	3K	+1	...
;;	8	...
;;	...
;;	3K-8
;;	6K
;;	...

;; Do 3584 six_complex_unfft operations
;;	distance between fft data elements is 512
;;	do 448 macros each processing 96 data values

	start_timer 13
	bump	rdi, -64*ZMM_SCD3
	zloop_init 448			;; 7 iterations of 64
	zloop_set_rw_prefetch ZDOUBLE	;; Prefetch 896 cache lines
	zr6_six_complex_djbunfft_preload
c2b:	zr6_six_complex_djbunfft rsi, 128, 2*dist32, rdi, ZMM_SCD3, 32, L1PREFETCH_ALL, 128
	zloop	32, c2b, -32*128+dist32	;; Test loop counter, next source pointer
	zloop	2, c2b, -2*dist32+12*dist32, rdi, -64*ZMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	7, c2b, -7*12*dist32	;; Test loop counter, restore source pointer
	end_timer 13

;; Do inverse FFT levels 1,2,3
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	3K	+1	...
;;	8	...
;;	...
;;	3K-8
;;	8K
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	21K	+1	...
;;	8	...
;;	...
;;	21K-8

;; Do 3072 seven_complex_unfft operations
;;	distance between fft data elements is 3K
;;	do 384 macros each processing 112 data values

	start_timer 14
	bump	rdi, -384*ZMM_SCD3
	zloop_init 384			;; 384 iterations
	zloop_set_rw_prefetch ZSINGLE	;; Prefetch 384 cache lines
	zr7_seven_complex_djbunfft_preload
c1b:	zr7_seven_complex_djbunfft rsi, 128, 12*dist32, rdi, ZMM_SCD3, 32, L1PREFETCH_ALL, 128
	zloop	32, c1b, -32*128+dist32	;; Test loop counter, next source pointer
	zloop	12, c1b			;; Test loop counter
	end_timer 14

zpass2_21504_complex_done:
	ENDM


; *************** radix8-pass2-24576-real macro ******************
; This macro takes 49142 real values and performs the final 15.585 levels of the FFT.
; The results are squared, and the first 15.585 levels of the inverse FFT are done.

zr4dwpn_pass2_24576_real MACRO
	LOCAL	b1b, b2a, b2b, b3a, b3b, b4a, b4b, b4c, b4d, c1b, c2a, c2b, c3a, c3b
	LOCAL	zpass2_24576_real_1, zpass2_24576_real_2
	LOCAL	zpass2_24576_real_3, zpass2_24576_real_4
	LOCAL	zpass2_24576_real_unfft, zpass2_24576_real_done

;; Type 4 FFTs skip the forward FFT process

	start_timer 2
	mov	rsi, DATA_ADDR		;; Load source address
	mov	rbx, DIST_TO_FFTSRCARG
	mov	rcx, DATA_PREFETCH	;; Prefetch pointer (we need to prefetch 6144 cache lines + 95 pad cache lines)
	cmp	ffttype, 4
	je	zpass2_24576_real_4

;; Do FFT levels 1,2,3,4
;;	0	+1	+1	+1	+1	+1	+1	+1	24K	+1	...
;;	8	...
;;	...
;;	24K-8
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	4K	+1	...
;;	8	...
;;	...
;;	4K-8
;;	8K
;;	...

;; Do 4096 twelve_real_fft operations
;;	distance between fft data elements is 4K
;;	do 512 macros each processing 96 data values

	mov	rdx, sincos3		;; Real sin/cos data
	mov	rdi, xsincos_complex	;; Complex sin/cos data
	zloop_init 512			;; 512 iterations
	zloop_set_rw_prefetch ZSINGLE	;; Prefetch 512 cache lines
	zr6f_2sc_twelve_reals_fft_preload
b1b:	zr6f_2sc_twelve_reals_fft rsi, 128, 16*dist32, rdx, ZMM_SCD3, rdi, ZMM_SCD3, 32
	zloop	32, b1b, -32*128+dist32	;; Test loop counter, next source pointer
	zloop	16, b1b, -16*dist32	;; Test loop counter, restore source pointer

;; Do FFT levels 5,6,7
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	4K	+1	...
;;	8	...
;;	...
;;	4K-8
;;	8K
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	512	+1	...
;;	8	...
;;	...
;;	504
;;	1K
;;	...

;; Do 512 sixteen_reals_fft operations
;; Do 2560 eight_complex_fft operations
;;	distance between fft data elements is 512
;;	do 384 macros each processing 128 data values

	zloop_init 384			;; 6 iterations of 64
	zloop_set_rw_prefetch ZDOUBLE	;; Prefetch 768 cache lines
	zr8_2sc_sixteen_reals_fft_preload
b2a:	zr8_2sc_sixteen_reals_fft rsi, 128, 2*dist32, 4*dist32, 8*dist32, rdx, ZMM_SCD4, rdi, ZMM_SCD4, 32
	zloop	32, b2a, -32*128+dist32	;; Test loop counter, next source pointer
	zloop	2, b2a, -2*dist32+16*dist32, rdi, -64*ZMM_SCD4 ;; Test loop counter, next source pointer, restore sin/cos pointer
	loops_reset			;; 5 more iters of 64
	zr8_eight_complex_djbfft_preload
b2b:	zr8_eight_complex_djbfft rsi, 128, 2*dist32, 4*dist32, 8*dist32, rdi, ZMM_SCD4, 32
	zloop	32, b2b, -32*128+dist32	;; Test loop counter, next source pointer
	zloop	2, b2b, -2*dist32+16*dist32, rdi, -64*ZMM_SCD4 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	6, b2b, -6*16*dist32, rdi, 64*ZMM_SCD4 ;; Test loop counter, restore source pointer, next sin/cos pointer

;; Do FFT levels 8,9,10
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	512	+1	...
;;	8	...
;;	...
;;	504
;;	1K
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	64	+1	...
;;	8	...
;;	...
;;	56
;;	128
;;	...

;; Do 64 sixteen_reals_fft operations
;; Do 3008 eight_complex_fft operations
;;	distance between fft data elements is 64
;;	do 384 macros each processing 128 data values

	zloop_init 384			;; 48 iterations of 8
	zloop_set_rw_prefetch ZDOUBLE	;; Prefetch 768 cache lines
	zr8_2sc_sixteen_reals_fft_preload
b3a:	zr8_2sc_sixteen_reals_fft rsi, 128, 8*128, 16*128, dist32, rdx, ZMM_SCD4, rdi, ZMM_SCD4, 8
	zloop	8, b3a, -8*128+2*dist32, rdi, -8*ZMM_SCD4 ;; Test loop counter, next source pointer, restore sin/cos pointer
	loops_reset			;; 47 more iters of 8
	zr8_eight_complex_djbfft_preload
b3b:	zr8_eight_complex_djbfft rsi, 128, 8*128, 16*128, dist32, rdi, ZMM_SCD4, 8
	zloop	8, b3b, -8*128+2*dist32, rdi, -8*ZMM_SCD4 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	48, b3b, -48*2*dist32, rdi, 8*ZMM_SCD4 ;; Test loop counter, restore source pointer, next sin/cos pointer

;; Do FFT levels 11,12,13,14,15,16 as well as inverse FFT levels 11,12,13,14,15,16
;; On input and output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	64	+1	...
;;	8	...
;;	...
;;	56
;;	128
;;	...

;; Do 1 hundredtwentyeight_reals_fft operation
;; Do 383 sixtyfour_complex_fft operations
;;	distance between fft data elements is 1 (8 for swizzling)
;;	do 384 macros each processing 128 data values

;; Execute the proper middle step

pfing	bump	rcx, -155*64		;; We will prefetch 155 cache lines too many
	cmp	ffttype, 2
	jl	zpass2_24576_real_1
	jg	zpass2_24576_real_3

zpass2_24576_real_2:
	mov	r8, DEST2ARG
;;	zr64_hundredtwentyeight_real_with_square_preload
;;	zr64_hundredtwentyeight_real_with_square rsi, 8*128, 1*128, 2*128, 4*128, rdx, 0*ZMM_SCD4, rdi, 0*ZMM_SCD4, 1
	CALLP	zreal_square
	zloop_init 384, 1, 4		;; 383 more iterations
	zloop_set_rw_prefetch ZSEXT	;; Prefetch 2298 cache lines
	zr64_sixtyfour_complex_with_square_preload
b4a:	zr64_sixtyfour_complex_with_square rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 1, L1PREFETCH_ALL, 8*128
	zloop	4, b4a, -4*8*128+dist32	;; Test loop counter, next source pointer
	zloop	96, b4a			;; Test loop counter
	add	rsi, r8
	jmp	zpass2_24576_real_unfft

zpass2_24576_real_1:
;;	zr64_hundredtwentyeight_real_fft_final_preload
;;	zr64_hundredtwentyeight_real_fft_final rsi, 8*128, 1*128, 2*128, 4*128, rdx, 0*ZMM_SCD4, rdi, 0*ZMM_SCD4, 1
	CALLP	zreal_fft_final
	zloop_init 384, 1, 4		;; 383 more iterations
	zloop_set_rw_prefetch ZSEXT	;; Prefetch 2298 cache lines
	zr64_sixtyfour_complex_fft_final_preload
b4b:	zr64_sixtyfour_complex_fft_final rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 1, L1PREFETCH_ALL, 8*128
	zloop	4, b4b, -4*8*128+dist32	;; Test loop counter, next source pointer
	zloop	96, b4b			;; Test loop counter
	jmp	zpass2_24576_real_done

zpass2_24576_real_3:
	mov	r8, DEST2ARG
	mov	rbp, DIST_TO_MULSRCARG
;;	zr64_hundredtwentyeight_real_with_mult_preload
;;	zr64_hundredtwentyeight_real_with_mult rsi, 8*128, 1*128, 2*128, 4*128, rdx, 0*ZMM_SCD4, rdi, 0*ZMM_SCD4, 1
	CALLP	zreal_mult
	zloop_init 384, 1, 4		;; 383 more iterations
	zloop_set_rw_prefetch ZSEXT	;; Prefetch 2298 cache lines
	zr64_sixtyfour_complex_with_mult_preload
b4c:	zr64_sixtyfour_complex_with_mult rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 1, L1PREFETCH_ALL, 8*128
	zloop	4, b4c, -4*8*128+dist32	;; Test loop counter, next source pointer
	zloop	96, b4c			;; Test loop counter
	add	rsi, r8
	jmp	zpass2_24576_real_unfft

zpass2_24576_real_4:
	mov	rbp, DIST_TO_MULSRCARG
	mov	rdx, sincos3		;; Real sin/cos data
	mov	rdi, xsincos_complex	;; Complex sin/cos data
	bump	rdx, 512*ZMM_SCD3+64*ZMM_SCD4+8*ZMM_SCD4
	bump	rdi, 512*ZMM_SCD3+64*ZMM_SCD4+8*ZMM_SCD4
;;	zr64f_hundredtwentyeight_real_with_mulf_preload
;;	zr64f_hundredtwentyeight_real_with_mulf rsi, 8*128, 1*128, 2*128, 4*128, rdx, 0*ZMM_SCD4, rdi, 0*ZMM_SCD4, 1
	CALLP	zreal_mulf
	zloop_init 384, 1, 4		;; 383 more iterations
	zloop_set_rw_prefetch ZSEXT	;; Prefetch 2298 cache lines
	zr64f_sixtyfour_complex_with_mulf_preload
b4d:	zr64f_sixtyfour_complex_with_mulf rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 1, L1PREFETCH_ALL, 8*128
	zloop	4, b4d, -4*8*128+dist32	;; Test loop counter, next source pointer
	zloop	96, b4d			;; Test loop counter
zpass2_24576_real_unfft:
	bump	rsi, -96*dist32		;; Restore source pointer

;; Do inverse FFT levels 8,9,10
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	64	+1	...
;;	8	...
;;	...
;;	56
;;	128
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	512	+1	...
;;	8	...
;;	...
;;	504
;;	1K
;;	...

;; Do 64 sixteen_reals_unfft operations
;; Do 3008 eight_complex_unfft operations
;;	distance between fft data elements is 64
;;	do 384 macros each processing 128 data values

	zloop_init 384			;; 48 iterations of 8
	zloop_set_rw_prefetch ZDOUBLE	;; Prefetch 768 cache lines
	bump	rdx, -8*ZMM_SCD4
	bump	rdi, -8*ZMM_SCD4
	zr8_2sc_sixteen_reals_unfft_preload
c3a:	zr8_2sc_sixteen_reals_unfft rsi, 128, 8*128, 16*128, dist32, rdx, ZMM_SCD4, rdi, ZMM_SCD4, 8
	zloop	8, c3a, -8*128+2*dist32, rdi, -8*ZMM_SCD4 ;; Test loop counter, next source pointer, restore sin/cos pointer
	loops_reset			;; 47 more iters of 8
	zr8_eight_complex_djbunfft_preload
c3b:	zr8_eight_complex_djbunfft rsi, 128, 8*128, 16*128, dist32, rdi, ZMM_SCD4, 8
	zloop	8, c3b, -8*128+2*dist32, rdi, -8*ZMM_SCD4 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	48, c3b, -48*2*dist32	;; Test loop counter, restore source pointer

;; Do inverse FFT levels 5,6,7
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	512	+1	...
;;	8	...
;;	...
;;	504
;;	1K
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	4K	+1	...
;;	8	...
;;	...
;;	4K-8
;;	8K
;;	...

;; Do 512 sixteen_reals_unfft operations
;; Do 2560 eight_complex_unfft operations
;;	distance between fft data elements is 512
;;	do 384 macros each processing 128 data values

	zloop_init 384			;; 6 iterations of 64
	zloop_set_rw_prefetch ZDOUBLE	;; Prefetch 768 cache lines
	bump	rdx, -8*ZMM_SCD4-64*ZMM_SCD4
	bump	rdi, -64*ZMM_SCD4
	zr8_2sc_sixteen_reals_unfft_preload
c2a:	zr8_2sc_sixteen_reals_unfft rsi, 128, 2*dist32, 4*dist32, 8*dist32, rdx, ZMM_SCD4, rdi, ZMM_SCD4, 32
	zloop	32, c2a, -32*128+dist32	;; Test loop counter, next source pointer
	zloop	2, c2a, -2*dist32+16*dist32, rdi, -64*ZMM_SCD4 ;; Test loop counter, next source pointer, restore sin/cos pointer
	loops_reset			;; 5 more iters of 64
	zr8_eight_complex_djbunfft_preload
c2b:	zr8_eight_complex_djbunfft rsi, 128, 2*dist32, 4*dist32, 8*dist32, rdi, ZMM_SCD4, 32
	zloop	32, c2b, -32*128+dist32	;; Test loop counter, next source pointer
	zloop	2, c2b, -2*dist32+16*dist32, rdi, -64*ZMM_SCD4 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	6, c2b, -6*16*dist32	;; Test loop counter, restore source pointer

;; Do inverse FFT levels 1,2,3,4
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	4K	+1	...
;;	8	...
;;	...
;;	4K-8
;;	8K
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	24K	+1	...
;;	8	...
;;	...
;;	24K-8

;; Do 4096 twelve_reals_last_unfft operations
;;	distance between fft data elements is 4K
;;	do 512 macros each processing 96 data values

	bump	rdx, -64*ZMM_SCD4-512*ZMM_SCD3
	bump	rdi, -512*ZMM_SCD3
	zloop_init 512			;; 512 iterations
	zloop_set_rw_prefetch ZSINGLE	;; Prefetch 512 cache lines
	zr6_2sc_twelve_reals_unfft_preload
c1b:	zr6_2sc_twelve_reals_unfft rsi, 128, 16*dist32, rdx, ZMM_SCD3, rdi, ZMM_SCD3, 32
	zloop	32, c1b, -32*128+dist32	;; Test loop counter, next source pointer
	zloop	16, c1b			;; Test loop counter

zpass2_24576_real_done:
	end_timer 2
	ENDM

; *************** radix8-pass2-24576-complex macro ******************
; This macro takes 24576 complex values and performs the final 14.585 levels of a standard
; radix-8 FFT, squares the results, and does the first 14.585 levels of the inverse FFT.

zr4dwpn_pass2_24576_complex MACRO
	LOCAL	b1b, b2b, b3b, b4a, b4b, b4c, b4d, c1b, c2b, c3b
	LOCAL	zpass2_24576_complex_1, zpass2_24576_complex_2
	LOCAL	zpass2_24576_complex_3, zpass2_24576_complex_4
	LOCAL	zpass2_24576_complex_unfft, zpass2_24576_complex_done

;; Type 4 FFTs skip the forward FFT process

	mov	rsi, DATA_ADDR		;; Load address of FFT data
	mov	rbx, DIST_TO_FFTSRCARG
	mov	rcx, DATA_PREFETCH	;; Prefetch pointer (we need to prefetch 6144 cache lines + 95 pad cache lines)
	cmp	ffttype, 4
	je	zpass2_24576_complex_4

;; Do FFT levels 1,2,3
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	24K	+1	...
;;	8	...
;;	...
;;	24K-8
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	4K	+1	...
;;	8	...
;;	...
;;	4K-8
;;	8K
;;	...

;; Do 4096 six_complex_fft operations
;;	distance between fft data elements is 4K
;;	do 512 macros each processing 96 data values

	start_timer 4
	mov	rdi, xsincos_complex	;; The sin/cos data
	zloop_init 512			;; 512 iterations
	zloop_set_rw_prefetch ZSINGLE	;; Prefetch 512 cache lines
	zr6f_six_complex_djbfft_preload
b1b:	zr6f_six_complex_djbfft rsi, 128, 16*dist32, rdi, ZMM_SCD3, 32, L1PREFETCH_ALL, 128
	zloop	32, b1b, -32*128+dist32	;; Test loop counter, next source pointer
	zloop	16, b1b, -16*dist32	;; Test loop counter, restore source pointer
	end_timer 4

;; Do FFT levels 4,5,6
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	4K	+1	...
;;	8	...
;;	...
;;	4K-8
;;	8K
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	512	+1	...
;;	8	...
;;	...
;;	504
;;	1K
;;	...

;; Do 3072 eight_complex_fft operations
;;	distance between fft data elements is 512
;;	do 384 macros each processing 128 data values

	start_timer 5
	zloop_init 384			;; 6 iterations of 64
	zloop_set_rw_prefetch ZDOUBLE	;; Prefetch 768 cache lines
	zr8_eight_complex_djbfft_preload
b2b:	zr8_eight_complex_djbfft rsi, 128, 2*dist32, 4*dist32, 8*dist32, rdi, ZMM_SCD4, 32, L1PREFETCH_ALL, 128
	zloop	32, b2b, -32*128+dist32	;; Test loop counter, next source pointer
	zloop	2, b2b, -2*dist32+16*dist32, rdi, -64*ZMM_SCD4 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	6, b2b, -6*16*dist32, rdi, 64*ZMM_SCD4 ;; Test loop counter, restore source pointer, next sin/cos pointer
	end_timer 5

;; Do FFT levels 7,8,9
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	512	+1	...
;;	8	...
;;	...
;;	504
;;	1K
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	64	+1	...
;;	8	...
;;	...
;;	56
;;	128
;;	...

;; Do 3072 eight_complex_fft operations
;;	distance between fft data elements is 64
;;	do 384 macros each processing 128 data values

	start_timer 6
	zloop_init 384			;; 48 iterations of 8
	zloop_set_rw_prefetch ZDOUBLE	;; Prefetch 768 cache lines
	zr8_eight_complex_djbfft_preload
b3b:	zr8_eight_complex_djbfft rsi, 128, 8*128, 16*128, dist32, rdi, ZMM_SCD4, 8, L1PREFETCH_ALL, 128
	zloop	8, b3b, -8*128+2*dist32, rdi, -8*ZMM_SCD4 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	48, b3b, -48*2*dist32, rdi, 8*ZMM_SCD4 ;; Test loop counter, restore source pointer, next sin/cos pointer
	end_timer 6

;; Do FFT levels 10,11,12,13,14,15 as well as inverse FFT levels 10,11,12,13,14,15
;; On input and output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	64	+1	...
;;	8	...
;;	...
;;	56
;;	128
;;	...

;; Do 384 sixtyfour_complex_fft operations
;;	distance between fft data elements is 1 (8 for swizzling)
;;	do 384 macros each processing 128 data values

;; Execute the right middle step

pfing	bump	rcx, -161*64		;; We will prefetch 161 cache lines too many
	cmp	ffttype, 2
	jl	zpass2_24576_complex_1
	jg	zpass2_24576_complex_3

zpass2_24576_complex_2:
	start_timer 9
	zloop_init 384			;; 384 iterations
	zloop_set_rw_prefetch ZSEXT	;; Prefetch 2304 cache lines
	mov	r8, DEST2ARG
	zr64_sixtyfour_complex_with_square_preload
b4a:	zr64_sixtyfour_complex_with_square rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 4, L1PREFETCH_ALL, 8*128
	zloop	4, b4a, -4*8*128+dist32	;; Test loop counter, next source pointer
	zloop	96, b4a			;; Test loop counter
	add	rsi, r8
	end_timer 9
	jmp	zpass2_24576_complex_unfft

zpass2_24576_complex_1:
	start_timer 9
	zloop_init 384			;; 384 iterations
	zloop_set_rw_prefetch ZSEXT	;; Prefetch 2304 cache lines
	zr64_sixtyfour_complex_fft_final_preload
b4b:	zr64_sixtyfour_complex_fft_final rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 4, L1PREFETCH_ALL, 8*128
	zloop	4, b4b, -4*8*128+dist32	;; Test loop counter, next source pointer
	zloop	96, b4b			;; Test loop counter
	end_timer 9
	jmp	zpass2_24576_complex_done

zpass2_24576_complex_3:
	start_timer 9
	zloop_init 384			;; 384 iterations
	zloop_set_rw_prefetch ZSEXT	;; Prefetch 2304 cache lines
	mov	r8, DEST2ARG
	mov	rbp, DIST_TO_MULSRCARG
	zr64_sixtyfour_complex_with_mult_preload
b4c:	zr64_sixtyfour_complex_with_mult rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 4, L1PREFETCH_ALL, 8*128
	zloop	4, b4c, -4*8*128+dist32	;; Test loop counter, next source pointer
	zloop	96, b4c			;; Test loop counter
	add	rsi, r8
	end_timer 9
	jmp	zpass2_24576_complex_unfft

zpass2_24576_complex_4:
	start_timer 9
	zloop_init 384			;; 384 iterations
	zloop_set_rw_prefetch ZSEXT	;; Prefetch 2304 cache lines
	mov	rbp, DIST_TO_MULSRCARG
	mov	rdi, xsincos_complex	;; The sin/cos data
	bump	rdi, 512*ZMM_SCD3+64*ZMM_SCD4+8*ZMM_SCD4
	zr64f_sixtyfour_complex_with_mulf_preload
b4d:	zr64f_sixtyfour_complex_with_mulf rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 4, L1PREFETCH_ALL, 8*128
	zloop	4, b4d, -4*8*128+dist32	;; Test loop counter, next source pointer
	zloop	96, b4d			;; Test loop counter
	end_timer 9
zpass2_24576_complex_unfft:
	bump	rsi, -96*dist32		;; Restore source pointer

;; Do inverse FFT levels 7,8,9
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	64	+1	...
;;	8	...
;;	...
;;	56
;;	128
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	512	+1	...
;;	8	...
;;	...
;;	504
;;	1K
;;	...

;; Do 3072 eight_complex_unfft operations
;;	distance between fft data elements is 64
;;	do 384 macros each processing 128 data values

	start_timer 12
	bump	rdi, -8*ZMM_SCD4
	zloop_init 384			;; 48 iterations of 8
	zloop_set_rw_prefetch ZDOUBLE	;; Prefetch 768 cache lines
	zr8_eight_complex_djbunfft_preload
c3b:	zr8_eight_complex_djbunfft rsi, 128, 8*128, 16*128, dist32, rdi, ZMM_SCD4, 8, L1PREFETCH_ALL, 128
	zloop	8, c3b, -8*128+2*dist32, rdi, -8*ZMM_SCD4 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	48, c3b, -48*2*dist32	;; Test loop counter, restore source pointer
	end_timer 12

;; Do inverse FFT levels 4,5,6
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	512	+1	...
;;	8	...
;;	...
;;	504
;;	1K
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	4K	+1	...
;;	8	...
;;	...
;;	4K-8
;;	8K
;;	...

;; Do 3072 eight_complex_unfft operations
;;	distance between fft data elements is 512
;;	do 384 macros each processing 128 data values

	start_timer 13
	bump	rdi, -64*ZMM_SCD4
	zloop_init 384			;; 6 iterations of 64
	zloop_set_rw_prefetch ZDOUBLE	;; Prefetch 768 cache lines
	zr8_eight_complex_djbunfft_preload
c2b:	zr8_eight_complex_djbunfft rsi, 128, 2*dist32, 4*dist32, 8*dist32, rdi, ZMM_SCD4, 32, L1PREFETCH_ALL, 128
	zloop	32, c2b, -32*128+dist32	;; Test loop counter, next source pointer
	zloop	2, c2b, -2*dist32+16*dist32, rdi, -64*ZMM_SCD4 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	6, c2b, -6*16*dist32	;; Test loop counter, restore source pointer
	end_timer 13

;; Do inverse FFT levels 1,2,3
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	4K	+1	...
;;	8	...
;;	...
;;	4K-8
;;	8K
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	24K	+1	...
;;	8	...
;;	...
;;	24K-8

;; Do 4096 six_complex_unfft operations
;;	distance between fft data elements is 4K
;;	do 512 macros each processing 96 data values

	start_timer 14
	bump	rdi, -512*ZMM_SCD3
	zloop_init 512			;; 512 iterations
	zloop_set_rw_prefetch ZSINGLE	;; Prefetch 512 cache lines
	zr6_six_complex_djbunfft_preload
c1b:	zr6_six_complex_djbunfft rsi, 128, 16*dist32, rdi, ZMM_SCD3, 32, L1PREFETCH_ALL, 128
	zloop	32, c1b, -32*128+dist32	;; Test loop counter, next source pointer
	zloop	16, c1b			;; Test loop counter
	end_timer 14

zpass2_24576_complex_done:
	ENDM


; *************** radix8-pass2-25088-real macro ******************
; This macro takes 50176 real values and performs the final 15.615 levels of the FFT.
; The results are squared, and the first 15.615 levels of the inverse FFT are done.

zr4dwpn_pass2_25088_real MACRO
	LOCAL	b1b, b2a, b2b, b3a, b3b, b4a, b4b, b4c, b4d, c1b, c2a, c2b, c3a, c3b
	LOCAL	zpass2_25088_real_1, zpass2_25088_real_2
	LOCAL	zpass2_25088_real_3, zpass2_25088_real_4
	LOCAL	zpass2_25088_real_unfft, zpass2_25088_real_done

;; Type 4 FFTs skip the forward FFT process

	start_timer 2
	mov	rsi, DATA_ADDR		;; Load source address
	mov	rbx, DIST_TO_FFTSRCARG
	mov	rcx, DATA_PREFETCH	;; Prefetch pointer (we need to prefetch 6272 cache lines + ??? pad cache lines)
	cmp	ffttype, 4
	je	zpass2_25088_real_4

;; Do FFT levels 1,2,3,4
;;	0	+1	+1	+1	+1	+1	+1	+1	24.5K	+1	...
;;	8	...
;;	...
;;	24.5K-8
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	3584	+1	...
;;	8	...
;;	...
;;	3576
;;	7K
;;	...

;; Do 3584 fourteen_real_fft operations
;;	distance between fft data elements is 3584
;;	do 448 macros each processing 112 data values

	mov	rdx, sincos3		;; Real sin/cos data
	mov	rdi, xsincos_complex	;; Complex sin/cos data
	zloop_init 448			;; 448 iterations
	zloop_set_rw_prefetch ZSINGLE	;; Prefetch 448 cache lines
	zr7f_2sc_fourteen_reals_fft_preload
b1b:	zr7f_2sc_fourteen_reals_fft rsi, 128, 14*dist32, rdx, ZMM_SCD3, rdi, ZMM_SCD3, 32
	zloop	32, b1b, -32*128+dist32	;; Test loop counter, next source pointer
	zloop	14, b1b, -14*dist32	;; Test loop counter, restore source pointer

;; Do FFT levels 5,6,7
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	3584	+1	...
;;	8	...
;;	...
;;	3576
;;	7K
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	512	+1	...
;;	8	...
;;	...
;;	504
;;	1K
;;	...

;; Do 512 fourteen_reals_fft operations
;; Do 3072 seven_complex_fft operations
;;	distance between fft data elements is 512
;;	do 448 macros each processing 112 data values

	zloop_init 448			;; 7 iterations of 64
	zloop_set_rw_prefetch ZDOUBLE	;; Prefetch 896 cache lines
	zr7_2sc_fourteen_reals_fft_preload
b2a:	zr7_2sc_fourteen_reals_fft rsi, 128, 2*dist32, rdx, ZMM_SCD3, rdi, ZMM_SCD3, 32
	zloop	32, b2a, -32*128+dist32	;; Test loop counter, next source pointer
	zloop	2, b2a, -2*dist32+14*dist32, rdi, -64*ZMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer
	loops_reset			;; 6 more iters of 64
	zr7_seven_complex_djbfft_preload
b2b:	zr7_seven_complex_djbfft rsi, 128, 2*dist32, rdi, ZMM_SCD3, 32
	zloop	32, b2b, -32*128+dist32	;; Test loop counter, next source pointer
	zloop	2, b2b, -2*dist32+14*dist32, rdi, -64*ZMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	7, b2b, -7*14*dist32, rdi, 64*ZMM_SCD3 ;; Test loop counter, restore source pointer, next sin/cos pointer

;; Do FFT levels 8,9,10
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	512	+1	...
;;	8	...
;;	...
;;	504
;;	1K
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	64	+1	...
;;	8	...
;;	...
;;	56
;;	128
;;	...

;; Do 64 sixteen_reals_fft operations
;; Do 3072 eight_complex_fft operations
;;	distance between fft data elements is 64
;;	do 392 macros each processing 128 data values

	zloop_init 392			;; 49 iterations of 8
	zloop_set_rw_prefetch ZDOUBLE	;; Prefetch 784 cache lines
	zr8_2sc_sixteen_reals_fft_preload
b3a:	zr8_2sc_sixteen_reals_fft rsi, 128, 8*128, 16*128, dist32, rdx, ZMM_SCD4, rdi, ZMM_SCD4, 8
	zloop	8, b3a, -8*128+2*dist32, rdi, -8*ZMM_SCD4 ;; Test loop counter, next source pointer, restore sin/cos pointer
	loops_reset			;; 48 more iters of 8
	zr8_eight_complex_djbfft_preload
b3b:	zr8_eight_complex_djbfft rsi, 128, 8*128, 16*128, dist32, rdi, ZMM_SCD4, 8
	zloop	8, b3b, -8*128+2*dist32, rdi, -8*ZMM_SCD4 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	49, b3b, -49*2*dist32, rdi, 8*ZMM_SCD4 ;; Test loop counter, restore source pointer, next sin/cos pointer

;; Do FFT levels 11,12,13,14,15,16 as well as inverse FFT levels 11,12,13,14,15,16
;; On input and output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	64	+1	...
;;	8	...
;;	...
;;	56
;;	128
;;	...

;; Do 1 hundredtwentyeight_reals_fft operation
;; Do 391 sixtyfour_complex_fft operations
;;	distance between fft data elements is 1 (8 for swizzling)
;;	do 392 macros each processing 128 data values

;; Execute the proper middle step

pfing	bump	rcx, -233*64		;; We will prefetch 233 cache lines too many
	cmp	ffttype, 2
	jl	zpass2_25088_real_1
	jg	zpass2_25088_real_3

zpass2_25088_real_2:
	mov	r8, DEST2ARG
;;	zr64_hundredtwentyeight_real_with_square_preload
;;	zr64_hundredtwentyeight_real_with_square rsi, 8*128, 1*128, 2*128, 4*128, rdx, 0*ZMM_SCD4, rdi, 0*ZMM_SCD4, 1
	CALLP	zreal_square
	zloop_init 392, 1, 4		;; 391 more iterations
	zloop_set_rw_prefetch ZSEXT	;; Prefetch 2352 cache lines
	zr64_sixtyfour_complex_with_square_preload
b4a:	zr64_sixtyfour_complex_with_square rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 1, L1PREFETCH_ALL, 8*128
	zloop	4, b4a, -4*8*128+dist32	;; Test loop counter, next source pointer
	zloop	98, b4a			;; Test loop counter
	add	rsi, r8
	jmp	zpass2_25088_real_unfft

zpass2_25088_real_1:
;;	zr64_hundredtwentyeight_real_fft_final_preload
;;	zr64_hundredtwentyeight_real_fft_final rsi, 8*128, 1*128, 2*128, 4*128, rdx, 0*ZMM_SCD4, rdi, 0*ZMM_SCD4, 1
	CALLP	zreal_fft_final
	zloop_init 392, 1, 4		;; 391 more iterations
	zloop_set_rw_prefetch ZSEXT	;; Prefetch 2352 cache lines
	zr64_sixtyfour_complex_fft_final_preload
b4b:	zr64_sixtyfour_complex_fft_final rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 1, L1PREFETCH_ALL, 8*128
	zloop	4, b4b, -4*8*128+dist32	;; Test loop counter, next source pointer
	zloop	98, b4b			;; Test loop counter
	jmp	zpass2_25088_real_done

zpass2_25088_real_3:
	mov	r8, DEST2ARG
	mov	rbp, DIST_TO_MULSRCARG
;;	zr64_hundredtwentyeight_real_with_mult_preload
;;	zr64_hundredtwentyeight_real_with_mult rsi, 8*128, 1*128, 2*128, 4*128, rdx, 0*ZMM_SCD4, rdi, 0*ZMM_SCD4, 1
	CALLP	zreal_mult
	zloop_init 392, 1, 4		;; 391 more iterations
	zloop_set_rw_prefetch ZSEXT	;; Prefetch 2352 cache lines
	zr64_sixtyfour_complex_with_mult_preload
b4c:	zr64_sixtyfour_complex_with_mult rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 1, L1PREFETCH_ALL, 8*128
	zloop	4, b4c, -4*8*128+dist32	;; Test loop counter, next source pointer
	zloop	98, b4c			;; Test loop counter
	add	rsi, r8
	jmp	zpass2_25088_real_unfft

zpass2_25088_real_4:
	mov	rbp, DIST_TO_MULSRCARG
	mov	rdx, sincos3		;; Real sin/cos data
	mov	rdi, xsincos_complex	;; Complex sin/cos data
	bump	rdx, 448*ZMM_SCD3+64*ZMM_SCD3+8*ZMM_SCD4
	bump	rdi, 448*ZMM_SCD3+64*ZMM_SCD3+8*ZMM_SCD4
;;	zr64f_hundredtwentyeight_real_with_mulf_preload
;;	zr64f_hundredtwentyeight_real_with_mulf rsi, 8*128, 1*128, 2*128, 4*128, rdx, 0*ZMM_SCD4, rdi, 0*ZMM_SCD4, 1
	CALLP	zreal_mulf
	zloop_init 392, 1, 4		;; 391 more iterations
	zloop_set_rw_prefetch ZSEXT	;; Prefetch 2352 cache lines
	zr64f_sixtyfour_complex_with_mulf_preload
b4d:	zr64f_sixtyfour_complex_with_mulf rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 1, L1PREFETCH_ALL, 8*128
	zloop	4, b4d, -4*8*128+dist32	;; Test loop counter, next source pointer
	zloop	98, b4d			;; Test loop counter
zpass2_25088_real_unfft:
	bump	rsi, -98*dist32		;; Restore source pointer

;; Do inverse FFT levels 8,9,10
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	64	+1	...
;;	8	...
;;	...
;;	56
;;	128
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	512	+1	...
;;	8	...
;;	...
;;	504
;;	1K
;;	...

;; Do 64 sixteen_reals_unfft operations
;; Do 3072 eight_complex_unfft operations
;;	distance between fft data elements is 64
;;	do 392 macros each processing 128 data values

	zloop_init 392			;; 49 iterations of 8
	zloop_set_rw_prefetch ZDOUBLE	;; Prefetch 784 cache lines
	bump	rdx, -8*ZMM_SCD4
	bump	rdi, -8*ZMM_SCD4
	zr8_2sc_sixteen_reals_unfft_preload
c3a:	zr8_2sc_sixteen_reals_unfft rsi, 128, 8*128, 16*128, dist32, rdx, ZMM_SCD4, rdi, ZMM_SCD4, 8
	zloop	8, c3a, -8*128+2*dist32, rdi, -8*ZMM_SCD4 ;; Test loop counter, next source pointer, restore sin/cos pointer
	loops_reset			;; 48 more iters of 8
	zr8_eight_complex_djbunfft_preload
c3b:	zr8_eight_complex_djbunfft rsi, 128, 8*128, 16*128, dist32, rdi, ZMM_SCD4, 8
	zloop	8, c3b, -8*128+2*dist32, rdi, -8*ZMM_SCD4 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	49, c3b, -49*2*dist32	;; Test loop counter, restore source pointer

;; Do inverse FFT levels 5,6,7
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	512	+1	...
;;	8	...
;;	...
;;	504
;;	1K
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	3584	+1	...
;;	8	...
;;	...
;;	3576
;;	7K
;;	...

;; Do 512 fourteen_reals_unfft operations
;; Do 3072 seven_complex_unfft operations
;;	distance between fft data elements is 512
;;	do 448 macros each processing 112 data values

	zloop_init 448			;; 7 iterations of 64
	zloop_set_rw_prefetch ZDOUBLE	;; Prefetch 896 cache lines
	bump	rdx, -8*ZMM_SCD4-64*ZMM_SCD3
	bump	rdi, -64*ZMM_SCD3
	zr7_2sc_fourteen_reals_unfft_preload
c2a:	zr7_2sc_fourteen_reals_unfft rsi, 128, 2*dist32, rdx, ZMM_SCD3, rdi, ZMM_SCD3, 32
	zloop	32, c2a, -32*128+dist32	;; Test loop counter, next source pointer
	zloop	2, c2a, -2*dist32+14*dist32, rdi, -64*ZMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer
	loops_reset			;; 6 more iters of 64
	zr7_seven_complex_djbunfft_preload
c2b:	zr7_seven_complex_djbunfft rsi, 128, 2*dist32, rdi, ZMM_SCD3, 32
	zloop	32, c2b, -32*128+dist32	;; Test loop counter, next source pointer
	zloop	2, c2b, -2*dist32+14*dist32, rdi, -64*ZMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	7, c2b, -7*14*dist32	;; Test loop counter, restore source pointer

;; Do inverse FFT levels 1,2,3,4
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	3584	+1	...
;;	8	...
;;	...
;;	3576
;;	7K
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	24.5K	+1	...
;;	8	...
;;	...
;;	24.5K-8

;; Do 3584 fourteen_reals_last_unfft operations
;;	distance between fft data elements is 3584
;;	do 448 macros each processing 112 data values

	bump	rdx, -64*ZMM_SCD3-448*ZMM_SCD3
	bump	rdi, -448*ZMM_SCD3
	zloop_init 448			;; 448 iterations
	zloop_set_rw_prefetch ZSINGLE	;; Prefetch 448 cache lines
	zr7_2sc_fourteen_reals_unfft_preload
c1b:	zr7_2sc_fourteen_reals_unfft rsi, 128, 14*dist32, rdx, ZMM_SCD3, rdi, ZMM_SCD3, 32
	zloop	32, c1b, -32*128+dist32	;; Test loop counter, next source pointer
	zloop	14, c1b			;; Test loop counter

zpass2_25088_real_done:
	end_timer 2
	ENDM

; *************** radix8-pass2-25088-complex macro ******************
; This macro takes 25088 complex values and performs the final 14.615 levels of a standard
; radix-8 FFT, squares the results, and does the first 14.615 levels of the inverse FFT.

zr4dwpn_pass2_25088_complex MACRO
	LOCAL	b1b, b2b, b3b, b4a, b4b, b4c, b4d, c1b, c2b, c3b
	LOCAL	zpass2_25088_complex_1, zpass2_25088_complex_2
	LOCAL	zpass2_25088_complex_3, zpass2_25088_complex_4
	LOCAL	zpass2_25088_complex_unfft, zpass2_25088_complex_done

;; Type 4 FFTs skip the forward FFT process

	mov	rsi, DATA_ADDR		;; Load address of FFT data
	mov	rbx, DIST_TO_FFTSRCARG
	mov	rcx, DATA_PREFETCH	;; Prefetch pointer (we need to prefetch 6272 cache lines + 97 pad cache lines)
	cmp	ffttype, 4
	je	zpass2_25088_complex_4

;; Do FFT levels 1,2,3
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	24.5K	+1	...
;;	8	...
;;	...
;;	24.5K-8
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	3584	+1	...
;;	8	...
;;	...
;;	3576
;;	7K
;;	...

;; Do 3584 seven_complex_fft operations
;;	distance between fft data elements is 3584
;;	do 448 macros each processing 112 data values

	start_timer 4
	mov	rdi, xsincos_complex	;; The sin/cos data
	zloop_init 448			;; 448 iterations
	zloop_set_rw_prefetch ZSINGLE	;; Prefetch 448 cache lines
	zr7f_seven_complex_djbfft_preload
b1b:	zr7f_seven_complex_djbfft rsi, 128, 14*dist32, rdi, ZMM_SCD3, 32, L1PREFETCH_ALL, 128
	zloop	32, b1b, -32*128+dist32	;; Test loop counter, next source pointer
	zloop	14, b1b, -14*dist32	;; Test loop counter, restore source pointer
	end_timer 4

;; Do FFT levels 4,5,6
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	3584	+1	...
;;	8	...
;;	...
;;	3576
;;	7K
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	512	+1	...
;;	8	...
;;	...
;;	504
;;	1K
;;	...

;; Do 3136 eight_complex_fft operations
;;	distance between fft data elements is 512
;;	do 448 macros each processing 112 data values

	start_timer 5
	zloop_init 448			;; 7 iterations of 64
	zloop_set_rw_prefetch ZDOUBLE	;; Prefetch 896 cache lines
	zr7_seven_complex_djbfft_preload
b2b:	zr7_seven_complex_djbfft rsi, 128, 2*dist32, rdi, ZMM_SCD3, 32, L1PREFETCH_ALL, 128
	zloop	32, b2b, -32*128+dist32	;; Test loop counter, next source pointer
	zloop	2, b2b, -2*dist32+14*dist32, rdi, -64*ZMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	7, b2b, -7*14*dist32, rdi, 64*ZMM_SCD3 ;; Test loop counter, restore source pointer, next sin/cos pointer
	end_timer 5

;; Do FFT levels 7,8,9
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	512	+1	...
;;	8	...
;;	...
;;	504
;;	1K
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	64	+1	...
;;	8	...
;;	...
;;	56
;;	128
;;	...

;; Do 3136 eight_complex_fft operations
;;	distance between fft data elements is 64
;;	do 392 macros each processing 128 data values

	start_timer 6
	zloop_init 392			;; 49 iterations of 8
	zloop_set_rw_prefetch ZDOUBLE	;; Prefetch 784 cache lines
	zr8_eight_complex_djbfft_preload
b3b:	zr8_eight_complex_djbfft rsi, 128, 8*128, 16*128, dist32, rdi, ZMM_SCD4, 8, L1PREFETCH_ALL, 128
	zloop	8, b3b, -8*128+2*dist32, rdi, -8*ZMM_SCD4 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	49, b3b, -49*2*dist32, rdi, 8*ZMM_SCD4 ;; Test loop counter, restore source pointer, next sin/cos pointer
	end_timer 6

;; Do FFT levels 10,11,12,13,14,15 as well as inverse FFT levels 10,11,12,13,14,15
;; On input and output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	64	+1	...
;;	8	...
;;	...
;;	56
;;	128
;;	...

;; Do 392 sixtyfour_complex_fft operations
;;	distance between fft data elements is 1 (8 for swizzling)
;;	do 392 macros each processing 128 data values

;; Execute the right middle step

pfing	bump	rcx, -239*64		;; We will prefetch 239 cache lines too many
	cmp	ffttype, 2
	jl	zpass2_25088_complex_1
	jg	zpass2_25088_complex_3

zpass2_25088_complex_2:
	start_timer 9
	zloop_init 392			;; 392 iterations
	zloop_set_rw_prefetch ZSEXT	;; Prefetch 2352 cache lines
	mov	r8, DEST2ARG
	zr64_sixtyfour_complex_with_square_preload
b4a:	zr64_sixtyfour_complex_with_square rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 4, L1PREFETCH_ALL, 8*128
	zloop	4, b4a, -4*8*128+dist32	;; Test loop counter, next source pointer
	zloop	98, b4a			;; Test loop counter
	add	rsi, r8
	end_timer 9
	jmp	zpass2_25088_complex_unfft

zpass2_25088_complex_1:
	start_timer 9
	zloop_init 392			;; 392 iterations
	zloop_set_rw_prefetch ZSEXT	;; Prefetch 2352 cache lines
	zr64_sixtyfour_complex_fft_final_preload
b4b:	zr64_sixtyfour_complex_fft_final rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 4, L1PREFETCH_ALL, 8*128
	zloop	4, b4b, -4*8*128+dist32	;; Test loop counter, next source pointer
	zloop	98, b4b			;; Test loop counter
	end_timer 9
	jmp	zpass2_25088_complex_done

zpass2_25088_complex_3:
	start_timer 9
	zloop_init 392			;; 392 iterations
	zloop_set_rw_prefetch ZSEXT	;; Prefetch 2352 cache lines
	mov	r8, DEST2ARG
	mov	rbp, DIST_TO_MULSRCARG
	zr64_sixtyfour_complex_with_mult_preload
b4c:	zr64_sixtyfour_complex_with_mult rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 4, L1PREFETCH_ALL, 8*128
	zloop	4, b4c, -4*8*128+dist32	;; Test loop counter, next source pointer
	zloop	98, b4c			;; Test loop counter
	add	rsi, r8
	end_timer 9
	jmp	zpass2_25088_complex_unfft

zpass2_25088_complex_4:
	start_timer 9
	zloop_init 392			;; 392 iterations
	zloop_set_rw_prefetch ZSEXT	;; Prefetch 2352 cache lines
	mov	rbp, DIST_TO_MULSRCARG
	mov	rdi, xsincos_complex	;; The sin/cos data
	bump	rdi, 448*ZMM_SCD3+64*ZMM_SCD3+8*ZMM_SCD4
	zr64f_sixtyfour_complex_with_mulf_preload
b4d:	zr64f_sixtyfour_complex_with_mulf rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 4, L1PREFETCH_ALL, 8*128
	zloop	4, b4d, -4*8*128+dist32	;; Test loop counter, next source pointer
	zloop	98, b4d			;; Test loop counter
	end_timer 9
zpass2_25088_complex_unfft:
	bump	rsi, -98*dist32		;; Restore source pointer

;; Do inverse FFT levels 7,8,9
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	64	+1	...
;;	8	...
;;	...
;;	56
;;	128
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	512	+1	...
;;	8	...
;;	...
;;	504
;;	1K
;;	...

;; Do 3136 eight_complex_unfft operations
;;	distance between fft data elements is 64
;;	do 392 macros each processing 128 data values

	start_timer 12
	bump	rdi, -8*ZMM_SCD4
	zloop_init 392			;; 49 iterations of 8
	zloop_set_rw_prefetch ZDOUBLE	;; Prefetch 784 cache lines
	zr8_eight_complex_djbunfft_preload
c3b:	zr8_eight_complex_djbunfft rsi, 128, 8*128, 16*128, dist32, rdi, ZMM_SCD4, 8, L1PREFETCH_ALL, 128
	zloop	8, c3b, -8*128+2*dist32, rdi, -8*ZMM_SCD4 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	49, c3b, -49*2*dist32	;; Test loop counter, restore source pointer
	end_timer 12

;; Do inverse FFT levels 4,5,6
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	512	+1	...
;;	8	...
;;	...
;;	504
;;	1K
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	3584	+1	...
;;	8	...
;;	...
;;	3576
;;	7K
;;	...

;; Do 3584 eight_complex_unfft operations
;;	distance between fft data elements is 512
;;	do 448 macros each processing 112 data values

	start_timer 13
	bump	rdi, -64*ZMM_SCD3
	zloop_init 448			;; 7 iterations of 64
	zloop_set_rw_prefetch ZDOUBLE	;; Prefetch 896 cache lines
	zr7_seven_complex_djbunfft_preload
c2b:	zr7_seven_complex_djbunfft rsi, 128, 2*dist32, rdi, ZMM_SCD3, 32, L1PREFETCH_ALL, 128
	zloop	32, c2b, -32*128+dist32	;; Test loop counter, next source pointer
	zloop	2, c2b, -2*dist32+14*dist32, rdi, -64*ZMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	7, c2b, -7*14*dist32	;; Test loop counter, restore source pointer
	end_timer 13

;; Do inverse FFT levels 1,2,3
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	3584	+1	...
;;	8	...
;;	...
;;	3576
;;	7K
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	24.5K	+1	...
;;	8	...
;;	...
;;	24.5K-8

;; Do 3584 seven_complex_unfft operations
;;	distance between fft data elements is 3584
;;	do 448 macros each processing 112 data values

	start_timer 14
	bump	rdi, -448*ZMM_SCD3
	zloop_init 448			;; 448 iterations
	zloop_set_rw_prefetch ZSINGLE	;; Prefetch 448 cache lines
	zr7_seven_complex_djbunfft_preload
c1b:	zr7_seven_complex_djbunfft rsi, 128, 14*dist32, rdi, ZMM_SCD3, 32, L1PREFETCH_ALL, 128
	zloop	32, c1b, -32*128+dist32	;; Test loop counter, next source pointer
	zloop	14, c1b			;; Test loop counter
	end_timer 14

zpass2_25088_complex_done:
	ENDM


; *************** radix8-pass2-28672-real macro ******************
; This macro takes 57344 real values and performs the final 15.807 levels of the FFT.
; The results are squared, and the first 15.807 levels of the inverse FFT are done.

zr4dwpn_pass2_28672_real MACRO
	LOCAL	b1b, b2a, b2b, b3a, b3b, b4a, b4b, b4c, b4d, c1b, c2a, c2b, c3a, c3b
	LOCAL	zpass2_28672_real_1, zpass2_28672_real_2
	LOCAL	zpass2_28672_real_3, zpass2_28672_real_4
	LOCAL	zpass2_28672_real_unfft, zpass2_28672_real_done

;; Type 4 FFTs skip the forward FFT process

	start_timer 2
	mov	rsi, DATA_ADDR		;; Load source address
	mov	rbx, DIST_TO_FFTSRCARG
	mov	rcx, DATA_PREFETCH	;; Prefetch pointer (we need to prefetch 7168 cache lines + ??? pad cache lines)
	cmp	ffttype, 4
	je	zpass2_28672_real_4

;; Do FFT levels 1,2,3,4
;;	0	+1	+1	+1	+1	+1	+1	+1	28K	+1	...
;;	8	...
;;	...
;;	28K-8
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	4K	+1	...
;;	8	...
;;	...
;;	4K-8
;;	8K
;;	...

;; Do 4096 fourteen_real_fft operations
;;	distance between fft data elements is 4K
;;	do 512 macros each processing 112 data values

	mov	rdx, sincos3		;; Real sin/cos data
	mov	rdi, xsincos_complex	;; Complex sin/cos data
	zloop_init 512			;; 512 iterations
	zloop_set_rw_prefetch ZSINGLE	;; Prefetch 512 cache lines
	zr7f_2sc_fourteen_reals_fft_preload
b1b:	zr7f_2sc_fourteen_reals_fft rsi, 128, 16*dist32, rdx, ZMM_SCD3, rdi, ZMM_SCD3, 32
	zloop	32, b1b, -32*128+dist32	;; Test loop counter, next source pointer
	zloop	16, b1b, -16*dist32	;; Test loop counter, restore source pointer

;; Do FFT levels 5,6,7
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	4K	+1	...
;;	8	...
;;	...
;;	4K-8
;;	8K
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	512	+1	...
;;	8	...
;;	...
;;	504
;;	1K
;;	...

;; Do 512 sixteen_reals_fft operations
;; Do 3072 eight_complex_fft operations
;;	distance between fft data elements is 512
;;	do 448 macros each processing 128 data values

	zloop_init 448			;; 7 iterations of 64
	zloop_set_rw_prefetch ZDOUBLE	;; Prefetch 896 cache lines
	zr8_2sc_sixteen_reals_fft_preload
b2a:	zr8_2sc_sixteen_reals_fft rsi, 128, 2*dist32, 4*dist32, 8*dist32, rdx, ZMM_SCD4, rdi, ZMM_SCD4, 32
	zloop	32, b2a, -32*128+dist32	;; Test loop counter, next source pointer
	zloop	2, b2a, -2*dist32+16*dist32, rdi, -64*ZMM_SCD4 ;; Test loop counter, next source pointer, restore sin/cos pointer
	loops_reset			;; 6 more iters of 64
	zr8_eight_complex_djbfft_preload
b2b:	zr8_eight_complex_djbfft rsi, 128, 2*dist32, 4*dist32, 8*dist32, rdi, ZMM_SCD4, 32
	zloop	32, b2b, -32*128+dist32	;; Test loop counter, next source pointer
	zloop	2, b2b, -2*dist32+16*dist32, rdi, -64*ZMM_SCD4 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	7, b2b, -7*16*dist32, rdi, 64*ZMM_SCD4 ;; Test loop counter, restore source pointer, next sin/cos pointer

;; Do FFT levels 8,9,10
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	512	+1	...
;;	8	...
;;	...
;;	504
;;	1K
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	64	+1	...
;;	8	...
;;	...
;;	56
;;	128
;;	...

;; Do 64 sixteen_reals_fft operations
;; Do 3520 eight_complex_fft operations
;;	distance between fft data elements is 64
;;	do 448 macros each processing 128 data values

	zloop_init 448			;; 56 iterations of 8
	zloop_set_rw_prefetch ZDOUBLE	;; Prefetch 896 cache lines
	zr8_2sc_sixteen_reals_fft_preload
b3a:	zr8_2sc_sixteen_reals_fft rsi, 128, 8*128, 16*128, dist32, rdx, ZMM_SCD4, rdi, ZMM_SCD4, 8
	zloop	8, b3a, -8*128+2*dist32, rdi, -8*ZMM_SCD4 ;; Test loop counter, next source pointer, restore sin/cos pointer
	loops_reset			;; 55 more iters of 8
	zr8_eight_complex_djbfft_preload
b3b:	zr8_eight_complex_djbfft rsi, 128, 8*128, 16*128, dist32, rdi, ZMM_SCD4, 8
	zloop	8, b3b, -8*128+2*dist32, rdi, -8*ZMM_SCD4 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	56, b3b, -56*2*dist32, rdi, 8*ZMM_SCD4 ;; Test loop counter, restore source pointer, next sin/cos pointer

;; Do FFT levels 11,12,13,14,15,16 as well as inverse FFT levels 11,12,13,14,15,16
;; On input and output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	64	+1	...
;;	8	...
;;	...
;;	56
;;	128
;;	...

;; Do 1 hundredtwentyeight_reals_fft operation
;; Do 447 sixtyfour_complex_fft operations
;;	distance between fft data elements is 1 (8 for swizzling)
;;	do 448 macros each processing 128 data values

;; Execute the proper middle step

pfing	bump	rcx, -11*64		;; We will prefetch 11 cache lines too many
	cmp	ffttype, 2
	jl	zpass2_28672_real_1
	jg	zpass2_28672_real_3

zpass2_28672_real_2:
	mov	r8, DEST2ARG
;;	zr64_hundredtwentyeight_real_with_square_preload
;;	zr64_hundredtwentyeight_real_with_square rsi, 8*128, 1*128, 2*128, 4*128, rdx, 0*ZMM_SCD4, rdi, 0*ZMM_SCD4, 1
	CALLP	zreal_square
	zloop_init 448, 1, 4		;; 447 more iterations
	zloop_set_rw_prefetch ZSEXT	;; Prefetch 2682 cache lines
	zr64_sixtyfour_complex_with_square_preload
b4a:	zr64_sixtyfour_complex_with_square rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 1, L1PREFETCH_ALL, 8*128
	zloop	4, b4a, -4*8*128+dist32	;; Test loop counter, next source pointer
	zloop	112, b4a		;; Test loop counter
	add	rsi, r8
	jmp	zpass2_28672_real_unfft

zpass2_28672_real_1:
;;	zr64_hundredtwentyeight_real_fft_final_preload
;;	zr64_hundredtwentyeight_real_fft_final rsi, 8*128, 1*128, 2*128, 4*128, rdx, 0*ZMM_SCD4, rdi, 0*ZMM_SCD4, 1
	CALLP	zreal_fft_final
	zloop_init 448, 1, 4		;; 447 more iterations
	zloop_set_rw_prefetch ZSEXT	;; Prefetch 2682 cache lines
	zr64_sixtyfour_complex_fft_final_preload
b4b:	zr64_sixtyfour_complex_fft_final rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 1, L1PREFETCH_ALL, 8*128
	zloop	4, b4b, -4*8*128+dist32	;; Test loop counter, next source pointer
	zloop	112, b4b		;; Test loop counter
	jmp	zpass2_28672_real_done

zpass2_28672_real_3:
	mov	r8, DEST2ARG
	mov	rbp, DIST_TO_MULSRCARG
;;	zr64_hundredtwentyeight_real_with_mult_preload
;;	zr64_hundredtwentyeight_real_with_mult rsi, 8*128, 1*128, 2*128, 4*128, rdx, 0*ZMM_SCD4, rdi, 0*ZMM_SCD4, 1
	CALLP	zreal_mult
	zloop_init 448, 1, 4		;; 447 more iterations
	zloop_set_rw_prefetch ZSEXT	;; Prefetch 2682 cache lines
	zr64_sixtyfour_complex_with_mult_preload
b4c:	zr64_sixtyfour_complex_with_mult rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 1, L1PREFETCH_ALL, 8*128
	zloop	4, b4c, -4*8*128+dist32	;; Test loop counter, next source pointer
	zloop	112, b4c		;; Test loop counter
	add	rsi, r8
	jmp	zpass2_28672_real_unfft

zpass2_28672_real_4:
	mov	rbp, DIST_TO_MULSRCARG
	mov	rdx, sincos3		;; Real sin/cos data
	mov	rdi, xsincos_complex	;; Complex sin/cos data
	bump	rdx, 512*ZMM_SCD3+64*ZMM_SCD4+8*ZMM_SCD4
	bump	rdi, 512*ZMM_SCD3+64*ZMM_SCD4+8*ZMM_SCD4
;;	zr64f_hundredtwentyeight_real_with_mulf_preload
;;	zr64f_hundredtwentyeight_real_with_mulf rsi, 8*128, 1*128, 2*128, 4*128, rdx, 0*ZMM_SCD4, rdi, 0*ZMM_SCD4, 1
	CALLP	zreal_mulf
	zloop_init 448, 1, 4		;; 447 more iterations
	zloop_set_rw_prefetch ZSEXT	;; Prefetch 2682 cache lines
	zr64f_sixtyfour_complex_with_mulf_preload
b4d:	zr64f_sixtyfour_complex_with_mulf rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 1, L1PREFETCH_ALL, 8*128
	zloop	4, b4d, -4*8*128+dist32	;; Test loop counter, next source pointer
	zloop	112, b4d		;; Test loop counter
zpass2_28672_real_unfft:
	bump	rsi, -112*dist32	;; Restore source pointer

;; Do inverse FFT levels 8,9,10
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	64	+1	...
;;	8	...
;;	...
;;	56
;;	128
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	512	+1	...
;;	8	...
;;	...
;;	504
;;	1K
;;	...

;; Do 64 sixteen_reals_unfft operations
;; Do 3520 eight_complex_unfft operations
;;	distance between fft data elements is 64
;;	do 448 macros each processing 128 data values

	zloop_init 448			;; 56 iterations of 8
	zloop_set_rw_prefetch ZDOUBLE	;; Prefetch 896 cache lines
	bump	rdx, -8*ZMM_SCD4
	bump	rdi, -8*ZMM_SCD4
	zr8_2sc_sixteen_reals_unfft_preload
c3a:	zr8_2sc_sixteen_reals_unfft rsi, 128, 8*128, 16*128, dist32, rdx, ZMM_SCD4, rdi, ZMM_SCD4, 8
	zloop	8, c3a, -8*128+2*dist32, rdi, -8*ZMM_SCD4 ;; Test loop counter, next source pointer, restore sin/cos pointer
	loops_reset			;; 55 more iters of 8
	zr8_eight_complex_djbunfft_preload
c3b:	zr8_eight_complex_djbunfft rsi, 128, 8*128, 16*128, dist32, rdi, ZMM_SCD4, 8
	zloop	8, c3b, -8*128+2*dist32, rdi, -8*ZMM_SCD4 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	56, c3b, -56*2*dist32	;; Test loop counter, restore source pointer

;; Do inverse FFT levels 5,6,7
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	512	+1	...
;;	8	...
;;	...
;;	504
;;	1K
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	4K	+1	...
;;	8	...
;;	...
;;	4K-8
;;	8K
;;	...

;; Do 512 sixteen_reals_unfft operations
;; Do 3072 eight_complex_unfft operations
;;	distance between fft data elements is 512
;;	do 448 macros each processing 128 data values

	zloop_init 448			;; 7 iterations of 64
	zloop_set_rw_prefetch ZDOUBLE	;; Prefetch 896 cache lines
	bump	rdx, -8*ZMM_SCD4-64*ZMM_SCD4
	bump	rdi, -64*ZMM_SCD4
	zr8_2sc_sixteen_reals_unfft_preload
c2a:	zr8_2sc_sixteen_reals_unfft rsi, 128, 2*dist32, 4*dist32, 8*dist32, rdx, ZMM_SCD4, rdi, ZMM_SCD4, 32
	zloop	32, c2a, -32*128+dist32	;; Test loop counter, next source pointer
	zloop	2, c2a, -2*dist32+16*dist32, rdi, -64*ZMM_SCD4 ;; Test loop counter, next source pointer, restore sin/cos pointer
	loops_reset			;; 6 more iters of 64
	zr8_eight_complex_djbunfft_preload
c2b:	zr8_eight_complex_djbunfft rsi, 128, 2*dist32, 4*dist32, 8*dist32, rdi, ZMM_SCD4, 32
	zloop	32, c2b, -32*128+dist32	;; Test loop counter, next source pointer
	zloop	2, c2b, -2*dist32+16*dist32, rdi, -64*ZMM_SCD4 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	7, c2b, -7*16*dist32	;; Test loop counter, restore source pointer

;; Do inverse FFT levels 1,2,3,4
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	4K	+1	...
;;	8	...
;;	...
;;	4K-8
;;	8K
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	28K	+1	...
;;	8	...
;;	...
;;	28K-8

;; Do 4096 fourteen_reals_last_unfft operations
;;	distance between fft data elements is 4K
;;	do 512 macros each processing 112 data values

	bump	rdx, -64*ZMM_SCD4-512*ZMM_SCD3
	bump	rdi, -512*ZMM_SCD3
	zloop_init 512			;; 512 iterations
	zloop_set_rw_prefetch ZSINGLE	;; Prefetch 512 cache lines
	zr7_2sc_fourteen_reals_unfft_preload
c1b:	zr7_2sc_fourteen_reals_unfft rsi, 128, 16*dist32, rdx, ZMM_SCD3, rdi, ZMM_SCD3, 32
	zloop	32, c1b, -32*128+dist32	;; Test loop counter, next source pointer
	zloop	16, c1b			;; Test loop counter

zpass2_28672_real_done:
	end_timer 2
	ENDM

; *************** radix8-pass2-28672-complex macro ******************
; This macro takes 28672 complex values and performs the final 14.807 levels of a standard
; radix-8 FFT, squares the results, and does the first 14.807 levels of the inverse FFT.

zr4dwpn_pass2_28672_complex MACRO
	LOCAL	b1b, b2b, b3b, b4a, b4b, b4c, b4d, c1b, c2b, c3b
	LOCAL	zpass2_28672_complex_1, zpass2_28672_complex_2
	LOCAL	zpass2_28672_complex_3, zpass2_28672_complex_4
	LOCAL	zpass2_28672_complex_unfft, zpass2_28672_complex_done

;; Type 4 FFTs skip the forward FFT process

	mov	rsi, DATA_ADDR		;; Load address of FFT data
	mov	rbx, DIST_TO_FFTSRCARG
	mov	rcx, DATA_PREFETCH	;; Prefetch pointer (we need to prefetch 7168 cache lines + 111 pad cache lines)
	cmp	ffttype, 4
	je	zpass2_28672_complex_4

;; Do FFT levels 1,2,3
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	28K	+1	...
;;	8	...
;;	...
;;	28K-8
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	4K	+1	...
;;	8	...
;;	...
;;	4K-8
;;	8K
;;	...

;; Do 4096 seven_complex_fft operations
;;	distance between fft data elements is 4K
;;	do 512 macros each processing 112 data values

	start_timer 4
	mov	rdi, xsincos_complex	;; The sin/cos data
	zloop_init 512			;; 512 iterations
	zloop_set_rw_prefetch ZSINGLE	;; Prefetch 512 cache lines
	zr7f_seven_complex_djbfft_preload
b1b:	zr7f_seven_complex_djbfft rsi, 128, 16*dist32, rdi, ZMM_SCD3, 32, L1PREFETCH_ALL, 128
	zloop	32, b1b, -32*128+dist32	;; Test loop counter, next source pointer
	zloop	16, b1b, -16*dist32	;; Test loop counter, restore source pointer
	end_timer 4

;; Do FFT levels 4,5,6
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	4K	+1	...
;;	8	...
;;	...
;;	4K-8
;;	8K
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	512	+1	...
;;	8	...
;;	...
;;	504
;;	1K
;;	...

;; Do 3584 eight_complex_fft operations
;;	distance between fft data elements is 512
;;	do 448 macros each processing 128 data values

	start_timer 5
	zloop_init 448			;; 7 iterations of 64
	zloop_set_rw_prefetch ZDOUBLE	;; Prefetch 896 cache lines
	zr8_eight_complex_djbfft_preload
b2b:	zr8_eight_complex_djbfft rsi, 128, 2*dist32, 4*dist32, 8*dist32, rdi, ZMM_SCD4, 32, L1PREFETCH_ALL, 128
	zloop	32, b2b, -32*128+dist32	;; Test loop counter, next source pointer
	zloop	2, b2b, -2*dist32+16*dist32, rdi, -64*ZMM_SCD4 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	7, b2b, -7*16*dist32, rdi, 64*ZMM_SCD4 ;; Test loop counter, restore source pointer, next sin/cos pointer
	end_timer 5

;; Do FFT levels 7,8,9
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	512	+1	...
;;	8	...
;;	...
;;	504
;;	1K
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	64	+1	...
;;	8	...
;;	...
;;	56
;;	128
;;	...

;; Do 3584 eight_complex_fft operations
;;	distance between fft data elements is 64
;;	do 448 macros each processing 128 data values

	start_timer 6
	zloop_init 448			;; 56 iterations of 8
	zloop_set_rw_prefetch ZDOUBLE	;; Prefetch 896 cache lines
	zr8_eight_complex_djbfft_preload
b3b:	zr8_eight_complex_djbfft rsi, 128, 8*128, 16*128, dist32, rdi, ZMM_SCD4, 8, L1PREFETCH_ALL, 128
	zloop	8, b3b, -8*128+2*dist32, rdi, -8*ZMM_SCD4 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	56, b3b, -56*2*dist32, rdi, 8*ZMM_SCD4 ;; Test loop counter, restore source pointer, next sin/cos pointer
	end_timer 6

;; Do FFT levels 10,11,12,13,14,15 as well as inverse FFT levels 10,11,12,13,14,15
;; On input and output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	64	+1	...
;;	8	...
;;	...
;;	56
;;	128
;;	...

;; Do 448 sixtyfour_complex_fft operations
;;	distance between fft data elements is 1 (8 for swizzling)
;;	do 448 macros each processing 128 data values

;; Execute the right middle step

pfing	bump	rcx, -17*64		;; We will prefetch 17 cache lines too many
	cmp	ffttype, 2
	jl	zpass2_28672_complex_1
	jg	zpass2_28672_complex_3

zpass2_28672_complex_2:
	start_timer 9
	zloop_init 448			;; 448 iterations
	zloop_set_rw_prefetch ZSEXT	;; Prefetch 2688 cache lines
	mov	r8, DEST2ARG
	zr64_sixtyfour_complex_with_square_preload
b4a:	zr64_sixtyfour_complex_with_square rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 4, L1PREFETCH_ALL, 8*128
	zloop	4, b4a, -4*8*128+dist32	;; Test loop counter, next source pointer
	zloop	112, b4a		;; Test loop counter
	add	rsi, r8
	end_timer 9
	jmp	zpass2_28672_complex_unfft

zpass2_28672_complex_1:
	start_timer 9
	zloop_init 448			;; 448 iterations
	zloop_set_rw_prefetch ZSEXT	;; Prefetch 2688 cache lines
	zr64_sixtyfour_complex_fft_final_preload
b4b:	zr64_sixtyfour_complex_fft_final rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 4, L1PREFETCH_ALL, 8*128
	zloop	4, b4b, -4*8*128+dist32	;; Test loop counter, next source pointer
	zloop	112, b4b		;; Test loop counter
	end_timer 9
	jmp	zpass2_28672_complex_done

zpass2_28672_complex_3:
	start_timer 9
	zloop_init 448			;; 448 iterations
	zloop_set_rw_prefetch ZSEXT	;; Prefetch 2688 cache lines
	mov	r8, DEST2ARG
	mov	rbp, DIST_TO_MULSRCARG
	zr64_sixtyfour_complex_with_mult_preload
b4c:	zr64_sixtyfour_complex_with_mult rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 4, L1PREFETCH_ALL, 8*128
	zloop	4, b4c, -4*8*128+dist32	;; Test loop counter, next source pointer
	zloop	112, b4c		;; Test loop counter
	add	rsi, r8
	end_timer 9
	jmp	zpass2_28672_complex_unfft

zpass2_28672_complex_4:
	start_timer 9
	zloop_init 448			;; 448 iterations
	zloop_set_rw_prefetch ZSEXT	;; Prefetch 2688 cache lines
	mov	rbp, DIST_TO_MULSRCARG
	mov	rdi, xsincos_complex	;; The sin/cos data
	bump	rdi, 512*ZMM_SCD3+64*ZMM_SCD4+8*ZMM_SCD4
	zr64f_sixtyfour_complex_with_mulf_preload
b4d:	zr64f_sixtyfour_complex_with_mulf rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 4, L1PREFETCH_ALL, 8*128
	zloop	4, b4d, -4*8*128+dist32	;; Test loop counter, next source pointer
	zloop	112, b4d		;; Test loop counter
	end_timer 9
zpass2_28672_complex_unfft:
	bump	rsi, -112*dist32	;; Restore source pointer

;; Do inverse FFT levels 7,8,9
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	64	+1	...
;;	8	...
;;	...
;;	56
;;	128
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	512	+1	...
;;	8	...
;;	...
;;	504
;;	1K
;;	...

;; Do 3584 eight_complex_unfft operations
;;	distance between fft data elements is 64
;;	do 448 macros each processing 128 data values

	start_timer 12
	bump	rdi, -8*ZMM_SCD4
	zloop_init 448			;; 56 iterations of 8
	zloop_set_rw_prefetch ZDOUBLE	;; Prefetch 896 cache lines
	zr8_eight_complex_djbunfft_preload
c3b:	zr8_eight_complex_djbunfft rsi, 128, 8*128, 16*128, dist32, rdi, ZMM_SCD4, 8, L1PREFETCH_ALL, 128
	zloop	8, c3b, -8*128+2*dist32, rdi, -8*ZMM_SCD4 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	56, c3b, -56*2*dist32	;; Test loop counter, restore source pointer
	end_timer 12

;; Do inverse FFT levels 4,5,6
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	512	+1	...
;;	8	...
;;	...
;;	504
;;	1K
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	4K	+1	...
;;	8	...
;;	...
;;	4K-8
;;	8K
;;	...

;; Do 3584 eight_complex_unfft operations
;;	distance between fft data elements is 512
;;	do 448 macros each processing 128 data values

	start_timer 13
	bump	rdi, -64*ZMM_SCD4
	zloop_init 448			;; 7 iterations of 64
	zloop_set_rw_prefetch ZDOUBLE	;; Prefetch 896 cache lines
	zr8_eight_complex_djbunfft_preload
c2b:	zr8_eight_complex_djbunfft rsi, 128, 2*dist32, 4*dist32, 8*dist32, rdi, ZMM_SCD4, 32, L1PREFETCH_ALL, 128
	zloop	32, c2b, -32*128+dist32	;; Test loop counter, next source pointer
	zloop	2, c2b, -2*dist32+16*dist32, rdi, -64*ZMM_SCD4 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	7, c2b, -7*16*dist32	;; Test loop counter, restore source pointer
	end_timer 13

;; Do inverse FFT levels 1,2,3
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	4K	+1	...
;;	8	...
;;	...
;;	4K-8
;;	8K
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	28K	+1	...
;;	8	...
;;	...
;;	28K-8

;; Do 4096 seven_complex_unfft operations
;;	distance between fft data elements is 4K
;;	do 512 macros each processing 112 data values

	start_timer 14
	bump	rdi, -512*ZMM_SCD3
	zloop_init 512			;; 512 iterations
	zloop_set_rw_prefetch ZSINGLE	;; Prefetch 512 cache lines
	zr7_seven_complex_djbunfft_preload
c1b:	zr7_seven_complex_djbunfft rsi, 128, 16*dist32, rdi, ZMM_SCD3, 32, L1PREFETCH_ALL, 128
	zloop	32, c1b, -32*128+dist32	;; Test loop counter, next source pointer
	zloop	16, c1b			;; Test loop counter
	end_timer 14

zpass2_28672_complex_done:
	ENDM


; *************** radix8-pass2-30720-real macro ******************
; This macro takes 61440 real values and performs the final 15.907 levels of the FFT.
; The results are squared, and the first 15.907 levels of the inverse FFT are done.

zr4dwpn_pass2_30720_real MACRO
	LOCAL	b1b, b2a, b2b, b3a, b3b, b4a, b4b, b4c, b4d, c1b, c2a, c2b, c3a, c3b
	LOCAL	zpass2_30720_real_1, zpass2_30720_real_2
	LOCAL	zpass2_30720_real_3, zpass2_30720_real_4
	LOCAL	zpass2_30720_real_unfft, zpass2_30720_real_done

;; Type 4 FFTs skip the forward FFT process

	start_timer 2
	mov	rsi, DATA_ADDR		;; Load source address
	mov	rbx, DIST_TO_FFTSRCARG
	mov	rcx, DATA_PREFETCH	;; Prefetch pointer (we need to prefetch 7680 cache lines + ??? pad cache lines)
	cmp	ffttype, 4
	je	zpass2_30720_real_4

;; Do FFT levels 1,2,3,4
;;	0	+1	+1	+1	+1	+1	+1	+1	30K	+1	...
;;	8	...
;;	...
;;	30K-8
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	6K	+1	...
;;	8	...
;;	...
;;	6K-8
;;	12K
;;	...

;; Do 6144 ten_real_fft operations
;;	distance between fft data elements is 6K
;;	do 768 macros each processing 80 data values

	mov	rdx, sincos3		;; Real sin/cos data
	mov	rdi, xsincos_complex	;; Complex sin/cos data
	zloop_init 768			;; 768 iterations
	zloop_set_rw_prefetch ZSINGLE	;; Prefetch 768 cache lines
	zr5f_2sc_ten_reals_fft_preload
b1b:	zr5f_2sc_ten_reals_fft rsi, 128, 24*dist32, rdx, ZMM_SCD2, rdi, ZMM_SCD2, 32
	zloop	32, b1b, -32*128+dist32	;; Test loop counter, next source pointer
	zloop	24, b1b, -24*dist32	;; Test loop counter, restore source pointer

;; Do FFT levels 5,6,7
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	6K	+1	...
;;	8	...
;;	...
;;	6K-8
;;	12K
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	512	+1	...
;;	8	...
;;	...
;;	504
;;	1K
;;	...

;; Do 512 twentyfour_reals_fft operations
;; Do 2048 twelve_complex_fft operations
;;	distance between fft data elements is 512
;;	do 320 macros each processing 192 data values

	zloop_init 320			;; 5 iterations of 64
	zloop_set_rw_prefetch ZDOUBLE	;; Prefetch 640 cache lines
	zr12_2sc_twentyfour_reals_fft_preload
b2a:	zr12_2sc_twentyfour_reals_fft rsi, 128, 2*dist32, 4*dist32, 8*dist32, rdx, ZMM_SCD6, rdi, ZMM_SCD6, 32
	zloop	32, b2a, -32*128+dist32	;; Test loop counter, next source pointer
	zloop	2, b2a, -2*dist32+24*dist32, rdi, -64*ZMM_SCD6 ;; Test loop counter, next source pointer, restore sin/cos pointer
	loops_reset			;; 4 more iters of 64
	zr12_twelve_complex_djbfft_preload
b2b:	zr12_twelve_complex_djbfft rsi, 128, 2*dist32, 4*dist32, 8*dist32, rdi, ZMM_SCD6, 32
	zloop	32, b2b, -32*128+dist32	;; Test loop counter, next source pointer
	zloop	2, b2b, -2*dist32+24*dist32, rdi, -64*ZMM_SCD6 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	5, b2b, -5*24*dist32, rdi, 64*ZMM_SCD6 ;; Test loop counter, restore source pointer, next sin/cos pointer

;; Do FFT levels 8,9,10
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	512	+1	...
;;	8	...
;;	...
;;	504
;;	1K
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	64	+1	...
;;	8	...
;;	...
;;	56
;;	128
;;	...

;; Do 64 sixteen_reals_fft operations
;; Do 3776 eight_complex_fft operations
;;	distance between fft data elements is 64
;;	do 480 macros each processing 128 data values

	zloop_init 480			;; 60 iterations of 8
	zloop_set_rw_prefetch ZDOUBLE	;; Prefetch 960 cache lines
	zr8_2sc_sixteen_reals_fft_preload
b3a:	zr8_2sc_sixteen_reals_fft rsi, 128, 8*128, 16*128, dist32, rdx, ZMM_SCD4, rdi, ZMM_SCD4, 8
	zloop	8, b3a, -8*128+2*dist32, rdi, -8*ZMM_SCD4 ;; Test loop counter, next source pointer, restore sin/cos pointer
	loops_reset			;; 59 more iters of 8
	zr8_eight_complex_djbfft_preload
b3b:	zr8_eight_complex_djbfft rsi, 128, 8*128, 16*128, dist32, rdi, ZMM_SCD4, 8
	zloop	8, b3b, -8*128+2*dist32, rdi, -8*ZMM_SCD4 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	60, b3b, -60*2*dist32, rdi, 8*ZMM_SCD4 ;; Test loop counter, restore source pointer, next sin/cos pointer

;; Do FFT levels 11,12,13,14,15,16 as well as inverse FFT levels 11,12,13,14,15,16
;; On input and output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	64	+1	...
;;	8	...
;;	...
;;	56
;;	128
;;	...

;; Do 1 hundredtwentyeight_reals_fft operation
;; Do 479 sixtyfour_complex_fft operations
;;	distance between fft data elements is 1 (8 for swizzling)
;;	do 480 macros each processing 128 data values

;; Execute the proper middle step

pfing	bump	rcx, -131*64		;; We will prefetch 131 cache lines too many
	cmp	ffttype, 2
	jl	zpass2_30720_real_1
	jg	zpass2_30720_real_3

zpass2_30720_real_2:
	mov	r8, DEST2ARG
;;	zr64_hundredtwentyeight_real_with_square_preload
;;	zr64_hundredtwentyeight_real_with_square rsi, 8*128, 1*128, 2*128, 4*128, rdx, 0*ZMM_SCD4, rdi, 0*ZMM_SCD4, 1
	CALLP	zreal_square
	zloop_init 480, 1, 4		;; 479 more iterations
	zloop_set_rw_prefetch ZSEXT	;; Prefetch 2874 cache lines
	zr64_sixtyfour_complex_with_square_preload
b4a:	zr64_sixtyfour_complex_with_square rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 1, L1PREFETCH_ALL, 8*128
	zloop	4, b4a, -4*8*128+dist32	;; Test loop counter, next source pointer
	zloop	120, b4a		;; Test loop counter
	add	rsi, r8
	jmp	zpass2_30720_real_unfft

zpass2_30720_real_1:
;;	zr64_hundredtwentyeight_real_fft_final_preload
;;	zr64_hundredtwentyeight_real_fft_final rsi, 8*128, 1*128, 2*128, 4*128, rdx, 0*ZMM_SCD4, rdi, 0*ZMM_SCD4, 1
	CALLP	zreal_fft_final
	zloop_init 480, 1, 4		;; 479 more iterations
	zloop_set_rw_prefetch ZSEXT	;; Prefetch 2874 cache lines
	zr64_sixtyfour_complex_fft_final_preload
b4b:	zr64_sixtyfour_complex_fft_final rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 1, L1PREFETCH_ALL, 8*128
	zloop	4, b4b, -4*8*128+dist32	;; Test loop counter, next source pointer
	zloop	120, b4b		;; Test loop counter
	jmp	zpass2_30720_real_done

zpass2_30720_real_3:
	mov	r8, DEST2ARG
	mov	rbp, DIST_TO_MULSRCARG
;;	zr64_hundredtwentyeight_real_with_mult_preload
;;	zr64_hundredtwentyeight_real_with_mult rsi, 8*128, 1*128, 2*128, 4*128, rdx, 0*ZMM_SCD4, rdi, 0*ZMM_SCD4, 1
	CALLP	zreal_mult
	zloop_init 480, 1, 4		;; 479 more iterations
	zloop_set_rw_prefetch ZSEXT	;; Prefetch 2874 cache lines
	zr64_sixtyfour_complex_with_mult_preload
b4c:	zr64_sixtyfour_complex_with_mult rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 1, L1PREFETCH_ALL, 8*128
	zloop	4, b4c, -4*8*128+dist32	;; Test loop counter, next source pointer
	zloop	120, b4c		;; Test loop counter
	add	rsi, r8
	jmp	zpass2_30720_real_unfft

zpass2_30720_real_4:
	mov	rbp, DIST_TO_MULSRCARG
	mov	rdx, sincos3		;; Real sin/cos data
	mov	rdi, xsincos_complex	;; Complex sin/cos data
	bump	rdx, 768*ZMM_SCD2+64*ZMM_SCD6+8*ZMM_SCD4
	bump	rdi, 768*ZMM_SCD2+64*ZMM_SCD6+8*ZMM_SCD4
;;	zr64f_hundredtwentyeight_real_with_mulf_preload
;;	zr64f_hundredtwentyeight_real_with_mulf rsi, 8*128, 1*128, 2*128, 4*128, rdx, 0*ZMM_SCD4, rdi, 0*ZMM_SCD4, 1
	CALLP	zreal_mulf
	zloop_init 480, 1, 4		;; 479 more iterations
	zloop_set_rw_prefetch ZSEXT	;; Prefetch 2874 cache lines
	zr64f_sixtyfour_complex_with_mulf_preload
b4d:	zr64f_sixtyfour_complex_with_mulf rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 1, L1PREFETCH_ALL, 8*128
	zloop	4, b4d, -4*8*128+dist32	;; Test loop counter, next source pointer
	zloop	120, b4d		;; Test loop counter
zpass2_30720_real_unfft:
	bump	rsi, -120*dist32	;; Restore source pointer

;; Do inverse FFT levels 8,9,10
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	64	+1	...
;;	8	...
;;	...
;;	56
;;	128
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	512	+1	...
;;	8	...
;;	...
;;	504
;;	1K
;;	...

;; Do 64 sixteen_reals_unfft operations
;; Do 3776 eight_complex_unfft operations
;;	distance between fft data elements is 64
;;	do 480 macros each processing 128 data values

	zloop_init 480			;; 60 iterations of 8
	zloop_set_rw_prefetch ZDOUBLE	;; Prefetch 960 cache lines
	bump	rdx, -8*ZMM_SCD4
	bump	rdi, -8*ZMM_SCD4
	zr8_2sc_sixteen_reals_unfft_preload
c3a:	zr8_2sc_sixteen_reals_unfft rsi, 128, 8*128, 16*128, dist32, rdx, ZMM_SCD4, rdi, ZMM_SCD4, 8
	zloop	8, c3a, -8*128+2*dist32, rdi, -8*ZMM_SCD4 ;; Test loop counter, next source pointer, restore sin/cos pointer
	loops_reset			;; 59 more iters of 8
	zr8_eight_complex_djbunfft_preload
c3b:	zr8_eight_complex_djbunfft rsi, 128, 8*128, 16*128, dist32, rdi, ZMM_SCD4, 8
	zloop	8, c3b, -8*128+2*dist32, rdi, -8*ZMM_SCD4 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	60, c3b, -60*2*dist32	;; Test loop counter, restore source pointer

;; Do inverse FFT levels 5,6,7
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	512	+1	...
;;	8	...
;;	...
;;	504
;;	1K
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	6K	+1	...
;;	8	...
;;	...
;;	6K-8
;;	12K
;;	...

;; Do 512 twentyfour_reals_unfft operations
;; Do 2048 twelve_complex_unfft operations
;;	distance between fft data elements is 512
;;	do 320 macros each processing 192 data values

	zloop_init 320			;; 5 iterations of 64
	zloop_set_rw_prefetch ZTRIPLE	;; Prefetch 960 cache lines
	bump	rdx, -8*ZMM_SCD4-64*ZMM_SCD6
	bump	rdi, -64*ZMM_SCD6
	zr12_2sc_twentyfour_reals_unfft_preload
c2a:	zr12_2sc_twentyfour_reals_unfft rsi, 128, 2*dist32, 4*dist32, 8*dist32, rdx, ZMM_SCD6, rdi, ZMM_SCD6, 32
	zloop	32, c2a, -32*128+dist32	;; Test loop counter, next source pointer
	zloop	2, c2a, -2*dist32+24*dist32, rdi, -64*ZMM_SCD6 ;; Test loop counter, next source pointer, restore sin/cos pointer
	loops_reset			;; 4 more iters of 64
	zr12_twelve_complex_djbunfft_preload
c2b:	zr12_twelve_complex_djbunfft rsi, 128, 2*dist32, 4*dist32, 8*dist32, rdi, ZMM_SCD6, 32
	zloop	32, c2b, -32*128+dist32	;; Test loop counter, next source pointer
	zloop	2, c2b, -2*dist32+24*dist32, rdi, -64*ZMM_SCD6 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	5, c2b, -5*24*dist32	;; Test loop counter, restore source pointer

;; Do inverse FFT levels 1,2,3,4
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	6K	+1	...
;;	8	...
;;	...
;;	6K-8
;;	12K
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	30K	+1	...
;;	8	...
;;	...
;;	30K-8

;; Do 6144 ten_reals_last_unfft operations
;;	distance between fft data elements is 6K
;;	do 768 macros each processing 80 data values

	bump	rdx, -64*ZMM_SCD6-768*ZMM_SCD2
	bump	rdi, -768*ZMM_SCD2
	zloop_init 768			;; 768 iterations
	zloop_set_rw_prefetch ZSINGLE	;; Prefetch 768 cache lines
	zr5_2sc_ten_reals_unfft_preload
c1b:	zr5_2sc_ten_reals_unfft rsi, 128, 24*dist32, rdx, ZMM_SCD2, rdi, ZMM_SCD2, 32
	zloop	32, c1b, -32*128+dist32	;; Test loop counter, next source pointer
	zloop	24, c1b			;; Test loop counter

zpass2_30720_real_done:
	end_timer 2
	ENDM

; *************** radix8-pass2-30720-complex macro ******************
; This macro takes 30720 complex values and performs the final 14.907 levels of a standard
; radix-8 FFT, squares the results, and does the first 14.907 levels of the inverse FFT.

zr4dwpn_pass2_30720_complex MACRO
	LOCAL	b1b, b2b, b3b, b4a, b4b, b4c, b4d, c1b, c2b, c3b
	LOCAL	zpass2_30720_complex_1, zpass2_30720_complex_2
	LOCAL	zpass2_30720_complex_3, zpass2_30720_complex_4
	LOCAL	zpass2_30720_complex_unfft, zpass2_30720_complex_done

;; Type 4 FFTs skip the forward FFT process

	mov	rsi, DATA_ADDR		;; Load address of FFT data
	mov	rbx, DIST_TO_FFTSRCARG
	mov	rcx, DATA_PREFETCH	;; Prefetch pointer (we need to prefetch 7680 cache lines + 119 pad cache lines)
	cmp	ffttype, 4
	je	zpass2_30720_complex_4

;; Do FFT levels 1,2,3
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	30K	+1	...
;;	8	...
;;	...
;;	30K-8
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	6K	+1	...
;;	8	...
;;	...
;;	6K-8
;;	12K
;;	...

;; Do 6144 five_complex_fft operations
;;	distance between fft data elements is 6K
;;	do 768 macros each processing 80 data values

	start_timer 4
	mov	rdi, xsincos_complex	;; The sin/cos data
	zloop_init 768			;; 768 iterations
	zloop_set_rw_prefetch ZSINGLE	;; Prefetch 768 cache lines
	zr5f_five_complex_djbfft_preload
b1b:	zr5f_five_complex_djbfft rsi, 128, 24*dist32, rdi, ZMM_SCD2, 32, L1PREFETCH_ALL, 128
	zloop	32, b1b, -32*128+dist32	;; Test loop counter, next source pointer
	zloop	24, b1b, -24*dist32	;; Test loop counter, restore source pointer
	end_timer 4

;; Do FFT levels 4,5,6
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	6K	+1	...
;;	8	...
;;	...
;;	6K-8
;;	12K
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	512	+1	...
;;	8	...
;;	...
;;	504
;;	1K
;;	...

;; Do 2560 twelve_complex_fft operations
;;	distance between fft data elements is 512
;;	do 320 macros each processing 192 data values

	start_timer 5
	zloop_init 320			;; 5 iterations of 64
	zloop_set_rw_prefetch ZDOUBLE	;; Prefetch 640 cache lines
	zr12_twelve_complex_djbfft_preload
b2b:	zr12_twelve_complex_djbfft rsi, 128, 2*dist32, 4*dist32, 8*dist32, rdi, ZMM_SCD6, 32, L1PREFETCH_ALL, 128
	zloop	32, b2b, -32*128+dist32	;; Test loop counter, next source pointer
	zloop	2, b2b, -2*dist32+24*dist32, rdi, -64*ZMM_SCD6 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	5, b2b, -5*24*dist32, rdi, 64*ZMM_SCD6 ;; Test loop counter, restore source pointer, next sin/cos pointer
	end_timer 5

;; Do FFT levels 7,8,9
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	512	+1	...
;;	8	...
;;	...
;;	504
;;	1K
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	64	+1	...
;;	8	...
;;	...
;;	56
;;	128
;;	...

;; Do 3840 eight_complex_fft operations
;;	distance between fft data elements is 64
;;	do 480 macros each processing 128 data values

	start_timer 6
	zloop_init 480			;; 60 iterations of 8
	zloop_set_rw_prefetch ZDOUBLE	;; Prefetch 960 cache lines
	zr8_eight_complex_djbfft_preload
b3b:	zr8_eight_complex_djbfft rsi, 128, 8*128, 16*128, dist32, rdi, ZMM_SCD4, 8, L1PREFETCH_ALL, 128
	zloop	8, b3b, -8*128+2*dist32, rdi, -8*ZMM_SCD4 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	60, b3b, -60*2*dist32, rdi, 8*ZMM_SCD4 ;; Test loop counter, restore source pointer, next sin/cos pointer
	end_timer 6

;; Do FFT levels 10,11,12,13,14,15 as well as inverse FFT levels 10,11,12,13,14,15
;; On input and output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	64	+1	...
;;	8	...
;;	...
;;	56
;;	128
;;	...

;; Do 480 sixtyfour_complex_fft operations
;;	distance between fft data elements is 1 (8 for swizzling)
;;	do 480 macros each processing 128 data values

;; Execute the right middle step

pfing	bump	rcx, -137*64		;; We will prefetch 137 cache lines too many
	cmp	ffttype, 2
	jl	zpass2_30720_complex_1
	jg	zpass2_30720_complex_3

zpass2_30720_complex_2:
	start_timer 9
	zloop_init 480			;; 480 iterations
	zloop_set_rw_prefetch ZSEXT	;; Prefetch 2880 cache lines
	mov	r8, DEST2ARG
	zr64_sixtyfour_complex_with_square_preload
b4a:	zr64_sixtyfour_complex_with_square rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 4, L1PREFETCH_ALL, 8*128
	zloop	4, b4a, -4*8*128+dist32	;; Test loop counter, next source pointer
	zloop	120, b4a		;; Test loop counter
	add	rsi, r8
	end_timer 9
	jmp	zpass2_30720_complex_unfft

zpass2_30720_complex_1:
	start_timer 9
	zloop_init 480			;; 480 iterations
	zloop_set_rw_prefetch ZSEXT	;; Prefetch 2880 cache lines
	zr64_sixtyfour_complex_fft_final_preload
b4b:	zr64_sixtyfour_complex_fft_final rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 4, L1PREFETCH_ALL, 8*128
	zloop	4, b4b, -4*8*128+dist32	;; Test loop counter, next source pointer
	zloop	120, b4b		;; Test loop counter
	end_timer 9
	jmp	zpass2_30720_complex_done

zpass2_30720_complex_3:
	start_timer 9
	zloop_init 480			;; 480 iterations
	zloop_set_rw_prefetch ZSEXT	;; Prefetch 2880 cache lines
	mov	r8, DEST2ARG
	mov	rbp, DIST_TO_MULSRCARG
	zr64_sixtyfour_complex_with_mult_preload
b4c:	zr64_sixtyfour_complex_with_mult rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 4, L1PREFETCH_ALL, 8*128
	zloop	4, b4c, -4*8*128+dist32	;; Test loop counter, next source pointer
	zloop	120, b4c		;; Test loop counter
	add	rsi, r8
	end_timer 9
	jmp	zpass2_30720_complex_unfft

zpass2_30720_complex_4:
	start_timer 9
	zloop_init 480			;; 480 iterations
	zloop_set_rw_prefetch ZSEXT	;; Prefetch 2880 cache lines
	mov	rbp, DIST_TO_MULSRCARG
	mov	rdi, xsincos_complex	;; The sin/cos data
	bump	rdi, 768*ZMM_SCD2+64*ZMM_SCD6+8*ZMM_SCD4
	zr64f_sixtyfour_complex_with_mulf_preload
b4d:	zr64f_sixtyfour_complex_with_mulf rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 4, L1PREFETCH_ALL, 8*128
	zloop	4, b4d, -4*8*128+dist32	;; Test loop counter, next source pointer
	zloop	120, b4d		;; Test loop counter
	end_timer 9
zpass2_30720_complex_unfft:
	bump	rsi, -120*dist32	;; Restore source pointer

;; Do inverse FFT levels 7,8,9
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	64	+1	...
;;	8	...
;;	...
;;	56
;;	128
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	512	+1	...
;;	8	...
;;	...
;;	504
;;	1K
;;	...

;; Do 3840 eight_complex_unfft operations
;;	distance between fft data elements is 64
;;	do 480 macros each processing 128 data values

	start_timer 12
	bump	rdi, -8*ZMM_SCD4
	zloop_init 480			;; 60 iterations of 8
	zloop_set_rw_prefetch ZDOUBLE	;; Prefetch 960 cache lines
	zr8_eight_complex_djbunfft_preload
c3b:	zr8_eight_complex_djbunfft rsi, 128, 8*128, 16*128, dist32, rdi, ZMM_SCD4, 8, L1PREFETCH_ALL, 128
	zloop	8, c3b, -8*128+2*dist32, rdi, -8*ZMM_SCD4 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	60, c3b, -60*2*dist32	;; Test loop counter, restore source pointer
	end_timer 12

;; Do inverse FFT levels 4,5,6
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	512	+1	...
;;	8	...
;;	...
;;	504
;;	1K
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	6K	+1	...
;;	8	...
;;	...
;;	6K-8
;;	12K
;;	...

;; Do 2560 twelve_complex_unfft operations
;;	distance between fft data elements is 512
;;	do 320 macros each processing 192 data values

	start_timer 13
	bump	rdi, -64*ZMM_SCD6
	zloop_init 320			;; 5 iterations of 64
	zloop_set_rw_prefetch ZTRIPLE	;; Prefetch 960 cache lines
	zr12_twelve_complex_djbunfft_preload
c2b:	zr12_twelve_complex_djbunfft rsi, 128, 2*dist32, 4*dist32, 8*dist32, rdi, ZMM_SCD6, 32, L1PREFETCH_ALL, 128
	zloop	32, c2b, -32*128+dist32	;; Test loop counter, next source pointer
	zloop	2, c2b, -2*dist32+24*dist32, rdi, -64*ZMM_SCD6 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	5, c2b, -5*24*dist32	;; Test loop counter, restore source pointer
	end_timer 13

;; Do inverse FFT levels 1,2,3
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	6K	+1	...
;;	8	...
;;	...
;;	6K-8
;;	12K
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	30K	+1	...
;;	8	...
;;	...
;;	30K-8

;; Do 6144 five_complex_unfft operations
;;	distance between fft data elements is 6K
;;	do 768 macros each processing 80 data values

	start_timer 14
	bump	rdi, -768*ZMM_SCD2
	zloop_init 768			;; 768 iterations
	zloop_set_rw_prefetch ZSINGLE	;; Prefetch 768 cache lines
	zr5_five_complex_djbunfft_preload
c1b:	zr5_five_complex_djbunfft rsi, 128, 24*dist32, rdi, ZMM_SCD2, 32, L1PREFETCH_ALL, 128
	zloop	32, c1b, -32*128+dist32	;; Test loop counter, next source pointer
	zloop	24, c1b			;; Test loop counter
	end_timer 14

zpass2_30720_complex_done:
	ENDM


; *************** radix8-pass2-32768-real macro ******************
; This macro takes 65536 real values and performs the final 16 levels of the FFT.
; The results are squared, and the first 16 levels of the inverse FFT are done.

zr4dwpn_pass2_32768_real MACRO
	LOCAL	b1b, b2a, b2b, b3a, b3b, b4a, b4b, b4c, b4d, c1b, c2a, c2b, c3a, c3b
	LOCAL	zpass2_32768_real_1, zpass2_32768_real_2
	LOCAL	zpass2_32768_real_3, zpass2_32768_real_4
	LOCAL	zpass2_32768_real_unfft, zpass2_32768_real_done

;; Type 4 FFTs skip the forward FFT process

	start_timer 2
	mov	rsi, DATA_ADDR		;; Load source address
	mov	rbx, DIST_TO_FFTSRCARG
	mov	rcx, DATA_PREFETCH	;; Prefetch pointer (we need to prefetch 8192 cache lines + 0 pad cache lines)
	cmp	ffttype, 4
	je	zpass2_32768_real_4

;; Do FFT levels 1,2,3,4
;;	0	+1	+1	+1	+1	+1	+1	+1	32K	+1	...
;;	8	...
;;	...
;;	32K-8
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	4K	+1	...
;;	8	...
;;	...
;;	4K-8
;;	8K
;;	...

;; Do 4096 sixteen_real_fft operations
;;	distance between fft data elements is 4K
;;	do 512 macros each processing 128 data values

	mov	rdx, sincos3		;; Real sin/cos data
	mov	rdi, xsincos_complex	;; Complex sin/cos data
	zloop_init 512			;; 512 iterations
	zloop_set_rw_prefetch ZSINGLE	;; Prefetch 512 cache lines
	zr8f_2sc_sixteen_reals_fft_preload
b1b:	zr8f_2sc_sixteen_reals_fft rsi, 128, 16*dist32, 32*dist32, 64*dist32, rdx, ZMM_SCD4, rdi, ZMM_SCD4, 32
	zloop	32, b1b, -32*128+dist32	;; Test loop counter, next source pointer
	zloop	16, b1b, -16*dist32	;; Test loop counter, restore source pointer

;; Do FFT levels 5,6,7
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	4K	+1	...
;;	8	...
;;	...
;;	4K-8
;;	8K
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	512	+1	...
;;	8	...
;;	...
;;	504
;;	1K
;;	...

;; Do 512 sixteen_reals_fft operations
;; Do 3584 eight_complex_fft operations
;;	distance between fft data elements is 512
;;	do 512 macros each processing 128 data values

	zloop_init 512			;; 8 iterations of 64
	zloop_set_rw_prefetch ZDOUBLE	;; Prefetch 1024 cache lines
	zr8_2sc_sixteen_reals_fft_preload
b2a:	zr8_2sc_sixteen_reals_fft rsi, 128, 2*dist32, 4*dist32, 8*dist32, rdx, ZMM_SCD4, rdi, ZMM_SCD4, 32
	zloop	32, b2a, -32*128+dist32	;; Test loop counter, next source pointer
	zloop	2, b2a, -2*dist32+16*dist32, rdi, -64*ZMM_SCD4 ;; Test loop counter, next source pointer, restore sin/cos pointer
	loops_reset			;; 7 more iters of 64
	zr8_eight_complex_djbfft_preload
b2b:	zr8_eight_complex_djbfft rsi, 128, 2*dist32, 4*dist32, 8*dist32, rdi, ZMM_SCD4, 32
	zloop	32, b2b, -32*128+dist32	;; Test loop counter, next source pointer
	zloop	2, b2b, -2*dist32+16*dist32, rdi, -64*ZMM_SCD4 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	8, b2b, -8*16*dist32, rdi, 64*ZMM_SCD4 ;; Test loop counter, restore source pointer, next sin/cos pointer

;; Do FFT levels 8,9,10
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	512	+1	...
;;	8	...
;;	...
;;	504
;;	1K
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	64	+1	...
;;	8	...
;;	...
;;	56
;;	128
;;	...

;; Do 64 sixteen_reals_fft operations
;; Do 4032 eight_complex_fft operations
;;	distance between fft data elements is 64
;;	do 512 macros each processing 128 data values

	zloop_init 512			;; 64 iterations of 8
	zloop_set_rw_prefetch ZDOUBLE	;; Prefetch 1024 cache lines
	zr8_2sc_sixteen_reals_fft_preload
b3a:	zr8_2sc_sixteen_reals_fft rsi, 128, 8*128, 16*128, dist32, rdx, ZMM_SCD4, rdi, ZMM_SCD4, 8
	zloop	8, b3a, -8*128+2*dist32, rdi, -8*ZMM_SCD4 ;; Test loop counter, next source pointer, restore sin/cos pointer
	loops_reset			;; 63 more iters of 8
	zr8_eight_complex_djbfft_preload
b3b:	zr8_eight_complex_djbfft rsi, 128, 8*128, 16*128, dist32, rdi, ZMM_SCD4, 8
	zloop	8, b3b, -8*128+2*dist32, rdi, -8*ZMM_SCD4 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	64, b3b, -64*2*dist32, rdi, 8*ZMM_SCD4 ;; Test loop counter, restore source pointer, next sin/cos pointer

;; Do FFT levels 11,12,13,14,15,16 as well as inverse FFT levels 11,12,13,14,15,16
;; On input and output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	64	+1	...
;;	8	...
;;	...
;;	56
;;	128
;;	...

;; Do 1 hundredtwentyeight_reals_fft operation
;; Do 511 sixtyfour_complex_fft operations
;;	distance between fft data elements is 1 (8 for swizzling)
;;	do 512 macros each processing 128 data values

;; Execute the proper middle step

pfing	bump	rcx, -378*64		;; We will prefetch 378 cache lines too many
	cmp	ffttype, 2
	jl	zpass2_32768_real_1
	jg	zpass2_32768_real_3

zpass2_32768_real_2:
	mov	r8, DEST2ARG
;;	zr64_hundredtwentyeight_real_with_square_preload
;;	zr64_hundredtwentyeight_real_with_square rsi, 8*128, 1*128, 2*128, 4*128, rdx, 0*ZMM_SCD4, rdi, 0*ZMM_SCD4, 1
	CALLP	zreal_square
	zloop_init 512, 1, 4		;; 511 more iterations
	zloop_set_rw_prefetch ZSEPT	;; Prefetch 3577 cache lines
	zr64_sixtyfour_complex_with_square_preload
b4a:	zr64_sixtyfour_complex_with_square rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 1, L1PREFETCH_ALL, 8*128
	zloop	4, b4a, -4*8*128+dist32	;; Test loop counter, next source pointer
	zloop	128, b4a		;; Test loop counter
	add	rsi, r8
	jmp	zpass2_32768_real_unfft

zpass2_32768_real_1:
;;	zr64_hundredtwentyeight_real_fft_final_preload
;;	zr64_hundredtwentyeight_real_fft_final rsi, 8*128, 1*128, 2*128, 4*128, rdx, 0*ZMM_SCD4, rdi, 0*ZMM_SCD4, 1
	CALLP	zreal_fft_final
	zloop_init 512, 1, 4		;; 511 more iterations
	zloop_set_rw_prefetch ZSEPT	;; Prefetch 3577 cache lines
	zr64_sixtyfour_complex_fft_final_preload
b4b:	zr64_sixtyfour_complex_fft_final rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 1, L1PREFETCH_ALL, 8*128
	zloop	4, b4b, -4*8*128+dist32	;; Test loop counter, next source pointer
	zloop	128, b4b		;; Test loop counter
	jmp	zpass2_32768_real_done

zpass2_32768_real_3:
	mov	r8, DEST2ARG
	mov	rbp, DIST_TO_MULSRCARG
;;	zr64_hundredtwentyeight_real_with_mult_preload
;;	zr64_hundredtwentyeight_real_with_mult rsi, 8*128, 1*128, 2*128, 4*128, rdx, 0*ZMM_SCD4, rdi, 0*ZMM_SCD4, 1
	CALLP	zreal_mult
	zloop_init 512, 1, 4		;; 511 more iterations
	zloop_set_rw_prefetch ZSEPT	;; Prefetch 3577 cache lines
	zr64_sixtyfour_complex_with_mult_preload
b4c:	zr64_sixtyfour_complex_with_mult rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 1, L1PREFETCH_ALL, 8*128
	zloop	4, b4c, -4*8*128+dist32	;; Test loop counter, next source pointer
	zloop	128, b4c		;; Test loop counter
	add	rsi, r8
	jmp	zpass2_32768_real_unfft

zpass2_32768_real_4:
	mov	rbp, DIST_TO_MULSRCARG
	mov	rdx, sincos3		;; Real sin/cos data
	mov	rdi, xsincos_complex	;; Complex sin/cos data
	bump	rdx, 512*ZMM_SCD4+64*ZMM_SCD4+8*ZMM_SCD4
	bump	rdi, 512*ZMM_SCD4+64*ZMM_SCD4+8*ZMM_SCD4
;;	zr64f_hundredtwentyeight_real_with_mulf_preload
;;	zr64f_hundredtwentyeight_real_with_mulf rsi, 8*128, 1*128, 2*128, 4*128, rdx, 0*ZMM_SCD4, rdi, 0*ZMM_SCD4, 1
	CALLP	zreal_mulf
	zloop_init 512, 1, 4		;; 511 more iterations
	zloop_set_rw_prefetch ZSEPT	;; Prefetch 3577 cache lines
	zr64f_sixtyfour_complex_with_mulf_preload
b4d:	zr64f_sixtyfour_complex_with_mulf rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 1, L1PREFETCH_ALL, 8*128
	zloop	4, b4d, -4*8*128+dist32	;; Test loop counter, next source pointer
	zloop	128, b4d		;; Test loop counter
zpass2_32768_real_unfft:
	bump	rsi, -128*dist32	;; Restore source pointer

;; Do inverse FFT levels 8,9,10
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	64	+1	...
;;	8	...
;;	...
;;	56
;;	128
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	512	+1	...
;;	8	...
;;	...
;;	504
;;	1K
;;	...

;; Do 64 sixteen_reals_unfft operations
;; Do 4032 eight_complex_unfft operations
;;	distance between fft data elements is 64
;;	do 512 macros each processing 128 data values

	zloop_init 512			;; 64 iterations of 8
	zloop_set_rw_prefetch ZSINGLE	;; Prefetch 512 cache lines
	bump	rdx, -8*ZMM_SCD4
	bump	rdi, -8*ZMM_SCD4
	zr8_2sc_sixteen_reals_unfft_preload
c3a:	zr8_2sc_sixteen_reals_unfft rsi, 128, 8*128, 16*128, dist32, rdx, ZMM_SCD4, rdi, ZMM_SCD4, 8
	zloop	8, c3a, -8*128+2*dist32, rdi, -8*ZMM_SCD4 ;; Test loop counter, next source pointer, restore sin/cos pointer
	loops_reset			;; 63 more iters of 8
	zr8_eight_complex_djbunfft_preload
c3b:	zr8_eight_complex_djbunfft rsi, 128, 8*128, 16*128, dist32, rdi, ZMM_SCD4, 8
	zloop	8, c3b, -8*128+2*dist32, rdi, -8*ZMM_SCD4 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	64, c3b, -64*2*dist32	;; Test loop counter, restore source pointer

;; Do inverse FFT levels 5,6,7
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	512	+1	...
;;	8	...
;;	...
;;	504
;;	1K
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	4K	+1	...
;;	8	...
;;	...
;;	4K-8
;;	8K
;;	...

;; Do 512 sixteen_reals_unfft operations
;; Do 3584 eight_complex_unfft operations
;;	distance between fft data elements is 512
;;	do 512 macros each processing 128 data values

	zloop_init 512			;; 8 iterations of 64
	zloop_set_rw_prefetch ZDOUBLE	;; Prefetch 1024 cache lines
	bump	rdx, -8*ZMM_SCD4-64*ZMM_SCD4
	bump	rdi, -64*ZMM_SCD4
	zr8_2sc_sixteen_reals_unfft_preload
c2a:	zr8_2sc_sixteen_reals_unfft rsi, 128, 2*dist32, 4*dist32, 8*dist32, rdx, ZMM_SCD4, rdi, ZMM_SCD4, 32
	zloop	32, c2a, -32*128+dist32	;; Test loop counter, next source pointer
	zloop	2, c2a, -2*dist32+16*dist32, rdi, -64*ZMM_SCD4 ;; Test loop counter, next source pointer, restore sin/cos pointer
	loops_reset			;; 7 more iters of 64
	zr8_eight_complex_djbunfft_preload
c2b:	zr8_eight_complex_djbunfft rsi, 128, 2*dist32, 4*dist32, 8*dist32, rdi, ZMM_SCD4, 32
	zloop	32, c2b, -32*128+dist32	;; Test loop counter, next source pointer
	zloop	2, c2b, -2*dist32+16*dist32, rdi, -64*ZMM_SCD4 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	8, c2b, -8*16*dist32	;; Test loop counter, restore source pointer

;; Do inverse FFT levels 1,2,3,4
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	4K	+1	...
;;	8	...
;;	...
;;	4K-8
;;	8K
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	32K	+1	...
;;	8	...
;;	...
;;	32K-8

;; Do 4096 sixteen_reals_last_unfft operations
;;	distance between fft data elements is 4K
;;	do 512 macros each processing 128 data values

	bump	rdx, -64*ZMM_SCD4-512*ZMM_SCD4
	bump	rdi, -512*ZMM_SCD4
	zloop_init 512			;; 512 iterations
	zloop_set_rw_prefetch ZDOUBLE	;; Prefetch 1024 cache lines
	zr8_2sc_sixteen_reals_unfft_preload
c1b:	zr8_2sc_sixteen_reals_unfft rsi, 128, 16*dist32, 32*dist32, 64*dist32, rdx, ZMM_SCD4, rdi, ZMM_SCD4, 32
	zloop	32, c1b, -32*128+dist32	;; Test loop counter, next source pointer
	zloop	16, c1b			;; Test loop counter

zpass2_32768_real_done:
	end_timer 2
	ENDM

; *************** radix8-pass2-32768-complex macro ******************
; This macro takes 32768 complex values and performs the final 15 levels of a standard
; radix-8 FFT, squares the results, and does the first 15 levels of the inverse FFT.

zr4dwpn_pass2_32768_complex MACRO
	LOCAL	b1b, b2b, b3b, b4a, b4b, b4c, b4d, c1b, c2b, c3b
	LOCAL	zpass2_32768_complex_1, zpass2_32768_complex_2
	LOCAL	zpass2_32768_complex_3, zpass2_32768_complex_4
	LOCAL	zpass2_32768_complex_unfft, zpass2_32768_complex_done

;; Type 4 FFTs skip the forward FFT process

	mov	rsi, DATA_ADDR		;; Load address of FFT data
	mov	rbx, DIST_TO_FFTSRCARG
	mov	rcx, DATA_PREFETCH	;; Prefetch pointer (we need to prefetch 8192 cache lines + 127 pad cache lines)
	cmp	ffttype, 4
	je	zpass2_32768_complex_4

;; Do FFT levels 1,2,3
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	32K	+1	...
;;	8	...
;;	...
;;	32K-8
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	4K	+1	...
;;	8	...
;;	...
;;	4K-8
;;	8K
;;	...

;; Do 4096 eight_complex_fft operations
;;	distance between fft data elements is 4K
;;	do 512 macros each processing 128 data values

	start_timer 4
	mov	rdi, xsincos_complex	;; The sin/cos data
	zloop_init 512			;; 512 iterations
	zloop_set_rw_prefetch ZSINGLE	;; Prefetch 512 cache lines
	zr8f_eight_complex_djbfft_preload
b1b:	zr8f_eight_complex_djbfft rsi, 128, 16*dist32, 32*dist32, 64*dist32, rdi, ZMM_SCD4, 32, L1PREFETCH_ALL, 128
	zloop	32, b1b, -32*128+dist32	;; Test loop counter, next source pointer
	zloop	16, b1b, -16*dist32	;; Test loop counter, restore source pointer
	end_timer 4

;; Do FFT levels 4,5,6
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	4K	+1	...
;;	8	...
;;	...
;;	4K-8
;;	8K
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	512	+1	...
;;	8	...
;;	...
;;	504
;;	1K
;;	...

;; Do 4096 eight_complex_fft operations
;;	distance between fft data elements is 512
;;	do 512 macros each processing 128 data values

	start_timer 5
	zloop_init 512			;; 8 iterations of 64
	zloop_set_rw_prefetch ZDOUBLE	;; Prefetch 1024 cache lines
	zr8_eight_complex_djbfft_preload
b2b:	zr8_eight_complex_djbfft rsi, 128, 2*dist32, 4*dist32, 8*dist32, rdi, ZMM_SCD4, 32, L1PREFETCH_ALL, 128
	zloop	32, b2b, -32*128+dist32	;; Test loop counter, next source pointer
	zloop	2, b2b, -2*dist32+16*dist32, rdi, -64*ZMM_SCD4 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	8, b2b, -8*16*dist32, rdi, 64*ZMM_SCD4 ;; Test loop counter, restore source pointer, next sin/cos pointer
	end_timer 5

;; Do FFT levels 7,8,9
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	512	+1	...
;;	8	...
;;	...
;;	504
;;	1K
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	64	+1	...
;;	8	...
;;	...
;;	56
;;	128
;;	...

;; Do 4096 eight_complex_fft operations
;;	distance between fft data elements is 64
;;	do 512 macros each processing 128 data values

	start_timer 6
	zloop_init 512			;; 64 iterations of 8
	zloop_set_rw_prefetch ZDOUBLE	;; Prefetch 1024 cache lines
	zr8_eight_complex_djbfft_preload
b3b:	zr8_eight_complex_djbfft rsi, 128, 8*128, 16*128, dist32, rdi, ZMM_SCD4, 8, L1PREFETCH_ALL, 128
	zloop	8, b3b, -8*128+2*dist32, rdi, -8*ZMM_SCD4 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	64, b3b, -64*2*dist32, rdi, 8*ZMM_SCD4 ;; Test loop counter, restore source pointer, next sin/cos pointer
	end_timer 6

;; Do FFT levels 10,11,12,13,14,15 as well as inverse FFT levels 10,11,12,13,14,15
;; On input and output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	64	+1	...
;;	8	...
;;	...
;;	56
;;	128
;;	...

;; Do 512 sixtyfour_complex_fft operations
;;	distance between fft data elements is 1 (8 for swizzling)
;;	do 512 macros each processing 128 data values

;; Execute the right middle step

pfing	bump	rcx, -385*64		;; We will prefetch 385 cache lines too many
	cmp	ffttype, 2
	jl	zpass2_32768_complex_1
	jg	zpass2_32768_complex_3

zpass2_32768_complex_2:
	start_timer 9
	zloop_init 512			;; 512 iterations
	zloop_set_rw_prefetch ZSEPT	;; Prefetch 3584 cache lines
	mov	r8, DEST2ARG
	zr64_sixtyfour_complex_with_square_preload
b4a:	zr64_sixtyfour_complex_with_square rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 4, L1PREFETCH_ALL, 8*128
	zloop	4, b4a, -4*8*128+dist32	;; Test loop counter, next source pointer
	zloop	128, b4a		;; Test loop counter
	add	rsi, r8
	end_timer 9
	jmp	zpass2_32768_complex_unfft

zpass2_32768_complex_1:
	start_timer 9
	zloop_init 512			;; 512 iterations
	zloop_set_rw_prefetch ZSEPT	;; Prefetch 3584 cache lines
	zr64_sixtyfour_complex_fft_final_preload
b4b:	zr64_sixtyfour_complex_fft_final rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 4, L1PREFETCH_ALL, 8*128
	zloop	4, b4b, -4*8*128+dist32	;; Test loop counter, next source pointer
	zloop	128, b4b		;; Test loop counter
	end_timer 9
	jmp	zpass2_32768_complex_done

zpass2_32768_complex_3:
	start_timer 9
	zloop_init 512			;; 512 iterations
	zloop_set_rw_prefetch ZSEPT	;; Prefetch 3584 cache lines
	mov	r8, DEST2ARG
	mov	rbp, DIST_TO_MULSRCARG
	zr64_sixtyfour_complex_with_mult_preload
b4c:	zr64_sixtyfour_complex_with_mult rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 4, L1PREFETCH_ALL, 8*128
	zloop	4, b4c, -4*8*128+dist32	;; Test loop counter, next source pointer
	zloop	128, b4c		;; Test loop counter
	add	rsi, r8
	end_timer 9
	jmp	zpass2_32768_complex_unfft

zpass2_32768_complex_4:
	start_timer 9
	zloop_init 512			;; 512 iterations
	zloop_set_rw_prefetch ZSEPT	;; Prefetch 3584 cache lines
	mov	rbp, DIST_TO_MULSRCARG
	mov	rdi, xsincos_complex	;; The sin/cos data
	bump	rdi, 512*ZMM_SCD4+64*ZMM_SCD4+8*ZMM_SCD4
	zr64f_sixtyfour_complex_with_mulf_preload
b4d:	zr64f_sixtyfour_complex_with_mulf rsi, 8*128, 1*128, 2*128, 4*128, rdi, 0*ZMM_SCD4, 4, L1PREFETCH_ALL, 8*128
	zloop	4, b4d, -4*8*128+dist32	;; Test loop counter, next source pointer
	zloop	128, b4d		;; Test loop counter
	end_timer 9
zpass2_32768_complex_unfft:
	bump	rsi, -128*dist32	;; Restore source pointer

;; Do inverse FFT levels 7,8,9
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	64	+1	...
;;	8	...
;;	...
;;	56
;;	128
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	512	+1	...
;;	8	...
;;	...
;;	504
;;	1K
;;	...

;; Do 4096 eight_complex_unfft operations
;;	distance between fft data elements is 64
;;	do 512 macros each processing 128 data values

	start_timer 12
	bump	rdi, -8*ZMM_SCD4
	zloop_init 512			;; 64 iterations of 8
	zloop_set_rw_prefetch ZSINGLE	;; Prefetch 512 cache lines
	zr8_eight_complex_djbunfft_preload
c3b:	zr8_eight_complex_djbunfft rsi, 128, 8*128, 16*128, dist32, rdi, ZMM_SCD4, 8, L1PREFETCH_ALL, 128
	zloop	8, c3b, -8*128+2*dist32, rdi, -8*ZMM_SCD4 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	64, c3b, -64*2*dist32	;; Test loop counter, restore source pointer
	end_timer 12

;; Do inverse FFT levels 4,5,6
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	512	+1	...
;;	8	...
;;	...
;;	504
;;	1K
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	4K	+1	...
;;	8	...
;;	...
;;	4K-8
;;	8K
;;	...

;; Do 4096 eight_complex_unfft operations
;;	distance between fft data elements is 512
;;	do 512 macros each processing 128 data values

	start_timer 13
	bump	rdi, -64*ZMM_SCD4
	zloop_init 512			;; 8 iterations of 64
	zloop_set_rw_prefetch ZDOUBLE	;; Prefetch 1024 cache lines
	zr8_eight_complex_djbunfft_preload
c2b:	zr8_eight_complex_djbunfft rsi, 128, 2*dist32, 4*dist32, 8*dist32, rdi, ZMM_SCD4, 32, L1PREFETCH_ALL, 128
	zloop	32, c2b, -32*128+dist32	;; Test loop counter, next source pointer
	zloop	2, c2b, -2*dist32+16*dist32, rdi, -64*ZMM_SCD4 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	8, c2b, -8*16*dist32	;; Test loop counter, restore source pointer
	end_timer 13

;; Do inverse FFT levels 1,2,3
;; On input the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	4K	+1	...
;;	8	...
;;	...
;;	4K-8
;;	8K
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0	+1	+1	+1	+1	+1	+1	+1	32K	+1	...
;;	8	...
;;	...
;;	32K-8

;; Do 4096 eight_complex_unfft operations
;;	distance between fft data elements is 4K
;;	do 512 macros each processing 128 data values

	start_timer 14
	bump	rdi, -512*ZMM_SCD4
	zloop_init 512			;; 512 iterations
	zloop_set_rw_prefetch ZDOUBLE	;; Prefetch 1024 cache lines
	zr8_eight_complex_djbunfft_preload
c1b:	zr8_eight_complex_djbunfft rsi, 128, 16*dist32, 32*dist32, 64*dist32, rdi, ZMM_SCD4, 32, L1PREFETCH_ALL, 128
	zloop	32, c1b, -32*128+dist32	;; Test loop counter, next source pointer
	zloop	16, c1b			;; Test loop counter
	end_timer 14

zpass2_32768_complex_done:
	ENDM


