; Copyright 2011-2023 - Mersenne Research, Inc.  All rights reserved
; Author:  George Woltman
; Email: woltman@alum.mit.edu
;
; These macros do the first pass of an AVX-512 primarily radix-8 DJB "r4dwpn" FFT.
;


; ********************************************************
; ********************************************************
; ******************  PASS 1 MACROS  *********************
; ********************************************************
; ********************************************************


;;***********************************************************************
;; clm = cache line multiplier (num cache lines processed each iteration)
;;***********************************************************************

;; Note: All memory examples are for 10 complex levels done in pass 2.


;; Perform a pass 1 FFT of length 2 reals.  Basically, this is a wrapper that lets
;; us provide small FFT sizes -- one for each 2nd pass FFT size.   This wrapper applies
;; weights and does swizzling to compensate for the different memory layouts required by
;; the normalization code and the 2nd pass FFT code.
;; NOTE: It may be more efficient to use a different memory layout, create custom code for
;; each one-pass FFT size, and create one-pass normalization code.  We do this for AVX.

zr4dwpn_pass1sc2 MACRO
	LOCAL	b1b, b1c, c1b, c1c, done, type4

	zfft_header

;; Do a multiply with pre-FFTed inputs

	cmp	ffttype, 4
	je	type4

;; Do zero-pad prep work.  Copy 7 words around the halfway point.

	pass1_forward_fft_setup

;; Do transformation from normalize-friendly layout to pass-2-friendly layout.
;; On input the 128-byte double cache lines hold these data values:
;;	0K	+8	+8	+8	+8	+8	+8	+8	1K	+8	...
;;	1
;;	...
;;	7
;;	64
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0K	+1	+1	+1	+1	+1	+1	+1	1K	+1	...
;;	8
;;	...

;; Do macros each processing 128 data values

	start_timer 20
	mov	rsi, DESTARG			;; FFT destination
 	mov	rbx, DIST_TO_FFTSRCARG		;; Distance to source FFT data
	mov	r15, sincos2			;; Addr of column multipliers
	lea	rdi, [r15+16*8]			;; Fudge flags pointer - after 8 column multipliers and 8 inverse column multipliers
	mov	rdx, norm_grp_mults		;; Addr of the group multipliers
	mov	ecx, addcount1			;; Loop FFTLEN / 128 times
	sub	rax, rax			;; Clear 4KB counter
	zsf_onepass_real_fft_wrapper_preload
b1b:	zsf_onepass_real_fft_wrapper rsi, 8*128, 128, 2*128, 4*128, r15, 0, rdi, 14, rdx, 128, 1, L1PREFETCH_ALL, 8*128
	add	al, 256/4			;; Check for every fourth iteration
	jnc	short b1c			;; Jump if not a 4KB boundary
	add	rsi, fourKBgapsize		;; Apply the 4KB padding
b1c:	dec	rcx				;; Decrement loop counter
	jnz	b1b				;; FFTLEN/128 iterations
	mov	DIST_TO_FFTSRCARG, 0		;; FFT data is now in DESTARG
	end_timer 20

;; Do pass 2 (the last 10 FFT levels if this is a 2K FFT)

type4:	start_timer 0
	mov	rax, ZMM_PASS2_ROUTINE
	call	rax
	end_timer 0
	cmp	ffttype, 1			;; We're done if FFTing only
	je	done

;; Do transformation from pass-2-friendly layout to normalize-friendly layout
;; On output the 128-byte double cache lines hold these data values:
;;	0K	+1	+1	+1	+1	+1	+1	+1	1K	+1	...
;;	8
;;	...
;; On input the 128-byte double cache lines hold these data values:
;;	0K	+8	+8	+8	+8	+8	+8	+8	1K	+8	...
;;	1
;;	...
;;	7
;;	64
;;	...

;; Do macros each processing 128 data values

	start_timer 21
	mov	rsi, DESTARG			;; Get address of FFT data
	add	rsi, DEST2ARG			;; Adjust for two destination case
	mov	DESTARG, rsi			;; Normally pass1_wake_up_threads does this adjustment
	mov	DATA_ADDR, rsi			;; Patch this address for zgw_carries
	mov	r15, sincos2			;; Addr of column multipliers
	lea	rdi, [r15+16*8]			;; Fudge flags pointer - after 8 column multipliers and 8 inverse column multipliers
	bump	r15, 8*8			;; Inverse column multipliers
	mov	ecx, addcount1			;; Loop FFTLEN / 128 times
	sub	rax, rax			;; Clear 4KB counter
	zs_onepass_real_unfft_wrapper_preload
c1b:	zs_onepass_real_unfft_wrapper rsi, 8*128, 128, 2*128, 4*128, r15, 0, rdi, 14, 1, L1PREFETCH_ALL, 8*128
	add	al, 256/4			;; Check for every fourth iteration
	jnc	short c1c			;; Jump if not a 4KB boundary
	add	rsi, fourKBgapsize		;; Apply the 4KB padding
c1c:	dec	rcx				;; Decrement loop counter
	jnz	c1b				;; FFTLEN/128 iterations
	mov	norm_ptr1, rdi			;; The big/lit flags are after the fudge flags (ptr used in ZMM_CARRIES_ROUTINE)
	end_timer 21

;; Normalize these values

	start_timer 28
	c_call	PASS1_PRE_CARRIES		;; Make pre-normalization callback (to subtract 7 ZPAD words)
	mov	rsi, DESTARG			;; Address of FFT data to normalize
	mov	edx, addcount1			;; Load count of "clmblks" to normalize where a "clmblk" is 8 double-cache lines
	imul	r10, rdx, 128			;; Calculate size of group multipliers
	add	r10, norm_grp_mults		;; Address of inverse group multipliers (they follow the group multipliers)
	mov	rcx, rsi			;; Disable prefetching by prefetching the data we are now operating on
	mov	ebx, 255			;; Big mask to disable large stride clm prefetching
	mov	rdi, norm_ptr1			;; Address of the big/lit flags
	mov	scratch_area, rsi		;; Set scratch area pointer in case we do a top_carry_adjust
	mov	rax, NORMRTN			;; Addr of normalization routine
	call	rax
	c_call	PASS1_POST_CARRIES		;; Make post-normalization callback (to process POSTADDIN_VALUE)
	end_timer 28

;; Propagate carries back into the FFT data.

	start_timer 29
	mov	rax, ZMM_CARRIES_ROUTINE
	call	rax
	end_timer 29

done:	zfft_footer
	ENDM


;; Perform a negacyclic pass 1 FFT of length 2 reals.  Basically, this is a wrapper that lets
;; us provide small FFT sizes -- one for each 2nd pass FFT size.   This wrapper applies
;; weights and does swizzling to compensate for the different memory layouts required by
;; the normalization code and the 2nd pass FFT code.
;; NOTE: It may be more efficient to use a different memory layout, create custom code for
;; each one-pass FFT size, and create one-pass normalization code.  We do this for AVX.

zr4dwpn_pass1sc2ac MACRO
	LOCAL	b1b, b1c, c1b, c1c, done, type4

	zfft_header

;; Do a multiply with pre-FFTed inputs

	cmp	ffttype, 4
	je	type4

;; Do zero-pad prep work.  Copy 7 words around the halfway point.

	pass1_forward_fft_setup

;; Do transformation from normalize-friendly layout to pass-2-friendly layout.
;; On input the 128-byte double cache lines hold these data values:
;;	0K	+8	+8	+8	+8	+8	+8	+8	1K	+8	...
;;	1
;;	...
;;	7
;;	64
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0K	+1	+1	+1	+1	+1	+1	+1	1K	+1	...
;;	8
;;	...

;; Do macros each processing 128 data values

	start_timer 20
	mov	rsi, DESTARG			;; FFT destination
 	mov	rbx, DIST_TO_FFTSRCARG		;; Distance to source FFT data
	mov	r15, sincos2			;; Addr of inverse column multipliers and complex premultipliers
	lea	r13, [r15+8*8]			;; Premultiplier data is after 8 inverse column multipliers
	mov	ecx, addcount1			;; Loop FFTLEN / 128 times
	imul	rdi, rcx, ZMM_SCD8		;; Compute size of premultiplier data
	add	rdi, r13			;; Fudge flags are after premultiplier data
	mov	rdx, norm_grp_mults		;; Addr of the group multipliers
	sub	rax, rax			;; Clear 4KB counter
	zsf_onepass_complex_fft_wrapper_preload
b1b:	zsf_onepass_complex_fft_wrapper rsi, 8*128, 128, 2*128, 4*128, r13, ZMM_SCD8, rdi, 14, rdx, 128, 1, L1PREFETCH_ALL, 8*128
	add	al, 256/4			;; Check for every fourth iteration
	jnc	short b1c			;; Jump if not a 4KB boundary
	add	rsi, fourKBgapsize		;; Apply the 4KB padding
b1c:	dec	rcx				;; Decrement loop counter
	jnz	b1b				;; FFTLEN/128 iterations
	mov	DIST_TO_FFTSRCARG, 0		;; FFT data is now in DESTARG
	end_timer 20

;; Do pass 2 (the last 10 FFT levels if this is a 2K FFT)

type4:	start_timer 0
	mov	rax, ZMM_PASS2_ROUTINE
	call	rax
	end_timer 0
	cmp	ffttype, 1			;; We're done if FFTing only
	je	done

;; Do transformation from pass-2-friendly layout to normalize-friendly layout
;; On output the 128-byte double cache lines hold these data values:
;;	0K	+1	+1	+1	+1	+1	+1	+1	1K	+1	...
;;	8
;;	...
;; On input the 128-byte double cache lines hold these data values:
;;	0K	+8	+8	+8	+8	+8	+8	+8	1K	+8	...
;;	1
;;	...
;;	7
;;	64
;;	...

;; Do macros each processing 128 data values

	start_timer 21
	mov	rsi, DESTARG			;; Get address of FFT data
	add	rsi, DEST2ARG			;; Adjust for two destination case
	mov	DESTARG, rsi			;; Normally pass1_wake_up_threads does this adjustment
	mov	DATA_ADDR, rsi			;; Patch this address for zgw_carries
	mov	r15, sincos2			;; Addr of inverse column multipliers and complex premultipliers
	lea	r13, [r15+8*8]			;; Premultiplier data is after 8 inverse column multipliers
	mov	ecx, addcount1			;; Loop FFTLEN / 128 times
	imul	rdi, rcx, ZMM_SCD8		;; Compute size of premultiplier data
	add	rdi, r13			;; Fudge flags are after premultiplier data
	sub	rax, rax			;; Clear 4KB counter
	zs_onepass_complex_unfft_wrapper_preload
c1b:	zs_onepass_complex_unfft_wrapper rsi, 8*128, 128, 2*128, 4*128, r13, ZMM_SCD8, r15, 0, rdi, 14, 1, L1PREFETCH_ALL, 8*128
	add	al, 256/4			;; Check for every fourth iteration
	jnc	short c1c			;; Jump if not a 4KB boundary
	add	rsi, fourKBgapsize		;; Apply the 4KB padding
c1c:	dec	rcx				;; Decrement loop counter
	jnz	c1b				;; FFTLEN/128 iterations
	mov	norm_ptr1, rdi			;; The big/lit flags are after the fudge flags (ptr used in ZMM_CARRIES_ROUTINE)
	end_timer 21

;; Normalize these values

	start_timer 28
	c_call	PASS1_PRE_CARRIES		;; Make pre-normalization callback (to subtract 7 ZPAD words)
	mov	rsi, DESTARG			;; Address of FFT data to normalize
	mov	edx, addcount1			;; Load count of "clmblks" to normalize where a "clmblk" is 8 double-cache lines
	imul	r10, rdx, 128			;; Calculate size of group multipliers
	add	r10, norm_grp_mults		;; Address of inverse group multipliers (they follow the group multipliers)
	mov	rcx, rsi			;; Disable prefetching by prefetching the data we are now operating on
	mov	ebx, 255			;; Big mask to disable large stride clm prefetching
	mov	rdi, norm_ptr1			;; Address of the big/lit flags
	mov	scratch_area, rsi		;; Set scratch area pointer in case we do a top_carry_adjust
	mov	rax, NORMRTN			;; Addr of normalization routine
	call	rax
	c_call	PASS1_POST_CARRIES		;; Make post-normalization callback (to process POSTADDIN_VALUE)
	end_timer 28

;; Propagate carries back into the FFT data.

	start_timer 29
	mov	rax, ZMM_CARRIES_ROUTINE
	call	rax
	end_timer 29

done:	zfft_footer
	ENDM



;; Perform a shared pass 1 FFT of length 128 doubles.
;; Memory examples are for a 128K FFT with 10 levels done in pass 2.
;; FFT data needed:
;;		8*clm*128 doubles					Total: 128*clm 64-byte cache lines
;; Fixed s/c data needed:
;;		ZMM_SCD7 (fixed s/c data),				Total: 14 64-byte cache lines
;; Group multipliers needed:
;;		16*64 (weights)
;;		16*64 (inverse weights),				Total: 32 64-byte cache lines
;; Variable data needed:
;;		clm*64	(column weights)
;;		clm*64	(column inverse weights)
;;		clm*ZMM_SCD4 (complex reduced s/c multipliers)
;;		clm*ZMM_SCD4 (real s/c multipliers)
;;		7*clm*ZMM_SCD1 (delayed s/c multipliers)
;;		8*clm*2 (group multiplier fudge flags)
;;		8*clm*2 (big/lit flags),				Total: 32.5*clm 64-byte cache lines
;; Total memory consumed (add for scratch area, prefetched data, carries):
;;		(128*3+32.5=416.5)*clm + (14+32+128/8=62) cache lines
;;		clm=1,2,4:  30KB, 56KB, 108KB

zr4dwpn_pass1sc128 MACRO
	LOCAL	b0b, b1b, b8a, b8b, b9aa, b9a, b9b, c0b, c1b, c8b, c9aa, c9a, c9b
	LOCAL	pass1, pass2, endlp, done, yes_fft

	zfft_header pass1

;; Do pass 2 (the last 10 FFT levels if this is a 128K FFT)

pass2:	start_timer 0
	mov	rax, ZMM_PASS2_ROUTINE
	call	rax
	end_timer 0
	cmp	ffttype, 1		;; We're done if FFTing only
	je	done

;; Do the inverse FFT

	pass1_inverse_fft_setup c0b

;; Do inverse FFT levels 5,6,7
;; On input the 128-byte double cache lines hold these data values:
;;	0K	+1	+1	+1	+1	+1	+1	+1	1K	+1	...
;;	2K
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0K	+1	+1	+1	+1	+1	+1	+1	8K	+1	...
;;	1K	...
;;	...
;; The output cache lines are swizzled to hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	8K	+1K	...
;;	16K
;;	...

;; Do 1 sixteen_reals_unfft operations
;;	distance between fft data elements is 1K
;;	do 0.125 macros each processing 128 data values
;; Do 7 eight_complex_unfft operations
;;	distance between fft data elements is 1K
;;	do 0.875 macros each processing 128 data values

c0b:	start_timer 16
	mov	rbp, DATA_ADDR			;; Get source address
	mov	rsi, scratch_area		;; Get address of scratch area
	set_data_prefetch_ptrs			;; Init prefetching pointers
	mov	r9, PREMULT_ADDR		;; Variable data
	bump	r9, clm*64			;; Inverse column weights
	lea	rbx, [r9+clm*64]		;; Complex data sin/cos ptr
	lea	rdi, [rbx+clm*ZMM_SCD4]		;; Real data sin/cos pointer, will become delayed sin/cos multipliers
	lea	rdx, ZMM_TMPS			;; Computed clm*SCD8 sin/cos values
	mov	r8, pass2blkdst			;; blkdst
	lea	r10, [2*blkdstreg+blkdstreg]	;; 3*blkdst
	lea	r13, [rbp+4*blkdstreg]		;; Source+4*blkdst pointer
	lea	r12, [rbp+8*blkdstreg]		;; L1 prefetch pointer part 1
	bump2	r12, (clm MOD 2), 8*blkdstreg	;; L1 prefetch pointer part 2
	lea	r14, [r12+4*blkdstreg]		;; L1 prefetch pointer + 4*blkdst

	zr8_rsc_complex_and_real_preload	;; Preload constants before calling zloop_init
	zloop_init 1*8*clm			;; 1 iter of 8*clm iterations
	zloop_set_clm_prefetch ZQUAD		;; Prefetch 2 of 64 blocks of clm*128 bytes
	zr8_rsc_wpn_eight_complex_calc_sincos_simple r9, rbx, clm ;; Generate clm*SCD8 sin/cos values from delayed and complex multipliers
c8b:	zr8_rsc_wpn_sgreg_2sc_sixteen_reals_unfft8 rbp, 128, blkdstreg, blkdst3reg, r13, rsi, 1024, 128, 256, 512, rdx, ZMM_SCD8, r9, 64, rdi, ZMM_SCD4, clm, L1PREFETCH_ALL, r12, r14 ;; Prefetch short dist ahead
	zloop	clm, c8b, -clm*1024+clmblkdst, rbp, -clm*128+8*blkdstreg, r13, -clm*128+8*blkdstreg, rdx, -clm*ZMM_SCD8, r9, -clm*64, r12, -clm*128+8*blkdstreg, r14, -clm*128+8*blkdstreg ;; Loop, next src/dest ptrs, restore sin/cos ptr
	;jmp	c9aa

	loops_reset
c9a:	zr8_rsc_wpn_eight_complex_calc_sincos r9, rdi, rbx, clm ;; Generate clm*SCD8 sin/cos values from delayed and complex multipliers
	bump	rdi, clm*ZMM_SCD1		;; Next set of delayed sin/cos multipliers
c9aa:	zloop_set_clm_prefetch ZQUAD		;; Prefetch 14 blocks of clm*128
c9b:	zr8_rsc_wpn_sgreg_eight_complex_unfft8 rbp, 128, blkdstreg, blkdst3reg, r13, rsi, 1024, 128, 256, 512, rdx, ZMM_SCD8, clm, L1PREFETCH_ALL, r12, r14 ;; Prefetch short dist ahead
	zloop	clm, c9b, -clm*1024+clmblkdst, rbp, -clm*128+8*blkdstreg, r13, -clm*128+8*blkdstreg, rdx, -clm*ZMM_SCD8, r12, -clm*128+8*blkdstreg, r14, -clm*128+8*blkdstreg ;; Loop, next src/dest ptrs, restore sin/cos ptr
	zloop	8, c9a, -8*clmblkdst		;; Test loop counter, restore source pointer
	end_timer 16

;; Do inverse FFT levels 1,2,3,4
;; On input the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	8K	+1K	...
;;	16K
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	64K	+1K	...
;;	8K
;;	...

;; Do 8 sixteen_reals_unfft operations
;;	distance between fft data elements is 8K
;;	do 1 macros each processing 128 data values

	start_timer 20
	mov	r15, sincos2			;; Load fixed premultiplier/sin/cos ptr
	sub	rbx, rbx			;; Clear register for loading fudge flags index
	mov	r12, compressed_fudges		;; Address of the compressed fudge flags
	mov	r13, r12			;; The XOR masks for fudge flags come from the compressed fudges
	zloop_init 1*8*clm			;; 1 iter of 8*clm
	zloop_set_clm_prefetch ZQUAD		;; Prefetch 16 of 64 blocks of clm*128 bytes
	zr8_wpn_sixteen_reals_last_unfft_preload
c1b:	zr8_wpn_sixteen_reals_last_unfft rsi, 128, clmblkdst, 2*clmblkdst, 4*clmblkdst, r15, 0, rdi, 2, 8*clm, L1PREFETCH_ALL, 128
	zloop	8*clm, c1b			;; 8*clm iterations
	end_timer 20

;; Normalize these values

	pass1_normalize yes_fft			;; Normalize while prefetching 8 of 64 blocks of clm*128 bytes

;; Copy data from the scratch area back to the FFT data area

	copy_scratch_data_to_fft
	jmp	endlp				;; Skip FFT code if flag not set

;; Fire up auxiliary threads to do pass 1 forward FFT

pass1:	pass1_forward_fft_setup b0b

;; Copy data from the FFT data area back to the scratch area

b0b:	copy_fft_data_to_scratch

;; Do FFT levels 1,2,3,4
;; On input the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	64K	+1K	...
;;	8K	...
;;	...
;;	56K
;; On output the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	8K	+1K	...
;;	16K
;;	...
;;	112K	...

;; Do 8 sixteen_reals_first_fft operations
;;	distance between fft data elements is 8K
;;	do 1 macros each processing 128 data values

yes_fft:start_timer 22
	mov	rsi, scratch_area		;; Get address of scratch area
	mov	r15, sincos2			;; Load fixed premultiplier/sin/cos ptr
	mov	rdi, PREMULT_ADDR		;; Calculate pointer to fudge flags
	bump	rdi, 7*clm*ZMM_SCD1+clm*ZMM_SCD4+clm*ZMM_SCD4+clm*64+clm*64 ;; Addr of the group multiplier fudge flags
	mov	rdx, norm_grp_mults		;; Addr of the group multipliers
	sub	rbx, rbx			;; Clear register for loading fudge flags index
	mov	r12, compressed_fudges		;; Address of the compressed fudge flags
	mov	r13, r12			;; The XOR masks for fudge flags come from the compressed fudges
	set_data_prefetch_ptrs			;; Init prefetching pointers
pfing	mov	r8, pass2blkdst			;; blkdst
pfing	imul	rax, blkdstreg, (40-8)		;; We've already prefetched 40 of 64 blocks (8 too many)
pfing	add	rcx, rax
	zloop_init 1*8*clm			;; 1 iter of 8*clm
	zloop_set_clm_prefetch ZOCT		;; Prefetch 32 of 64 blocks of clm*128 bytes
	zr8_wpn_sixteen_reals_first_fft_preload
b1b:	zr8_wpn_sixteen_reals_first_fft rsi, 128, clmblkdst, 2*clmblkdst, 4*clmblkdst, r15, 0, rdi, 2, rdx, 0, 8*clm, L1PREFETCH_ALL, 128
	zloop	8*clm, b1b, -8*clm*128		;; 8*clm iters prefetching 8*clm cache lines, restore source pointer
	end_timer 22

;; Do FFT levels 5,6,7
;; On input the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	8K	+1K	...
;;	16K
;;	...
;; The input is swizzled to hold these data values:
;;	0K	+1	+1	+1	+1	+1	+1	+1	8K	+1	...
;;	1K	...
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0K	+1	+1	+1	+1	+1	+1	+1	1K	+1	...
;;	2K
;;	...

;; Do 1 sixteen_reals_fft operations
;;	distance between fft data elements is 1K
;;	do 0.125 macros each processing 128 data values
;; Do 8 eight_complex_fft operations
;;	distance between fft data elements is 1K
;;	do 0.875 macros each processing 128 data values

	start_timer 26
	bump	rdi, -8*clm*2-7*clm*ZMM_SCD1-clm*ZMM_SCD4 ;; Real data sin/cos pointer, will become delayed sin/cos multipliers
	lea	rbx, [rdi-clm*ZMM_SCD4]		;; Complex data sin/cos ptr
	lea	r9, [rbx-clm*64-clm*64]		;; Column weights
	lea	rdx, ZMM_TMPS			;; Computed clm*SCD4 sin/cos values
	mov	rbp, DATA_ADDR			;; Destination pointer
	mov	r8, pass2blkdst			;; blkdst
	lea	r10, [2*blkdstreg+blkdstreg]	;; 3*blkdst
	lea	r13, [rbp+4*blkdstreg]		;; Dest+4*blkdst pointer
pfing	mov	rcx, PREMULT_PREFETCH		;; Load premult/sin/cos prefetch pointer

	zr8_rsc_complex_and_real_preload	;; Preload constants before calling zloop_init
	zloop_init 1*8*clm			;; 1 iter of 8*clm
	zloop_set_prefetch ZQUAD		;; Prefetch 4*clm cache lines (0.5 short of optimal)
	zr8_rsc_wpn_eight_complex_calc_sincos_simple r9, rbx, clm ;; Generate clm*SCD8 sin/cos values from delayed and complex multipliers
b8b:	zr8_rsc_wpn_sgreg_2sc_sixteen_reals_fft8 rsi, 1024, 128, 256, 512, rbp, 128, blkdstreg, blkdst3reg, r13, rdx, ZMM_SCD8, r9, 64, rdi, ZMM_SCD4, clm, L1PREFETCH_DEST_ALL
	zloop	clm, b8b, -clm*1024+clmblkdst, rbp, -clm*128+8*blkdstreg, r13, -clm*128+8*blkdstreg, rdx, -clm*ZMM_SCD8, r9, -clm*64 ;; Loop, next source/dest pointers, restore premult/sine/cosine pointer
	;jmp	b9aa

	loops_reset
b9a:	zr8_rsc_wpn_eight_complex_calc_sincos r9, rdi, rbx, clm ;; Generate clm*SCD8 sin/cos values from delayed and complex multipliers
	bump	rdi, clm*ZMM_SCD1		;; Next set of delayed multipliers
b9aa:	zloop_set_prefetch ZQUAD		;; Prefetch 28*clm cache lines (0.5 short of optimal)
b9b:	zr8_rsc_wpn_sgreg_eight_complex_fft8 rsi, 1024, 128, 256, 512, rbp, 128, blkdstreg, blkdst3reg, r13, rdx, ZMM_SCD8, clm, L1PREFETCH_DEST_ALL
	zloop	clm, b9b, -clm*1024+clmblkdst, rbp, -clm*128+8*blkdstreg, r13, -clm*128+8*blkdstreg, rdx, -clm*ZMM_SCD8 ;; Loop, next source/dest pointers, restore premult/sine/cosine pointer
	zloop	8, b9a				;; Test loop counter
	end_timer 26

;; Work on next 8*clm sets of 128 values

endlp:	zpass1_get_next_block pass2, c0b, b0b ;; Get next block and jump
done:	zfft_footer
	ENDM


;; Perform a shared pass 1 negacyclic FFT of length 128 doubles.
;; Memory examples are for a 128K FFT with 10 levels done in pass 2.
;; FFT data needed:
;;		8*clm*128 doubles					Total: 128*clm 64-byte cache lines
;; Fixed s/c data needed:
;;		ZMM_CD8SCD4 (complex premultipliers and fixed s/c data), Total: 16 64-byte cache lines
;; Group multipliers needed:
;;		16*64 (weights)
;;		16*64 (inverse weights),				Total: 32 64-byte cache lines
;; Variable data needed:
;;		clm*64	(column weights)
;;		clm*64	(column inverse weights)
;;		clm*ZMM_SCD4 (complex reduced s/c multipliers)
;;		8*clm*ZMM_SCD1 (delayed s/c multipliers)
;;		8*clm*2 (group multiplier fudge flags)
;;		8*clm*2 (big/lit flags),				Total: 26.5*clm 64-byte cache lines
;; Total memory consumed (add for scratch area, prefetched data, carries):
;;		(128*3+26.5=410.5)*clm + (16+32+128/8=64) cache lines
;;		clm=1,2,4:  30KB, 56KB, 107KB

zr4dwpn_pass1sc128ac MACRO
	LOCAL	b0b, b1b, b9a, b9b, c0b, c1b, c9a, c9b
	LOCAL	pass1, pass2, endlp, done, yes_fft

	zfft_header pass1

;; Do pass 2 (the last 10 FFT levels if this is a 128K FFT)

pass2:	start_timer 0
	mov	rax, ZMM_PASS2_ROUTINE
	call	rax
	end_timer 0
	cmp	ffttype, 1		;; We're done if FFTing only
	je	done

;; Do the inverse FFT

	pass1_inverse_fft_setup c0b

;; Do inverse FFT levels 5,6,7
;; On input the 128-byte double cache lines hold these data values:
;;	0K	+1	+1	+1	+1	+1	+1	+1	1K	+1	...
;;	2K
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0K	+1	+1	+1	+1	+1	+1	+1	8K	+1	...
;;	1K	...
;;	...
;; The output cache lines are swizzled to hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	8K	+1K	...
;;	16K
;;	...

;; Do 8 eight_complex_unfft operations
;;	distance between fft data elements is 1K
;;	do 1 macros each processing 128 data values

c0b:	start_timer 16
	mov	rbp, DATA_ADDR			;; Get source address
	mov	rsi, scratch_area		;; Get address of scratch area
	set_data_prefetch_ptrs			;; Init prefetching pointers
	mov	r9, PREMULT_ADDR		;; Variable data
	bump	r9, clm*64			;; Inverse column weights
	lea	rbx, [r9+clm*64]		;; Complex data sin/cos ptr
	lea	rdi, [rbx+clm*ZMM_SCD4]		;; Delayed sin/cos multipliers
	lea	rdx, ZMM_TMPS			;; Computed clm*SCD8 sin/cos values
	mov	r8, pass2blkdst			;; blkdst
	lea	r10, [2*blkdstreg+blkdstreg]	;; 3*blkdst
	lea	r13, [rbp+4*blkdstreg]		;; Source+4*blkdst pointer
	lea	r12, [rbp+8*blkdstreg]		;; L1 prefetch pointer part 1
	bump2	r12, (clm MOD 2), 8*blkdstreg	;; L1 prefetch pointer part 2
	lea	r14, [r12+4*blkdstreg]		;; L1 prefetch pointer + 4*blkdst

	zr8_rsc_complex_only_preload		;; Preload constants before calling zloop_init
	zloop_init 1*8*clm			;; 1 iter of 8*clm iterations
	zloop_set_clm_prefetch ZQUAD		;; Prefetch 16 blocks of clm*128
c9a:	zr8_rsc_wpn_eight_complex_calc_sincos r9, rdi, rbx, clm ;; Generate clm*SCD8 sin/cos values from weights, delayed, and complex multipliers
	bump	rdi, clm*ZMM_SCD1		;; Next set of delayed sin/cos multipliers
c9b:	zr8_rsc_wpn_sgreg_eight_complex_unfft8 rbp, 128, blkdstreg, blkdst3reg, r13, rsi, 1024, 128, 256, 512, rdx, ZMM_SCD8, clm, L1PREFETCH_ALL, r12, r14 ;; Prefetch short dist ahead
	zloop	clm, c9b, -clm*1024+clmblkdst, rbp, -clm*128+8*blkdstreg, r13, -clm*128+8*blkdstreg, rdx, -clm*ZMM_SCD8, r12, -clm*128+8*blkdstreg, r14, -clm*128+8*blkdstreg ;; Loop, next src/dest ptrs, restore sin/cos ptr
	zloop	8, c9a, -8*clmblkdst		;; Test loop counter, restore source pointer
	end_timer 16

;; Do inverse FFT levels 1,2,3,4
;; On input the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	8K	+1K	...
;;	16K
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	64K	+1K	...
;;	8K
;;	...

;; Do 8 eight_complex_unfft operations
;;	distance between fft data elements is 8K
;;	do 1 macros each processing 128 data values

	start_timer 20
	mov	r15, sincos2			;; Load fixed premultiplier/sin/cos ptr
	sub	rbx, rbx			;; Clear register for loading fudge flags index
	mov	r12, compressed_fudges		;; Address of the compressed fudge flags
	mov	r13, r12			;; The XOR masks for fudge flags come from the compressed fudges
	zloop_init 1*8*clm			;; 1 iter of 8*clm
	zloop_set_clm_prefetch ZQUAD		;; Prefetch 16 blocks of clm*128
	zr8_csc_wpn_eight_complex_last_djbunfft_preload
c1b:	zr8_csc_wpn_eight_complex_last_djbunfft rsi, 128, clmblkdst, 2*clmblkdst, 4*clmblkdst, r15, 0, rdi, 2, 8*clm, L1PREFETCH_ALL, 128
	zloop	8*clm, c1b			;; 8*clm iterations
	end_timer 20

;; Normalize these values

	pass1_normalize yes_fft			;; Normalize while prefetching 8 of 64 blocks of clm*128 bytes

;; Copy data from the scratch area back to the FFT data area

	copy_scratch_data_to_fft
	jmp	endlp				;; Skip FFT code if flag not set

;; Fire up auxiliary threads to do pass 1 forward FFT

pass1:	pass1_forward_fft_setup b0b

;; Copy data from the FFT data area back to the scratch area

b0b:	copy_fft_data_to_scratch

;; Do FFT levels 1,2,3,4
;; On input the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	64K	+1K	...
;;	8K	...
;;	...
;;	56K
;; On output the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	8K	+1K	...
;;	16K
;;	...
;;	112K	...

;; Do 8 eight_complex_first_fft operations
;;	distance between fft data elements is 8K
;;	do 1 macros each processing 128 data values

yes_fft:start_timer 22
	mov	rsi, scratch_area		;; Get address of scratch area
	mov	r15, sincos2			;; Load fixed premultiplier/sin/cos ptr
	mov	rdi, PREMULT_ADDR		;; Calculate pointer to fudge flags
	bump	rdi, 8*clm*ZMM_SCD1+clm*ZMM_SCD4+clm*64+clm*64 ;; Addr of the group multiplier fudge flags
	mov	rdx, norm_grp_mults		;; Addr of the group multipliers
	sub	rbx, rbx			;; Clear register for loading fudge flags index
	mov	r12, compressed_fudges		;; Address of the compressed fudge flags
	mov	r13, r12			;; The XOR masks for fudge flags come from the compressed fudges
	set_data_prefetch_ptrs			;; Init prefetching pointers
pfing	mov	r8, pass2blkdst			;; blkdst
pfing	imul	rax, blkdstreg, (40-8)		;; We've already prefetched 40 of 64 blocks (8 too many)
pfing	add	rcx, rax
	zloop_init 1*8*clm			;; 1 iter of 8*clm
	zloop_set_clm_prefetch ZOCT		;; Prefetch 32 of 64 blocks of clm*128 bytes
	zr8_csc_wpn_eight_complex_first_djbfft_preload
b1b:	zr8_csc_wpn_eight_complex_first_djbfft rsi, 128, clmblkdst, 2*clmblkdst, 4*clmblkdst, r15, 0, rdi, 2, rdx, 0, 8*clm, L1PREFETCH_ALL, 128
	zloop	8*clm, b1b, -8*clm*128		;; 8*clm iters prefetching 8*clm cache lines, restore source pointer
	end_timer 22

;; Do FFT levels 5,6,7
;; On input the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	8K	+1K	...
;;	16K
;;	...
;; The input is swizzled to hold these data values:
;;	0K	+1	+1	+1	+1	+1	+1	+1	8K	+1	...
;;	1K	...
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0K	+1	+1	+1	+1	+1	+1	+1	1K	+1	...
;;	2K
;;	...

;; Do 8 eight_complex_fft operations
;;	distance between fft data elements is 1K
;;	do 1 macros each processing 128 data values

	start_timer 26
	mov	rbp, DATA_ADDR			;; Destination pointer
	bump	rdi, -8*clm*2-8*clm*ZMM_SCD1	;; Delayed sin/cos multipliers
	lea	rbx, [rdi-clm*ZMM_SCD4]		;; Complex data sin/cos ptr
	lea	r9, [rbx-clm*64-clm*64]		;; Column weights
	lea	rdx, ZMM_TMPS			;; Computed clm*SCD4 sin/cos values
	mov	r8, pass2blkdst			;; blkdst
	lea	r10, [2*blkdstreg+blkdstreg]	;; 3*blkdst
	lea	r13, [rbp+4*blkdstreg]		;; Dest+4*blkdst pointer
pfing	mov	rcx, PREMULT_PREFETCH		;; Load premult/sin/cos prefetch pointer

	zr8_rsc_complex_only_preload		;; Preload constants before calling zloop_init
	zloop_init 1*8*clm			;; 1 iter of 8*clm
	zloop_set_prefetch ZQUAD		;; Prefetch 32*clm times
	zloop_set_alternate_prefetch_increment 26*2+1, 32*2  ;; Prefetch approximately 26.5*clm cache lines in those 32*clm prefetches
b9a:	zr8_rsc_wpn_eight_complex_calc_sincos r9, rdi, rbx, clm ;; Generate clm*SCD8 sin/cos values from weights, delayed and complex multipliers
	bump	rdi, clm*ZMM_SCD1		;; Next set of delayed multipliers
b9b:	zr8_rsc_wpn_sgreg_eight_complex_fft8 rsi, 1024, 128, 256, 512, rbp, 128, blkdstreg, blkdst3reg, r13, rdx, ZMM_SCD8, clm, L1PREFETCH_DEST_ALL
	zloop	clm, b9b, -clm*1024+clmblkdst, rbp, -clm*128+8*blkdstreg, r13, -clm*128+8*blkdstreg, rdx, -clm*ZMM_SCD8 ;; Loop, next source/dest pointers, restore premult/sine/cosine pointer
	zloop	8, b9a				;; Test loop counter
	end_timer 26

;; Work on next 8*clm sets of 128 values

endlp:	zpass1_get_next_block pass2, c0b, b0b ;; Get next block and jump
done:	zfft_footer
	ENDM


;; Perform a shared pass 1 FFT of length 192 doubles.
;; Memory examples are for a 192K FFT with 10 levels done in pass 2.
;; FFT data needed:
;;		8*clm*192 doubles					Total: 192*clm 64-byte cache lines
;; Fixed s/c data needed:
;;		ZMM_SCD11 (fixed s/c data),				Total: 22 64-byte cache lines
;; Group multipliers needed:
;;		24*64 (weights)
;;		24*64 (inverse weights),				Total: 48 64-byte cache lines
;; Variable data needed:
;;		clm*64	(column weights)
;;		clm*64	(column inverse weights)
;;		clm*ZMM_SCD4 (complex reduced s/c multipliers)
;;		clm*ZMM_SCD4 (real s/c multipliers)
;;		11*clm*ZMM_SCD1 (delayed s/c multipliers)
;;		8*clm*3 (group multiplier fudge flags)
;;		8*clm*3 (biglit flags),					Total: 49.75*clm 64-byte cache lines
;; Total memory consumed (add for scratch area, prefetched data, carries):
;;		(192*3+49.75=625.75)*clm + (22+48+192/8=94) cache lines
;;		clm=1,2,4:  45KB, 85KB, 163KB

zr4dwpn_pass1sc192 MACRO
	LOCAL	b0b, b1b, b8a, b8b, b9aa, b9a, b9b, c0b, c1b, c8b, c9aa, c9a, c9b
	LOCAL	pass1, pass2, endlp, done, yes_fft

	zfft_header pass1

;; Do pass 2 (the last 10 FFT levels if this is a 192K FFT)

pass2:	start_timer 0
	mov	rax, ZMM_PASS2_ROUTINE
	call	rax
	end_timer 0
	cmp	ffttype, 1		;; We're done if FFTing only
	je	done

;; Do the inverse FFT

	pass1_inverse_fft_setup c0b

;; Do inverse FFT levels 5,6,7
;; On input the 128-byte double cache lines hold these data values:
;;	0K	+1	+1	+1	+1	+1	+1	+1	1K	+1	...
;;	2K
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0K	+1	+1	+1	+1	+1	+1	+1	8K	+1	...
;;	1K	...
;;	...
;; The output cache lines are swizzled to hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	8K	+1K	...
;;	16K
;;	...

;; Do 1 sixteen_reals_unfft operations
;;	distance between fft data elements is 1K
;;	do 0.125 macros each processing 128 data values
;; Do 11 eight_complex_unfft operations
;;	distance between fft data elements is 1K
;;	do 1.375 macros each processing 128 data values

c0b:	start_timer 16
	mov	rbp, DATA_ADDR			;; Get source address
	mov	rsi, scratch_area		;; Get address of scratch area
	set_data_prefetch_ptrs			;; Init prefetching pointers
	mov	r9, PREMULT_ADDR		;; Variable data
	bump	r9, clm*64			;; Inverse column weights
	lea	rbx, [r9+clm*64]		;; Complex data sin/cos ptr
	lea	rdi, [rbx+clm*ZMM_SCD4]		;; Real data sin/cos pointer, will become delayed sin/cos multipliers
	lea	rdx, ZMM_TMPS			;; Computed clm*SCD8 sin/cos values
	mov	r8, pass2blkdst			;; blkdst
	lea	r10, [2*blkdstreg+blkdstreg]	;; 3*blkdst
	lea	r13, [rbp+4*blkdstreg]		;; Source+4*blkdst pointer
	lea	r12, [rbp+8*blkdstreg]		;; L1 prefetch pointer part 1
	bump2	r12, (clm MOD 2), 8*blkdstreg	;; L1 prefetch pointer part 2
	lea	r14, [r12+4*blkdstreg]		;; L1 prefetch pointer + 4*blkdst

	zr8_rsc_complex_and_real_preload	;; Preload constants before calling zloop_init
	zloop_init 12*clm			;; 12 iters of clm
	zr8_rsc_wpn_eight_complex_calc_sincos_simple r9, rbx, clm ;; Generate clm*SCD8 sin/cos values from delayed and complex multipliers
	zloop_set_clm_prefetch ZQUAD		;; Prefetch 2 of 96 blocks of clm*128 bytes
c8b:	zr8_rsc_wpn_sgreg_2sc_sixteen_reals_unfft8 rbp, 128, blkdstreg, blkdst3reg, r13, rsi, 1024, 128, 256, 512, rdx, ZMM_SCD8, r9, 64, rdi, ZMM_SCD4, clm, L1PREFETCH_ALL, r12, r14 ;; Prefetch short dist ahead
	zloop	clm, c8b, -clm*1024+clmblkdst, rbp, -clm*128+8*blkdstreg, r13, -clm*128+8*blkdstreg, rdx, -clm*ZMM_SCD8, r9, -clm*64, r12, -clm*128+8*blkdstreg, r14, -clm*128+8*blkdstreg ;; Loop, next src/dest ptrs, restore sin/cos ptr
	;jmp	c9aa

	loops_reset
c9a:	zr8_rsc_wpn_eight_complex_calc_sincos r9, rdi, rbx, clm ;; Generate clm*SCD8 sin/cos values from delayed and complex multipliers
	bump	rdi, clm*ZMM_SCD1		;; Next set of delayed sin/cos multipliers
c9aa:	zloop_set_clm_prefetch ZQUAD		;; Prefetch 22 of 96 blocks of clm*128 bytes
c9b:	zr8_rsc_wpn_sgreg_eight_complex_unfft8 rbp, 128, blkdstreg, blkdst3reg, r13, rsi, 1024, 128, 256, 512, rdx, ZMM_SCD8, clm, L1PREFETCH_ALL, r12, r14 ;; Prefetch short dist ahead
	zloop	clm, c9b, -clm*1024+clmblkdst, rbp, -clm*128+8*blkdstreg, r13, -clm*128+8*blkdstreg, rdx, -clm*ZMM_SCD8, r12, -clm*128+8*blkdstreg, r14, -clm*128+8*blkdstreg ;; Loop, next src/dest ptrs, restore sin/cos ptr
	zloop	12, c9a, -12*clmblkdst		;; Test loop counter, restore source pointer
	end_timer 16

;; Do inverse FFT levels 1,2,3,4
;; On input the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	8K	+1K	...
;;	16K
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	96K	+1K	...
;;	8K
;;	...

;; Do 8 twentyfour_reals_unfft operations
;;	distance between fft data elements is 8K
;;	do 1 macros each processing 192 data values

	start_timer 20
	mov	r15, sincos2			;; Load fixed premultiplier/sin/cos ptr
	sub	rbx, rbx			;; Clear register for loading fudge flags index
	mov	r12, compressed_fudges		;; Address of the compressed fudge flags
	mov	r13, r12			;; The XOR masks for fudge flags come from the compressed fudges
	zloop_init 1*8*clm			;; 1 iter of 8*clm
	zloop_set_clm_prefetch ZOCT		;; Prefetch 32 of 96 blocks of clm*128 bytes
	zr12_wpn_twentyfour_reals_last_unfft_preload
c1b:	zr12_wpn_twentyfour_reals_last_unfft rsi, 128, clmblkdst, 2*clmblkdst, 4*clmblkdst, r15, 0, rdi, 3, 8*clm, L1PREFETCH_ALL, 128
	zloop	8*clm, c1b			;; 8*clm iterations
	end_timer 20

;; Normalize these values

	pass1_normalize yes_fft			;; Normalize while prefetching 12 of 96 blocks of clm*128 bytes

;; Copy data from the scratch area back to the FFT data area

	copy_scratch_data_to_fft
	jmp	endlp				;; Skip FFT code if flag not set

;; Fire up auxiliary threads to do pass 1 forward FFT

pass1:	pass1_forward_fft_setup b0b

;; Copy data from the FFT data area back to the scratch area

b0b:	copy_fft_data_to_scratch

;; Do FFT levels 1,2,3,4
;; On input the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	96K	+1K	...
;;	8K	...
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	8K	+1K	...
;;	16K
;;	...

;; Do 8 twentyfour_reals_first_fft operations
;;	distance between fft data elements is 8K
;;	do 1 macros each processing 192 data values

yes_fft:start_timer 22
	mov	rsi, scratch_area		;; Get address of scratch area
	mov	r15, sincos2			;; Load fixed premultiplier/sin/cos ptr
	mov	rdi, PREMULT_ADDR		;; Calculate pointer to fudge flags
	bump	rdi, 11*clm*ZMM_SCD1+clm*ZMM_SCD4+clm*ZMM_SCD4+clm*64+clm*64 ;; Addr of the group multiplier fudge flags
	mov	rdx, norm_grp_mults		;; Addr of the group multipliers
	sub	rbx, rbx			;; Clear register for loading fudge flags index
	mov	r12, compressed_fudges		;; Address of the compressed fudge flags
	mov	r13, r12			;; The XOR masks for fudge flags come from the compressed fudges
	set_data_prefetch_ptrs			;; Init prefetching pointers
pfing	mov	r8, pass2blkdst			;; blkdst
pfing	imul	rax, blkdstreg, (68-4)		;; We've already prefetched 68 of 96 blocks (4 too many)
pfing	add	rcx, rax
	zloop_init 1*8*clm			;; 1 iter of 8*clm
	zloop_set_clm_prefetch ZOCT		;; Prefetch 32 of 96 blocks of clm*128 bytes
	zr12_wpn_twentyfour_reals_first_fft_preload
b1b:	zr12_wpn_twentyfour_reals_first_fft rsi, 128, clmblkdst, 2*clmblkdst, 4*clmblkdst, r15, 0, rdi, 3, rdx, 0, 8*clm, L1PREFETCH_ALL, 128
	zloop	8*clm, b1b, -8*clm*128		;; 8*clm iters, restore source pointer
	end_timer 22

;; Do FFT levels 5,6,7
;; On input the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	8K	+1K	...
;;	16K
;;	...
;; The input is swizzled to hold these data values:
;;	0K	+1	+1	+1	+1	+1	+1	+1	8K	+1	...
;;	1K	...
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0K	+1	+1	+1	+1	+1	+1	+1	1K	+1	...
;;	2K
;;	...

;; Do 1 sixteen_reals_fft operations
;;	distance between fft data elements is 1K
;;	do 0.125 macros each processing 128 data values
;; Do 11 eight_complex_fft operations
;;	distance between fft data elements is 1K
;;	do 1.375 macros each processing 128 data values

	start_timer 26
	bump	rdi, -8*clm*3-11*clm*ZMM_SCD1-clm*ZMM_SCD4 ;; Real data sin/cos pointer, will become delayed sin/cos multipliers
	lea	rbx, [rdi-clm*ZMM_SCD4]		;; Complex data sin/cos ptr
	lea	r9, [rbx-clm*64-clm*64]		;; Column weights
	lea	rdx, ZMM_TMPS			;; Computed clm*SCD4 sin/cos values
	mov	rbp, DATA_ADDR			;; Destination pointer
	mov	r8, pass2blkdst			;; blkdst
	lea	r10, [2*blkdstreg+blkdstreg]	;; 3*blkdst
	lea	r13, [rbp+4*blkdstreg]		;; Dest+4*blkdst pointer
pfing	mov	rcx, PREMULT_PREFETCH		;; Load premult/sin/cos prefetch pointer

	zr8_rsc_complex_and_real_preload	;; Preload constants before calling zloop_init
	zloop_init 12*clm			;; 12 iters of clm
	zloop_set_prefetch ZQUAD		;; Prefetch 4*clm cache lines (1.75 short of optimal)
	zr8_rsc_wpn_eight_complex_calc_sincos_simple r9, rbx, clm ;; Generate clm*SCD8 sin/cos values from delayed and complex multipliers
b8b:	zr8_rsc_wpn_sgreg_2sc_sixteen_reals_fft8 rsi, 1024, 128, 256, 512, rbp, 128, blkdstreg, blkdst3reg, r13, rdx, ZMM_SCD8, r9, 64, rdi, ZMM_SCD4, clm, L1PREFETCH_DEST_ALL
	zloop	clm, b8b, -clm*1024+clmblkdst, rbp, -clm*128+8*blkdstreg, r13, -clm*128+8*blkdstreg, rdx, -clm*ZMM_SCD8, r9, -clm*64 ;; Loop, next source/dest pointers, restore premult/sine/cosine pointer
	;jmp	b9aa

	loops_reset
b9a:	zr8_rsc_wpn_eight_complex_calc_sincos r9, rdi, rbx, clm ;; Generate clm*SCD8 sin/cos values from delayed and complex multipliers
	bump	rdi, clm*ZMM_SCD1		;; Next set of delayed multipliers
b9aa:	zloop_set_prefetch ZQUAD		;; Prefetch 44*clm cache lines (1.75 short of optimal)
b9b:	zr8_rsc_wpn_sgreg_eight_complex_fft8 rsi, 1024, 128, 256, 512, rbp, 128, blkdstreg, blkdst3reg, r13, rdx, ZMM_SCD8, clm, L1PREFETCH_DEST_ALL
	zloop	clm, b9b, -clm*1024+clmblkdst, rbp, -clm*128+8*blkdstreg, r13, -clm*128+8*blkdstreg, rdx, -clm*ZMM_SCD8 ;; Loop, next source/dest pointers, restore premult/sine/cosine pointer
	zloop	12, b9a				;; Test loop counter
	end_timer 26

;; Work on next 8*clm sets of 192 values

endlp:	zpass1_get_next_block pass2, c0b, b0b ;; Get next block and jump
done:	zfft_footer
	ENDM


;; Perform a shared pass 1 negacyclic FFT of length 192 doubles.
;; Memory examples are for a 192K FFT with 10 levels done in pass 2.
;; FFT data needed:
;;		8*clm*192 doubles					Total: 192*clm 64-byte cache lines
;; Fixed s/c data needed:
;;		ZMM_CD12SCD6 (complex premultipliers and fixed s/c data), Total: 24 64-byte cache lines
;; Group multipliers needed:
;;		24*64 (weights)
;;		24*64 (inverse weights),				Total: 48 64-byte cache lines
;; Variable data needed:
;;		clm*64	(column weights)
;;		clm*64	(column inverse weights)
;;		clm*ZMM_SCD4 (complex reduced s/c multipliers)
;;		12*clm*ZMM_SCD1 (delayed s/c multipliers)
;;		8*clm*3 (group multiplier fudge flags)
;;		8*clm*3 (biglit flags),					Total: 34.75*clm 64-byte cache lines
;; Total memory consumed (add for scratch area, prefetched data, carries):
;;		(192*3+34.75=610.75)*clm + (24+48+192/8=96) cache lines
;;		clm=1,2,4:  45KB, 83KB, 159KB

zr4dwpn_pass1sc192ac MACRO
	LOCAL	b0b, b1b, b9a, b9b, c0b, c1b, c9a, c9b
	LOCAL	pass1, pass2, endlp, done, yes_fft

	zfft_header pass1

;; Do pass 2 (the last 10 FFT levels if this is a 192K FFT)

pass2:	start_timer 0
	mov	rax, ZMM_PASS2_ROUTINE
	call	rax
	end_timer 0
	cmp	ffttype, 1		;; We're done if FFTing only
	je	done

;; Do the inverse FFT

	pass1_inverse_fft_setup c0b

;; Do inverse FFT levels 5,6,7
;; On input the 128-byte double cache lines hold these data values:
;;	0K	+1	+1	+1	+1	+1	+1	+1	1K	+1	...
;;	2K
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0K	+1	+1	+1	+1	+1	+1	+1	8K	+1	...
;;	1K	...
;;	...
;; The output cache lines are swizzled to hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	8K	+1K	...
;;	16K
;;	...

;; Do 12 eight_complex_unfft operations
;;	distance between fft data elements is 1K
;;	do 1.5 macros each processing 128 data values

c0b:	start_timer 16
	mov	rbp, DATA_ADDR			;; Get source address
	mov	rsi, scratch_area		;; Get address of scratch area
	set_data_prefetch_ptrs			;; Init prefetching pointers
	mov	r9, PREMULT_ADDR		;; Variable data
	bump	r9, clm*64			;; Inverse column weights
	lea	rbx, [r9+clm*64]		;; Complex data sin/cos ptr
	lea	rdi, [rbx+clm*ZMM_SCD4]		;; Delayed sin/cos multipliers
	lea	rdx, ZMM_TMPS			;; Computed clm*SCD8 sin/cos values
	mov	r8, pass2blkdst			;; blkdst
	lea	r10, [2*blkdstreg+blkdstreg]	;; 3*blkdst
	lea	r13, [rbp+4*blkdstreg]		;; Source+4*blkdst pointer
	lea	r12, [rbp+8*blkdstreg]		;; L1 prefetch pointer part 1
	bump2	r12, (clm MOD 2), 8*blkdstreg	;; L1 prefetch pointer part 2
	lea	r14, [r12+4*blkdstreg]		;; L1 prefetch pointer + 4*blkdst

	zr8_rsc_complex_only_preload		;; Preload constants before calling zloop_init
	zloop_init 12*clm			;; 12 iters of clm
	zloop_set_clm_prefetch ZQUAD		;; Prefetch 24 of 96 blocks of clm*128 bytes
c9a:	zr8_rsc_wpn_eight_complex_calc_sincos r9, rdi, rbx, clm ;; Generate clm*SCD8 sin/cos values from weights, delayed, and complex multipliers
	bump	rdi, clm*ZMM_SCD1		;; Next set of delayed sin/cos multipliers
c9b:	zr8_rsc_wpn_sgreg_eight_complex_unfft8 rbp, 128, blkdstreg, blkdst3reg, r13, rsi, 1024, 128, 256, 512, rdx, ZMM_SCD8, clm, L1PREFETCH_ALL, r12, r14 ;; Prefetch short dist ahead
	zloop	clm, c9b, -clm*1024+clmblkdst, rbp, -clm*128+8*blkdstreg, r13, -clm*128+8*blkdstreg, rdx, -clm*ZMM_SCD8, r12, -clm*128+8*blkdstreg, r14, -clm*128+8*blkdstreg ;; Loop, next src/dest ptrs, restore sin/cos ptr
	zloop	12, c9a, -12*clmblkdst		;; Test loop counter, restore source pointer
	end_timer 16

;; Do inverse FFT levels 1,2,3,4
;; On input the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	8K	+1K	...
;;	16K
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	96K	+1K	...
;;	8K
;;	...

;; Do 8 twelve_complex_unfft operations
;;	distance between fft data elements is 8K
;;	do 1 macros each processing 192 data values

	start_timer 20
	mov	r15, sincos2			;; Load fixed premultiplier/sin/cos ptr
	sub	rbx, rbx			;; Clear register for loading fudge flags index
	mov	r12, compressed_fudges		;; Address of the compressed fudge flags
	mov	r13, r12			;; The XOR masks for fudge flags come from the compressed fudges
	zloop_init 1*8*clm			;; 1 iter of 8*clm
	zloop_set_clm_prefetch ZOCT		;; Prefetch 32 of 96 blocks of clm*128 bytes
	zr12_csc_wpn_twelve_complex_last_djbunfft_preload
c1b:	zr12_csc_wpn_twelve_complex_last_djbunfft rsi, 128, clmblkdst, 2*clmblkdst, 4*clmblkdst, r15, 0, rdi, 3, 8*clm, L1PREFETCH_ALL, 128
	zloop	8*clm, c1b			;; 8*clm iterations
	end_timer 20

;; Normalize these values

	pass1_normalize yes_fft			;; Normalize while prefetching 12 of 96 blocks of clm*128 bytes

;; Copy data from the scratch area back to the FFT data area

	copy_scratch_data_to_fft
	jmp	endlp				;; Skip FFT code if flag not set

;; Fire up auxiliary threads to do pass 1 forward FFT

pass1:	pass1_forward_fft_setup b0b

;; Copy data from the FFT data area back to the scratch area

b0b:	copy_fft_data_to_scratch

;; Do FFT levels 1,2,3,4
;; On input the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	96K	+1K	...
;;	8K	...
;;	...
;;	88K
;; On output the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	8K	+1K	...
;;	16K
;;	...
;;	176K	...

;; Do 8 twelve_complex_first_fft operations
;;	distance between fft data elements is 8K
;;	do 1 macros each processing 192 data values

yes_fft:start_timer 22
	mov	rsi, scratch_area		;; Get address of scratch area
	mov	r15, sincos2			;; Load fixed premultiplier/sin/cos ptr
	mov	rdi, PREMULT_ADDR		;; Calculate pointer to fudge flags
	bump	rdi, 12*clm*ZMM_SCD1+clm*ZMM_SCD4+clm*64+clm*64 ;; Addr of the group multiplier fudge flags
	mov	rdx, norm_grp_mults		;; Addr of the group multipliers
	sub	rbx, rbx			;; Clear register for loading fudge flags index
	mov	r12, compressed_fudges		;; Address of the compressed fudge flags
	mov	r13, r12			;; The XOR masks for fudge flags come from the compressed fudges
	set_data_prefetch_ptrs			;; Init prefetching pointers
pfing	mov	r8, pass2blkdst			;; blkdst
pfing	imul	rax, blkdstreg, (68-4)		;; We've already prefetched 68 of 96 blocks (4 too many)
pfing	add	rcx, rax
	zloop_init 1*8*clm			;; 1 iter of 8*clm
	zloop_set_clm_prefetch ZOCT		;; Prefetch 32 of 96 blocks of clm*128 bytes
	zr12_csc_wpn_twelve_complex_first_djbfft_preload
b1b:	zr12_csc_wpn_twelve_complex_first_djbfft rsi, 128, clmblkdst, 2*clmblkdst, 4*clmblkdst, r15, 0, rdi, 3, rdx, 0, 8*clm, L1PREFETCH_ALL, 128
	zloop	8*clm, b1b, -8*clm*128		;; 8*clm iters prefetching 8*clm cache lines, restore source pointer
	end_timer 22

;; Do FFT levels 5,6,7
;; On input the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	8K	+1K	...
;;	16K
;;	...
;; The input is swizzled to hold these data values:
;;	0K	+1	+1	+1	+1	+1	+1	+1	8K	+1	...
;;	1K	...
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0K	+1	+1	+1	+1	+1	+1	+1	1K	+1	...
;;	2K
;;	...

;; Do 8 eight_complex_fft operations
;;	distance between fft data elements is 1K
;;	do 1 macros each processing 128 data values

	start_timer 26
	mov	rbp, DATA_ADDR			;; Destination pointer
	bump	rdi, -8*clm*3-12*clm*ZMM_SCD1	;; Delayed sin/cos multipliers
	lea	rbx, [rdi-clm*ZMM_SCD4]		;; Complex data sin/cos ptr
	lea	r9, [rbx-clm*64-clm*64]		;; Column weights
	lea	rdx, ZMM_TMPS			;; Computed clm*SCD4 sin/cos values
	mov	r8, pass2blkdst			;; blkdst
	lea	r10, [2*blkdstreg+blkdstreg]	;; 3*blkdst
	lea	r13, [rbp+4*blkdstreg]		;; Dest+4*blkdst pointer
pfing	mov	rcx, PREMULT_PREFETCH		;; Load premult/sin/cos prefetch pointer

	zr8_rsc_complex_only_preload		;; Preload constants before calling zloop_init
	zloop_init 12*clm			;; 12 iters of clm
	zloop_set_prefetch ZTRIPLE		;; Prefetch 36*clm cache lines (??? too many)
	zloop_set_alternate_prefetch_increment 34*4+3, 36*4  ;; Prefetch approximately 34.75*clm cache lines in those 36*clm prefetches
b9a:	zr8_rsc_wpn_eight_complex_calc_sincos r9, rdi, rbx, clm ;; Generate clm*SCD8 sin/cos values from weights, delayed and complex multipliers
	bump	rdi, clm*ZMM_SCD1		;; Next set of delayed multipliers
b9b:	zr8_rsc_wpn_sgreg_eight_complex_fft8 rsi, 1024, 128, 256, 512, rbp, 128, blkdstreg, blkdst3reg, r13, rdx, ZMM_SCD8, clm, L1PREFETCH_DEST_ALL
	zloop	clm, b9b, -clm*1024+clmblkdst, rbp, -clm*128+8*blkdstreg, r13, -clm*128+8*blkdstreg, rdx, -clm*ZMM_SCD8 ;; Loop, next source/dest pointers, restore premult/sine/cosine pointer
	zloop	12, b9a				;; Test loop counter
	end_timer 26

;; Work on next 8*clm sets of 192 values

endlp:	zpass1_get_next_block pass2, c0b, b0b ;; Get next block and jump
done:	zfft_footer
	ENDM


;; Perform a shared pass 1 FFT of length 640 doubles.
;; Memory examples are for a 640K FFT with 10 levels done in pass 2.
;; FFT data needed:
;;		8*clm*640 doubles					Total: 640*clm 64-byte cache lines
;; Fixed s/c data needed:
;;		5*ZMM_SCD7 (fixed s/c data),				Total: 70 64-byte cache lines
;; Group multipliers needed:
;;		5*16*64 (weights)
;;		5*16*64 (inverse weights),				Total: 160 64-byte cache lines
;; Variable data needed:
;;		clm*64	(column weights)
;;		clm*64	(column inverse weights)
;;		clm*ZMM_SCD4 (complex reduced s/c multipliers)
;;		clm*ZMM_SCD4 (real s/c multipliers)
;;		7*clm*ZMM_SCD1 (delayed s/c multipliers)
;;		8*clm*ZMM_SCD4 (5-complex 10-real sin/cos data)
;;		40*clm*2 (group multiplier fudge flags)
;;		40*clm*2 (biglit flags),				Total: 98.5*clm 64-byte cache lines
;; Total memory consumed (add for scratch area, prefetched data, carries):
;;		(640*3+98.5=2018.5)*clm + (70+160+640/8=310) cache lines
;;		clm=1,2,4:  146KB, 272KB, 524KB

zr4dwpn_pass1sc640 MACRO
	LOCAL	b0b, b1b, b2a, b2b, b2c, b8a, b8b, b9aa, b9a, b9b, c0b, c1b, c2a, c2b, c2c, c8b, c9aa, c9a, c9b
	LOCAL	pass1, pass2, endlp, done, yes_fft

	zfft_header pass1

;; Do pass 2 (the last 10 FFT levels if this is a 640K FFT)

pass2:	start_timer 0
	mov	rax, ZMM_PASS2_ROUTINE
	call	rax
	end_timer 0
	cmp	ffttype, 1		;; We're done if FFTing only
	je	done

;; Do the inverse FFT

	pass1_inverse_fft_setup c0b

;; Do inverse FFT levels 8,9,10
;; On input the 128-byte double cache lines hold these data values:
;;	0K	+1	+1	+1	+1	+1	+1	+1	1K	+1	...
;;	2K
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0K	+1	+1	+1	+1	+1	+1	+1	8K	+1	...
;;	1K	...
;;	...
;; The output cache lines are swizzled to hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	8K	+1K	...
;;	16K
;;	...

;; Do 1 sixteen_reals_unfft operations
;;	distance between fft data elements is 1K
;;	do 0.125 macros each processing 128 data values
;; Do 39 eight_complex_unfft operations
;;	distance between fft data elements is 1K
;;	do 4.875 macros each processing 128 data values

c0b:	start_timer 16
	mov	rbp, DATA_ADDR			;; Get source address
	mov	rsi, scratch_area		;; Get address of scratch area
	set_data_prefetch_ptrs			;; Init prefetching pointers
	mov	r9, PREMULT_ADDR		;; Variable data
	bump	r9, clm*64			;; Inverse column weights
	lea	rbx, [r9+clm*64]		;; Complex data sin/cos ptr
	lea	rdi, [rbx+clm*ZMM_SCD4]		;; Real data sin/cos pointer, will become delayed sin/cos multipliers
	lea	rdx, ZMM_TMPS			;; Computed clm*SCD8 sin/cos values
	mov	r8, pass2blkdst			;; blkdst
	lea	r10, [2*blkdstreg+blkdstreg]	;; 3*blkdst
	lea	r13, [rbp+4*blkdstreg]		;; Source+4*blkdst pointer
	lea	r12, [rbp+8*blkdstreg]		;; L1 prefetch pointer part 1
	bump2	r12, (clm MOD 2), 8*blkdstreg	;; L1 prefetch pointer part 2
	lea	r14, [r12+4*blkdstreg]		;; L1 prefetch pointer + 4*blkdst

	zr8_rsc_complex_and_real_preload	;; Preload constants before calling zloop_init
	zloop_init 5*8*clm			;; 5 iters of 8*clm iterations
	zr8_rsc_wpn_eight_complex_calc_sincos_simple r9, rbx, clm ;; Generate clm*SCD8 sin/cos values from delayed and complex multipliers
	zloop_set_clm_prefetch ZDOUBLE		;; Prefetch 40 of 320 blocks of clm*128 bytes
c8b:	zr8_rsc_wpn_sgreg_2sc_sixteen_reals_unfft8 rbp, 128, blkdstreg, blkdst3reg, r13, rsi, 1024, 128, 256, 512, rdx, ZMM_SCD8, r9, 64, rdi, ZMM_SCD4, clm, L1PREFETCH_ALL, r12, r14 ;; Prefetch short dist ahead
	zloop	clm, c8b, -clm*1024+clmblkdst, rbp, -clm*128+8*blkdstreg, r13, -clm*128+8*blkdstreg, rdx, -clm*ZMM_SCD8, r9, -clm*64, r12, -clm*128+8*blkdstreg, r14, -clm*128+8*blkdstreg ;; Loop, next src/dest ptrs, restore sin/cos ptr
	jmp	c9aa

	loops_reset
c9a:	zr8_rsc_wpn_eight_complex_calc_sincos r9, rdi, rbx, clm ;; Generate clm*SCD8 sin/cos values from delayed and complex multipliers
	bump	rdi, clm*ZMM_SCD1		;; Next set of delayed sin/cos multipliers
c9aa:	zloop_set_clm_prefetch ZDOUBLE		;; Prefetch 40 of 320 blocks of clm*128 bytes
c9b:	zr8_rsc_wpn_sgreg_eight_complex_unfft8 rbp, 128, blkdstreg, blkdst3reg, r13, rsi, 1024, 128, 256, 512, rdx, ZMM_SCD8, clm, L1PREFETCH_ALL, r12, r14 ;; Prefetch short dist ahead
	zloop	clm, c9b, -clm*1024+clmblkdst, rbp, -clm*128+8*blkdstreg, r13, -clm*128+8*blkdstreg, rdx, -clm*ZMM_SCD8, r12, -clm*128+8*blkdstreg, r14, -clm*128+8*blkdstreg ;; Loop, next src/dest ptrs, restore sin/cos ptr
	zloop	5, c9b				;; Test loop counter
	zloop	8, c9a, -8*5*clmblkdst		;; Test loop counter, restore source pointer
	end_timer 16

;; Do inverse FFT levels 5,6,7
;; On input the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	8K	+1K	...
;;	16K
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	40K	+1K	...
;;	8K
;;	...
;;	32K
;;	80K
;;	...

;; Do 8 ten_reals_unfft operations
;; Do 56 five_complex_unfft operations
;;	distance between fft data elements is 8K
;;	do 8 macros each processing 80 data values

	start_timer 18
	zloop_init  8*8*clm			;; 8 iterations of 8*clm
	zloop_set_clm_prefetch ZDOUBLE		;; Prefetch 64 of 320 blocks of clm*128 bytes
IFDEF OLDDD
	zr5_csc_ten_reals_unfft_preload
c2a:	zr5_csc_ten_reals_unfft rsi, 128, clmblkdst, rdi, ZMM_SCD4, 8*clm, L1PREFETCH_ALL, 128
	zloop	8*clm, c2a, -8*clm*128+5*clmblkdst, rdi, -8*clm*ZMM_SCD4 ;; Test loop counter, next source pointer, restore sin/cos pointer
	loops_reset				;; 7 more iters of 8*clm
	zr5_five_complex_djbunfft_preload
c2b:	zr5_five_complex_djbunfft rsi, 128, clmblkdst, rdi, ZMM_SCD4, 8*clm, L1PREFETCH_ALL, 128
	zloop	8*clm, c2b, -8*clm*128+5*clmblkdst, rdi, -8*clm*ZMM_SCD4 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	8, c2b, -8*5*clmblkdst, rdi, 8*clm*ZMM_SCD4 ;; Test loop counter, restore source pointer, next sin/cos pointer
ELSE
c2a:	zr5_csc_ten_reals_unfft_preload
	zr5_csc_ten_reals_unfft rsi, 5*clmblkdst, clmblkdst, rdi, 0, 1, L1PREFETCH_ALL, 5*clmblkdst
	zr5_five_complex_djbunfft_preload
	jmp	c2c				;; 7 more iters
c2b:	zr5_five_complex_djbunfft rsi, 5*clmblkdst, clmblkdst, rdi, 0, 1, L1PREFETCH_ALL, 5*clmblkdst
c2c:	zloop	8, c2b, -8*5*clmblkdst+128, rdi, ZMM_SCD4 ;; Test loop counter, next source pointer, next sin/cos pointer
	zloop	8*clm, c2a, -8*clm*128		;; Test loop counter, restore source pointer
ENDIF
	end_timer 18

;; Do inverse FFT levels 1,2,3,4
;; On input the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	40K	+1K	...
;;	8K
;;	...
;;	32K
;;	80K
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	320K	+1K	...
;;	8K
;;	...
;;	312K

;; Do 40 sixteen_reals_unfft operations
;;	distance between fft data elements is 40K
;;	do 5 macros each processing 128 data values

	start_timer 20
	mov	r15, sincos2			;; Load fixed premultiplier/sin/cos ptr
	sub	rbx, rbx			;; Clear register for loading fudge flags index
	mov	r12, compressed_fudges		;; Address of the compressed fudge flags
	mov	r13, r12			;; The XOR masks for fudge flags come from the compressed fudges
	zloop_init 5*8*clm			;; 5 iters of 8*clm
	zloop_set_clm_prefetch ZDOUBLE		;; Prefetch 40 of 320 blocks of clm*128 bytes
	zr8_wpn_sixteen_reals_last_unfft_preload
c1b:	zr8_wpn_sixteen_reals_last_unfft rsi, 128, 5*clmblkdst, 10*clmblkdst, 20*clmblkdst, r15, 0, rdi, 2, 8*clm, L1PREFETCH_ALL, 128
	zloop	8*clm, c1b, -8*clm*128+clmblkdst, r15, ZMM_SCD7, rdx, 16*64, r13, 16 ;; 8*clm iterations, next grp multipliers, next XOR masks
	zloop	5, c1b				;; Test loop counter
	end_timer 20

;; Normalize these values

	pass1_normalize yes_fft			;; Normalize while prefetching 40 of 320 blocks of clm*128 bytes

;; Copy data from the scratch area back to the FFT data area

	copy_scratch_data_to_fft
	jmp	endlp				;; Skip FFT code if flag not set

;; Fire up auxiliary threads to do pass 1 forward FFT

pass1:	pass1_forward_fft_setup b0b

;; Copy data from the FFT data area back to the scratch area

b0b:	copy_fft_data_to_scratch

;; Do FFT levels 1,2,3,4
;; On input the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	320K	+1K	...
;;	8K	...
;;	...
;;	312K
;; On output the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	40K	+1K	...
;;	8K
;;	...
;;	32K
;;	80K
;;	...

;; Do 40 sixteen_reals_first_fft operations
;;	distance between fft data elements is 40K
;;	do 5 macros each processing 128 data values

yes_fft:start_timer 22
	mov	rsi, scratch_area		;; Get address of scratch area
	mov	r15, sincos2			;; Load fixed premultiplier/sin/cos ptr
	mov	rdi, PREMULT_ADDR		;; Calculate pointer to fudge flags
	bump	rdi, 8*clm*ZMM_SCD4+7*clm*ZMM_SCD1+clm*ZMM_SCD4+clm*ZMM_SCD4+clm*64+clm*64 ;; Addr of the group multiplier fudge flags
	mov	rdx, norm_grp_mults		;; Addr of the group multipliers
	sub	rbx, rbx			;; Clear register for loading fudge flags index
	mov	r12, compressed_fudges		;; Address of the compressed fudge flags
	mov	r13, r12			;; The XOR masks for fudge flags come from the compressed fudges
	set_data_prefetch_ptrs			;; Init prefetching pointers
pfing	mov	r8, pass2blkdst			;; blkdst
pfing	imul	rax, blkdstreg, (184-8)		;; We've already prefetched 184 of 320 blocks (8 too many)
pfing	add	rcx, rax
	zloop_init 5*8*clm			;; 5 iters of 8*clm
	zloop_set_clm_prefetch ZQUAD		;; Prefetch 80 of 320 blocks of clm*128 bytes
	zr8_wpn_sixteen_reals_first_fft_preload
b1b:	zr8_wpn_sixteen_reals_first_fft rsi, 128, 5*clmblkdst, 10*clmblkdst, 20*clmblkdst, r15, 0, rdi, 2, rdx, 0, 8*clm, L1PREFETCH_ALL, 128
	zloop	8*clm, b1b, -8*clm*128+clmblkdst, r15, ZMM_SCD7, rdx, 16*64, r13, 16 ;; 8*clm iters prefetching 8*clm cache lines, next source, fixed s/c, grp multipliers pointer, next XOR masks
	zloop	5, b1b, -5*clmblkdst		;; Test loop counter, restore source pointer
	end_timer 22

;; Do FFT levels 5,6,7
;; On input the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	40K	+1K	...
;;	8K
;;	...
;;	32K
;;	80K
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	8K	+1K	...
;;	16K
;;	...

;; Do 8 ten_reals_fft operations
;; Do 56 five_complex_fft operations
;;	distance between fft data elements is 8K
;;	do 8 macros each processing 80 data values

	start_timer 24
	bump	rdi, -40*clm*2-8*clm*ZMM_SCD4	;; Sin/cos pointer
	zloop_init  8*8*clm			;; 8 iterations of 8*clm
	zloop_set_clm_prefetch ZDOUBLE		;; Prefetch 64 of 320 blocks of clm*128 bytes
IFDEF OLDDD
	zr5_csc_ten_reals_fft_preload
b2a:	zr5_csc_ten_reals_fft rsi, 128, clmblkdst, rdi, ZMM_SCD4, 8*clm, L1PREFETCH_ALL, 128
	zloop	8*clm, b2a, -8*clm*128+5*clmblkdst, rdi, -8*clm*ZMM_SCD4 ;; Test loop counter, next source pointer, restore sin/cos pointer
	loops_reset				;; 7 more iters of 8*clm
	zr5_five_complex_djbfft_preload
b2b:	zr5_five_complex_djbfft rsi, 128, clmblkdst, rdi, ZMM_SCD4, 8*clm, L1PREFETCH_ALL, 128
	zloop	8*clm, b2b, -8*clm*128+5*clmblkdst, rdi, -8*clm*ZMM_SCD4 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	8, b2b, -8*5*clmblkdst		;; Test loop counter, restore source pointer
ELSE
b2a:	zr5_csc_ten_reals_fft_preload
	zr5_csc_ten_reals_fft rsi, 5*clmblkdst, clmblkdst, rdi, 0, 1, L1PREFETCH_ALL, 5*clmblkdst
	zr5_five_complex_djbfft_preload
	jmp	b2c				;; 7 more iters
b2b:	zr5_five_complex_djbfft rsi, 5*clmblkdst, clmblkdst, rdi, 0, 1, L1PREFETCH_ALL, 5*clmblkdst
b2c:	zloop	8, b2b, -8*5*clmblkdst+128, rdi, ZMM_SCD4 ;; Test loop counter, next source pointer, next sin/cos pointer
	zloop	8*clm, b2a, -8*clm*128		;; Test loop counter, restore source pointer
ENDIF
	end_timer 24

;; Do FFT levels 8,9,10
;; On input the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	8K	+1K	...
;;	16K
;;	...
;; The input is swizzled to hold these data values:
;;	0K	+1	+1	+1	+1	+1	+1	+1	8K	+1	...
;;	1K	...
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0K	+1	+1	+1	+1	+1	+1	+1	1K	+1	...
;;	2K
;;	...

;; Do 1 sixteen_reals_fft operations
;;	distance between fft data elements is 1K
;;	do 0.125 macros each processing 128 data values
;; Do 39 eight_complex_fft operations
;;	distance between fft data elements is 1K
;;	do 4.875 macros each processing 128 data values

	start_timer 26
IFDEF OLDDD
	bump	rdi, -7*clm*ZMM_SCD1-clm*ZMM_SCD4 ;; Real data sin/cos pointer, will become delayed sin/cos multipliers
ELSE
	bump	rdi, -8*clm*ZMM_SCD4-7*clm*ZMM_SCD1-clm*ZMM_SCD4 ;; Real data sin/cos pointer, will become delayed sin/cos multipliers
ENDIF
	lea	rbx, [rdi-clm*ZMM_SCD4]		;; Complex data sin/cos ptr
	lea	r9, [rbx-clm*64-clm*64]		;; Column weights
	lea	rdx, ZMM_TMPS			;; Computed clm*SCD4 sin/cos values
	mov	rbp, DATA_ADDR			;; Destination pointer
	mov	r8, pass2blkdst			;; blkdst
	lea	r10, [2*blkdstreg+blkdstreg]	;; 3*blkdst
	lea	r13, [rbp+4*blkdstreg]		;; Dest+4*blkdst pointer
pfing	mov	rcx, PREMULT_PREFETCH		;; Load premult/sin/cos prefetch pointer

	zr8_rsc_complex_and_real_preload	;; Preload constants before calling zloop_init
	zloop_init 5*8*clm			;; 5 iters of 8*clm
	zloop_set_prefetch ZTRIPLE		;; Prefetch 120*clm times
	zloop_set_alternate_prefetch_increment 98*2+1, 120*2  ;; Prefetch approximately 98.5*clm cache lines in those 120*clm prefetches
	zr8_rsc_wpn_eight_complex_calc_sincos_simple r9, rbx, clm ;; Generate clm*SCD8 sin/cos values from delayed and complex multipliers
b8b:	zr8_rsc_wpn_sgreg_2sc_sixteen_reals_fft8 rsi, 1024, 128, 256, 512, rbp, 128, blkdstreg, blkdst3reg, r13, rdx, ZMM_SCD8, r9, 64, rdi, ZMM_SCD4, clm, L1PREFETCH_DEST_ALL
	zloop	clm, b8b, -clm*1024+clmblkdst, rbp, -clm*128+8*blkdstreg, r13, -clm*128+8*blkdstreg, rdx, -clm*ZMM_SCD8, r9, -clm*64 ;; Loop, next source/dest pointers, restore premult/sine/cosine pointer
	jmp	b9aa

	loops_reset
b9a:	zr8_rsc_wpn_eight_complex_calc_sincos r9, rdi, rbx, clm ;; Generate clm*SCD8 sin/cos values from delayed and complex multipliers
	bump	rdi, clm*ZMM_SCD1		;; Next set of delayed multipliers
b9aa:
b9b:	zr8_rsc_wpn_sgreg_eight_complex_fft8 rsi, 1024, 128, 256, 512, rbp, 128, blkdstreg, blkdst3reg, r13, rdx, ZMM_SCD8, clm, L1PREFETCH_DEST_ALL
	zloop	clm, b9b, -clm*1024+clmblkdst, rbp, -clm*128+8*blkdstreg, r13, -clm*128+8*blkdstreg, rdx, -clm*ZMM_SCD8 ;; Loop, next source/dest pointers, restore premult/sine/cosine pointer
	zloop	5, b9b				;; Test loop counter
	zloop	8, b9a				;; Test loop counter
	end_timer 26

;; Work on next 8*clm sets of 640 values

endlp:	zpass1_get_next_block pass2, c0b, b0b ;; Get next block and jump
done:	zfft_footer
	ENDM


;; Perform a shared pass 1 negacyclic FFT of length 640 doubles.
;; Memory examples are for a 640K FFT with 10 levels done in pass 2.
;; FFT data needed:
;;		8*clm*640 doubles					Total: 640*clm 64-byte cache lines
;; Fixed s/c data needed:
;;		5*ZMM_CD8SCD4 (complex premultipliers and fixed s/c data), Total: 80 64-byte cache lines
;; Group multipliers needed:
;;		5*16*64 (weights)
;;		5*16*64 (inverse weights),				Total: 160 64-byte cache lines
;; Variable data needed:
;;		clm*64	(column weights)
;;		clm*64	(column inverse weights)
;;		clm*ZMM_SCD4 (complex reduced s/c multipliers)
;;		8*clm*ZMM_SCD1 (delayed s/c multipliers)
;;		8*clm*ZMM_SCD2 (5-complex sin/cos data)
;;		40*clm*2 (group multiplier fudge flags)
;;		40*clm*2 (biglit flags),				Total: 60.5*clm 64-byte cache lines
;; Total memory consumed (add for scratch area, prefetched data, carries):
;;		(640*3+60.5=1980.5)*clm + (80+160+640/8=320) cache lines
;;		clm=1,2,4:  144KB, 268KB, 516KB

zr4dwpn_pass1sc640ac MACRO
	LOCAL	b0b, b1b, b2b, b9a, b9b, c0b, c1b, c2b, c9a, c9b
	LOCAL	pass1, pass2, endlp, done, yes_fft

	zfft_header pass1

;; Do pass 2 (the last 10 FFT levels if this is a 640K FFT)

pass2:	start_timer 0
	mov	rax, ZMM_PASS2_ROUTINE
	call	rax
	end_timer 0
	cmp	ffttype, 1		;; We're done if FFTing only
	je	done

;; Do the inverse FFT

	pass1_inverse_fft_setup c0b

;; Do inverse FFT levels 8.9.10
;; On input the 128-byte double cache lines hold these data values:
;;	0K	+1	+1	+1	+1	+1	+1	+1	1K	+1	...
;;	2K
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0K	+1	+1	+1	+1	+1	+1	+1	8K	+1	...
;;	1K	...
;;	...
;; The output cache lines are swizzled to hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	8K	+1K	...
;;	16K
;;	...

;; Do 40 eight_complex_unfft operations
;;	distance between fft data elements is 1K
;;	do 5 macros each processing 128 data values

c0b:	start_timer 16
	mov	rbp, DATA_ADDR			;; Get source address
	mov	rsi, scratch_area		;; Get address of scratch area
	set_data_prefetch_ptrs			;; Init prefetching pointers
	mov	r9, PREMULT_ADDR		;; Variable data
	bump	r9, clm*64			;; Inverse column weights
	lea	rbx, [r9+clm*64]		;; Complex data sin/cos ptr
	lea	rdi, [rbx+clm*ZMM_SCD4]		;; Delayed sin/cos multipliers
	lea	rdx, ZMM_TMPS			;; Computed clm*SCD8 sin/cos values
	mov	r8, pass2blkdst			;; blkdst
	lea	r10, [2*blkdstreg+blkdstreg]	;; 3*blkdst
	lea	r13, [rbp+4*blkdstreg]		;; Source+4*blkdst pointer
	lea	r12, [rbp+8*blkdstreg]		;; L1 prefetch pointer part 1
	bump2	r12, (clm MOD 2), 8*blkdstreg	;; L1 prefetch pointer part 2
	lea	r14, [r12+4*blkdstreg]		;; L1 prefetch pointer + 4*blkdst

	zr8_rsc_complex_only_preload		;; Preload constants before calling zloop_init
	zloop_init 5*8*clm			;; 5 iters of 8*clm iterations
	zloop_set_clm_prefetch ZDOUBLE		;; Prefetch 40 of 320 blocks of clm*128 bytes
c9a:	zr8_rsc_wpn_eight_complex_calc_sincos r9, rdi, rbx, clm ;; Generate clm*SCD8 sin/cos values from weights, delayed, and complex multipliers
	bump	rdi, clm*ZMM_SCD1		;; Next set of delayed sin/cos multipliers
c9b:	zr8_rsc_wpn_sgreg_eight_complex_unfft8 rbp, 128, blkdstreg, blkdst3reg, r13, rsi, 1024, 128, 256, 512, rdx, ZMM_SCD8, clm, L1PREFETCH_ALL, r12, r14 ;; Prefetch short dist ahead
	zloop	clm, c9b, -clm*1024+clmblkdst, rbp, -clm*128+8*blkdstreg, r13, -clm*128+8*blkdstreg, rdx, -clm*ZMM_SCD8, r12, -clm*128+8*blkdstreg, r14, -clm*128+8*blkdstreg ;; Loop, next src/dest ptrs, restore sin/cos ptr
	zloop	5, c9b				;; Test loop counter
	zloop	8, c9a, -8*5*clmblkdst		;; Test loop counter, restore source pointer
	end_timer 16

;; Do inverse FFT levels 5,6,7
;; On input the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	8K	+1K	...
;;	16K
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	40K	+1K	...
;;	8K
;;	...
;;	32K
;;	80K
;;	...

;; Do 64 five_complex_unfft operations
;;	distance between fft data elements is 8K
;;	do 8 macros each processing 80 data values

	start_timer 18
	zloop_init  8*8*clm			;; 8 iterations of 8*clm
	zloop_set_clm_prefetch ZDOUBLE		;; Prefetch 64 of 320 blocks of clm*128 bytes
	zr5_five_complex_djbunfft_preload
IFDEF OLDDD
c2b:	zr5_five_complex_djbunfft rsi, 128, clmblkdst, rdi, ZMM_SCD2, 8*clm, L1PREFETCH_ALL, 128
	zloop	8*clm, c2b, -8*clm*128+5*clmblkdst, rdi, -8*clm*ZMM_SCD2 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	8, c2b, -8*5*clmblkdst, rdi, 8*clm*ZMM_SCD2 ;; Test loop counter, restore source pointer, next sin/cos pointer
ELSE
c2b:	zr5_five_complex_djbunfft rsi, 5*clmblkdst, clmblkdst, rdi, 0, 8, L1PREFETCH_ALL, 5*clmblkdst
	zloop	8, c2b, -8*5*clmblkdst+128, rdi, ZMM_SCD2 ;; Test loop counter, next source pointer, next sin/cos pointer
	zloop	8*clm, c2b, -8*clm*128		;; Test loop counter, restore source pointer
ENDIF
	end_timer 18

;; Do inverse FFT levels 1,2,3,4
;; On input the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	40K	+1K	...
;;	8K
;;	...
;;	32K
;;	80K
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	320K	+1K	...
;;	8K	...
;;	...
;;	312K

;; Do 40 eight_complex_unfft operations
;;	distance between fft data elements is 40K
;;	do 5 macros each processing 128 data values

	start_timer 20
	mov	r15, sincos2			;; Load fixed premultiplier/sin/cos ptr
	sub	rbx, rbx			;; Clear register for loading fudge flags index
	mov	r12, compressed_fudges		;; Address of the compressed fudge flags
	mov	r13, r12			;; The XOR masks for fudge flags come from the compressed fudges
	zloop_init 5*8*clm			;; 5 iters of 8*clm
	zloop_set_clm_prefetch ZDOUBLE		;; Prefetch 40 of 320 blocks of clm*128 bytes
	zr8_csc_wpn_eight_complex_last_djbunfft_preload
c1b:	zr8_csc_wpn_eight_complex_last_djbunfft rsi, 128, 5*clmblkdst, 10*clmblkdst, 20*clmblkdst, r15, 0, rdi, 2, 8*clm, L1PREFETCH_ALL, 128
	zloop	8*clm, c1b, -8*clm*128+clmblkdst, r15, ZMM_CD8SCD4, rdx, 16*64, r13, 16 ;; 8*clm iterations, next fixed s/c, next grp multipliers, next XOR masks
	zloop	5, c1b				;; 5 iterations
	end_timer 20

;; Normalize these values

	pass1_normalize yes_fft			;; Normalize while prefetching 40 of 320 blocks of clm*128 bytes

;; Copy data from the scratch area back to the FFT data area

	copy_scratch_data_to_fft
	jmp	endlp				;; Skip FFT code if flag not set

;; Fire up auxiliary threads to do pass 1 forward FFT

pass1:	pass1_forward_fft_setup b0b

;; Copy data from the FFT data area back to the scratch area

b0b:	copy_fft_data_to_scratch

;; Do FFT levels 1,2,3,4
;; On input the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	320K	+1K	...
;;	8K	...
;;	...
;;	312K
;; On output the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	40K	+1K	...
;;	8K
;;	...
;;	32K
;;	80K
;;	...

;; Do 40 eight_complex_first_fft operations
;;	distance between fft data elements is 40K
;;	do 5 macros each processing 128 data values

yes_fft:start_timer 22
	mov	rsi, scratch_area		;; Get address of scratch area
	mov	r15, sincos2			;; Load fixed premultiplier/sin/cos ptr
	mov	rdi, PREMULT_ADDR		;; Calculate pointer to fudge flags
	bump	rdi, 8*clm*ZMM_SCD2+8*clm*ZMM_SCD1+clm*ZMM_SCD4+clm*64+clm*64 ;; Addr of the group multiplier fudge flags
	mov	rdx, norm_grp_mults		;; Addr of the group multipliers
	sub	rbx, rbx			;; Clear register for loading fudge flags index
	mov	r12, compressed_fudges		;; Address of the compressed fudge flags
	mov	r13, r12			;; The XOR masks for fudge flags come from the compressed fudges
	set_data_prefetch_ptrs			;; Init prefetching pointers
pfing	mov	r8, pass2blkdst			;; blkdst
pfing	imul	rax, blkdstreg, (184-8)		;; We've already prefetched 184 of 320 blocks (8 too many)
pfing	add	rcx, rax
	zloop_init 5*8*clm			;; 5 iters of 8*clm
	zloop_set_clm_prefetch ZQUAD		;; Prefetch 80 of 320 blocks of clm*128 bytes
	zr8_csc_wpn_eight_complex_first_djbfft_preload
b1b:	zr8_csc_wpn_eight_complex_first_djbfft rsi, 128, 5*clmblkdst, 10*clmblkdst, 20*clmblkdst, r15, 0, rdi, 2, rdx, 0, 8*clm, L1PREFETCH_ALL, 128
	zloop	8*clm, b1b, -8*clm*128+clmblkdst, r15, ZMM_CD8SCD4, rdx, 16*64, r13, 16 ;; 8*clm iters, next source, fixed s/c, grp multipliers pointer, next XOR masks
	zloop	5, b1b, -5*clmblkdst		;; Test loop counter, restore source pointer
	end_timer 22

;; Do FFT levels 5,6,7
;; On input the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	40K	+1K	...
;;	8K
;;	...
;;	32K
;;	80K
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	8K	+1K	...
;;	16K
;;	...

;; Do 64 five_complex_fft operations
;;	distance between fft data elements is 8K
;;	do 8 macros each processing 80 data values

	start_timer 24
	bump	rdi, -40*clm*2-8*clm*ZMM_SCD2	;; Sin/cos pointer
	zloop_init  8*8*clm			;; 8 iterations of 8*clm
	zloop_set_clm_prefetch ZDOUBLE		;; Prefetch 64 of 320 blocks of clm*128 bytes
	zr5_five_complex_djbfft_preload
IFDEF OLDDD
b2b:	zr5_five_complex_djbfft rsi, 128, clmblkdst, rdi, ZMM_SCD2, 8*clm, L1PREFETCH_ALL, 128
	zloop	8*clm, b2b, -8*clm*128+5*clmblkdst, rdi, -8*clm*ZMM_SCD2 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	8, b2b, -8*5*clmblkdst		;; Test loop counter, restore source pointer
ELSE
b2b:	zr5_five_complex_djbfft rsi, 5*clmblkdst, clmblkdst, rdi, 0, 8, L1PREFETCH_ALL, 5*clmblkdst
	zloop	8, b2b, -8*5*clmblkdst+128, rdi, ZMM_SCD2 ;; Test loop counter, next source pointer, next sin/cos pointer
	zloop	8*clm, b2b, -8*clm*128		;; Test loop counter, restore source pointer
ENDIF
	end_timer 24

;; Do FFT levels 8,9,10
;; On input the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	8K	+1K	...
;;	16K
;;	...
;; The input is swizzled to hold these data values:
;;	0K	+1	+1	+1	+1	+1	+1	+1	8K	+1	...
;;	1K	...
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0K	+1	+1	+1	+1	+1	+1	+1	1K	+1	...
;;	2K
;;	...

;; Do 40 eight_complex_fft operations
;;	distance between fft data elements is 1K
;;	do 5 macros each processing 128 data values

	start_timer 26
	mov	rbp, DATA_ADDR			;; Destination pointer
IFDEF OLDDD
	bump	rdi, -8*clm*ZMM_SCD1		;; Delayed sin/cos multipliers
ELSE
	bump	rdi, -8*clm*ZMM_SCD2-8*clm*ZMM_SCD1 ;; Delayed sin/cos multipliers
ENDIF
	lea	rbx, [rdi-clm*ZMM_SCD4]		;; Complex data sin/cos ptr
	lea	r9, [rbx-clm*64-clm*64]		;; Column weights
	lea	rdx, ZMM_TMPS			;; Computed clm*SCD4 sin/cos values
	mov	r8, pass2blkdst			;; blkdst
	lea	r10, [2*blkdstreg+blkdstreg]	;; 3*blkdst
	lea	r13, [rbp+4*blkdstreg]		;; Dest+4*blkdst pointer
pfing	mov	rcx, PREMULT_PREFETCH		;; Load premult/sin/cos prefetch pointer

	zr8_rsc_complex_only_preload		;; Preload constants before calling zloop_init
	zloop_init 5*8*clm			;; 5 iters of 8*clm
	zloop_set_prefetch ZDOUBLE		;; Prefetch 80*clm cache lines
	zloop_set_alternate_prefetch_increment 60*2+1, 80*2  ;; Prefetch approximately 60.5*clm cache lines in those 80*clm prefetches
b9a:	zr8_rsc_wpn_eight_complex_calc_sincos r9, rdi, rbx, clm ;; Generate clm*SCD8 sin/cos values from weights, delayed and complex multipliers
	bump	rdi, clm*ZMM_SCD1		;; Next set of delayed multipliers
b9b:	zr8_rsc_wpn_sgreg_eight_complex_fft8 rsi, 1024, 128, 256, 512, rbp, 128, blkdstreg, blkdst3reg, r13, rdx, ZMM_SCD8, clm, L1PREFETCH_DEST_ALL
	zloop	clm, b9b, -clm*1024+clmblkdst, rbp, -clm*128+8*blkdstreg, r13, -clm*128+8*blkdstreg, rdx, -clm*ZMM_SCD8 ;; Loop, next source/dest pointers, restore premult/sine/cosine pointer
	zloop	5, b9b				;; Test loop counter
	zloop	8, b9a				;; Test loop counter
	end_timer 26

;; Work on next 8*clm sets of 640 values

endlp:	zpass1_get_next_block pass2, c0b, b0b ;; Get next block and jump
done:	zfft_footer
	ENDM



;; Perform a shared pass 1 FFT of length 768 doubles.
;; Memory examples are for a 768K FFT with 10 levels done in pass 2.
;; FFT data needed:
;;		8*clm*768 doubles					Total: 768*clm 64-byte cache lines
;; Fixed s/c data needed:
;;		6*ZMM_SCD7 (fixed s/c data),				Total: 84 64-byte cache lines
;; Group multipliers needed:
;;		6*16*64 (weights)
;;		6*16*64 (inverse weights),				Total: 192 64-byte cache lines
;; Variable data needed:
;;		clm*64	(column weights)
;;		clm*64	(column inverse weights)
;;		clm*ZMM_SCD4 (complex reduced s/c multipliers)
;;		clm*ZMM_SCD4 (real s/c multipliers)
;;		7*clm*ZMM_SCD1 (delayed s/c multipliers)
;;		8*clm*ZMM_SCD6 (6-complex 12-real sin/cos data)
;;		48*clm*2 (group multiplier fudge flags)
;;		48*clm*2 (biglit flags),				Total: 131*clm 64-byte cache lines
;; Total memory consumed (add for scratch area, prefetched data, carries):
;;		(768*3+131=2435)*clm + (84+192+768/8=372) cache lines
;;		clm=1,2,4:  176KB, 328KB, 632KB

zr4dwpn_pass1sc768 MACRO
	LOCAL	b0b, b1b, b2a, b2b, b2c, b8a, b8b, b9aa, b9a, b9b, c0b, c1b, c2a, c2b, c2c, c8b, c9aa, c9a, c9b
	LOCAL	pass1, pass2, endlp, done, yes_fft

	zfft_header pass1

;; Do pass 2 (the last 10 FFT levels if this is a 768K FFT)

pass2:	start_timer 0
	mov	rax, ZMM_PASS2_ROUTINE
	call	rax
	end_timer 0
	cmp	ffttype, 1		;; We're done if FFTing only
	je	done

;; Do the inverse FFT

	pass1_inverse_fft_setup c0b

;; Do inverse FFT levels 8,9,10
;; On input the 128-byte double cache lines hold these data values:
;;	0K	+1	+1	+1	+1	+1	+1	+1	1K	+1	...
;;	2K
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0K	+1	+1	+1	+1	+1	+1	+1	8K	+1	...
;;	1K	...
;;	...
;; The output cache lines are swizzled to hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	8K	+1K	...
;;	16K
;;	...

;; Do 1 sixteen_reals_unfft operations
;;	distance between fft data elements is 1K
;;	do 0.125 macros each processing 128 data values
;; Do 47 eight_complex_unfft operations
;;	distance between fft data elements is 1K
;;	do 5.875 macros each processing 128 data values

c0b:	start_timer 16
	mov	rbp, DATA_ADDR			;; Get source address
	mov	rsi, scratch_area		;; Get address of scratch area
	set_data_prefetch_ptrs			;; Init prefetching pointers
	mov	r9, PREMULT_ADDR		;; Variable data
	bump	r9, clm*64			;; Inverse column weights
	lea	rbx, [r9+clm*64]		;; Complex data sin/cos ptr
	lea	rdi, [rbx+clm*ZMM_SCD4]		;; Real data sin/cos pointer, will become delayed sin/cos multipliers
	lea	rdx, ZMM_TMPS			;; Computed clm*SCD8 sin/cos values
	mov	r8, pass2blkdst			;; blkdst
	lea	r10, [2*blkdstreg+blkdstreg]	;; 3*blkdst
	lea	r13, [rbp+4*blkdstreg]		;; Source+4*blkdst pointer
	lea	r12, [rbp+8*blkdstreg]		;; L1 prefetch pointer part 1
	bump2	r12, (clm MOD 2), 8*blkdstreg	;; L1 prefetch pointer part 2
	lea	r14, [r12+4*blkdstreg]		;; L1 prefetch pointer + 4*blkdst

	zr8_rsc_complex_and_real_preload	;; Preload constants before calling zloop_init
	zloop_init 6*8*clm			;; 6 iters of 8*clm iterations
	zr8_rsc_wpn_eight_complex_calc_sincos_simple r9, rbx, clm ;; Generate clm*SCD8 sin/cos values from delayed and complex multipliers
	zloop_set_clm_prefetch ZDOUBLE		;; Prefetch 48 of 384 blocks of clm*128 bytes
c8b:	zr8_rsc_wpn_sgreg_2sc_sixteen_reals_unfft8 rbp, 128, blkdstreg, blkdst3reg, r13, rsi, 1024, 128, 256, 512, rdx, ZMM_SCD8, r9, 64, rdi, ZMM_SCD4, clm, L1PREFETCH_ALL, r12, r14 ;; Prefetch short dist ahead
	zloop	clm, c8b, -clm*1024+clmblkdst, rbp, -clm*128+8*blkdstreg, r13, -clm*128+8*blkdstreg, rdx, -clm*ZMM_SCD8, r9, -clm*64, r12, -clm*128+8*blkdstreg, r14, -clm*128+8*blkdstreg ;; Loop, next src/dest ptrs, restore sin/cos ptr
	jmp	c9aa

	loops_reset
c9a:	zr8_rsc_wpn_eight_complex_calc_sincos r9, rdi, rbx, clm ;; Generate clm*SCD8 sin/cos values from delayed and complex multipliers
	bump	rdi, clm*ZMM_SCD1		;; Next set of delayed sin/cos multipliers
c9aa:	zloop_set_clm_prefetch ZDOUBLE		;; Prefetch 48 of 384 blocks of clm*128 bytes
c9b:	zr8_rsc_wpn_sgreg_eight_complex_unfft8 rbp, 128, blkdstreg, blkdst3reg, r13, rsi, 1024, 128, 256, 512, rdx, ZMM_SCD8, clm, L1PREFETCH_ALL, r12, r14 ;; Prefetch short dist ahead
	zloop	clm, c9b, -clm*1024+clmblkdst, rbp, -clm*128+8*blkdstreg, r13, -clm*128+8*blkdstreg, rdx, -clm*ZMM_SCD8, r12, -clm*128+8*blkdstreg, r14, -clm*128+8*blkdstreg ;; Loop, next src/dest ptrs, restore sin/cos ptr
	zloop	6, c9b				;; Test loop counter
	zloop	8, c9a, -8*6*clmblkdst		;; Test loop counter, restore source pointer
	end_timer 16

;; Do inverse FFT levels 5,6,7
;; On input the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	8K	+1K	...
;;	16K
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	48K	+1K	...
;;	8K
;;	...
;;	40K
;;	96K
;;	...

;; Do 8 twelve_reals_unfft operations
;; Do 56 six_complex_unfft operations
;;	distance between fft data elements is 8K
;;	do 8 macros each processing 96 data values

	start_timer 18
	zloop_init  8*8*clm			;; 8 iterations of 8*clm
	zloop_set_clm_prefetch ZDOUBLE		;; Prefetch 64 of 384 blocks of clm*128 bytes
IFDEF OLDDD
	zr6_csc_twelve_reals_unfft_preload
c2a:	zr6_csc_twelve_reals_unfft rsi, 128, clmblkdst, rdi, ZMM_SCD6, 8*clm, L1PREFETCH_ALL, 128
	zloop	8*clm, c2a, -8*clm*128+6*clmblkdst, rdi, -8*clm*ZMM_SCD6 ;; Test loop counter, next source pointer, restore sin/cos pointer
	loops_reset				;; 7 more iters of 8*clm
	zr6_six_complex_djbunfft_preload
c2b:	zr6_six_complex_djbunfft rsi, 128, clmblkdst, rdi, ZMM_SCD6, 8*clm, L1PREFETCH_ALL, 128
	zloop	8*clm, c2b, -8*clm*128+6*clmblkdst, rdi, -8*clm*ZMM_SCD6 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	8, c2b, -8*6*clmblkdst, rdi, 8*clm*ZMM_SCD6 ;; Test loop counter, restore source pointer, next sin/cos pointer
ELSE
c2a:	zr6_csc_twelve_reals_unfft_preload
	zr6_csc_twelve_reals_unfft rsi, 6*clmblkdst, clmblkdst, rdi, 0, 1, L1PREFETCH_ALL, 6*clmblkdst
	zr6_six_complex_djbunfft_preload
	jmp	c2c				;; 7 more iters
c2b:	zr6_six_complex_djbunfft rsi, 6*clmblkdst, clmblkdst, rdi, 0, 1, L1PREFETCH_ALL, 6*clmblkdst
c2c:	zloop	8, c2b, -8*6*clmblkdst+128, rdi, ZMM_SCD6 ;; Test loop counter, next source pointer, next sin/cos pointer
	zloop	8*clm, c2a, -8*clm*128		;; Test loop counter, restore source pointer
ENDIF
	end_timer 18

;; Do inverse FFT levels 1,2,3,4
;; On input the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	48K	+1K	...
;;	8K
;;	...
;;	40K
;;	96K
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	384K	+1K	...
;;	8K
;;	...
;;	376K

;; Do 48 sixteen_reals_unfft operations
;;	distance between fft data elements is 48K
;;	do 6 macros each processing 128 data values

	start_timer 20
	mov	r15, sincos2			;; Load fixed premultiplier/sin/cos ptr
	sub	rbx, rbx			;; Clear register for loading fudge flags index
	mov	r12, compressed_fudges		;; Address of the compressed fudge flags
	mov	r13, r12			;; The XOR masks for fudge flags come from the compressed fudges
	zloop_init 6*8*clm			;; 6 iters of 8*clm
	zloop_set_clm_prefetch ZQUAD		;; Prefetch 96 of 384 blocks of clm*128 bytes
	zr8_wpn_sixteen_reals_last_unfft_preload
c1b:	zr8_wpn_sixteen_reals_last_unfft rsi, 128, 6*clmblkdst, 12*clmblkdst, 24*clmblkdst, r15, 0, rdi, 2, 8*clm, L1PREFETCH_ALL, 128
	zloop	8*clm, c1b, -8*clm*128+clmblkdst, r15, ZMM_SCD7, rdx, 16*64, r13, 16 ;; 8*clm iters, next source, fixed s/c, grp multipliers pointer, next xor mask
	zloop	6, c1b				;; Test loop counter
	end_timer 20

;; Normalize these values

	pass1_normalize yes_fft			;; Normalize while prefetching 48 of 384 blocks of clm*128 bytes

;; Copy data from the scratch area back to the FFT data area

	copy_scratch_data_to_fft
	jmp	endlp				;; Skip FFT code if flag not set

;; Fire up auxiliary threads to do pass 1 forward FFT

pass1:	pass1_forward_fft_setup b0b

;; Copy data from the FFT data area back to the scratch area

b0b:	copy_fft_data_to_scratch

;; Do FFT levels 1,2,3,4
;; On input the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	384K	+1K	...
;;	8K	...
;;	...
;;	376K
;; On output the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	48K	+1K	...
;;	8K
;;	...
;;	40K
;;	96K
;;	...

;; Do 48 sixteen_reals_first_fft operations
;;	distance between fft data elements is 48K
;;	do 6 macros each processing 128 data values

yes_fft:start_timer 22
	mov	rsi, scratch_area		;; Get address of scratch area
	mov	r15, sincos2			;; Load fixed premultiplier/sin/cos ptr
	mov	rdi, PREMULT_ADDR		;; Calculate pointer to fudge flags
	bump	rdi, 8*clm*ZMM_SCD6+7*clm*ZMM_SCD1+clm*ZMM_SCD4+clm*ZMM_SCD4+clm*64+clm*64 ;; Addr of the group multiplier fudge flags
	mov	rdx, norm_grp_mults		;; Addr of the group multipliers
	sub	rbx, rbx			;; Clear register for loading fudge flags index
	mov	r12, compressed_fudges		;; Address of the compressed fudge flags
	mov	r13, r12			;; The XOR masks for fudge flags come from the compressed fudges
	set_data_prefetch_ptrs			;; Init prefetching pointers
pfing	mov	r8, pass2blkdst			;; blkdst
pfing	imul	rax, blkdstreg, (256-0)		;; We've already prefetched 256 of 384 blocks (zero too many)
pfing	add	rcx, rax
	zloop_init 6*8*clm			;; 6 iters of 8*clm
	zloop_set_clm_prefetch ZQUAD		;; Prefetch 96 of 384 blocks of clm*128 bytes
	zr8_wpn_sixteen_reals_first_fft_preload
b1b:	zr8_wpn_sixteen_reals_first_fft rsi, 128, 6*clmblkdst, 12*clmblkdst, 24*clmblkdst, r15, 0, rdi, 2, rdx, 0, 8*clm, L1PREFETCH_ALL, 128
	zloop	8*clm, b1b, -8*clm*128+clmblkdst, r15, ZMM_SCD7, rdx, 16*64, r13, 16 ;; 8*clm iters prefetching 8*clm cache lines, next source, fixed s/c, grp multipliers pointer, next xor mask
	zloop	6, b1b, -6*clmblkdst		;; Test loop counter, restore source pointer
	end_timer 22

;; Do FFT levels 5,6,7
;; On input the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	48K	+1K	...
;;	8K
;;	...
;;	40K
;;	96K
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	8K	+1K	...
;;	16K
;;	...

;; Do 8 twelve_reals_fft operations
;; Do 56 six_complex_fft operations
;;	distance between fft data elements is 8K
;;	do 8 macros each processing 96 data values

	start_timer 24
	bump	rdi, -48*clm*2-8*clm*ZMM_SCD6	;; Sin/cos pointer
	zloop_init  8*8*clm			;; 8 iterations of 8*clm
	zloop_set_clm_prefetch ZSINGLE		;; Prefetch 32 of 384 blocks of clm*128 bytes
IFDEF OLDDD
	zr6_csc_twelve_reals_fft_preload
b2a:	zr6_csc_twelve_reals_fft rsi, 128, clmblkdst, rdi, ZMM_SCD6, 8*clm, L1PREFETCH_ALL, 128
	zloop	8*clm, b2a, -8*clm*128+6*clmblkdst, rdi, -8*clm*ZMM_SCD6 ;; Test loop counter, next source pointer, restore sin/cos pointer
	loops_reset				;; 7 more iters of 8*clm
	zr6_six_complex_djbfft_preload
b2b:	zr6_six_complex_djbfft rsi, 128, clmblkdst, rdi, ZMM_SCD6, 8*clm, L1PREFETCH_ALL, 128
	zloop	8*clm, b2b, -8*clm*128+6*clmblkdst, rdi, -8*clm*ZMM_SCD6 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	8, b2b, -8*6*clmblkdst		;; Test loop counter, restore source pointer
ELSE
b2a:	zr6_csc_twelve_reals_fft_preload
	zr6_csc_twelve_reals_fft rsi, 6*clmblkdst, clmblkdst, rdi, 0, 1, L1PREFETCH_ALL, 6*clmblkdst
	zr6_six_complex_djbfft_preload
	jmp	b2c				;; 7 more iters
b2b:	zr6_six_complex_djbfft rsi, 6*clmblkdst, clmblkdst, rdi, 0, 1, L1PREFETCH_ALL, 6*clmblkdst
b2c:	zloop	8, b2b, -8*6*clmblkdst+128, rdi, ZMM_SCD6 ;; Test loop counter, next source pointer, next sin/cos pointer
	zloop	8*clm, b2a, -8*clm*128		;; Test loop counter, restore source pointer
ENDIF
	end_timer 24

;; Do FFT levels 8,9,10
;; On input the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	8K	+1K	...
;;	16K
;;	...
;; The input is swizzled to hold these data values:
;;	0K	+1	+1	+1	+1	+1	+1	+1	8K	+1	...
;;	1K	...
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0K	+1	+1	+1	+1	+1	+1	+1	1K	+1	...
;;	2K
;;	...

;; Do 1 sixteen_reals_fft operations
;;	distance between fft data elements is 1K
;;	do 0.125 macros each processing 128 data values
;; Do 47 eight_complex_fft operations
;;	distance between fft data elements is 1K
;;	do 5.875 macros each processing 128 data values

	start_timer 26
IFDEF OLDDD
	bump	rdi, -7*clm*ZMM_SCD1-clm*ZMM_SCD4 ;; Real data sin/cos pointer, will become delayed sin/cos multipliers
ELSE
	bump	rdi, -8*clm*ZMM_SCD6-7*clm*ZMM_SCD1-clm*ZMM_SCD4 ;; Real data sin/cos pointer, will become delayed sin/cos multipliers
ENDIF
	lea	rbx, [rdi-clm*ZMM_SCD4]		;; Complex data sin/cos ptr
	lea	r9, [rbx-clm*64-clm*64]		;; Column weights
	lea	rdx, ZMM_TMPS			;; Computed clm*SCD4 sin/cos values
	mov	rbp, DATA_ADDR			;; Destination pointer
	mov	r8, pass2blkdst			;; blkdst
	lea	r10, [2*blkdstreg+blkdstreg]	;; 3*blkdst
	lea	r13, [rbp+4*blkdstreg]		;; Dest+4*blkdst pointer
pfing	mov	rcx, PREMULT_PREFETCH		;; Load premult/sin/cos prefetch pointer

	zr8_rsc_complex_and_real_preload	;; Preload constants before calling zloop_init
	zloop_init 6*8*clm			;; 6 iters of 8*clm
	zloop_set_prefetch ZTRIPLE		;; Prefetch 144*clm times
	zloop_set_alternate_prefetch_increment 131, 144  ;; Prefetch approximately 131*clm cache lines in those 144*clm prefetches
	zr8_rsc_wpn_eight_complex_calc_sincos_simple r9, rbx, clm ;; Generate clm*SCD8 sin/cos values from delayed and complex multipliers
b8b:	zr8_rsc_wpn_sgreg_2sc_sixteen_reals_fft8 rsi, 1024, 128, 256, 512, rbp, 128, blkdstreg, blkdst3reg, r13, rdx, ZMM_SCD8, r9, 64, rdi, ZMM_SCD4, clm, L1PREFETCH_DEST_ALL
	zloop	clm, b8b, -clm*1024+clmblkdst, rbp, -clm*128+8*blkdstreg, r13, -clm*128+8*blkdstreg, rdx, -clm*ZMM_SCD8, r9, -clm*64 ;; Loop, next source/dest pointers, restore premult/sine/cosine pointer
	jmp	b9aa

	loops_reset
b9a:	zr8_rsc_wpn_eight_complex_calc_sincos r9, rdi, rbx, clm ;; Generate clm*SCD8 sin/cos values from delayed and complex multipliers
	bump	rdi, clm*ZMM_SCD1		;; Next set of delayed multipliers
b9aa:
b9b:	zr8_rsc_wpn_sgreg_eight_complex_fft8 rsi, 1024, 128, 256, 512, rbp, 128, blkdstreg, blkdst3reg, r13, rdx, ZMM_SCD8, clm, L1PREFETCH_DEST_ALL
	zloop	clm, b9b, -clm*1024+clmblkdst, rbp, -clm*128+8*blkdstreg, r13, -clm*128+8*blkdstreg, rdx, -clm*ZMM_SCD8 ;; Loop, next source/dest pointers, restore premult/sine/cosine pointer
	zloop	6, b9b				;; Test loop counter
	zloop	8, b9a				;; Test loop counter
	end_timer 26

;; Work on next 8*clm sets of 768 values

endlp:	zpass1_get_next_block pass2, c0b, b0b ;; Get next block and jump
done:	zfft_footer
	ENDM


;; Perform a shared pass 1 negacyclic FFT of length 768 doubles.
;; Memory examples are for a 768K FFT with 10 levels done in pass 2.
;; FFT data needed:
;;		8*clm*768 doubles					Total: 768*clm 64-byte cache lines
;; Fixed s/c data needed:
;;		6*ZMM_CD8SCD4 (complex premultipliers and fixed s/c data), Total: 96 64-byte cache lines
;; Group multipliers needed:
;;		6*16*64 (weights)
;;		6*16*64 (inverse weights),				Total: 192 64-byte cache lines
;; Variable data needed:
;;		clm*64	(column weights)
;;		clm*64	(column inverse weights)
;;		clm*ZMM_SCD4 (complex reduced s/c multipliers)
;;		8*clm*ZMM_SCD1 (delayed s/c multipliers)
;;		8*clm*ZMM_SCD3 (6-complex sin/cos data)
;;		48*clm*2 (group multiplier fudge flags)
;;		48*clm*2 (biglit flags),				Total: 77*clm 64-byte cache lines
;; Total memory consumed (add for scratch area, prefetched data, carries):
;;		(768*3+77=2381)*clm + (96+192+768/8=384) cache lines
;;		clm=1,2,4:  173KB, 322KB, 620KB

zr4dwpn_pass1sc768ac MACRO
	LOCAL	b0b, b1b, b2b, b9a, b9b, c0b, c1b, c2b, c9a, c9b
	LOCAL	pass1, pass2, endlp, done, yes_fft

	zfft_header pass1

;; Do pass 2 (the last 10 FFT levels if this is a 768K FFT)

pass2:	start_timer 0
	mov	rax, ZMM_PASS2_ROUTINE
	call	rax
	end_timer 0
	cmp	ffttype, 1		;; We're done if FFTing only
	je	done

;; Do the inverse FFT

	pass1_inverse_fft_setup c0b

;; Do inverse FFT levels 8.9.10
;; On input the 128-byte double cache lines hold these data values:
;;	0K	+1	+1	+1	+1	+1	+1	+1	1K	+1	...
;;	2K
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0K	+1	+1	+1	+1	+1	+1	+1	8K	+1	...
;;	1K	...
;;	...
;; The output cache lines are swizzled to hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	8K	+1K	...
;;	16K
;;	...

;; Do 48 eight_complex_unfft operations
;;	distance between fft data elements is 1K
;;	do 6 macros each processing 128 data values

c0b:	start_timer 16
	mov	rbp, DATA_ADDR			;; Get source address
	mov	rsi, scratch_area		;; Get address of scratch area
	set_data_prefetch_ptrs			;; Init prefetching pointers
	mov	r9, PREMULT_ADDR		;; Variable data
	bump	r9, clm*64			;; Inverse column weights
	lea	rbx, [r9+clm*64]		;; Complex data sin/cos ptr
	lea	rdi, [rbx+clm*ZMM_SCD4]		;; Delayed sin/cos multipliers
	lea	rdx, ZMM_TMPS			;; Computed clm*SCD8 sin/cos values
	mov	r8, pass2blkdst			;; blkdst
	lea	r10, [2*blkdstreg+blkdstreg]	;; 3*blkdst
	lea	r13, [rbp+4*blkdstreg]		;; Source+4*blkdst pointer
	lea	r12, [rbp+8*blkdstreg]		;; L1 prefetch pointer part 1
	bump2	r12, (clm MOD 2), 8*blkdstreg	;; L1 prefetch pointer part 2
	lea	r14, [r12+4*blkdstreg]		;; L1 prefetch pointer + 4*blkdst

	zr8_rsc_complex_only_preload		;; Preload constants before calling zloop_init
	zloop_init 6*8*clm			;; 6 iters of 8*clm iterations
	zloop_set_clm_prefetch ZDOUBLE		;; Prefetch 48 of 384 blocks of clm*128 bytes
c9a:	zr8_rsc_wpn_eight_complex_calc_sincos r9, rdi, rbx, clm ;; Generate clm*SCD8 sin/cos values from weights, delayed, and complex multipliers
	bump	rdi, clm*ZMM_SCD1		;; Next set of delayed sin/cos multipliers
c9b:	zr8_rsc_wpn_sgreg_eight_complex_unfft8 rbp, 128, blkdstreg, blkdst3reg, r13, rsi, 1024, 128, 256, 512, rdx, ZMM_SCD8, clm, L1PREFETCH_ALL, r12, r14 ;; Prefetch short dist ahead
	zloop	clm, c9b, -clm*1024+clmblkdst, rbp, -clm*128+8*blkdstreg, r13, -clm*128+8*blkdstreg, rdx, -clm*ZMM_SCD8, r12, -clm*128+8*blkdstreg, r14, -clm*128+8*blkdstreg ;; Loop, next src/dest ptrs, restore sin/cos ptr
	zloop	6, c9b				;; Test loop counter
	zloop	8, c9a, -8*6*clmblkdst		;; Test loop counter, restore source pointer
	end_timer 16

;; Do inverse FFT levels 5,6,7
;; On input the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	8K	+1K	...
;;	16K
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	48K	+1K	...
;;	8K
;;	...
;;	40K
;;	96K
;;	...

;; Do 64 six_complex_unfft operations
;;	distance between fft data elements is 8K
;;	do 8 macros each processing 96 data values

	start_timer 18
	zloop_init  8*8*clm			;; 8 iterations of 8*clm
	zloop_set_clm_prefetch ZDOUBLE		;; Prefetch 64 of 384 blocks of clm*128 bytes
	zr6_six_complex_djbunfft_preload
IFDEF OLDDD
c2b:	zr6_six_complex_djbunfft rsi, 128, clmblkdst, rdi, ZMM_SCD3, 8*clm, L1PREFETCH_ALL, 128
	zloop	8*clm, c2b, -8*clm*128+6*clmblkdst, rdi, -8*clm*ZMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	8, c2b, -8*6*clmblkdst, rdi, 8*clm*ZMM_SCD3 ;; Test loop counter, restore source pointer, next sin/cos pointer
ELSE
c2b:	zr6_six_complex_djbunfft rsi, 6*clmblkdst, clmblkdst, rdi, 0, 8, L1PREFETCH_ALL, 6*clmblkdst
	zloop	8, c2b, -8*6*clmblkdst+128, rdi, ZMM_SCD3 ;; Test loop counter, next source pointer, next sin/cos pointer
	zloop	8*clm, c2b, -8*clm*128		;; Test loop counter, restore source pointer
ENDIF
	end_timer 18

;; Do inverse FFT levels 1,2,3,4
;; On input the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	48K	+1K	...
;;	8K
;;	...
;;	40K
;;	96K
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	384K	+1K	...
;;	8K	...
;;	...
;;	376K

;; Do 48 eight_complex_unfft operations
;;	distance between fft data elements is 48K
;;	do 6 macros each processing 128 data values

	start_timer 20
	mov	r15, sincos2			;; Load fixed premultiplier/sin/cos ptr
	sub	rbx, rbx			;; Clear register for loading fudge flags index
	mov	r12, compressed_fudges		;; Address of the compressed fudge flags
	mov	r13, r12			;; The XOR masks for fudge flags come from the compressed fudges
	zloop_init 6*8*clm			;; 6 iters of 8*clm
	zloop_set_clm_prefetch ZQUAD		;; Prefetch 96 of 384 blocks of clm*128 bytes
	zr8_csc_wpn_eight_complex_last_djbunfft_preload
c1b:	zr8_csc_wpn_eight_complex_last_djbunfft rsi, 128, 6*clmblkdst, 12*clmblkdst, 24*clmblkdst, r15, 0, rdi, 2, 8*clm, L1PREFETCH_ALL, 128
	zloop	8*clm, c1b, -8*clm*128+clmblkdst, r15, ZMM_CD8SCD4, rdx, 16*64, r13, 16 ;; 8*clm iterations, next fixed s/c, next grp multipliers, next xor mask
	zloop	6, c1b				;; 6 iterations
	end_timer 20

;; Normalize these values

	pass1_normalize yes_fft			;; Normalize while prefetching 48 of 384 blocks of clm*128 bytes

;; Copy data from the scratch area back to the FFT data area

	copy_scratch_data_to_fft
	jmp	endlp				;; Skip FFT code if flag not set

;; Fire up auxiliary threads to do pass 1 forward FFT

pass1:	pass1_forward_fft_setup b0b

;; Copy data from the FFT data area back to the scratch area

b0b:	copy_fft_data_to_scratch

;; Do FFT levels 1,2,3,4
;; On input the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	384K	+1K	...
;;	8K	...
;;	...
;;	376K
;; On output the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	48K	+1K	...
;;	8K
;;	...
;;	40K
;;	96K
;;	...

;; Do 48 eight_complex_first_fft operations
;;	distance between fft data elements is 48K
;;	do 6 macros each processing 128 data values

yes_fft:start_timer 22
	mov	rsi, scratch_area		;; Get address of scratch area
	mov	r15, sincos2			;; Load fixed premultiplier/sin/cos ptr
	mov	rdi, PREMULT_ADDR		;; Calculate pointer to fudge flags
	bump	rdi, 8*clm*ZMM_SCD3+8*clm*ZMM_SCD1+clm*ZMM_SCD4+clm*64+clm*64 ;; Addr of the group multiplier fudge flags
	mov	rdx, norm_grp_mults		;; Addr of the group multipliers
	sub	rbx, rbx			;; Clear register for loading fudge flags index
	mov	r12, compressed_fudges		;; Address of the compressed fudge flags
	mov	r13, r12			;; The XOR masks for fudge flags come from the compressed fudges
	set_data_prefetch_ptrs			;; Init prefetching pointers
pfing	mov	r8, pass2blkdst			;; blkdst
pfing	imul	rax, blkdstreg, (256-0)		;; We've already prefetched 256 of 384 blocks (zero too many)
pfing	add	rcx, rax
	zloop_init 6*8*clm			;; 6 iters of 8*clm
	zloop_set_clm_prefetch ZQUAD		;; Prefetch 96 of 384 blocks of clm*128 bytes
	zr8_csc_wpn_eight_complex_first_djbfft_preload
b1b:	zr8_csc_wpn_eight_complex_first_djbfft rsi, 128, 6*clmblkdst, 12*clmblkdst, 24*clmblkdst, r15, 0, rdi, 2, rdx, 0, 8*clm, L1PREFETCH_ALL, 128
	zloop	8*clm, b1b, -8*clm*128+clmblkdst, r15, ZMM_CD8SCD4, rdx, 16*64, r13, 16 ;; 8*clm iters, next source, fixed s/c, grp multipliers pointer, next xor mask
	zloop	6, b1b, -6*clmblkdst		;; Test loop counter, restore source pointer
	end_timer 22

;; Do FFT levels 5,6,7
;; On input the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	48K	+1K	...
;;	8K
;;	...
;;	50K
;;	96K
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	8K	+1K	...
;;	16K
;;	...

;; Do 64 six_complex_fft operations
;;	distance between fft data elements is 8K
;;	do 8 macros each processing 80 data values

	start_timer 24
	bump	rdi, -48*clm*2-8*clm*ZMM_SCD3	;; Sin/cos pointer
	zloop_init  8*8*clm			;; 8 iterations of 8*clm
	zloop_set_clm_prefetch ZSINGLE		;; Prefetch 32 of 384 blocks of clm*128 bytes
	zr6_six_complex_djbfft_preload
IFDEF OLDDD
b2b:	zr6_six_complex_djbfft rsi, 128, clmblkdst, rdi, ZMM_SCD3, 8*clm, L1PREFETCH_ALL, 128
	zloop	8*clm, b2b, -8*clm*128+6*clmblkdst, rdi, -8*clm*ZMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	8, b2b, -8*6*clmblkdst		;; Test loop counter, restore source pointer
ELSE
b2b:	zr6_six_complex_djbfft rsi, 6*clmblkdst, clmblkdst, rdi, 0, 8, L1PREFETCH_ALL, 6*clmblkdst
	zloop	8, b2b, -8*6*clmblkdst+128, rdi, ZMM_SCD3 ;; Test loop counter, next source pointer, next sin/cos pointer
	zloop	8*clm, b2b, -8*clm*128		;; Test loop counter, restore source pointer
ENDIF
	end_timer 24

;; Do FFT levels 8,9,10
;; On input the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	8K	+1K	...
;;	16K
;;	...
;; The input is swizzled to hold these data values:
;;	0K	+1	+1	+1	+1	+1	+1	+1	8K	+1	...
;;	1K	...
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0K	+1	+1	+1	+1	+1	+1	+1	1K	+1	...
;;	2K
;;	...

;; Do 48 eight_complex_fft operations
;;	distance between fft data elements is 1K
;;	do 6 macros each processing 128 data values

	start_timer 26
	mov	rbp, DATA_ADDR			;; Destination pointer
IFDEF OLDDD
	bump	rdi, -8*clm*ZMM_SCD1		;; Delayed sin/cos multipliers
ELSE
	bump	rdi, -8*clm*ZMM_SCD3-8*clm*ZMM_SCD1 ;; Delayed sin/cos multipliers
ENDIF
	lea	rbx, [rdi-clm*ZMM_SCD4]		;; Complex data sin/cos ptr
	lea	r9, [rbx-clm*64-clm*64]		;; Column weights
	lea	rdx, ZMM_TMPS			;; Computed clm*SCD4 sin/cos values
	mov	r8, pass2blkdst			;; blkdst
	lea	r10, [2*blkdstreg+blkdstreg]	;; 3*blkdst
	lea	r13, [rbp+4*blkdstreg]		;; Dest+4*blkdst pointer
pfing	mov	rcx, PREMULT_PREFETCH		;; Load premult/sin/cos prefetch pointer

	zr8_rsc_complex_only_preload		;; Preload constants before calling zloop_init
	zloop_init 6*8*clm			;; 6 iters of 8*clm
	zloop_set_prefetch ZDOUBLE		;; Prefetch 96*clm cache lines
	zloop_set_alternate_prefetch_increment 77, 96  ;; Prefetch approximately 77*clm cache lines in those 96*clm prefetches
b9a:	zr8_rsc_wpn_eight_complex_calc_sincos r9, rdi, rbx, clm ;; Generate clm*SCD8 sin/cos values from weights, delayed and complex multipliers
	bump	rdi, clm*ZMM_SCD1		;; Next set of delayed multipliers
b9b:	zr8_rsc_wpn_sgreg_eight_complex_fft8 rsi, 1024, 128, 256, 512, rbp, 128, blkdstreg, blkdst3reg, r13, rdx, ZMM_SCD8, clm, L1PREFETCH_DEST_ALL
	zloop	clm, b9b, -clm*1024+clmblkdst, rbp, -clm*128+8*blkdstreg, r13, -clm*128+8*blkdstreg, rdx, -clm*ZMM_SCD8 ;; Loop, next source/dest pointers, restore premult/sine/cosine pointer
	zloop	6, b9b				;; Test loop counter
	zloop	8, b9a				;; Test loop counter
	end_timer 26

;; Work on next 8*clm sets of 768 values

endlp:	zpass1_get_next_block pass2, c0b, b0b ;; Get next block and jump
done:	zfft_footer
	ENDM



;; Perform a shared pass 1 FFT of length 896 doubles.
;; Memory examples are for a 896K FFT with 10 levels done in pass 2.
;; FFT data needed:
;;		8*clm*896 doubles					Total: 896*clm 64-byte cache lines
;; Fixed s/c data needed:
;;		7*ZMM_SCD7 (fixed s/c data),				Total: 98 64-byte cache lines
;; Group multipliers needed:
;;		7*16*64 (weights)
;;		7*16*64 (inverse weights),				Total: 224 64-byte cache lines
;; Variable data needed:
;;		clm*64	(column weights)
;;		clm*64	(column inverse weights)
;;		clm*ZMM_SCD4 (complex reduced s/c multipliers)
;;		clm*ZMM_SCD4 (real s/c multipliers)
;;		7*clm*ZMM_SCD1 (delayed s/c multipliers)
;;		8*clm*ZMM_SCD6 (7-complex 14-real sin/cos data)
;;		56*clm*2 (group multiplier fudge flags)
;;		56*clm*2 (biglit flags),				Total: 131.5*clm 64-byte cache lines
;; Total memory consumed (add for scratch area, prefetched data, carries):
;;		(896*3+131.5=2819.5)*clm + (98+224+896/8=434) cache lines
;;		clm=1,2,4:  204KB, 380KB, 732KB

zr4dwpn_pass1sc896 MACRO
	LOCAL	b0b, b1b, b2a, b2b, b2c, b8a, b8b, b9aa, b9a, b9b, c0b, c1b, c2a, c2b, c2c, c8b, c9aa, c9a, c9b
	LOCAL	pass1, pass2, endlp, done, yes_fft

	zfft_header pass1

;; Do pass 2 (the last 10 FFT levels if this is a 896K FFT)

pass2:	start_timer 0
	mov	rax, ZMM_PASS2_ROUTINE
	call	rax
	end_timer 0
	cmp	ffttype, 1		;; We're done if FFTing only
	je	done

;; Do the inverse FFT

	pass1_inverse_fft_setup c0b

;; Do inverse FFT levels 8,9,10
;; On input the 128-byte double cache lines hold these data values:
;;	0K	+1	+1	+1	+1	+1	+1	+1	1K	+1	...
;;	2K
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0K	+1	+1	+1	+1	+1	+1	+1	8K	+1	...
;;	1K	...
;;	...
;; The output cache lines are swizzled to hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	8K	+1K	...
;;	16K
;;	...

;; Do 1 sixteen_reals_unfft operations
;;	distance between fft data elements is 1K
;;	do 0.125 macros each processing 128 data values
;; Do 55 eight_complex_unfft operations
;;	distance between fft data elements is 1K
;;	do 6.875 macros each processing 128 data values

c0b:	start_timer 16
	mov	rbp, DATA_ADDR			;; Get source address
	mov	rsi, scratch_area		;; Get address of scratch area
	set_data_prefetch_ptrs			;; Init prefetching pointers
	mov	r9, PREMULT_ADDR		;; Variable data
	bump	r9, clm*64			;; Inverse column weights
	lea	rbx, [r9+clm*64]		;; Complex data sin/cos ptr
	lea	rdi, [rbx+clm*ZMM_SCD4]		;; Real data sin/cos pointer, will become delayed sin/cos multipliers
	lea	rdx, ZMM_TMPS			;; Computed clm*SCD8 sin/cos values
	mov	r8, pass2blkdst			;; blkdst
	lea	r10, [2*blkdstreg+blkdstreg]	;; 3*blkdst
	lea	r13, [rbp+4*blkdstreg]		;; Source+4*blkdst pointer
	lea	r12, [rbp+8*blkdstreg]		;; L1 prefetch pointer part 1
	bump2	r12, (clm MOD 2), 8*blkdstreg	;; L1 prefetch pointer part 2
	lea	r14, [r12+4*blkdstreg]		;; L1 prefetch pointer + 4*blkdst

	zr8_rsc_complex_and_real_preload	;; Preload constants before calling zloop_init
	zloop_init 7*8*clm			;; 7 iters of 8*clm iterations
	zloop_set_clm_prefetch ZDOUBLE		;; Prefetch 56 of 448 blocks of clm*128 bytes
	zr8_rsc_wpn_eight_complex_calc_sincos_simple r9, rbx, clm ;; Generate clm*SCD8 sin/cos values from delayed and complex multipliers
c8b:	zr8_rsc_wpn_sgreg_2sc_sixteen_reals_unfft8 rbp, 128, blkdstreg, blkdst3reg, r13, rsi, 1024, 128, 256, 512, rdx, ZMM_SCD8, r9, 64, rdi, ZMM_SCD4, clm, L1PREFETCH_ALL, r12, r14 ;; Prefetch short dist ahead
	zloop	clm, c8b, -clm*1024+clmblkdst, rbp, -clm*128+8*blkdstreg, r13, -clm*128+8*blkdstreg, rdx, -clm*ZMM_SCD8, r9, -clm*64, r12, -clm*128+8*blkdstreg, r14, -clm*128+8*blkdstreg ;; Loop, next src/dest ptrs, restore sin/cos ptr
	jmp	c9aa

	loops_reset
c9a:	zr8_rsc_wpn_eight_complex_calc_sincos r9, rdi, rbx, clm ;; Generate clm*SCD8 sin/cos values from delayed and complex multipliers
	bump	rdi, clm*ZMM_SCD1		;; Next set of delayed sin/cos multipliers
c9aa:
c9b:	zr8_rsc_wpn_sgreg_eight_complex_unfft8 rbp, 128, blkdstreg, blkdst3reg, r13, rsi, 1024, 128, 256, 512, rdx, ZMM_SCD8, clm, L1PREFETCH_ALL, r12, r14 ;; Prefetch short dist ahead
	zloop	clm, c9b, -clm*1024+clmblkdst, rbp, -clm*128+8*blkdstreg, r13, -clm*128+8*blkdstreg, rdx, -clm*ZMM_SCD8, r12, -clm*128+8*blkdstreg, r14, -clm*128+8*blkdstreg ;; Loop, next src/dest ptrs, restore sin/cos ptr
	zloop	7, c9b				;; Test loop counter
	zloop	8, c9a, -8*7*clmblkdst		;; Test loop counter, restore source pointer
	end_timer 16

;; Do inverse FFT levels 5,6,7
;; On input the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	8K	+1K	...
;;	16K
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	56K	+1K	...
;;	8K
;;	...
;;	48K
;;	112K
;;	...

;; Do 8 fourteen_reals_unfft operations
;; Do 56 seven_complex_unfft operations
;;	distance between fft data elements is 8K
;;	do 8 macros each processing 112 data values

	start_timer 18
	zloop_init  8*8*clm			;; 8 iterations of 8*clm
	zloop_set_clm_prefetch ZQUAD		;; Prefetch 128 of 448 blocks of clm*128 bytes
IFDEF OLDDD
	zr7_csc_fourteen_reals_unfft_preload
c2a:	zr7_csc_fourteen_reals_unfft rsi, 128, clmblkdst, rdi, ZMM_SCD6, 8*clm, L1PREFETCH_ALL, 128
	zloop	8*clm, c2a, -8*clm*128+7*clmblkdst, rdi, -8*clm*ZMM_SCD6 ;; Test loop counter, next source pointer, restore sin/cos pointer
	loops_reset				;; 7 more iters of 8*clm
	zr7_seven_complex_djbunfft_preload
c2b:	zr7_seven_complex_djbunfft rsi, 128, clmblkdst, rdi, ZMM_SCD6, 8*clm, L1PREFETCH_ALL, 128
	zloop	8*clm, c2b, -8*clm*128+7*clmblkdst, rdi, -8*clm*ZMM_SCD6 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	8, c2b, -8*7*clmblkdst, rdi, 8*clm*ZMM_SCD6 ;; Test loop counter, restore source pointer, next sin/cos pointer
ELSE
c2a:	zr7_csc_fourteen_reals_unfft_preload
	zr7_csc_fourteen_reals_unfft rsi, 7*clmblkdst, clmblkdst, rdi, 0, 1, L1PREFETCH_ALL, 7*clmblkdst
	zr7_seven_complex_djbunfft_preload
	jmp	c2c				;; 7 more iters
c2b:	zr7_seven_complex_djbunfft rsi, 7*clmblkdst, clmblkdst, rdi, 0, 1, L1PREFETCH_ALL, 7*clmblkdst
c2c:	zloop	8, c2b, -8*7*clmblkdst+128, rdi, ZMM_SCD6 ;; Test loop counter, next source pointer, next sin/cos pointer
	zloop	8*clm, c2a, -8*clm*128		;; Test loop counter, restore source pointer
ENDIF
	end_timer 18

;; Do inverse FFT levels 1,2,3,4
;; On input the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	56K	+1K	...
;;	8K
;;	...
;;	48K
;;	112K
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	448K	+1K	...
;;	8K
;;	...
;;	440K

;; Do 56 sixteen_reals_unfft operations
;;	distance between fft data elements is 56K
;;	do 7 macros each processing 128 data values

	start_timer 20
	mov	r15, sincos2			;; Load fixed premultiplier/sin/cos ptr
	sub	rbx, rbx			;; Clear register for loading fudge flags index
	mov	r12, compressed_fudges		;; Address of the compressed fudge flags
	mov	r13, r12			;; The XOR masks for fudge flags come from the compressed fudges
	zloop_init 7*8*clm			;; 7 iters of 8*clm
	zloop_set_clm_prefetch ZDOUBLE		;; Prefetch 56 of 448 blocks of clm*128 bytes
	zr8_wpn_sixteen_reals_last_unfft_preload
c1b:	zr8_wpn_sixteen_reals_last_unfft rsi, 128, 7*clmblkdst, 14*clmblkdst, 28*clmblkdst, r15, 0, rdi, 2, 8*clm, L1PREFETCH_ALL, 128
	zloop	8*clm, c1b, -8*clm*128+clmblkdst, r15, ZMM_SCD7, rdx, 16*64, r13, 16 ;; 8*clm iterations, next source, fixed s/c, grp multipliers pointer, next xor mask
	zloop	7, c1b				;; Test loop counter
	end_timer 20

;; Normalize these values

	pass1_normalize yes_fft			;; Normalize while prefetching 56 of 448 blocks of clm*128 bytes

;; Copy data from the scratch area back to the FFT data area

	copy_scratch_data_to_fft
	jmp	endlp				;; Skip FFT code if flag not set

;; Fire up auxiliary threads to do pass 1 forward FFT

pass1:	pass1_forward_fft_setup b0b

;; Copy data from the FFT data area back to the scratch area

b0b:	copy_fft_data_to_scratch

;; Do FFT levels 1,2,3,4
;; On input the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	448K	+1K	...
;;	8K	...
;;	...
;;	440K
;; On output the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	56K	+1K	...
;;	8K
;;	...
;;	48K
;;	112K
;;	...

;; Do 56 sixteen_reals_first_fft operations
;;	distance between fft data elements is 56K
;;	do 7 macros each processing 128 data values

yes_fft:start_timer 22
	mov	rsi, scratch_area		;; Get address of scratch area
	mov	r15, sincos2			;; Load fixed premultiplier/sin/cos ptr
	mov	rdi, PREMULT_ADDR		;; Calculate pointer to fudge flags
	bump	rdi, 8*clm*ZMM_SCD6+7*clm*ZMM_SCD1+clm*ZMM_SCD4+clm*ZMM_SCD4+clm*64+clm*64 ;; Addr of the group multiplier fudge flags
	mov	rdx, norm_grp_mults		;; Addr of the group multipliers
	sub	rbx, rbx			;; Clear register for loading fudge flags index
	mov	r12, compressed_fudges		;; Address of the compressed fudge flags
	mov	r13, r12			;; The XOR masks for fudge flags come from the compressed fudges
	set_data_prefetch_ptrs			;; Init prefetching pointers
pfing	mov	r8, pass2blkdst			;; blkdst
pfing	imul	rax, blkdstreg, (296-24)	;; We've already prefetched 296 of 448 blocks (24 too many)
pfing	add	rcx, rax
	zloop_init 7*8*clm			;; 7 iters of 8*clm
	zloop_set_clm_prefetch ZQUAD		;; Prefetch 112 of 448 blocks of clm*128 bytes
	zr8_wpn_sixteen_reals_first_fft_preload
b1b:	zr8_wpn_sixteen_reals_first_fft rsi, 128, 7*clmblkdst, 14*clmblkdst, 28*clmblkdst, r15, 0, rdi, 2, rdx, 0, 8*clm, L1PREFETCH_ALL, 128
	zloop	8*clm, b1b, -8*clm*128+clmblkdst, r15, ZMM_SCD7, rdx, 16*64, r13, 16 ;; 8*clm iters prefetching 8*clm cache lines, next source, fixed s/c, grp multipliers pointer, next xor mask
	zloop	7, b1b, -7*clmblkdst		;; Test loop counter, restore source pointer
	end_timer 22

;; Do FFT levels 5,6,7
;; On input the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	56K	+1K	...
;;	8K
;;	...
;;	48K
;;	112K
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	8K	+1K	...
;;	16K
;;	...

;; Do 8 fourteen_reals_fft operations
;; Do 56 seven_complex_fft operations
;;	distance between fft data elements is 8K
;;	do 8 macros each processing 112 data values

	start_timer 24
	bump	rdi, -56*clm*2-8*clm*ZMM_SCD6	;; Sin/cos pointer
	zloop_init  8*8*clm			;; 8 iterations of 8*clm
	zloop_set_clm_prefetch ZDOUBLE		;; Prefetch 64 of 448 blocks of clm*128 bytes
IFDEF OLDDD
	zr7_csc_fourteen_reals_fft_preload
b2a:	zr7_csc_fourteen_reals_fft rsi, 128, clmblkdst, rdi, ZMM_SCD6, 8*clm, L1PREFETCH_ALL, 128
	zloop	8*clm, b2a, -8*clm*128+7*clmblkdst, rdi, -8*clm*ZMM_SCD6 ;; Test loop counter, next source pointer, restore sin/cos pointer
	loops_reset				;; 7 more iters of 8*clm
	zr7_seven_complex_djbfft_preload
b2b:	zr7_seven_complex_djbfft rsi, 128, clmblkdst, rdi, ZMM_SCD6, 8*clm, L1PREFETCH_ALL, 128
	zloop	8*clm, b2b, -8*clm*128+7*clmblkdst, rdi, -8*clm*ZMM_SCD6 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	8, b2b, -8*7*clmblkdst		;; Test loop counter, restore source pointer
ELSE
b2a:	zr7_csc_fourteen_reals_fft_preload
	zr7_csc_fourteen_reals_fft rsi, 7*clmblkdst, clmblkdst, rdi, 0, 1, L1PREFETCH_ALL, 7*clmblkdst
	zr7_seven_complex_djbfft_preload
	jmp	b2c				;; 7 more iters
b2b:	zr7_seven_complex_djbfft rsi, 7*clmblkdst, clmblkdst, rdi, 0, 1, L1PREFETCH_ALL, 7*clmblkdst
b2c:	zloop	8, b2b, -8*7*clmblkdst+128, rdi, ZMM_SCD6 ;; Test loop counter, next source pointer, next sin/cos pointer
	zloop	8*clm, b2a, -8*clm*128		;; Test loop counter, restore source pointer
ENDIF
	end_timer 24

;; Do FFT levels 8,9,10
;; On input the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	8K	+1K	...
;;	16K
;;	...
;; The input is swizzled to hold these data values:
;;	0K	+1	+1	+1	+1	+1	+1	+1	8K	+1	...
;;	1K	...
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0K	+1	+1	+1	+1	+1	+1	+1	1K	+1	...
;;	2K
;;	...

;; Do 1 sixteen_reals_fft operations
;;	distance between fft data elements is 1K
;;	do 0.125 macros each processing 128 data values
;; Do 55 eight_complex_fft operations
;;	distance between fft data elements is 1K
;;	do 6.875 macros each processing 128 data values

	start_timer 26
IFDEF OLDDD
	bump	rdi, -7*clm*ZMM_SCD1-clm*ZMM_SCD4 ;; Real data sin/cos pointer, will become delayed sin/cos multipliers
ELSE
	bump	rdi, -8*clm*ZMM_SCD6-7*clm*ZMM_SCD1-clm*ZMM_SCD4 ;; Real data sin/cos pointer, will become delayed sin/cos multipliers
ENDIF
	lea	rbx, [rdi-clm*ZMM_SCD4]		;; Complex data sin/cos ptr
	lea	r9, [rbx-clm*64-clm*64]		;; Column weights
	lea	rdx, ZMM_TMPS			;; Computed clm*SCD4 sin/cos values
	mov	rbp, DATA_ADDR			;; Destination pointer
	mov	r8, pass2blkdst			;; blkdst
	lea	r10, [2*blkdstreg+blkdstreg]	;; 3*blkdst
	lea	r13, [rbp+4*blkdstreg]		;; Dest+4*blkdst pointer
pfing	mov	rcx, PREMULT_PREFETCH		;; Load premult/sin/cos prefetch pointer

	zr8_rsc_complex_and_real_preload	;; Preload constants before calling zloop_init
	zloop_init 7*8*clm			;; 7 iters of 8*clm
	zloop_set_prefetch ZTRIPLE		;; Prefetch 168*clm times
	zloop_set_alternate_prefetch_increment 131*2+1, 168*2  ;; Prefetch approximately 131.5*clm cache lines in those 168*clm prefetches
	zr8_rsc_wpn_eight_complex_calc_sincos_simple r9, rbx, clm ;; Generate clm*SCD8 sin/cos values from delayed and complex multipliers
b8b:	zr8_rsc_wpn_sgreg_2sc_sixteen_reals_fft8 rsi, 1024, 128, 256, 512, rbp, 128, blkdstreg, blkdst3reg, r13, rdx, ZMM_SCD8, r9, 64, rdi, ZMM_SCD4, clm, L1PREFETCH_DEST_ALL
	zloop	clm, b8b, -clm*1024+clmblkdst, rbp, -clm*128+8*blkdstreg, r13, -clm*128+8*blkdstreg, rdx, -clm*ZMM_SCD8, r9, -clm*64 ;; Loop, next source/dest pointers, restore premult/sine/cosine pointer
	jmp	b9aa

	loops_reset
b9a:	zr8_rsc_wpn_eight_complex_calc_sincos r9, rdi, rbx, clm ;; Generate clm*SCD8 sin/cos values from delayed and complex multipliers
	bump	rdi, clm*ZMM_SCD1		;; Next set of delayed multipliers
b9aa:
b9b:	zr8_rsc_wpn_sgreg_eight_complex_fft8 rsi, 1024, 128, 256, 512, rbp, 128, blkdstreg, blkdst3reg, r13, rdx, ZMM_SCD8, clm, L1PREFETCH_DEST_ALL
	zloop	clm, b9b, -clm*1024+clmblkdst, rbp, -clm*128+8*blkdstreg, r13, -clm*128+8*blkdstreg, rdx, -clm*ZMM_SCD8 ;; Loop, next source/dest pointers, restore premult/sine/cosine pointer
	zloop	7, b9b				;; Test loop counter
	zloop	8, b9a				;; Test loop counter
	end_timer 26

;; Work on next 8*clm sets of 896 values

endlp:	zpass1_get_next_block pass2, c0b, b0b ;; Get next block and jump
done:	zfft_footer
	ENDM


;; Perform a shared pass 1 negacyclic FFT of length 896 doubles.
;; Memory examples are for a 896K FFT with 10 levels done in pass 2.
;; FFT data needed:
;;		8*clm*896 doubles					Total: 896*clm 64-byte cache lines
;; Fixed s/c data needed:
;;		7*ZMM_CD8SCD4 (complex premultipliers and fixed s/c data), Total: 112 64-byte cache lines
;; Group multipliers needed:
;;		7*16*64 (weights)
;;		7*16*64 (inverse weights),				Total: 224 64-byte cache lines
;; Variable data needed:
;;		clm*64	(column weights)
;;		clm*64	(column inverse weights)
;;		clm*ZMM_SCD4 (complex reduced s/c multipliers)
;;		8*clm*ZMM_SCD1 (delayed s/c multipliers)
;;		8*clm*ZMM_SCD3 (7-complex sin/cos data)
;;		56*clm*2 (group multiplier fudge flags)
;;		56*clm*2 (biglit flags),				Total: 77.5*clm 64-byte cache lines
;; Total memory consumed (add for scratch area, prefetched data, carries):
;;		(896*3+77.5=2765.5)*clm + (112+224+896/8=448) cache lines
;;		clm=1,2,4:  201KB, 374KB, 720KB

zr4dwpn_pass1sc896ac MACRO
	LOCAL	b0b, b1b, b2b, b9a, b9b, c0b, c1b, c2b, c9a, c9b
	LOCAL	pass1, pass2, endlp, done, yes_fft

	zfft_header pass1

;; Do pass 2 (the last 10 FFT levels if this is a 896K FFT)

pass2:	start_timer 0
	mov	rax, ZMM_PASS2_ROUTINE
	call	rax
	end_timer 0
	cmp	ffttype, 1		;; We're done if FFTing only
	je	done

;; Do the inverse FFT

	pass1_inverse_fft_setup c0b

;; Do inverse FFT levels 8.9.10
;; On input the 128-byte double cache lines hold these data values:
;;	0K	+1	+1	+1	+1	+1	+1	+1	1K	+1	...
;;	2K
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0K	+1	+1	+1	+1	+1	+1	+1	8K	+1	...
;;	1K	...
;;	...
;; The output cache lines are swizzled to hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	8K	+1K	...
;;	16K
;;	...

;; Do 56 eight_complex_unfft operations
;;	distance between fft data elements is 1K
;;	do 7 macros each processing 128 data values

c0b:	start_timer 16
	mov	rbp, DATA_ADDR			;; Get source address
	mov	rsi, scratch_area		;; Get address of scratch area
	set_data_prefetch_ptrs			;; Init prefetching pointers
	mov	r9, PREMULT_ADDR		;; Variable data
	bump	r9, clm*64			;; Inverse column weights
	lea	rbx, [r9+clm*64]		;; Complex data sin/cos ptr
	lea	rdi, [rbx+clm*ZMM_SCD4]		;; Delayed sin/cos multipliers
	lea	rdx, ZMM_TMPS			;; Computed clm*SCD8 sin/cos values
	mov	r8, pass2blkdst			;; blkdst
	lea	r10, [2*blkdstreg+blkdstreg]	;; 3*blkdst
	lea	r13, [rbp+4*blkdstreg]		;; Source+4*blkdst pointer
	lea	r12, [rbp+8*blkdstreg]		;; L1 prefetch pointer part 1
	bump2	r12, (clm MOD 2), 8*blkdstreg	;; L1 prefetch pointer part 2
	lea	r14, [r12+4*blkdstreg]		;; L1 prefetch pointer + 4*blkdst

	zr8_rsc_complex_only_preload		;; Preload constants before calling zloop_init
	zloop_init 7*8*clm			;; 7 iters of 8*clm iterations
	zloop_set_clm_prefetch ZDOUBLE		;; Prefetch 56 of 448 blocks of clm*128 bytes
c9a:	zr8_rsc_wpn_eight_complex_calc_sincos r9, rdi, rbx, clm ;; Generate clm*SCD8 sin/cos values from weights, delayed, and complex multipliers
	bump	rdi, clm*ZMM_SCD1		;; Next set of delayed sin/cos multipliers
c9b:	zr8_rsc_wpn_sgreg_eight_complex_unfft8 rbp, 128, blkdstreg, blkdst3reg, r13, rsi, 1024, 128, 256, 512, rdx, ZMM_SCD8, clm, L1PREFETCH_ALL, r12, r14 ;; Prefetch short dist ahead
	zloop	clm, c9b, -clm*1024+clmblkdst, rbp, -clm*128+8*blkdstreg, r13, -clm*128+8*blkdstreg, rdx, -clm*ZMM_SCD8, r12, -clm*128+8*blkdstreg, r14, -clm*128+8*blkdstreg ;; Loop, next src/dest ptrs, restore sin/cos ptr
	zloop	7, c9b				;; Test loop counter
	zloop	8, c9a, -8*7*clmblkdst		;; Test loop counter, restore source pointer
	end_timer 16

;; Do inverse FFT levels 5,6,7
;; On input the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	8K	+1K	...
;;	16K
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	56K	+1K	...
;;	8K
;;	...
;;	48K
;;	112K
;;	...

;; Do 64 seven_complex_unfft operations
;;	distance between fft data elements is 8K
;;	do 8 macros each processing 112 data values

	start_timer 18
	zloop_init  8*8*clm			;; 8 iterations of 8*clm
	zloop_set_clm_prefetch ZQUAD		;; Prefetch 128 of 448 blocks of clm*128 bytes
	zr7_seven_complex_djbunfft_preload
IFDEF OLDDD
c2b:	zr7_seven_complex_djbunfft rsi, 128, clmblkdst, rdi, ZMM_SCD3, 8*clm, L1PREFETCH_ALL, 128
	zloop	8*clm, c2b, -8*clm*128+7*clmblkdst, rdi, -8*clm*ZMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	8, c2b, -8*7*clmblkdst, rdi, 8*clm*ZMM_SCD3 ;; Test loop counter, restore source pointer, next sin/cos pointer
ELSE
c2b:	zr7_seven_complex_djbunfft rsi, 7*clmblkdst, clmblkdst, rdi, 0, 8, L1PREFETCH_ALL, 7*clmblkdst
	zloop	8, c2b, -8*7*clmblkdst+128, rdi, ZMM_SCD3 ;; Test loop counter, next source pointer, next sin/cos pointer
	zloop	8*clm, c2b, -8*clm*128		;; Test loop counter, restore source pointer
ENDIF
	end_timer 18

;; Do inverse FFT levels 1,2,3,4
;; On input the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	56K	+1K	...
;;	8K
;;	...
;;	48K
;;	112K
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	448K	+1K	...
;;	8K	...
;;	...
;;	440K

;; Do 56 eight_complex_unfft operations
;;	distance between fft data elements is 56K
;;	do 7 macros each processing 128 data values

	start_timer 20
	mov	r15, sincos2			;; Load fixed premultiplier/sin/cos ptr
	sub	rbx, rbx			;; Clear register for loading fudge flags index
	mov	r12, compressed_fudges		;; Address of the compressed fudge flags
	mov	r13, r12			;; The XOR masks for fudge flags come from the compressed fudges
	zloop_init 7*8*clm			;; 7 iters of 8*clm
	zloop_set_clm_prefetch ZDOUBLE		;; Prefetch 56 of 448 blocks of clm*128 bytes
	zr8_csc_wpn_eight_complex_last_djbunfft_preload
c1b:	zr8_csc_wpn_eight_complex_last_djbunfft rsi, 128, 7*clmblkdst, 14*clmblkdst, 28*clmblkdst, r15, 0, rdi, 2, 8*clm, L1PREFETCH_ALL, 128
	zloop	8*clm, c1b, -8*clm*128+clmblkdst, r15, ZMM_CD8SCD4, rdx, 16*64, r13, 16 ;; 8*clm iterations, next fixed s/c, next grp multipliers, next xor mask
	zloop	7, c1b				;; 7 iterations
	end_timer 20

;; Normalize these values

	pass1_normalize yes_fft			;; Normalize while prefetching 56 of 448 blocks of clm*128 bytes

;; Copy data from the scratch area back to the FFT data area

	copy_scratch_data_to_fft
	jmp	endlp				;; Skip FFT code if flag not set

;; Fire up auxiliary threads to do pass 1 forward FFT

pass1:	pass1_forward_fft_setup b0b

;; Copy data from the FFT data area back to the scratch area

b0b:	copy_fft_data_to_scratch

;; Do FFT levels 1,2,3,4
;; On input the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	448K	+1K	...
;;	8K	...
;;	...
;;	440K
;; On output the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	56K	+1K	...
;;	8K
;;	...
;;	48K
;;	112K
;;	...

;; Do 56 eight_complex_first_fft operations
;;	distance between fft data elements is 56K
;;	do 7 macros each processing 128 data values

yes_fft:start_timer 22
	mov	rsi, scratch_area		;; Get address of scratch area
	mov	r15, sincos2			;; Load fixed premultiplier/sin/cos ptr
	mov	rdi, PREMULT_ADDR		;; Calculate pointer to fudge flags
	bump	rdi, 8*clm*ZMM_SCD3+8*clm*ZMM_SCD1+clm*ZMM_SCD4+clm*64+clm*64 ;; Addr of the group multiplier fudge flags
	mov	rdx, norm_grp_mults		;; Addr of the group multipliers
	sub	rbx, rbx			;; Clear register for loading fudge flags index
	mov	r12, compressed_fudges		;; Address of the compressed fudge flags
	mov	r13, r12			;; The XOR masks for fudge flags come from the compressed fudges
	set_data_prefetch_ptrs			;; Init prefetching pointers
pfing	mov	r8, pass2blkdst			;; blkdst
pfing	imul	rax, blkdstreg, (296-24)	;; We've already prefetched 296 of 448 blocks (24 too many)
pfing	add	rcx, rax
	zloop_init 7*8*clm			;; 7 iters of 8*clm
	zloop_set_clm_prefetch ZQUAD		;; Prefetch 112 of 448 blocks of clm*128 bytes
	zr8_csc_wpn_eight_complex_first_djbfft_preload
b1b:	zr8_csc_wpn_eight_complex_first_djbfft rsi, 128, 7*clmblkdst, 14*clmblkdst, 28*clmblkdst, r15, 0, rdi, 2, rdx, 0, 8*clm, L1PREFETCH_ALL, 128
	zloop	8*clm, b1b, -8*clm*128+clmblkdst, r15, ZMM_CD8SCD4, rdx, 16*64, r13, 16 ;; 8*clm iters, next source, fixed s/c, grp multipliers pointer, next xor mask
	zloop	7, b1b, -7*clmblkdst		;; Test loop counter, restore source pointer
	end_timer 22

;; Do FFT levels 5,6,7
;; On input the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	56K	+1K	...
;;	8K
;;	...
;;	48K
;;	112K
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	8K	+1K	...
;;	16K
;;	...

;; Do 64 seven_complex_fft operations
;;	distance between fft data elements is 8K
;;	do 8 macros each processing 112 data values

	start_timer 24
	bump	rdi, -56*clm*2-8*clm*ZMM_SCD3	;; Sin/cos pointer
	zloop_init  8*8*clm			;; 8 iterations of 8*clm
	zloop_set_clm_prefetch ZDOUBLE		;; Prefetch 64 of 448 blocks of clm*128 bytes
	zr7_seven_complex_djbfft_preload
IFDEF OLDDD
b2b:	zr7_seven_complex_djbfft rsi, 128, clmblkdst, rdi, ZMM_SCD3, 8*clm, L1PREFETCH_ALL, 128
	zloop	8*clm, b2b, -8*clm*128+7*clmblkdst, rdi, -8*clm*ZMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	8, b2b, -8*7*clmblkdst		;; Test loop counter, restore source pointer
ELSE
b2b:	zr7_seven_complex_djbfft rsi, 7*clmblkdst, clmblkdst, rdi, 0, 8, L1PREFETCH_ALL, 7*clmblkdst
	zloop	8, b2b, -8*7*clmblkdst+128, rdi, ZMM_SCD3 ;; Test loop counter, next source pointer, next sin/cos pointer
	zloop	8*clm, b2b, -8*clm*128		;; Test loop counter, restore source pointer
ENDIF
	end_timer 24

;; Do FFT levels 8,9,10
;; On input the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	8K	+1K	...
;;	16K
;;	...
;; The input is swizzled to hold these data values:
;;	0K	+1	+1	+1	+1	+1	+1	+1	8K	+1	...
;;	1K	...
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0K	+1	+1	+1	+1	+1	+1	+1	1K	+1	...
;;	2K
;;	...

;; Do 56 eight_complex_fft operations
;;	distance between fft data elements is 1K
;;	do 7 macros each processing 128 data values

	start_timer 26
	mov	rbp, DATA_ADDR			;; Destination pointer
IFDEF OLDDD
	bump	rdi, -8*clm*ZMM_SCD1		;; Delayed sin/cos multipliers
ELSE
	bump	rdi, -8*clm*ZMM_SCD3-8*clm*ZMM_SCD1 ;; Delayed sin/cos multipliers
ENDIF
	lea	rbx, [rdi-clm*ZMM_SCD4]		;; Complex data sin/cos ptr
	lea	r9, [rbx-clm*64-clm*64]		;; Column weights
	lea	rdx, ZMM_TMPS			;; Computed clm*SCD4 sin/cos values
	mov	r8, pass2blkdst			;; blkdst
	lea	r10, [2*blkdstreg+blkdstreg]	;; 3*blkdst
	lea	r13, [rbp+4*blkdstreg]		;; Dest+4*blkdst pointer
pfing	mov	rcx, PREMULT_PREFETCH		;; Load premult/sin/cos prefetch pointer

	zr8_rsc_complex_only_preload		;; Preload constants before calling zloop_init
	zloop_init 7*8*clm			;; 7 iters of 8*clm
	zloop_set_prefetch ZDOUBLE		;; Prefetch 112*clm cache lines
	zloop_set_alternate_prefetch_increment 77*2+1, 112*2  ;; Prefetch approximately 77.5*clm cache lines in those 112*clm prefetches
b9a:	zr8_rsc_wpn_eight_complex_calc_sincos r9, rdi, rbx, clm ;; Generate clm*SCD8 sin/cos values from weights, delayed and complex multipliers
	bump	rdi, clm*ZMM_SCD1		;; Next set of delayed multipliers
b9b:	zr8_rsc_wpn_sgreg_eight_complex_fft8 rsi, 1024, 128, 256, 512, rbp, 128, blkdstreg, blkdst3reg, r13, rdx, ZMM_SCD8, clm, L1PREFETCH_DEST_ALL
	zloop	clm, b9b, -clm*1024+clmblkdst, rbp, -clm*128+8*blkdstreg, r13, -clm*128+8*blkdstreg, rdx, -clm*ZMM_SCD8 ;; Loop, next source/dest pointers, restore premult/sine/cosine pointer
	zloop	7, b9b				;; Test loop counter
	zloop	8, b9a				;; Test loop counter
	end_timer 26

;; Work on next 8*clm sets of 896 values

endlp:	zpass1_get_next_block pass2, c0b, b0b ;; Get next block and jump
done:	zfft_footer
	ENDM



;; Perform a shared pass 1 FFT of length 960 doubles.
;; Memory examples are for a 960K FFT with 10 levels done in pass 2.
;; FFT data needed:
;;		8*clm*960 doubles					Total: 960*clm 64-byte cache lines
;; Fixed s/c data needed:
;;		5*ZMM_SCD11 (fixed s/c data),				Total: 132 64-byte cache lines
;; Group multipliers needed:
;;		5*24*64 (weights)
;;		5*24*64 (inverse weights),				Total: 240 64-byte cache lines
;; Variable data needed:
;;		clm*64	(column weights)
;;		clm*64	(column inverse weights)
;;		clm*ZMM_SCD4 (complex reduced s/c multipliers)
;;		clm*ZMM_SCD4 (real s/c multipliers)
;;		11*clm*ZMM_SCD1 (delayed s/c multipliers)
;;		8*clm*ZMM_SCD4 (5-complex 10-real sin/cos data)
;;		40*clm*3 (group multiplier fudge flags)
;;		40*clm*3 (biglit flags),				Total: 107.75*clm 64-byte cache lines
;; Total memory consumed (add for scratch area, prefetched data, carries):
;;		(960*3+107.75=2987.75)*clm + (132+240+960/8=492) cache lines
;;		clm=1,2,4:  218KB, 405KB, 778KB

zr4dwpn_pass1sc960 MACRO
	LOCAL	b0b, b1b, b2a, b2b, b2c, b8a, b8b, b9aa, b9a, b9b, c0b, c1b, c2a, c2b, c2c, c8b, c9aa, c9a, c9b
	LOCAL	pass1, pass2, endlp, done, yes_fft

	zfft_header pass1

;; Do pass 2 (the last 10 FFT levels if this is a 960K FFT)

pass2:	start_timer 0
	mov	rax, ZMM_PASS2_ROUTINE
	call	rax
	end_timer 0
	cmp	ffttype, 1		;; We're done if FFTing only
	je	done

;; Do the inverse FFT

	pass1_inverse_fft_setup c0b

;; Do inverse FFT levels 8,9,10
;; On input the 128-byte double cache lines hold these data values:
;;	0K	+1	+1	+1	+1	+1	+1	+1	1K	+1	...
;;	2K
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0K	+1	+1	+1	+1	+1	+1	+1	8K	+1	...
;;	1K	...
;;	...
;; The output cache lines are swizzled to hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	8K	+1K	...
;;	16K
;;	...

;; Do 1 sixteen_reals_unfft operations
;;	distance between fft data elements is 1K
;;	do 0.125 macros each processing 128 data values
;; Do 59 eight_complex_unfft operations
;;	distance between fft data elements is 1K
;;	do 7.375 macros each processing 128 data values

c0b:	start_timer 16
	mov	rbp, DATA_ADDR			;; Get source address
	mov	rsi, scratch_area		;; Get address of scratch area
	set_data_prefetch_ptrs			;; Init prefetching pointers
	mov	r9, PREMULT_ADDR		;; Variable data
	bump	r9, clm*64			;; Inverse column weights
	lea	rbx, [r9+clm*64]		;; Complex data sin/cos ptr
	lea	rdi, [rbx+clm*ZMM_SCD4]		;; Real data sin/cos pointer, will become delayed sin/cos multipliers
	lea	rdx, ZMM_TMPS			;; Computed clm*SCD8 sin/cos values
	mov	r8, pass2blkdst			;; blkdst
	lea	r10, [2*blkdstreg+blkdstreg]	;; 3*blkdst
	lea	r13, [rbp+4*blkdstreg]		;; Source+4*blkdst pointer
	lea	r12, [rbp+8*blkdstreg]		;; L1 prefetch pointer part 1
	bump2	r12, (clm MOD 2), 8*blkdstreg	;; L1 prefetch pointer part 2
	lea	r14, [r12+4*blkdstreg]		;; L1 prefetch pointer + 4*blkdst

	zr8_rsc_complex_and_real_preload	;; Preload constants before calling zloop_init
;;	zloop_init 12*5*clm			;; 12 iters of 5*clm iterations
	zloop_init 12*5*clm  *2			;; 12 iters of 5*clm iterations	(work around loops "add al, 1" bug)
	zr8_rsc_wpn_eight_complex_calc_sincos_simple r9, rbx, clm ;; Generate clm*SCD8 sin/cos values from delayed and complex multipliers
	zloop_set_clm_prefetch ZQUAD		;; Prefetch 120 of 480 blocks of clm*128 bytes
c8b:	zr8_rsc_wpn_sgreg_2sc_sixteen_reals_unfft8 rbp, 128, blkdstreg, blkdst3reg, r13, rsi, 1024, 128, 256, 512, rdx, ZMM_SCD8, r9, 64, rdi, ZMM_SCD4, clm, L1PREFETCH_ALL, r12, r14 ;; Prefetch short dist ahead
	zloop	clm, c8b, -clm*1024+clmblkdst, rbp, -clm*128+8*blkdstreg, r13, -clm*128+8*blkdstreg, rdx, -clm*ZMM_SCD8, r9, -clm*64, r12, -clm*128+8*blkdstreg, r14, -clm*128+8*blkdstreg ;; Loop, next src/dest ptrs, restore sin/cos ptr
	jmp	c9aa

	loops_reset
c9a:	zr8_rsc_wpn_eight_complex_calc_sincos r9, rdi, rbx, clm ;; Generate clm*SCD8 sin/cos values from delayed and complex multipliers
	bump	rdi, clm*ZMM_SCD1		;; Next set of delayed sin/cos multipliers
c9aa:	zloop_set_clm_prefetch ZQUAD		;; Prefetch 120 of 480 blocks of clm*128 bytes
c9b:	zr8_rsc_wpn_sgreg_eight_complex_unfft8 rbp, 128, blkdstreg, blkdst3reg, r13, rsi, 1024, 128, 256, 512, rdx, ZMM_SCD8, clm, L1PREFETCH_ALL, r12, r14 ;; Prefetch short dist ahead
	zloop	clm, c9b, -clm*1024+clmblkdst, rbp, -clm*128+8*blkdstreg, r13, -clm*128+8*blkdstreg, rdx, -clm*ZMM_SCD8, r12, -clm*128+8*blkdstreg, r14, -clm*128+8*blkdstreg ;; Loop, next src/dest ptrs, restore sin/cos ptr
	zloop	5, c9b				;; Test loop counter
	zloop	12, c9a, -12*5*clmblkdst	;; Test loop counter, restore source pointer
	end_timer 16

;; Do inverse FFT levels 5,6,7
;; On input the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	8K	+1K	...
;;	16K
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	40K	+1K	...
;;	8K
;;	...
;;	32K
;;	80K
;;	...

;; Do 8 ten_reals_unfft operations
;; Do 88 five_complex_unfft operations
;;	distance between fft data elements is 8K
;;	do 12 macros each processing 80 data values

	start_timer 18
	zloop_init  12*8*clm			;; 12 iterations of 8*clm
	zloop_set_clm_prefetch ZDOUBLE		;; Prefetch 96 of 480 blocks of clm*128 bytes
IFDEF OLDDD
	zr5_csc_ten_reals_unfft_preload
c2a:	zr5_csc_ten_reals_unfft rsi, 128, clmblkdst, rdi, ZMM_SCD4, 8*clm, L1PREFETCH_ALL, 128
	zloop	8*clm, c2a, -8*clm*128+5*clmblkdst, rdi, -8*clm*ZMM_SCD4 ;; Test loop counter, next source pointer, restore sin/cos pointer
	loops_reset				;; 11 more iters of 8*clm
	zr5_five_complex_djbunfft_preload
c2b:	zr5_five_complex_djbunfft rsi, 128, clmblkdst, rdi, ZMM_SCD4, 8*clm, L1PREFETCH_ALL, 128
	zloop	8*clm, c2b, -8*clm*128+5*clmblkdst, rdi, -8*clm*ZMM_SCD4 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	12, c2b, -12*5*clmblkdst, rdi, 8*clm*ZMM_SCD4 ;; Test loop counter, restore source pointer, next sin/cos pointer
ELSE
c2a:	zr5_csc_ten_reals_unfft_preload
	zr5_csc_ten_reals_unfft rsi, 5*clmblkdst, clmblkdst, rdi, 0, 1, L1PREFETCH_ALL, 5*clmblkdst
	zr5_five_complex_djbunfft_preload
	jmp	c2c				;; 11 more iters
c2b:	zr5_five_complex_djbunfft rsi, 5*clmblkdst, clmblkdst, rdi, 0, 1, L1PREFETCH_ALL, 5*clmblkdst
c2c:	zloop	12, c2b, -12*5*clmblkdst+128, rdi, ZMM_SCD4 ;; Test loop counter, next source pointer, next sin/cos pointer
	zloop	8*clm, c2a, -8*clm*128		;; Test loop counter, restore source pointer
ENDIF
	end_timer 18

;; Do inverse FFT levels 1,2,3,4
;; On input the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	40K	+1K	...
;;	8K
;;	...
;;	32K
;;	80K
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	480K	+1K	...
;;	8K
;;	...
;;	472K

;; Do 40 twentyfour_reals_unfft operations
;;	distance between fft data elements is 40K
;;	do 5 macros each processing 192 data values

	start_timer 20
	mov	r15, sincos2			;; Load fixed premultiplier/sin/cos ptr
	sub	rbx, rbx			;; Clear register for loading fudge flags index
	mov	r12, compressed_fudges		;; Address of the compressed fudge flags
	mov	r13, r12			;; The XOR masks for fudge flags come from the compressed fudges
	zloop_init 5*8*clm			;; 5 iters of 8*clm
	zloop_set_clm_prefetch ZDOUBLE		;; Prefetch 40 of 480 blocks of clm*128 bytes
	zr12_wpn_twentyfour_reals_last_unfft_preload
c1b:	zr12_wpn_twentyfour_reals_last_unfft rsi, 128, 5*clmblkdst, 10*clmblkdst, 20*clmblkdst, r15, 0, rdi, 3, 8*clm, L1PREFETCH_ALL, 128
	zloop	8*clm, c1b, -8*clm*128+clmblkdst, r15, ZMM_SCD11, rdx, 24*64, r13, 24 ;; 8*clm iterations, next grp multipliers, next XOR masks
	zloop	5, c1b				;; Test loop counter
	end_timer 20

;; Normalize these values

	pass1_normalize yes_fft			;; Normalize while prefetching 60 of 480 blocks of clm*128 bytes

;; Copy data from the scratch area back to the FFT data area

	copy_scratch_data_to_fft
	jmp	endlp				;; Skip FFT code if flag not set

;; Fire up auxiliary threads to do pass 1 forward FFT

pass1:	pass1_forward_fft_setup b0b

;; Copy data from the FFT data area back to the scratch area

b0b:	copy_fft_data_to_scratch

;; Do FFT levels 1,2,3,4
;; On input the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	480K	+1K	...
;;	8K	...
;;	...
;;	472K
;; On output the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	40K	+1K	...
;;	8K
;;	...
;;	32K
;;	80K
;;	...

;; Do 40 twentyfour_reals_first_fft operations
;;	distance between fft data elements is 40K
;;	do 5 macros each processing 192 data values

yes_fft:start_timer 22
	mov	rsi, scratch_area		;; Get address of scratch area
	mov	r15, sincos2			;; Load fixed premultiplier/sin/cos ptr
	mov	rdi, PREMULT_ADDR		;; Calculate pointer to fudge flags
	bump	rdi, 8*clm*ZMM_SCD4+11*clm*ZMM_SCD1+clm*ZMM_SCD4+clm*ZMM_SCD4+clm*64+clm*64 ;; Addr of the group multiplier fudge flags
	mov	rdx, norm_grp_mults		;; Addr of the group multipliers
	sub	rbx, rbx			;; Clear register for loading fudge flags index
	mov	r12, compressed_fudges		;; Address of the compressed fudge flags
	mov	r13, r12			;; The XOR masks for fudge flags come from the compressed fudges
	set_data_prefetch_ptrs			;; Init prefetching pointers
pfing	mov	r8, pass2blkdst			;; blkdst
pfing	imul	rax, blkdstreg, (316-12)	;; We've already prefetched 316 of 480 blocks (12 too many)
pfing	add	rcx, rax
	zloop_init 5*8*clm			;; 5 iters of 8*clm
	zloop_set_clm_prefetch ZQUAD		;; Prefetch 80 of 480 blocks of clm*128 bytes
	zr12_wpn_twentyfour_reals_first_fft_preload
b1b:	zr12_wpn_twentyfour_reals_first_fft rsi, 128, 5*clmblkdst, 10*clmblkdst, 20*clmblkdst, r15, 0, rdi, 3, rdx, 0, 8*clm, L1PREFETCH_ALL, 128
	zloop	8*clm, b1b, -8*clm*128+clmblkdst, r15, ZMM_SCD11, rdx, 24*64, r13, 24 ;; 8*clm iters prefetching 8*clm cache lines, next source, fixed s/c, grp multipliers pointer, next XOR masks
	zloop	5, b1b, -5*clmblkdst		;; Test loop counter, restore source pointer
	end_timer 22

;; Do FFT levels 5,6,7
;; On input the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	40K	+1K	...
;;	8K
;;	...
;;	32K
;;	80K
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	8K	+1K	...
;;	16K
;;	...

;; Do 8 ten_reals_fft operations
;; Do 88 five_complex_fft operations
;;	distance between fft data elements is 8K
;;	do 12 macros each processing 80 data values

	start_timer 24
	bump	rdi, -40*clm*3-8*clm*ZMM_SCD4	;; Sin/cos pointer
	zloop_init  12*8*clm			;; 8 iterations of 8*clm
	zloop_set_clm_prefetch ZDOUBLE		;; Prefetch 96 of 480 blocks of clm*128 bytes
IFDEF OLDDD
	zr5_csc_ten_reals_fft_preload
b2a:	zr5_csc_ten_reals_fft rsi, 128, clmblkdst, rdi, ZMM_SCD4, 8*clm, L1PREFETCH_ALL, 128
	zloop	8*clm, b2a, -8*clm*128+5*clmblkdst, rdi, -8*clm*ZMM_SCD4 ;; Test loop counter, next source pointer, restore sin/cos pointer
	loops_reset				;; 11 more iters of 8*clm
	zr5_five_complex_djbfft_preload
b2b:	zr5_five_complex_djbfft rsi, 128, clmblkdst, rdi, ZMM_SCD4, 8*clm, L1PREFETCH_ALL, 128
	zloop	8*clm, b2b, -8*clm*128+5*clmblkdst, rdi, -8*clm*ZMM_SCD4 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	12, b2b, -12*5*clmblkdst	;; Test loop counter, restore source pointer
ELSE
b2a:	zr5_csc_ten_reals_fft_preload
	zr5_csc_ten_reals_fft rsi, 5*clmblkdst, clmblkdst, rdi, 0, 1, L1PREFETCH_ALL, 5*clmblkdst
	zr5_five_complex_djbfft_preload
	jmp	b2c				;; 11 more iters
b2b:	zr5_five_complex_djbfft rsi, 5*clmblkdst, clmblkdst, rdi, 0, 1, L1PREFETCH_ALL, 5*clmblkdst
b2c:	zloop	12, b2b, -12*5*clmblkdst+128, rdi, ZMM_SCD4 ;; Test loop counter, next source pointer, next sin/cos pointer
	zloop	8*clm, b2a, -8*clm*128		;; Test loop counter, restore source pointer
ENDIF
	end_timer 24

;; Do FFT levels 8,9,10
;; On input the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	8K	+1K	...
;;	16K
;;	...
;; The input is swizzled to hold these data values:
;;	0K	+1	+1	+1	+1	+1	+1	+1	8K	+1	...
;;	1K	...
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0K	+1	+1	+1	+1	+1	+1	+1	1K	+1	...
;;	2K
;;	...

;; Do 1 sixteen_reals_fft operations
;;	distance between fft data elements is 1K
;;	do 0.125 macros each processing 128 data values
;; Do 59 eight_complex_fft operations
;;	distance between fft data elements is 1K
;;	do 7.375 macros each processing 128 data values

	start_timer 26
IFDEF OLDDD
	bump	rdi, -11*clm*ZMM_SCD1-clm*ZMM_SCD4 ;; Real data sin/cos pointer, will become delayed sin/cos multipliers
ELSE
	bump	rdi, -8*clm*ZMM_SCD4-11*clm*ZMM_SCD1-clm*ZMM_SCD4 ;; Real data sin/cos pointer, will become delayed sin/cos multipliers
ENDIF
	lea	rbx, [rdi-clm*ZMM_SCD4]		;; Complex data sin/cos ptr
	lea	r9, [rbx-clm*64-clm*64]		;; Column weights
	lea	rdx, ZMM_TMPS			;; Computed clm*SCD4 sin/cos values
	mov	rbp, DATA_ADDR			;; Destination pointer
	mov	r8, pass2blkdst			;; blkdst
	lea	r10, [2*blkdstreg+blkdstreg]	;; 3*blkdst
	lea	r13, [rbp+4*blkdstreg]		;; Dest+4*blkdst pointer
pfing	mov	rcx, PREMULT_PREFETCH		;; Load premult/sin/cos prefetch pointer

	zr8_rsc_complex_and_real_preload	;; Preload constants before calling zloop_init
;;	zloop_init 12*5*clm			;; 12 iters of 5*clm
	zloop_init 12*5*clm  *2			;; 12 iters of 5*clm iterations	(work around loops "add al, 1" bug)
	zloop_set_prefetch ZDOUBLE		;; Prefetch 120*clm times
	zloop_set_alternate_prefetch_increment 107*4+3, 120*4  ;; Prefetch approximately 107.75*clm cache lines in those 120*clm prefetches
	zr8_rsc_wpn_eight_complex_calc_sincos_simple r9, rbx, clm ;; Generate clm*SCD8 sin/cos values from delayed and complex multipliers
b8b:	zr8_rsc_wpn_sgreg_2sc_sixteen_reals_fft8 rsi, 1024, 128, 256, 512, rbp, 128, blkdstreg, blkdst3reg, r13, rdx, ZMM_SCD8, r9, 64, rdi, ZMM_SCD4, clm, L1PREFETCH_DEST_ALL
	zloop	clm, b8b, -clm*1024+clmblkdst, rbp, -clm*128+8*blkdstreg, r13, -clm*128+8*blkdstreg, rdx, -clm*ZMM_SCD8, r9, -clm*64 ;; Loop, next source/dest pointers, restore premult/sine/cosine pointer
	jmp	b9aa

	loops_reset
b9a:	zr8_rsc_wpn_eight_complex_calc_sincos r9, rdi, rbx, clm ;; Generate clm*SCD8 sin/cos values from delayed and complex multipliers
	bump	rdi, clm*ZMM_SCD1		;; Next set of delayed multipliers
b9aa:
b9b:	zr8_rsc_wpn_sgreg_eight_complex_fft8 rsi, 1024, 128, 256, 512, rbp, 128, blkdstreg, blkdst3reg, r13, rdx, ZMM_SCD8, clm, L1PREFETCH_DEST_ALL
	zloop	clm, b9b, -clm*1024+clmblkdst, rbp, -clm*128+8*blkdstreg, r13, -clm*128+8*blkdstreg, rdx, -clm*ZMM_SCD8 ;; Loop, next source/dest pointers, restore premult/sine/cosine pointer
	zloop	5, b9b				;; Test loop counter
	zloop	12, b9a				;; Test loop counter
	end_timer 26

;; Work on next 8*clm sets of 960 values

endlp:	zpass1_get_next_block pass2, c0b, b0b ;; Get next block and jump
done:	zfft_footer
	ENDM


;; Perform a shared pass 1 negacyclic FFT of length 960 doubles.
;; Memory examples are for a 640K FFT with 10 levels done in pass 2.
;; FFT data needed:
;;		8*clm*960 doubles					Total: 960*clm 64-byte cache lines
;; Fixed s/c data needed:
;;		5*ZMM_CD12SCD6 (complex premultipliers and fixed s/c data), Total: 120 64-byte cache lines
;; Group multipliers needed:
;;		5*24*64 (weights)
;;		5*24*64 (inverse weights),				Total: 240 64-byte cache lines
;; Variable data needed:
;;		clm*64	(column weights)
;;		clm*64	(column inverse weights)
;;		clm*ZMM_SCD4 (complex reduced s/c multipliers)
;;		12*clm*ZMM_SCD1 (delayed s/c multipliers)
;;		8*clm*ZMM_SCD2 (5-complex sin/cos data)
;;		40*clm*3 (group multiplier fudge flags)
;;		40*clm*3 (biglit flags),				Total: 69.75*clm 64-byte cache lines
;; Total memory consumed (add for scratch area, prefetched data, carries):
;;		(960*3+69.75=2949.75)*clm + (120+240+960/8=480) cache lines
;;		clm=1,2,4:  215KB, 399KB, 768KB

zr4dwpn_pass1sc960ac MACRO
	LOCAL	b0b, b1b, b2b, b9a, b9b, c0b, c1b, c2b, c9a, c9b
	LOCAL	pass1, pass2, endlp, done, yes_fft

	zfft_header pass1

;; Do pass 2 (the last 10 FFT levels if this is a 960K FFT)

pass2:	start_timer 0
	mov	rax, ZMM_PASS2_ROUTINE
	call	rax
	end_timer 0
	cmp	ffttype, 1		;; We're done if FFTing only
	je	done

;; Do the inverse FFT

	pass1_inverse_fft_setup c0b

;; Do inverse FFT levels 8.9.10
;; On input the 128-byte double cache lines hold these data values:
;;	0K	+1	+1	+1	+1	+1	+1	+1	1K	+1	...
;;	2K
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0K	+1	+1	+1	+1	+1	+1	+1	8K	+1	...
;;	1K	...
;;	...
;; The output cache lines are swizzled to hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	8K	+1K	...
;;	16K
;;	...

;; Do 40 eight_complex_unfft operations
;;	distance between fft data elements is 1K
;;	do 7.5 macros each processing 128 data values

c0b:	start_timer 16
	mov	rbp, DATA_ADDR			;; Get source address
	mov	rsi, scratch_area		;; Get address of scratch area
	set_data_prefetch_ptrs			;; Init prefetching pointers
	mov	r9, PREMULT_ADDR		;; Variable data
	bump	r9, clm*64			;; Inverse column weights
	lea	rbx, [r9+clm*64]		;; Complex data sin/cos ptr
	lea	rdi, [rbx+clm*ZMM_SCD4]		;; Delayed sin/cos multipliers
	lea	rdx, ZMM_TMPS			;; Computed clm*SCD8 sin/cos values
	mov	r8, pass2blkdst			;; blkdst
	lea	r10, [2*blkdstreg+blkdstreg]	;; 3*blkdst
	lea	r13, [rbp+4*blkdstreg]		;; Source+4*blkdst pointer
	lea	r12, [rbp+8*blkdstreg]		;; L1 prefetch pointer part 1
	bump2	r12, (clm MOD 2), 8*blkdstreg	;; L1 prefetch pointer part 2
	lea	r14, [r12+4*blkdstreg]		;; L1 prefetch pointer + 4*blkdst

	zr8_rsc_complex_only_preload		;; Preload constants before calling zloop_init
;;	zloop_init 12*5*clm			;; 12 iters of 5*clm iterations
	zloop_init 12*5*clm  *2			;; 12 iters of 5*clm iterations	(work around loops "add al, 1" bug)
	zloop_set_clm_prefetch ZQUAD		;; Prefetch 120 of 480 blocks of clm*128 bytes
c9a:	zr8_rsc_wpn_eight_complex_calc_sincos r9, rdi, rbx, clm ;; Generate clm*SCD8 sin/cos values from weights, delayed, and complex multipliers
	bump	rdi, clm*ZMM_SCD1		;; Next set of delayed sin/cos multipliers
c9b:	zr8_rsc_wpn_sgreg_eight_complex_unfft8 rbp, 128, blkdstreg, blkdst3reg, r13, rsi, 1024, 128, 256, 512, rdx, ZMM_SCD8, clm, L1PREFETCH_ALL, r12, r14 ;; Prefetch short dist ahead
	zloop	clm, c9b, -clm*1024+clmblkdst, rbp, -clm*128+8*blkdstreg, r13, -clm*128+8*blkdstreg, rdx, -clm*ZMM_SCD8, r12, -clm*128+8*blkdstreg, r14, -clm*128+8*blkdstreg ;; Loop, next src/dest ptrs, restore sin/cos ptr
	zloop	5, c9b				;; Test loop counter
	zloop	12, c9a, -12*5*clmblkdst	;; Test loop counter, restore source pointer
	end_timer 16

;; Do inverse FFT levels 5,6,7
;; On input the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	8K	+1K	...
;;	16K
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	40K	+1K	...
;;	8K
;;	...
;;	32K
;;	80K
;;	...

;; Do 96 five_complex_unfft operations
;;	distance between fft data elements is 8K
;;	do 12 macros each processing 80 data values

	start_timer 18
	zloop_init  12*8*clm			;; 12 iterations of 8*clm
	zloop_set_clm_prefetch ZDOUBLE		;; Prefetch 96 of 480 blocks of clm*128 bytes
	zr5_five_complex_djbunfft_preload
IFDEF OLDDD
c2b:	zr5_five_complex_djbunfft rsi, 128, clmblkdst, rdi, ZMM_SCD2, 8*clm, L1PREFETCH_ALL, 128
	zloop	8*clm, c2b, -8*clm*128+5*clmblkdst, rdi, -8*clm*ZMM_SCD2 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	12, c2b, -12*5*clmblkdst, rdi, 8*clm*ZMM_SCD2 ;; Test loop counter, restore source pointer, next sin/cos pointer
ELSE
c2b:	zr5_five_complex_djbunfft rsi, 5*clmblkdst, clmblkdst, rdi, 0, 8, L1PREFETCH_ALL, 5*clmblkdst
	zloop	12, c2b, -12*5*clmblkdst+128, rdi, ZMM_SCD2 ;; Test loop counter, next source pointer, next sin/cos pointer
	zloop	8*clm, c2b, -8*clm*128		;; Test loop counter, restore source pointer
ENDIF
	end_timer 18

;; Do inverse FFT levels 1,2,3,4
;; On input the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	40K	+1K	...
;;	8K
;;	...
;;	32K
;;	80K
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	480K	+1K	...
;;	8K	...
;;	...
;;	472K

;; Do 40 twelve_complex_unfft operations
;;	distance between fft data elements is 40K
;;	do 5 macros each processing 192 data values

	start_timer 20
	mov	r15, sincos2			;; Load fixed premultiplier/sin/cos ptr
	sub	rbx, rbx			;; Clear register for loading fudge flags index
	mov	r12, compressed_fudges		;; Address of the compressed fudge flags
	mov	r13, r12			;; The XOR masks for fudge flags come from the compressed fudges
	zloop_init 5*8*clm			;; 5 iters of 8*clm
	zloop_set_clm_prefetch ZDOUBLE		;; Prefetch 40 of 480 blocks of clm*128 bytes
	zr12_csc_wpn_twelve_complex_last_djbunfft_preload
c1b:	zr12_csc_wpn_twelve_complex_last_djbunfft rsi, 128, 5*clmblkdst, 10*clmblkdst, 20*clmblkdst, r15, 0, rdi, 3, 8*clm, L1PREFETCH_ALL, 128
	zloop	8*clm, c1b, -8*clm*128+clmblkdst, r15, ZMM_CD12SCD6, rdx, 24*64, r13, 24 ;; 8*clm iterations, next fixed s/c, next grp multipliers, next XOR masks
	zloop	5, c1b				;; 5 iterations
	end_timer 20

;; Normalize these values

	pass1_normalize yes_fft			;; Normalize while prefetching 60 of 480 blocks of clm*128 bytes

;; Copy data from the scratch area back to the FFT data area

	copy_scratch_data_to_fft
	jmp	endlp				;; Skip FFT code if flag not set

;; Fire up auxiliary threads to do pass 1 forward FFT

pass1:	pass1_forward_fft_setup b0b

;; Copy data from the FFT data area back to the scratch area

b0b:	copy_fft_data_to_scratch

;; Do FFT levels 1,2,3,4
;; On input the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	480K	+1K	...
;;	8K	...
;;	...
;;	472K
;; On output the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	40K	+1K	...
;;	8K
;;	...
;;	32K
;;	80K
;;	...

;; Do 40 twelve_complex_first_fft operations
;;	distance between fft data elements is 40K
;;	do 5 macros each processing 192 data values

yes_fft:start_timer 22
	mov	rsi, scratch_area		;; Get address of scratch area
	mov	r15, sincos2			;; Load fixed premultiplier/sin/cos ptr
	mov	rdi, PREMULT_ADDR		;; Calculate pointer to fudge flags
	bump	rdi, 8*clm*ZMM_SCD2+12*clm*ZMM_SCD1+clm*ZMM_SCD4+clm*64+clm*64 ;; Addr of the group multiplier fudge flags
	mov	rdx, norm_grp_mults		;; Addr of the group multipliers
	sub	rbx, rbx			;; Clear register for loading fudge flags index
	mov	r12, compressed_fudges		;; Address of the compressed fudge flags
	mov	r13, r12			;; The XOR masks for fudge flags come from the compressed fudges
	set_data_prefetch_ptrs			;; Init prefetching pointers
pfing	mov	r8, pass2blkdst			;; blkdst
pfing	imul	rax, blkdstreg, (316-12)	;; We've already prefetched 316 of 480 blocks (12 too many)
pfing	add	rcx, rax
	zloop_init 5*8*clm			;; 5 iters of 8*clm
	zloop_set_clm_prefetch ZQUAD		;; Prefetch 80 of 480 blocks of clm*128 bytes
	zr12_csc_wpn_twelve_complex_first_djbfft_preload
b1b:	zr12_csc_wpn_twelve_complex_first_djbfft rsi, 128, 5*clmblkdst, 10*clmblkdst, 20*clmblkdst, r15, 0, rdi, 3, rdx, 0, 8*clm, L1PREFETCH_ALL, 128
	zloop	8*clm, b1b, -8*clm*128+clmblkdst, r15, ZMM_CD12SCD6, rdx, 24*64, r13, 24 ;; 8*clm iters, next source, fixed s/c, grp multipliers pointer, next XOR masks
	zloop	5, b1b, -5*clmblkdst		;; Test loop counter, restore source pointer
	end_timer 22

;; Do FFT levels 5,6,7
;; On input the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	40K	+1K	...
;;	8K
;;	...
;;	32K
;;	80K
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	8K	+1K	...
;;	16K
;;	...

;; Do 96 five_complex_fft operations
;;	distance between fft data elements is 8K
;;	do 12 macros each processing 80 data values

	start_timer 24
	bump	rdi, -40*clm*3-8*clm*ZMM_SCD2	;; Sin/cos pointer
	zloop_init  12*8*clm			;; 12 iterations of 8*clm
	zloop_set_clm_prefetch ZDOUBLE		;; Prefetch 96 of 480 blocks of clm*128 bytes
	zr5_five_complex_djbfft_preload
IFDEF OLDDD
b2b:	zr5_five_complex_djbfft rsi, 128, clmblkdst, rdi, ZMM_SCD2, 8*clm, L1PREFETCH_ALL, 128
	zloop	8*clm, b2b, -8*clm*128+5*clmblkdst, rdi, -8*clm*ZMM_SCD2 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	12, b2b, -12*5*clmblkdst	;; Test loop counter, restore source pointer
ELSE
b2b:	zr5_five_complex_djbfft rsi, 5*clmblkdst, clmblkdst, rdi, 0, 8, L1PREFETCH_ALL, 5*clmblkdst
	zloop	12, b2b, -12*5*clmblkdst+128, rdi, ZMM_SCD2 ;; Test loop counter, next source pointer, next sin/cos pointer
	zloop	8*clm, b2b, -8*clm*128		;; Test loop counter, restore source pointer
ENDIF
	end_timer 24

;; Do FFT levels 8,9,10
;; On input the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	8K	+1K	...
;;	16K
;;	...
;; The input is swizzled to hold these data values:
;;	0K	+1	+1	+1	+1	+1	+1	+1	8K	+1	...
;;	1K	...
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0K	+1	+1	+1	+1	+1	+1	+1	1K	+1	...
;;	2K
;;	...

;; Do 60 eight_complex_fft operations
;;	distance between fft data elements is 1K
;;	do 7.5 macros each processing 128 data values

	start_timer 26
	mov	rbp, DATA_ADDR			;; Destination pointer
IFDEF OLDDD
	bump	rdi, -12*clm*ZMM_SCD1		;; Delayed sin/cos multipliers
ELSE
	bump	rdi, -8*clm*ZMM_SCD2-12*clm*ZMM_SCD1 ;; Delayed sin/cos multipliers
ENDIF
	lea	rbx, [rdi-clm*ZMM_SCD4]		;; Complex data sin/cos ptr
	lea	r9, [rbx-clm*64-clm*64]		;; Column weights
	lea	rdx, ZMM_TMPS			;; Computed clm*SCD4 sin/cos values
	mov	r8, pass2blkdst			;; blkdst
	lea	r10, [2*blkdstreg+blkdstreg]	;; 3*blkdst
	lea	r13, [rbp+4*blkdstreg]		;; Dest+4*blkdst pointer
pfing	mov	rcx, PREMULT_PREFETCH		;; Load premult/sin/cos prefetch pointer

	zr8_rsc_complex_only_preload		;; Preload constants before calling zloop_init
;;	zloop_init 12*5*clm			;; 12 iters of 5*clm
	zloop_init 12*5*clm  *2			;; 12 iters of 5*clm iterations	(work around loops "add al, 1" bug)
	zloop_set_prefetch ZDOUBLE		;; Prefetch 120*clm cache lines
	zloop_set_alternate_prefetch_increment 69*4+3, 120*4  ;; Prefetch approximately 69.75*clm cache lines in those 120*clm prefetches
b9a:	zr8_rsc_wpn_eight_complex_calc_sincos r9, rdi, rbx, clm ;; Generate clm*SCD8 sin/cos values from weights, delayed and complex multipliers
	bump	rdi, clm*ZMM_SCD1		;; Next set of delayed multipliers
b9b:	zr8_rsc_wpn_sgreg_eight_complex_fft8 rsi, 1024, 128, 256, 512, rbp, 128, blkdstreg, blkdst3reg, r13, rdx, ZMM_SCD8, clm, L1PREFETCH_DEST_ALL
	zloop	clm, b9b, -clm*1024+clmblkdst, rbp, -clm*128+8*blkdstreg, r13, -clm*128+8*blkdstreg, rdx, -clm*ZMM_SCD8 ;; Loop, next source/dest pointers, restore premult/sine/cosine pointer
	zloop	5, b9b				;; Test loop counter
	zloop	12, b9a				;; Test loop counter
	end_timer 26

;; Work on next 8*clm sets of 960 values

endlp:	zpass1_get_next_block pass2, c0b, b0b ;; Get next block and jump
done:	zfft_footer
	ENDM



;; Perform a shared pass 1 FFT of length 1024 doubles.
;; Memory examples are for a 1M FFT with 10 levels done in pass 2.
;; FFT data needed:
;;		8*clm*1024 doubles					Total: 1024*clm 64-byte cache lines
;; Fixed s/c data needed:
;;		8*ZMM_SCD7 (fixed s/c data),				Total: 112 64-byte cache lines
;; Group multipliers needed:
;;		8*16*64 (weights)
;;		8*16*64 (inverse weights),				Total: 256 64-byte cache lines
;; Variable data needed:
;;		clm*64	(column weights)
;;		clm*64	(column inverse weights)
;;		clm*ZMM_SCD4 (complex reduced s/c multipliers)
;;		clm*ZMM_SCD4 (real s/c multipliers)
;;		7*clm*ZMM_SCD1 (delayed s/c multipliers)
;;		8*clm*ZMM_SCD8 (8-complex 16-real sin/cos data)
;;		64*clm*2 (group multiplier fudge flags)
;;		64*clm*2 (biglit flags),				Total: 164*clm 64-byte cache lines
;; Total memory consumed (add for scratch area, prefetched data, carries):
;;		(1024*3+164=3236)*clm + (112+256+1024/8=496) cache lines
;;		clm=1,2,4:  234KB, 436KB, 840KB

zr4dwpn_pass1sc1024 MACRO
	LOCAL	b0b, b1b, b2a, b2b, b2c, b8a, b8b, b9aa, b9a, b9b, c0b, c1b, c2a, c2b, c2c, c8b, c9aa, c9a, c9b
	LOCAL	pass1, pass2, endlp, done, yes_fft

	zfft_header pass1

;; Do pass 2 (the last 10 FFT levels if this is a 1M FFT)

pass2:	start_timer 0
	mov	rax, ZMM_PASS2_ROUTINE
	call	rax
	end_timer 0
	cmp	ffttype, 1		;; We're done if FFTing only
	je	done

;; Do the inverse FFT

	pass1_inverse_fft_setup c0b

;; Do inverse FFT levels 8,9,10
;; On input the 128-byte double cache lines hold these data values:
;;	0K	+1	+1	+1	+1	+1	+1	+1	1K	+1	...
;;	2K
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0K	+1	+1	+1	+1	+1	+1	+1	8K	+1	...
;;	1K	...
;;	...
;; The output cache lines are swizzled to hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	8K	+1K	...
;;	16K
;;	...

;; Do 1 sixteen_reals_unfft operations
;;	distance between fft data elements is 1K
;;	do 0.125 macros each processing 128 data values
;; Do 63 eight_complex_unfft operations
;;	distance between fft data elements is 1K
;;	do 7.875 macros each processing 128 data values

c0b:	start_timer 16
	mov	rbp, DATA_ADDR			;; Get source address
	mov	rsi, scratch_area		;; Get address of scratch area
	set_data_prefetch_ptrs			;; Init prefetching pointers
	mov	r9, PREMULT_ADDR		;; Variable data
	bump	r9, clm*64			;; Inverse column weights
	lea	rbx, [r9+clm*64]		;; Complex data sin/cos ptr
	lea	rdi, [rbx+clm*ZMM_SCD4]		;; Real data sin/cos pointer, will become delayed sin/cos multipliers
	lea	rdx, ZMM_TMPS			;; Computed clm*SCD8 sin/cos values
	mov	r8, pass2blkdst			;; blkdst
	lea	r10, [2*blkdstreg+blkdstreg]	;; 3*blkdst
	lea	r13, [rbp+4*blkdstreg]		;; Source+4*blkdst pointer
	lea	r12, [rbp+8*blkdstreg]		;; L1 prefetch pointer part 1
	bump2	r12, (clm MOD 2), 8*blkdstreg	;; L1 prefetch pointer part 2
	lea	r14, [r12+4*blkdstreg]		;; L1 prefetch pointer + 4*blkdst

	zr8_rsc_complex_and_real_preload	;; Preload constants before calling zloop_init
	zloop_init 8*8*clm			;; 8 iters of 8*clm iterations
	zloop_set_clm_prefetch ZDOUBLE		;; Prefetch 64 of 512 blocks of clm*128 bytes
	zr8_rsc_wpn_eight_complex_calc_sincos_simple r9, rbx, clm ;; Generate clm*SCD8 sin/cos values from delayed and complex multipliers
c8b:	zr8_rsc_wpn_sgreg_2sc_sixteen_reals_unfft8 rbp, 128, blkdstreg, blkdst3reg, r13, rsi, 1024, 128, 256, 512, rdx, ZMM_SCD8, r9, 64, rdi, ZMM_SCD4, clm, L1PREFETCH_ALL, r12, r14 ;; Prefetch short dist ahead
	zloop	clm, c8b, -clm*1024+clmblkdst, rbp, -clm*128+8*blkdstreg, r13, -clm*128+8*blkdstreg, rdx, -clm*ZMM_SCD8, r9, -clm*64, r12, -clm*128+8*blkdstreg, r14, -clm*128+8*blkdstreg ;; Loop, next src/dest ptrs, restore sin/cos ptr
	jmp	c9aa

	loops_reset
c9a:	zr8_rsc_wpn_eight_complex_calc_sincos r9, rdi, rbx, clm ;; Generate clm*SCD8 sin/cos values from delayed and complex multipliers
	bump	rdi, clm*ZMM_SCD1		;; Next set of delayed sin/cos multipliers
c9aa:
c9b:	zr8_rsc_wpn_sgreg_eight_complex_unfft8 rbp, 128, blkdstreg, blkdst3reg, r13, rsi, 1024, 128, 256, 512, rdx, ZMM_SCD8, clm, L1PREFETCH_ALL, r12, r14 ;; Prefetch short dist ahead
	zloop	clm, c9b, -clm*1024+clmblkdst, rbp, -clm*128+8*blkdstreg, r13, -clm*128+8*blkdstreg, rdx, -clm*ZMM_SCD8, r12, -clm*128+8*blkdstreg, r14, -clm*128+8*blkdstreg ;; Loop, next src/dest ptrs, restore sin/cos ptr
	zloop	8, c9b, -8*clmblkdst+clmblkdst8	;; Test loop counter, next source pointer
	zloop	8, c9a, -8*clmblkdst8		;; Test loop counter, restore source pointer
	end_timer 16

;; Do inverse FFT levels 5,6,7
;; On input the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	8K	+1K	...
;;	16K
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	64K	+1K	...
;;	8K
;;	...
;;	56K
;;	128K
;;	...

;; Do 8 sixteen_reals_unfft operations
;; Do 56 eight_complex_unfft operations
;;	distance between fft data elements is 8K
;;	do 8 macros each processing 128 data values

	start_timer 18
	zloop_init  8*8*clm			;; 8 iterations of 8*clm
	zloop_set_clm_prefetch ZQUAD		;; Prefetch 128 of 512 blocks of clm*128 bytes
IFDEF OLDDD
	zr8_csc_sixteen_reals_unfft_preload
c2a:	zr8_csc_sixteen_reals_unfft rsi, 128, clmblkdst, 2*clmblkdst, 4*clmblkdst, rdi, ZMM_SCD8, 8*clm, L1PREFETCH_ALL, 128
	zloop	8*clm, c2a, -8*clm*128+clmblkdst8, rdi, -8*clm*ZMM_SCD8 ;; Test loop counter, next source pointer, restore sin/cos pointer
	loops_reset				;; 7 more iters of 8*clm
	zr8_eight_complex_djbunfft_preload
c2b:	zr8_eight_complex_djbunfft rsi, 128, clmblkdst, 2*clmblkdst, 4*clmblkdst, rdi, ZMM_SCD8, 8*clm, L1PREFETCH_ALL, 128
	zloop	8*clm, c2b, -8*clm*128+clmblkdst8, rdi, -8*clm*ZMM_SCD8 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	8, c2b, -8*clmblkdst8, rdi, 8*clm*ZMM_SCD8 ;; Test loop counter, restore source pointer, next sin/cos pointer
ELSE
c2a:	zr8_csc_sixteen_reals_unfft_preload
	zr8_csc_sixteen_reals_unfft rsi, clmblkdst8, clmblkdst, 2*clmblkdst, 4*clmblkdst, rdi, 0, 1, L1PREFETCH_ALL, clmblkdst8
	zr8_eight_complex_djbunfft_preload
	jmp	c2c				;; 7 more iters
c2b:	zr8_eight_complex_djbunfft rsi, clmblkdst8, clmblkdst, 2*clmblkdst, 4*clmblkdst, rdi, 0, 1, L1PREFETCH_ALL, clmblkdst8
c2c:	zloop	8, c2b, -8*clmblkdst8+128, rdi, ZMM_SCD8 ;; Test loop counter, next source pointer, next sin/cos pointer
	zloop	8*clm, c2a, -8*clm*128		;; Test loop counter, restore source pointer
ENDIF
	end_timer 18

;; Do inverse FFT levels 1,2,3,4
;; On input the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	64K	+1K	...
;;	8K
;;	...
;;	56K
;;	128K
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	512K	+1K	...
;;	8K
;;	...
;;	504K

;; Do 64 sixteen_reals_unfft operations
;;	distance between fft data elements is 64K
;;	do 8 macros each processing 128 data values

	start_timer 20
	mov	r15, sincos2			;; Load fixed premultiplier/sin/cos ptr
	sub	rbx, rbx			;; Clear register for loading fudge flags index
	mov	r12, compressed_fudges		;; Address of the compressed fudge flags
	mov	r13, r12			;; The XOR masks for fudge flags come from the compressed fudges
	zloop_init 8*8*clm			;; 8 iters of 8*clm
	zloop_set_clm_prefetch ZDOUBLE		;; Prefetch 64 of 512 blocks of clm*128 bytes
	zr8_wpn_sixteen_reals_last_unfft_preload
c1b:	zr8_wpn_sixteen_reals_last_unfft rsi, 128, clmblkdst8, 2*clmblkdst8, 4*clmblkdst8, r15, 0, rdi, 2, 8*clm, L1PREFETCH_ALL, 128
	zloop	8*clm, c1b, -8*clm*128+clmblkdst, r15, ZMM_SCD7, rdx, 16*64, r13, 16 ;; 8*clm iters, next source, fixed s/c, grp multipliers pointer, next xor mask
	zloop	8, c1b				;; Test loop counter
	end_timer 20

;; Normalize these values

	pass1_normalize yes_fft			;; Normalize while prefetching 64 of 512 blocks of clm*128 bytes

;; Copy data from the scratch area back to the FFT data area

	copy_scratch_data_to_fft
	jmp	endlp				;; Skip FFT code if flag not set

;; Fire up auxiliary threads to do pass 1 forward FFT

pass1:	pass1_forward_fft_setup b0b

;; Copy data from the FFT data area back to the scratch area

b0b:	copy_fft_data_to_scratch

;; Do FFT levels 1,2,3,4
;; On input the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	512K	+1K	...
;;	8K	...
;;	...
;;	504K
;; On output the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	64K	+1K	...
;;	8K
;;	...
;;	56K
;;	128K
;;	...

;; Do 64 sixteen_reals_first_fft operations
;;	distance between fft data elements is 64K
;;	do 8 macros each processing 128 data values

yes_fft:start_timer 22
	mov	rsi, scratch_area		;; Get address of scratch area
	mov	r15, sincos2			;; Load fixed premultiplier/sin/cos ptr
	mov	rdi, PREMULT_ADDR		;; Calculate pointer to fudge flags
	bump	rdi, 8*clm*ZMM_SCD8+7*clm*ZMM_SCD1+clm*ZMM_SCD4+clm*ZMM_SCD4+clm*64+clm*64 ;; Addr of the group multiplier fudge flags
	mov	rdx, norm_grp_mults		;; Addr of the group multipliers
	sub	rbx, rbx			;; Clear register for loading fudge flags index
	mov	r12, compressed_fudges		;; Address of the compressed fudge flags
	mov	r13, r12			;; The XOR masks for fudge flags come from the compressed fudges
	set_data_prefetch_ptrs			;; Init prefetching pointers
pfing	mov	r8, pass2blkdst			;; blkdst
pfing	imul	rax, blkdstreg, (320-0)		;; We've already prefetched 320 of 512 blocks (zero too many)
pfing	add	rcx, rax
	zloop_init 8*8*clm			;; 8 iters of 8*clm
	zloop_set_clm_prefetch ZQUAD		;; Prefetch 128 of 512 blocks of clm*128 bytes
	zr8_wpn_sixteen_reals_first_fft_preload
b1b:	zr8_wpn_sixteen_reals_first_fft rsi, 128, clmblkdst8, 2*clmblkdst8, 4*clmblkdst8, r15, 0, rdi, 2, rdx, 0, 8*clm, L1PREFETCH_ALL, 128
	zloop	8*clm, b1b, -8*clm*128+clmblkdst, r15, ZMM_SCD7, rdx, 16*64, r13, 16 ;; 8*clm iters prefetching 8*clm cache lines, next source, fixed s/c, grp multipliers pointer, next xor mask
	zloop	8, b1b, -8*clmblkdst		;; Test loop counter, restore source pointer
	end_timer 22

;; Do FFT levels 5,6,7
;; On input the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	64K	+1K	...
;;	8K
;;	...
;;	56K
;;	128K
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	8K	+1K	...
;;	16K
;;	...

;; Do 8 sixteen_reals_fft operations
;; Do 56 eight_complex_fft operations
;;	distance between fft data elements is 8K
;;	do 8 macros each processing 128 data values

	start_timer 24
	bump	rdi, -64*clm*2-8*clm*ZMM_SCD8	;; Sin/cos pointer
	zloop_init  8*8*clm			;; 8 iterations of 8*clm
	zloop_set_clm_prefetch ZDOUBLE		;; Prefetch 64 of 512 blocks of clm*128 bytes
IFDEF OLDDD
	zr8_csc_sixteen_reals_fft_preload
b2a:	zr8_csc_sixteen_reals_fft rsi, 128, clmblkdst, 2*clmblkdst, 4*clmblkdst, rdi, ZMM_SCD8, 8*clm, L1PREFETCH_ALL, 128
	zloop	8*clm, b2a, -8*clm*128+clmblkdst8, rdi, -8*clm*ZMM_SCD8 ;; Test loop counter, next source pointer, restore sin/cos pointer
	loops_reset				;; 7 more iters of 8*clm
	zr8_eight_complex_djbfft_preload
b2b:	zr8_eight_complex_djbfft rsi, 128, clmblkdst, 2*clmblkdst, 4*clmblkdst, rdi, ZMM_SCD8, 8*clm, L1PREFETCH_ALL, 128
	zloop	8*clm, b2b, -8*clm*128+clmblkdst8, rdi, -8*clm*ZMM_SCD8 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	8, b2b, -8*clmblkdst8		;; Test loop counter, restore source pointer
ELSE
b2a:	zr8_csc_sixteen_reals_fft_preload
	zr8_csc_sixteen_reals_fft rsi, clmblkdst8, clmblkdst, 2*clmblkdst, 4*clmblkdst, rdi, 0, 1, L1PREFETCH_ALL, clmblkdst8
	zr8_eight_complex_djbfft_preload
	jmp	b2c				;; 7 more iters
b2b:	zr8_eight_complex_djbfft rsi, clmblkdst8, clmblkdst, 2*clmblkdst, 4*clmblkdst, rdi, 0, 1, L1PREFETCH_ALL, clmblkdst8
b2c:	zloop	8, b2b, -8*clmblkdst8+128, rdi, ZMM_SCD8 ;; Test loop counter,  source pointer, next sin/cos pointer
	zloop	8*clm, b2a, -8*clm*128		;; Test loop counter, restore source pointer
ENDIF
	end_timer 24

;; Do FFT levels 8,9,10
;; On input the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	8K	+1K	...
;;	16K
;;	...
;; The input is swizzled to hold these data values:
;;	0K	+1	+1	+1	+1	+1	+1	+1	8K	+1	...
;;	1K	...
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0K	+1	+1	+1	+1	+1	+1	+1	1K	+1	...
;;	2K
;;	...

;; Do 1 sixteen_reals_fft operations
;;	distance between fft data elements is 1K
;;	do 0.125 macros each processing 128 data values
;; Do 63 eight_complex_fft operations
;;	distance between fft data elements is 1K
;;	do 7.875 macros each processing 128 data values

	start_timer 26
IFDEF OLDDD
	bump	rdi, -7*clm*ZMM_SCD1-clm*ZMM_SCD4 ;; Real data sin/cos pointer, will become delayed sin/cos multipliers
ELSE
	bump	rdi, -8*clm*ZMM_SCD8-7*clm*ZMM_SCD1-clm*ZMM_SCD4 ;; Real data sin/cos pointer, will become delayed sin/cos multipliers
ENDIF
	lea	rbx, [rdi-clm*ZMM_SCD4]		;; Complex data sin/cos ptr
	lea	r9, [rbx-clm*64-clm*64]		;; Column weights
	lea	rdx, ZMM_TMPS			;; Computed clm*SCD4 sin/cos values
	mov	rbp, DATA_ADDR			;; Destination pointer
	mov	r8, pass2blkdst			;; blkdst
	lea	r10, [2*blkdstreg+blkdstreg]	;; 3*blkdst
	lea	r13, [rbp+4*blkdstreg]		;; Dest+4*blkdst pointer
pfing	mov	rcx, PREMULT_PREFETCH		;; Load premult/sin/cos prefetch pointer

	zr8_rsc_complex_and_real_preload	;; Preload constants before calling zloop_init
	zloop_init 8*8*clm			;; 8 iters of 8*clm
	zloop_set_prefetch ZTRIPLE		;; Prefetch 192*clm times
	zloop_set_alternate_prefetch_increment 164, 192  ;; Prefetch approximately 164*clm cache lines in those 192*clm prefetches
	zr8_rsc_wpn_eight_complex_calc_sincos_simple r9, rbx, clm ;; Generate clm*SCD8 sin/cos values from delayed and complex multipliers
b8b:	zr8_rsc_wpn_sgreg_2sc_sixteen_reals_fft8 rsi, 1024, 128, 256, 512, rbp, 128, blkdstreg, blkdst3reg, r13, rdx, ZMM_SCD8, r9, 64, rdi, ZMM_SCD4, clm, L1PREFETCH_DEST_ALL
	zloop	clm, b8b, -clm*1024+clmblkdst, rbp, -clm*128+8*blkdstreg, r13, -clm*128+8*blkdstreg, rdx, -clm*ZMM_SCD8, r9, -clm*64 ;; Loop, next source/dest pointers, restore premult/sine/cosine pointer
	jmp	b9aa

	loops_reset
b9a:	zr8_rsc_wpn_eight_complex_calc_sincos r9, rdi, rbx, clm ;; Generate clm*SCD8 sin/cos values from delayed and complex multipliers
	bump	rdi, clm*ZMM_SCD1		;; Next set of delayed multipliers
b9aa:
b9b:	zr8_rsc_wpn_sgreg_eight_complex_fft8 rsi, 1024, 128, 256, 512, rbp, 128, blkdstreg, blkdst3reg, r13, rdx, ZMM_SCD8, clm, L1PREFETCH_DEST_ALL
	zloop	clm, b9b, -clm*1024+clmblkdst, rbp, -clm*128+8*blkdstreg, r13, -clm*128+8*blkdstreg, rdx, -clm*ZMM_SCD8 ;; Loop, next source/dest pointers, restore premult/sine/cosine pointer
	zloop	8, b9b, -8*clmblkdst+clmblkdst8	;; Test loop counter, next source pointer
	zloop	8, b9a				;; Test loop counter
	end_timer 26

;; Work on next 8*clm sets of 1024 values

endlp:	zpass1_get_next_block pass2, c0b, b0b ;; Get next block and jump
done:	zfft_footer
	ENDM


;; Perform a shared pass 1 negacyclic FFT of length 1024 doubles.
;; Memory examples are for a 1M FFT with 10 levels done in pass 2.
;; FFT data needed:
;;		8*clm*1024 doubles					Total: 1024*clm 64-byte cache lines
;; Fixed s/c data needed:
;;		8*ZMM_CD8SCD4 (complex premultipliers and fixed s/c data), Total: 128 64-byte cache lines
;; Group multipliers needed:
;;		8*16*64 (weights)
;;		8*16*64 (inverse weights),				Total: 256 64-byte cache lines
;; Variable data needed:
;;		clm*64	(column weights)
;;		clm*64	(column inverse weights)
;;		clm*ZMM_SCD4 (complex reduced s/c multipliers)
;;		8*clm*ZMM_SCD1 (delayed s/c multipliers)
;;		8*clm*ZMM_SCD4 (8-complex sin/cos data)
;;		64*clm*2 (group multiplier fudge flags)
;;		64*clm*2 (biglit flags),				Total: 94*clm 64-byte cache lines
;; Total memory consumed (add for scratch area, prefetched data, carries):
;;		(1024*3+94=3166)*clm + (128+256+1024/8=512) cache lines
;;		clm=1,2,4:  230KB, 428KB, 824KB

zr4dwpn_pass1sc1024ac MACRO
	LOCAL	b0b, b1b, b2b, b9a, b9b, c0b, c1b, c2b, c9a, c9b
	LOCAL	pass1, pass2, endlp, done, yes_fft

	zfft_header pass1

;; Do pass 2 (the last 10 FFT levels if this is a 1M FFT)

pass2:	start_timer 0
	mov	rax, ZMM_PASS2_ROUTINE
	call	rax
	end_timer 0
	cmp	ffttype, 1		;; We're done if FFTing only
	je	done

;; Do the inverse FFT

	pass1_inverse_fft_setup c0b

;; Do inverse FFT levels 8.9.10
;; On input the 128-byte double cache lines hold these data values:
;;	0K	+1	+1	+1	+1	+1	+1	+1	1K	+1	...
;;	2K
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0K	+1	+1	+1	+1	+1	+1	+1	8K	+1	...
;;	1K	...
;;	...
;; The output cache lines are swizzled to hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	8K	+1K	...
;;	16K
;;	...

;; Do 64 eight_complex_unfft operations
;;	distance between fft data elements is 1K
;;	do 8 macros each processing 128 data values

c0b:	start_timer 16
	mov	rbp, DATA_ADDR			;; Get source address
	mov	rsi, scratch_area		;; Get address of scratch area
	set_data_prefetch_ptrs			;; Init prefetching pointers
	mov	r9, PREMULT_ADDR		;; Variable data
	bump	r9, clm*64			;; Inverse column weights
	lea	rbx, [r9+clm*64]		;; Complex data sin/cos ptr
	lea	rdi, [rbx+clm*ZMM_SCD4]		;; Delayed sin/cos multipliers
	lea	rdx, ZMM_TMPS			;; Computed clm*SCD8 sin/cos values
	mov	r8, pass2blkdst			;; blkdst
	lea	r10, [2*blkdstreg+blkdstreg]	;; 3*blkdst
	lea	r13, [rbp+4*blkdstreg]		;; Source+4*blkdst pointer
	lea	r12, [rbp+8*blkdstreg]		;; L1 prefetch pointer part 1
	bump2	r12, (clm MOD 2), 8*blkdstreg	;; L1 prefetch pointer part 2
	lea	r14, [r12+4*blkdstreg]		;; L1 prefetch pointer + 4*blkdst

	zr8_rsc_complex_only_preload		;; Preload constants before calling zloop_init
	zloop_init 8*8*clm			;; 8 iters of 8*clm iterations
	zloop_set_clm_prefetch ZDOUBLE		;; Prefetch 64 of 512 blocks of clm*128 bytes
c9a:	zr8_rsc_wpn_eight_complex_calc_sincos r9, rdi, rbx, clm ;; Generate clm*SCD8 sin/cos values from weights, delayed, and complex multipliers
	bump	rdi, clm*ZMM_SCD1		;; Next set of delayed sin/cos multipliers
c9b:	zr8_rsc_wpn_sgreg_eight_complex_unfft8 rbp, 128, blkdstreg, blkdst3reg, r13, rsi, 1024, 128, 256, 512, rdx, ZMM_SCD8, clm, L1PREFETCH_ALL, r12, r14 ;; Prefetch short dist ahead
	zloop	clm, c9b, -clm*1024+clmblkdst, rbp, -clm*128+8*blkdstreg, r13, -clm*128+8*blkdstreg, rdx, -clm*ZMM_SCD8, r12, -clm*128+8*blkdstreg, r14, -clm*128+8*blkdstreg ;; Loop, next src/dest ptrs, restore sin/cos ptr
	zloop	8, c9b, -8*clmblkdst+clmblkdst8	;; Test loop counter, next source pointer
	zloop	8, c9a, -8*clmblkdst8		;; Test loop counter, restore source pointer
	end_timer 16

;; Do inverse FFT levels 5,6,7
;; On input the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	8K	+1K	...
;;	16K
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	64K	+1K	...
;;	8K
;;	...
;;	56K
;;	128K
;;	...

;; Do 64 eight_complex_unfft operations
;;	distance between fft data elements is 8K
;;	do 8 macros each processing 128 data values

	start_timer 18
	zloop_init  8*8*clm			;; 8 iterations of 8*clm
	zloop_set_clm_prefetch ZQUAD		;; Prefetch 128 of 512 blocks of clm*128 bytes
	zr8_eight_complex_djbunfft_preload
IFDEF OLDDD
c2b:	zr8_eight_complex_djbunfft rsi, 128, clmblkdst, 2*clmblkdst, 4*clmblkdst, rdi, ZMM_SCD4, 8*clm, L1PREFETCH_ALL, 128
	zloop	8*clm, c2b, -8*clm*128+clmblkdst8, rdi, -8*clm*ZMM_SCD4 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	8, c2b, -8*clmblkdst8, rdi, 8*clm*ZMM_SCD4 ;; Test loop counter, restore source pointer, next sin/cos pointer
ELSE
c2b:	zr8_eight_complex_djbunfft rsi, clmblkdst8, clmblkdst, 2*clmblkdst, 4*clmblkdst, rdi, 0, 8, L1PREFETCH_ALL, clmblkdst8
	zloop	8, c2b, -8*clmblkdst8+128, rdi, ZMM_SCD4 ;; Test loop counter, next source pointer, next sin/cos pointer
	zloop	8*clm, c2b, -8*clm*128		;; Test loop counter, restore source pointer
ENDIF
	end_timer 18

;; Do inverse FFT levels 1,2,3,4
;; On input the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	64K	+1K	...
;;	8K
;;	...
;;	56K
;;	128K
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	512K	+1K	...
;;	8K	...
;;	...
;;	504K

;; Do 64 eight_complex_unfft operations
;;	distance between fft data elements is 64K
;;	do 8 macros each processing 128 data values

	start_timer 20
	mov	r15, sincos2			;; Load fixed premultiplier/sin/cos ptr
	sub	rbx, rbx			;; Clear register for loading fudge flags index
	mov	r12, compressed_fudges		;; Address of the compressed fudge flags
	mov	r13, r12			;; The XOR masks for fudge flags come from the compressed fudges
	zloop_init 8*8*clm			;; 8 iters of 8*clm
	zloop_set_clm_prefetch ZDOUBLE		;; Prefetch 64 of 512 blocks of clm*128 bytes
	zr8_csc_wpn_eight_complex_last_djbunfft_preload
c1b:	zr8_csc_wpn_eight_complex_last_djbunfft rsi, 128, clmblkdst8, 2*clmblkdst8, 4*clmblkdst8, r15, 0, rdi, 2, 8*clm, L1PREFETCH_ALL, 128
	zloop	8*clm, c1b, -8*clm*128+clmblkdst, r15, ZMM_CD8SCD4, rdx, 16*64, r13, 16 ;; 8*clm iterations, next fixed s/c, next grp multipliers, next xor mask
	zloop	8, c1b				;; 8 iterations
	end_timer 20

;; Normalize these values

	pass1_normalize yes_fft			;; Normalize while prefetching 64 of 512 blocks of clm*128 bytes

;; Copy data from the scratch area back to the FFT data area

	copy_scratch_data_to_fft
	jmp	endlp				;; Skip FFT code if flag not set

;; Fire up auxiliary threads to do pass 1 forward FFT

pass1:	pass1_forward_fft_setup b0b

;; Copy data from the FFT data area back to the scratch area

b0b:	copy_fft_data_to_scratch

;; Do FFT levels 1,2,3,4
;; On input the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	512K	+1K	...
;;	8K	...
;;	...
;;	504K
;; On output the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	64K	+1K	...
;;	8K
;;	...
;;	56K
;;	128K
;;	...

;; Do 64 eight_complex_first_fft operations
;;	distance between fft data elements is 64K
;;	do 8 macros each processing 128 data values

yes_fft:start_timer 22
	mov	rsi, scratch_area		;; Get address of scratch area
	mov	r15, sincos2			;; Load fixed premultiplier/sin/cos ptr
	mov	rdi, PREMULT_ADDR		;; Calculate pointer to fudge flags
	bump	rdi, 8*clm*ZMM_SCD4+8*clm*ZMM_SCD1+clm*ZMM_SCD4+clm*64+clm*64 ;; Addr of the group multiplier fudge flags
	mov	rdx, norm_grp_mults		;; Addr of the group multipliers
	sub	rbx, rbx			;; Clear register for loading fudge flags index
	mov	r12, compressed_fudges		;; Address of the compressed fudge flags
	mov	r13, r12			;; The XOR masks for fudge flags come from the compressed fudges
	set_data_prefetch_ptrs			;; Init prefetching pointers
pfing	mov	r8, pass2blkdst			;; blkdst
pfing	imul	rax, blkdstreg, (320-0)		;; We've already prefetched 320 of 512 blocks (zero too many)
pfing	add	rcx, rax
	zloop_init 8*8*clm			;; 8 iters of 8*clm
	zloop_set_clm_prefetch ZQUAD		;; Prefetch 128 of 512 blocks of clm*128 bytes
	zr8_csc_wpn_eight_complex_first_djbfft_preload
b1b:	zr8_csc_wpn_eight_complex_first_djbfft rsi, 128, clmblkdst8, 2*clmblkdst8, 4*clmblkdst8, r15, 0, rdi, 2, rdx, 0, 8*clm, L1PREFETCH_ALL, 128
	zloop	8*clm, b1b, -8*clm*128+clmblkdst, r15, ZMM_CD8SCD4, rdx, 16*64, r13, 16 ;; 8*clm iters, next source, fixed s/c, grp multipliers pointer, next xor mask
	zloop	8, b1b, -8*clmblkdst		;; Test loop counter, restore source pointer
	end_timer 22

;; Do FFT levels 5,6,7
;; On input the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	64K	+1K	...
;;	8K
;;	...
;;	56K
;;	128K
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	8K	+1K	...
;;	16K
;;	...

;; Do 64 eight_complex_fft operations
;;	distance between fft data elements is 8K
;;	do 8 macros each processing 128 data values

	start_timer 24
	bump	rdi, -64*clm*2-8*clm*ZMM_SCD4	;; Sin/cos pointer
	zloop_init  8*8*clm			;; 8 iterations of 8*clm
	zloop_set_clm_prefetch ZDOUBLE		;; Prefetch 64 of 512 blocks of clm*128 bytes
	zr8_eight_complex_djbfft_preload
IFDEF OLDDD
b2b:	zr8_eight_complex_djbfft rsi, 128, clmblkdst, 2*clmblkdst, 4*clmblkdst, rdi, ZMM_SCD4, 8*clm, L1PREFETCH_ALL, 128
	zloop	8*clm, b2b, -8*clm*128+clmblkdst8, rdi, -8*clm*ZMM_SCD4 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	8, b2b, -8*clmblkdst8		;; Test loop counter, restore source pointer
ELSE
b2b:	zr8_eight_complex_djbfft rsi, clmblkdst8, clmblkdst, 2*clmblkdst, 4*clmblkdst, rdi, 0, 8, L1PREFETCH_ALL, clmblkdst8
	zloop	8, b2b, -8*clmblkdst8+128, rdi, ZMM_SCD4 ;; Test loop counter, next source pointer, next sin/cos pointer
	zloop	8*clm, b2b, -8*clm*128		;; Test loop counter, restore source pointer
ENDIF
	end_timer 24

;; Do FFT levels 8,9,10
;; On input the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	8K	+1K	...
;;	16K
;;	...
;; The input is swizzled to hold these data values:
;;	0K	+1	+1	+1	+1	+1	+1	+1	8K	+1	...
;;	1K	...
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0K	+1	+1	+1	+1	+1	+1	+1	1K	+1	...
;;	2K
;;	...

;; Do 64 eight_complex_fft operations
;;	distance between fft data elements is 1K
;;	do 8 macros each processing 128 data values

	start_timer 26
	mov	rbp, DATA_ADDR			;; Destination pointer
IFDEF OLDDD
	bump	rdi, -8*clm*ZMM_SCD1		;; Delayed sin/cos multipliers
ELSE
	bump	rdi, -8*clm*ZMM_SCD4-8*clm*ZMM_SCD1 ;; Delayed sin/cos multipliers
ENDIF
	lea	rbx, [rdi-clm*ZMM_SCD4]		;; Complex data sin/cos ptr
	lea	r9, [rbx-clm*64-clm*64]		;; Column weights
	lea	rdx, ZMM_TMPS			;; Computed clm*SCD4 sin/cos values
	mov	r8, pass2blkdst			;; blkdst
	lea	r10, [2*blkdstreg+blkdstreg]	;; 3*blkdst
	lea	r13, [rbp+4*blkdstreg]		;; Dest+4*blkdst pointer
pfing	mov	rcx, PREMULT_PREFETCH		;; Load premult/sin/cos prefetch pointer

	zr8_rsc_complex_only_preload		;; Preload constants before calling zloop_init
	zloop_init 8*8*clm			;; 8 iters of 8*clm
	zloop_set_prefetch ZDOUBLE		;; Prefetch 128*clm cache lines
	zloop_set_alternate_prefetch_increment 94, 128  ;; Prefetch approximately 94*clm cache lines in those 128*clm prefetches
b9a:	zr8_rsc_wpn_eight_complex_calc_sincos r9, rdi, rbx, clm ;; Generate clm*SCD8 sin/cos values from weights, delayed and complex multipliers
	bump	rdi, clm*ZMM_SCD1		;; Next set of delayed multipliers
b9b:	zr8_rsc_wpn_sgreg_eight_complex_fft8 rsi, 1024, 128, 256, 512, rbp, 128, blkdstreg, blkdst3reg, r13, rdx, ZMM_SCD8, clm, L1PREFETCH_DEST_ALL
	zloop	clm, b9b, -clm*1024+clmblkdst, rbp, -clm*128+8*blkdstreg, r13, -clm*128+8*blkdstreg, rdx, -clm*ZMM_SCD8 ;; Loop, next source/dest pointers, restore premult/sine/cosine pointer
	zloop	8, b9b, -8*clmblkdst+clmblkdst8	;; Test loop counter, next source pointer
	zloop	8, b9a				;; Test loop counter
	end_timer 26

;; Work on next 8*clm sets of 1024 values

endlp:	zpass1_get_next_block pass2, c0b, b0b ;; Get next block and jump
done:	zfft_footer
	ENDM



;; Perform a shared pass 1 FFT of length 1152 doubles.
;; Memory examples are for a 1152K FFT with 10 levels done in pass 2.
;; FFT data needed:
;;		8*clm*1152 doubles					Total: 1152*clm 64-byte cache lines
;; Fixed s/c data needed:
;;		6*ZMM_SCD11 (fixed s/c data),				Total: 132 64-byte cache lines
;; Group multipliers needed:
;;		6*24*64 (weights)
;;		6*24*64 (inverse weights),				Total: 288 64-byte cache lines
;; Variable data needed:
;;		clm*64	(column weights)
;;		clm*64	(column inverse weights)
;;		clm*ZMM_SCD4 (complex reduced s/c multipliers)
;;		clm*ZMM_SCD4 (real s/c multipliers)
;;		11*clm*ZMM_SCD1 (delayed s/c multipliers)
;;		8*clm*ZMM_SCD6 (6-complex 12-real sin/cos data)
;;		48*clm*3 (group multiplier fudge flags)
;;		48*clm*3 (biglit flags),				Total: 140.5*clm 64-byte cache lines
;; Total memory consumed (add for scratch area, prefetched data, carries):
;;		(1152*3+140.5=3596.5)*clm + (132+288+1152/8=564) cache lines
;;		clm=1,2,4:  261KB, 485KB, 935KB

zr4dwpn_pass1sc1152 MACRO
	LOCAL	b0b, b1b, b2a, b2b, b2c, b8a, b8b, b9aa, b9a, b9b, c0b, c1b, c2a, c2b, c2c, c8b, c9aa, c9a, c9b
	LOCAL	pass1, pass2, endlp, done, yes_fft

	zfft_header pass1

;; Do pass 2 (the last 10 FFT levels if this is a 1152K FFT)

pass2:	start_timer 0
	mov	rax, ZMM_PASS2_ROUTINE
	call	rax
	end_timer 0
	cmp	ffttype, 1		;; We're done if FFTing only
	je	done

;; Do the inverse FFT

	pass1_inverse_fft_setup c0b

;; Do inverse FFT levels 8,9,10
;; On input the 128-byte double cache lines hold these data values:
;;	0K	+1	+1	+1	+1	+1	+1	+1	1K	+1	...
;;	2K
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0K	+1	+1	+1	+1	+1	+1	+1	8K	+1	...
;;	1K	...
;;	...
;; The output cache lines are swizzled to hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	8K	+1K	...
;;	16K
;;	...

;; Do 1 sixteen_reals_unfft operations
;;	distance between fft data elements is 1K
;;	do 0.125 macros each processing 128 data values
;; Do 71 eight_complex_unfft operations
;;	distance between fft data elements is 1K
;;	do 8.875 macros each processing 128 data values

c0b:	start_timer 16
	mov	rbp, DATA_ADDR			;; Get source address
	mov	rsi, scratch_area		;; Get address of scratch area
	set_data_prefetch_ptrs			;; Init prefetching pointers
	mov	r9, PREMULT_ADDR		;; Variable data
	bump	r9, clm*64			;; Inverse column weights
	lea	rbx, [r9+clm*64]		;; Complex data sin/cos ptr
	lea	rdi, [rbx+clm*ZMM_SCD4]		;; Real data sin/cos pointer, will become delayed sin/cos multipliers
	lea	rdx, ZMM_TMPS			;; Computed clm*SCD8 sin/cos values
	mov	r8, pass2blkdst			;; blkdst
	lea	r10, [2*blkdstreg+blkdstreg]	;; 3*blkdst
	lea	r13, [rbp+4*blkdstreg]		;; Source+4*blkdst pointer
	lea	r12, [rbp+8*blkdstreg]		;; L1 prefetch pointer part 1
	bump2	r12, (clm MOD 2), 8*blkdstreg	;; L1 prefetch pointer part 2
	lea	r14, [r12+4*blkdstreg]		;; L1 prefetch pointer + 4*blkdst

	zr8_rsc_complex_and_real_preload	;; Preload constants before calling zloop_init
;;	zloop_init 12*6*clm			;; 12 iters of 6*clm iterations
	zloop_init 12*6*clm  *2			;; 12 iters of 6*clm iterations	(work around loops "add al, 1" bug)
	zr8_rsc_wpn_eight_complex_calc_sincos_simple r9, rbx, clm ;; Generate clm*SCD8 sin/cos values from delayed and complex multipliers
	zloop_set_clm_prefetch ZQUAD		;; Prefetch 144 of 576 blocks of clm*128 bytes
c8b:	zr8_rsc_wpn_sgreg_2sc_sixteen_reals_unfft8 rbp, 128, blkdstreg, blkdst3reg, r13, rsi, 1024, 128, 256, 512, rdx, ZMM_SCD8, r9, 64, rdi, ZMM_SCD4, clm, L1PREFETCH_ALL, r12, r14 ;; Prefetch short dist ahead
	zloop	clm, c8b, -clm*1024+clmblkdst, rbp, -clm*128+8*blkdstreg, r13, -clm*128+8*blkdstreg, rdx, -clm*ZMM_SCD8, r9, -clm*64, r12, -clm*128+8*blkdstreg, r14, -clm*128+8*blkdstreg ;; Loop, next src/dest ptrs, restore sin/cos ptr
	jmp	c9aa

	loops_reset
c9a:	zr8_rsc_wpn_eight_complex_calc_sincos r9, rdi, rbx, clm ;; Generate clm*SCD8 sin/cos values from delayed and complex multipliers
	bump	rdi, clm*ZMM_SCD1		;; Next set of delayed sin/cos multipliers
c9aa:	zloop_set_clm_prefetch ZQUAD		;; Prefetch 144 of 576 blocks of clm*128 bytes
c9b:	zr8_rsc_wpn_sgreg_eight_complex_unfft8 rbp, 128, blkdstreg, blkdst3reg, r13, rsi, 1024, 128, 256, 512, rdx, ZMM_SCD8, clm, L1PREFETCH_ALL, r12, r14 ;; Prefetch short dist ahead
	zloop	clm, c9b, -clm*1024+clmblkdst, rbp, -clm*128+8*blkdstreg, r13, -clm*128+8*blkdstreg, rdx, -clm*ZMM_SCD8, r12, -clm*128+8*blkdstreg, r14, -clm*128+8*blkdstreg ;; Loop, next src/dest ptrs, restore sin/cos ptr
	zloop	6, c9b				;; Test loop counter
	zloop	12, c9a, -12*6*clmblkdst	;; Test loop counter, restore source pointer
	end_timer 16

;; Do inverse FFT levels 5,6,7
;; On input the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	8K	+1K	...
;;	16K
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	48K	+1K	...
;;	8K
;;	...
;;	40K
;;	96K
;;	...

;; Do 8 twelve_reals_unfft operations
;; Do 88 six_complex_unfft operations
;;	distance between fft data elements is 8K
;;	do 12 macros each processing 96 data values

	start_timer 18
	zloop_init  12*8*clm			;; 12 iterations of 8*clm
	zloop_set_clm_prefetch ZDOUBLE		;; Prefetch 96 of 576 blocks of clm*128 bytes
IFDEF OLDDD
	zr6_csc_twelve_reals_unfft_preload
c2a:	zr6_csc_twelve_reals_unfft rsi, 128, clmblkdst, rdi, ZMM_SCD6, 8*clm, L1PREFETCH_ALL, 128
	zloop	8*clm, c2a, -8*clm*128+6*clmblkdst, rdi, -8*clm*ZMM_SCD6 ;; Test loop counter, next source pointer, restore sin/cos pointer
	loops_reset				;; 11 more iters of 8*clm
	zr6_six_complex_djbunfft_preload
c2b:	zr6_six_complex_djbunfft rsi, 128, clmblkdst, rdi, ZMM_SCD6, 8*clm, L1PREFETCH_ALL, 128
	zloop	8*clm, c2b, -8*clm*128+6*clmblkdst, rdi, -8*clm*ZMM_SCD6 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	12, c2b, -12*6*clmblkdst, rdi, 8*clm*ZMM_SCD6 ;; Test loop counter, restore source pointer, next sin/cos pointer
ELSE
c2a:	zr6_csc_twelve_reals_unfft_preload
	zr6_csc_twelve_reals_unfft rsi, 6*clmblkdst, clmblkdst, rdi, 0, 1, L1PREFETCH_ALL, 6*clmblkdst
	zr6_six_complex_djbunfft_preload
	jmp	c2c				;; 11 more iters
c2b:	zr6_six_complex_djbunfft rsi, 6*clmblkdst, clmblkdst, rdi, 0, 1, L1PREFETCH_ALL, 6*clmblkdst
c2c:	zloop	12, c2b, -12*6*clmblkdst+128, rdi, ZMM_SCD6 ;; Test loop counter, next source pointer, next sin/cos pointer
	zloop	8*clm, c2a, -8*clm*128		;; Test loop counter, restore source pointer
ENDIF
	end_timer 18

;; Do inverse FFT levels 1,2,3,4
;; On input the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	48K	+1K	...
;;	8K
;;	...
;;	40K
;;	96K
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	576K	+1K	...
;;	8K
;;	...
;;	568K

;; Do 48 twentyfour_reals_unfft operations
;;	distance between fft data elements is 48K
;;	do 6 macros each processing 192 data values

	start_timer 20
	mov	r15, sincos2			;; Load fixed premultiplier/sin/cos ptr
	sub	rbx, rbx			;; Clear register for loading fudge flags index
	mov	r12, compressed_fudges		;; Address of the compressed fudge flags
	mov	r13, r12			;; The XOR masks for fudge flags come from the compressed fudges
	zloop_init 6*8*clm			;; 6 iters of 8*clm
	zloop_set_clm_prefetch ZQUAD		;; Prefetch 96 of 576 blocks of clm*128 bytes
	zr12_wpn_twentyfour_reals_last_unfft_preload
c1b:	zr12_wpn_twentyfour_reals_last_unfft rsi, 128, 6*clmblkdst, 12*clmblkdst, 24*clmblkdst, r15, 0, rdi, 3, 8*clm, L1PREFETCH_ALL, 128
	zloop	8*clm, c1b, -8*clm*128+clmblkdst, r15, ZMM_SCD11, rdx, 24*64, r13, 24 ;; 8*clm iters, next source, fixed s/c, grp multipliers pointer, next xor mask
	zloop	6, c1b				;; Test loop counter
	end_timer 20

;; Normalize these values

	pass1_normalize yes_fft			;; Normalize while prefetching 72 of 576 blocks of clm*128 bytes

;; Copy data from the scratch area back to the FFT data area

	copy_scratch_data_to_fft
	jmp	endlp				;; Skip FFT code if flag not set

;; Fire up auxiliary threads to do pass 1 forward FFT

pass1:	pass1_forward_fft_setup b0b

;; Copy data from the FFT data area back to the scratch area

b0b:	copy_fft_data_to_scratch

;; Do FFT levels 1,2,3,4
;; On input the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	576K	+1K	...
;;	8K	...
;;	...
;;	568K
;; On output the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	48K	+1K	...
;;	8K
;;	...
;;	40K
;;	96K
;;	...

;; Do 48 twentyfour_reals_first_fft operations
;;	distance between fft data elements is 48K
;;	do 6 macros each processing 192 data values

yes_fft:start_timer 22
	mov	rsi, scratch_area		;; Get address of scratch area
	mov	r15, sincos2			;; Load fixed premultiplier/sin/cos ptr
	mov	rdi, PREMULT_ADDR		;; Calculate pointer to fudge flags
	bump	rdi, 8*clm*ZMM_SCD6+11*clm*ZMM_SCD1+clm*ZMM_SCD4+clm*ZMM_SCD4+clm*64+clm*64 ;; Addr of the group multiplier fudge flags
	mov	rdx, norm_grp_mults		;; Addr of the group multipliers
	sub	rbx, rbx			;; Clear register for loading fudge flags index
	mov	r12, compressed_fudges		;; Address of the compressed fudge flags
	mov	r13, r12			;; The XOR masks for fudge flags come from the compressed fudges
	set_data_prefetch_ptrs			;; Init prefetching pointers
pfing	mov	r8, pass2blkdst			;; blkdst
pfing	imul	rax, blkdstreg, (408-24)	;; We've already prefetched 408 of 576 blocks (24 too many)
pfing	add	rcx, rax
	zloop_init 6*8*clm			;; 6 iters of 8*clm
	zloop_set_clm_prefetch ZQUAD		;; Prefetch 96 of 576 blocks of clm*128 bytes
	zr12_wpn_twentyfour_reals_first_fft_preload
b1b:	zr12_wpn_twentyfour_reals_first_fft rsi, 128, 6*clmblkdst, 12*clmblkdst, 24*clmblkdst, r15, 0, rdi, 3, rdx, 0, 8*clm, L1PREFETCH_ALL, 128
	zloop	8*clm, b1b, -8*clm*128+clmblkdst, r15, ZMM_SCD11, rdx, 24*64, r13, 24 ;; 8*clm iters prefetching 8*clm cache lines, next source, fixed s/c, grp multipliers pointer, next xor mask
	zloop	6, b1b, -6*clmblkdst		;; Test loop counter, restore source pointer
	end_timer 22

;; Do FFT levels 5,6,7
;; On input the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	48K	+1K	...
;;	8K
;;	...
;;	40K
;;	96K
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	8K	+1K	...
;;	16K
;;	...

;; Do 8 twelve_reals_fft operations
;; Do 88 six_complex_fft operations
;;	distance between fft data elements is 8K
;;	do 12 macros each processing 96 data values

	start_timer 24
	bump	rdi, -48*clm*3-8*clm*ZMM_SCD6	;; Sin/cos pointer
	zloop_init  12*8*clm			;; 8 iterations of 8*clm
	zloop_set_clm_prefetch ZDOUBLE		;; Prefetch 96 of 576 blocks of clm*128 bytes
IFDEF OLDDD
	zr6_csc_twelve_reals_fft_preload
b2a:	zr6_csc_twelve_reals_fft rsi, 128, clmblkdst, rdi, ZMM_SCD6, 8*clm, L1PREFETCH_ALL, 128
	zloop	8*clm, b2a, -8*clm*128+6*clmblkdst, rdi, -8*clm*ZMM_SCD6 ;; Test loop counter, next source pointer, restore sin/cos pointer
	loops_reset				;; 11 more iters of 8*clm
	zr6_six_complex_djbfft_preload
b2b:	zr6_six_complex_djbfft rsi, 128, clmblkdst, rdi, ZMM_SCD6, 8*clm, L1PREFETCH_ALL, 128
	zloop	8*clm, b2b, -8*clm*128+6*clmblkdst, rdi, -8*clm*ZMM_SCD6 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	12, b2b, -12*6*clmblkdst	;; Test loop counter, restore source pointer
ELSE
b2a:	zr6_csc_twelve_reals_fft_preload
	zr6_csc_twelve_reals_fft rsi, 6*clmblkdst, clmblkdst, rdi, 0, 1, L1PREFETCH_ALL, 6*clmblkdst
	zr6_six_complex_djbfft_preload
	jmp	b2c				;; 11 more iters
b2b:	zr6_six_complex_djbfft rsi, 6*clmblkdst, clmblkdst, rdi, 0, 1, L1PREFETCH_ALL, 6*clmblkdst
b2c:	zloop	12, b2b, -12*6*clmblkdst+128, rdi, ZMM_SCD6 ;; Test loop counter, next source pointer, next sin/cos pointer
	zloop	8*clm, b2a, -8*clm*128		;; Test loop counter, restore source pointer
ENDIF
	end_timer 24

;; Do FFT levels 8,9,10
;; On input the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	8K	+1K	...
;;	16K
;;	...
;; The input is swizzled to hold these data values:
;;	0K	+1	+1	+1	+1	+1	+1	+1	8K	+1	...
;;	1K	...
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0K	+1	+1	+1	+1	+1	+1	+1	1K	+1	...
;;	2K
;;	...

;; Do 1 sixteen_reals_fft operations
;;	distance between fft data elements is 1K
;;	do 0.125 macros each processing 128 data values
;; Do 71 eight_complex_fft operations
;;	distance between fft data elements is 1K
;;	do 8.875 macros each processing 128 data values

	start_timer 26
IFDEF OLDDD
	bump	rdi, -11*clm*ZMM_SCD1-clm*ZMM_SCD4 ;; Real data sin/cos pointer, will become delayed sin/cos multipliers
ELSE
	bump	rdi, -8*clm*ZMM_SCD6-11*clm*ZMM_SCD1-clm*ZMM_SCD4 ;; Real data sin/cos pointer, will become delayed sin/cos multipliers
ENDIF
	lea	rbx, [rdi-clm*ZMM_SCD4]		;; Complex data sin/cos ptr
	lea	r9, [rbx-clm*64-clm*64]		;; Column weights
	lea	rdx, ZMM_TMPS			;; Computed clm*SCD4 sin/cos values
	mov	rbp, DATA_ADDR			;; Destination pointer
	mov	r8, pass2blkdst			;; blkdst
	lea	r10, [2*blkdstreg+blkdstreg]	;; 3*blkdst
	lea	r13, [rbp+4*blkdstreg]		;; Dest+4*blkdst pointer
pfing	mov	rcx, PREMULT_PREFETCH		;; Load premult/sin/cos prefetch pointer

	zr8_rsc_complex_and_real_preload	;; Preload constants before calling zloop_init
;;	zloop_init 12*6*clm			;; 12 iters of 6*clm
	zloop_init 12*6*clm  *2			;; 12 iters of 6*clm iterations	(work around loops "add al, 1" bug)
	zloop_set_prefetch ZDOUBLE		;; Prefetch 144*clm times
	zloop_set_alternate_prefetch_increment 140*2+1, 144*2  ;; Prefetch approximately 140.5*clm cache lines in those 144*clm prefetches
	zr8_rsc_wpn_eight_complex_calc_sincos_simple r9, rbx, clm ;; Generate clm*SCD8 sin/cos values from delayed and complex multipliers
b8b:	zr8_rsc_wpn_sgreg_2sc_sixteen_reals_fft8 rsi, 1024, 128, 256, 512, rbp, 128, blkdstreg, blkdst3reg, r13, rdx, ZMM_SCD8, r9, 64, rdi, ZMM_SCD4, clm, L1PREFETCH_DEST_ALL
	zloop	clm, b8b, -clm*1024+clmblkdst, rbp, -clm*128+8*blkdstreg, r13, -clm*128+8*blkdstreg, rdx, -clm*ZMM_SCD8, r9, -clm*64 ;; Loop, next source/dest pointers, restore premult/sine/cosine pointer
	jmp	b9aa

	loops_reset
b9a:	zr8_rsc_wpn_eight_complex_calc_sincos r9, rdi, rbx, clm ;; Generate clm*SCD8 sin/cos values from delayed and complex multipliers
	bump	rdi, clm*ZMM_SCD1		;; Next set of delayed multipliers
b9aa:
b9b:	zr8_rsc_wpn_sgreg_eight_complex_fft8 rsi, 1024, 128, 256, 512, rbp, 128, blkdstreg, blkdst3reg, r13, rdx, ZMM_SCD8, clm, L1PREFETCH_DEST_ALL
	zloop	clm, b9b, -clm*1024+clmblkdst, rbp, -clm*128+8*blkdstreg, r13, -clm*128+8*blkdstreg, rdx, -clm*ZMM_SCD8 ;; Loop, next source/dest pointers, restore premult/sine/cosine pointer
	zloop	6, b9b				;; Test loop counter
	zloop	12, b9a				;; Test loop counter
	end_timer 26

;; Work on next 8*clm sets of 1152 values

endlp:	zpass1_get_next_block pass2, c0b, b0b ;; Get next block and jump
done:	zfft_footer
	ENDM


;; Perform a shared pass 1 negacyclic FFT of length 1152 doubles.
;; Memory examples are for a 1152K FFT with 10 levels done in pass 2.
;; FFT data needed:
;;		8*clm*1152 doubles					Total: 1152*clm 64-byte cache lines
;; Fixed s/c data needed:
;;		6*ZMM_CD12SCD6 (complex premultipliers and fixed s/c data), Total: 144 64-byte cache lines
;; Group multipliers needed:
;;		6*24*64 (weights)
;;		6*24*64 (inverse weights),				Total: 288 64-byte cache lines
;; Variable data needed:
;;		clm*64	(column weights)
;;		clm*64	(column inverse weights)
;;		clm*ZMM_SCD4 (complex reduced s/c multipliers)
;;		12*clm*ZMM_SCD1 (delayed s/c multipliers)
;;		8*clm*ZMM_SCD3 (6-complex sin/cos data)
;;		48*clm*3 (group multiplier fudge flags)
;;		48*clm*3 (biglit flags),				Total: 86.5*clm 64-byte cache lines
;; Total memory consumed (add for scratch area, prefetched data, carries):
;;		(1152*3+86.5=3542.5)*clm + (144+288+1152/8=576) cache lines
;;		clm=1,2,4:  258KB, 479KB, 922KB

zr4dwpn_pass1sc1152ac MACRO
	LOCAL	b0b, b1b, b2b, b9a, b9b, c0b, c1b, c2b, c9a, c9b
	LOCAL	pass1, pass2, endlp, done, yes_fft

	zfft_header pass1

;; Do pass 2 (the last 10 FFT levels if this is a 1152K FFT)

pass2:	start_timer 0
	mov	rax, ZMM_PASS2_ROUTINE
	call	rax
	end_timer 0
	cmp	ffttype, 1		;; We're done if FFTing only
	je	done

;; Do the inverse FFT

	pass1_inverse_fft_setup c0b

;; Do inverse FFT levels 8.9.10
;; On input the 128-byte double cache lines hold these data values:
;;	0K	+1	+1	+1	+1	+1	+1	+1	1K	+1	...
;;	2K
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0K	+1	+1	+1	+1	+1	+1	+1	8K	+1	...
;;	1K	...
;;	...
;; The output cache lines are swizzled to hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	8K	+1K	...
;;	16K
;;	...

;; Do 72 eight_complex_unfft operations
;;	distance between fft data elements is 1K
;;	do 9 macros each processing 128 data values

c0b:	start_timer 16
	mov	rbp, DATA_ADDR			;; Get source address
	mov	rsi, scratch_area		;; Get address of scratch area
	set_data_prefetch_ptrs			;; Init prefetching pointers
	mov	r9, PREMULT_ADDR		;; Variable data
	bump	r9, clm*64			;; Inverse column weights
	lea	rbx, [r9+clm*64]		;; Complex data sin/cos ptr
	lea	rdi, [rbx+clm*ZMM_SCD4]		;; Delayed sin/cos multipliers
	lea	rdx, ZMM_TMPS			;; Computed clm*SCD8 sin/cos values
	mov	r8, pass2blkdst			;; blkdst
	lea	r10, [2*blkdstreg+blkdstreg]	;; 3*blkdst
	lea	r13, [rbp+4*blkdstreg]		;; Source+4*blkdst pointer
	lea	r12, [rbp+8*blkdstreg]		;; L1 prefetch pointer part 1
	bump2	r12, (clm MOD 2), 8*blkdstreg	;; L1 prefetch pointer part 2
	lea	r14, [r12+4*blkdstreg]		;; L1 prefetch pointer + 4*blkdst

	zr8_rsc_complex_only_preload		;; Preload constants before calling zloop_init
;;	zloop_init 12*6*clm			;; 12 iters of 6*clm iterations
	zloop_init 12*6*clm  *2			;; 12 iters of 6*clm iterations	(work around loops "add al, 1" bug)
	zloop_set_clm_prefetch ZQUAD		;; Prefetch 144 of 576 blocks of clm*128 bytes
c9a:	zr8_rsc_wpn_eight_complex_calc_sincos r9, rdi, rbx, clm ;; Generate clm*SCD8 sin/cos values from weights, delayed, and complex multipliers
	bump	rdi, clm*ZMM_SCD1		;; Next set of delayed sin/cos multipliers
c9b:	zr8_rsc_wpn_sgreg_eight_complex_unfft8 rbp, 128, blkdstreg, blkdst3reg, r13, rsi, 1024, 128, 256, 512, rdx, ZMM_SCD8, clm, L1PREFETCH_ALL, r12, r14 ;; Prefetch short dist ahead
	zloop	clm, c9b, -clm*1024+clmblkdst, rbp, -clm*128+8*blkdstreg, r13, -clm*128+8*blkdstreg, rdx, -clm*ZMM_SCD8, r12, -clm*128+8*blkdstreg, r14, -clm*128+8*blkdstreg ;; Loop, next src/dest ptrs, restore sin/cos ptr
	zloop	6, c9b				;; Test loop counter
	zloop	12, c9a, -12*6*clmblkdst	;; Test loop counter, restore source pointer
	end_timer 16

;; Do inverse FFT levels 5,6,7
;; On input the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	8K	+1K	...
;;	16K
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	48K	+1K	...
;;	8K
;;	...
;;	40K
;;	96K
;;	...

;; Do 96 six_complex_unfft operations
;;	distance between fft data elements is 8K
;;	do 12 macros each processing 96 data values

	start_timer 18
	zloop_init  12*8*clm			;; 12 iterations of 8*clm
	zloop_set_clm_prefetch ZDOUBLE		;; Prefetch 96 of 576 blocks of clm*128 bytes
	zr6_six_complex_djbunfft_preload
IFDEF OLDDD
c2b:	zr6_six_complex_djbunfft rsi, 128, clmblkdst, rdi, ZMM_SCD3, 8*clm, L1PREFETCH_ALL, 128
	zloop	8*clm, c2b, -8*clm*128+6*clmblkdst, rdi, -8*clm*ZMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	12, c2b, -12*6*clmblkdst, rdi, 8*clm*ZMM_SCD3 ;; Test loop counter, restore source pointer, next sin/cos pointer
ELSE
c2b:	zr6_six_complex_djbunfft rsi, 6*clmblkdst, clmblkdst, rdi, 0, 8, L1PREFETCH_ALL, 6*clmblkdst
	zloop	12, c2b, -12*6*clmblkdst+128, rdi, ZMM_SCD3 ;; Test loop counter, next source pointer, next sin/cos pointer
	zloop	8*clm, c2b, -8*clm*128		;; Test loop counter, restore source pointer
ENDIF
	end_timer 18

;; Do inverse FFT levels 1,2,3,4
;; On input the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	48K	+1K	...
;;	8K
;;	...
;;	40K
;;	96K
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	576K	+1K	...
;;	8K	...
;;	...
;;	568K

;; Do 48 twelve_complex_unfft operations
;;	distance between fft data elements is 48K
;;	do 6 macros each processing 192 data values

	start_timer 20
	mov	r15, sincos2			;; Load fixed premultiplier/sin/cos ptr
	sub	rbx, rbx			;; Clear register for loading fudge flags index
	mov	r12, compressed_fudges		;; Address of the compressed fudge flags
	mov	r13, r12			;; The XOR masks for fudge flags come from the compressed fudges
	zloop_init 6*8*clm			;; 6 iters of 8*clm
	zloop_set_clm_prefetch ZQUAD		;; Prefetch 96 of 576 blocks of clm*128 bytes
	zr12_csc_wpn_twelve_complex_last_djbunfft_preload
c1b:	zr12_csc_wpn_twelve_complex_last_djbunfft rsi, 128, 6*clmblkdst, 12*clmblkdst, 24*clmblkdst, r15, 0, rdi, 3, 8*clm, L1PREFETCH_ALL, 128
	zloop	8*clm, c1b, -8*clm*128+clmblkdst, r15, ZMM_CD12SCD6, rdx, 24*64, r13, 24 ;; 8*clm iterations, next fixed s/c, next grp multipliers, next xor mask
	zloop	6, c1b				;; 6 iterations
	end_timer 20

;; Normalize these values

	pass1_normalize yes_fft			;; Normalize while prefetching 72 of 384 blocks of clm*128 bytes

;; Copy data from the scratch area back to the FFT data area

	copy_scratch_data_to_fft
	jmp	endlp				;; Skip FFT code if flag not set

;; Fire up auxiliary threads to do pass 1 forward FFT

pass1:	pass1_forward_fft_setup b0b

;; Copy data from the FFT data area back to the scratch area

b0b:	copy_fft_data_to_scratch

;; Do FFT levels 1,2,3,4
;; On input the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	576K	+1K	...
;;	8K	...
;;	...
;;	568K
;; On output the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	48K	+1K	...
;;	8K
;;	...
;;	40K
;;	96K
;;	...

;; Do 48 twelve_complex_first_fft operations
;;	distance between fft data elements is 48K
;;	do 6 macros each processing 192 data values

yes_fft:start_timer 22
	mov	rsi, scratch_area		;; Get address of scratch area
	mov	r15, sincos2			;; Load fixed premultiplier/sin/cos ptr
	mov	rdi, PREMULT_ADDR		;; Calculate pointer to fudge flags
	bump	rdi, 8*clm*ZMM_SCD3+12*clm*ZMM_SCD1+clm*ZMM_SCD4+clm*64+clm*64 ;; Addr of the group multiplier fudge flags
	mov	rdx, norm_grp_mults		;; Addr of the group multipliers
	sub	rbx, rbx			;; Clear register for loading fudge flags index
	mov	r12, compressed_fudges		;; Address of the compressed fudge flags
	mov	r13, r12			;; The XOR masks for fudge flags come from the compressed fudges
	set_data_prefetch_ptrs			;; Init prefetching pointers
pfing	mov	r8, pass2blkdst			;; blkdst
pfing	imul	rax, blkdstreg, (408-24)	;; We've already prefetched 408 of 576 blocks (24 too many)
pfing	add	rcx, rax
	zloop_init 6*8*clm			;; 6 iters of 8*clm
	zloop_set_clm_prefetch ZQUAD		;; Prefetch 96 of 576 blocks of clm*128 bytes
	zr12_csc_wpn_twelve_complex_first_djbfft_preload
b1b:	zr12_csc_wpn_twelve_complex_first_djbfft rsi, 128, 6*clmblkdst, 12*clmblkdst, 24*clmblkdst, r15, 0, rdi, 3, rdx, 0, 8*clm, L1PREFETCH_ALL, 128
	zloop	8*clm, b1b, -8*clm*128+clmblkdst, r15, ZMM_CD12SCD6, rdx, 24*64, r13, 24 ;; 8*clm iters, next source, fixed s/c, grp multipliers pointer, next xor mask
	zloop	6, b1b, -6*clmblkdst		;; Test loop counter, restore source pointer
	end_timer 22

;; Do FFT levels 5,6,7
;; On input the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	48K	+1K	...
;;	8K
;;	...
;;	50K
;;	96K
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	8K	+1K	...
;;	16K
;;	...

;; Do 96 six_complex_fft operations
;;	distance between fft data elements is 8K
;;	do 12 macros each processing 80 data values

	start_timer 24
	bump	rdi, -48*clm*3-8*clm*ZMM_SCD3	;; Sin/cos pointer
	zloop_init  12*8*clm			;; 8 iterations of 8*clm
	zloop_set_clm_prefetch ZDOUBLE		;; Prefetch 96 of 384 blocks of clm*128 bytes
	zr6_six_complex_djbfft_preload
IFDEF OLDDD
b2b:	zr6_six_complex_djbfft rsi, 128, clmblkdst, rdi, ZMM_SCD3, 8*clm, L1PREFETCH_ALL, 128
	zloop	8*clm, b2b, -8*clm*128+6*clmblkdst, rdi, -8*clm*ZMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	12, b2b, -12*6*clmblkdst	;; Test loop counter, restore source pointer
ELSE
b2b:	zr6_six_complex_djbfft rsi, 6*clmblkdst, clmblkdst, rdi, 0, 8, L1PREFETCH_ALL, 6*clmblkdst
	zloop	12, b2b, -12*6*clmblkdst+128, rdi, ZMM_SCD3 ;; Test loop counter, next source pointer, next sin/cos pointer
	zloop	8*clm, b2b, -8*clm*128		;; Test loop counter, restore source pointer
ENDIF
	end_timer 24

;; Do FFT levels 8,9,10
;; On input the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	8K	+1K	...
;;	16K
;;	...
;; The input is swizzled to hold these data values:
;;	0K	+1	+1	+1	+1	+1	+1	+1	8K	+1	...
;;	1K	...
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0K	+1	+1	+1	+1	+1	+1	+1	1K	+1	...
;;	2K
;;	...

;; Do 72 eight_complex_fft operations
;;	distance between fft data elements is 1K
;;	do 6 macros each processing 128 data values

	start_timer 26
	mov	rbp, DATA_ADDR			;; Destination pointer
IFDEF OLDDD
	bump	rdi, -12*clm*ZMM_SCD1		;; Delayed sin/cos multipliers
ELSE
	bump	rdi, -8*clm*ZMM_SCD3-12*clm*ZMM_SCD1 ;; Delayed sin/cos multipliers
ENDIF
	lea	rbx, [rdi-clm*ZMM_SCD4]		;; Complex data sin/cos ptr
	lea	r9, [rbx-clm*64-clm*64]		;; Column weights
	lea	rdx, ZMM_TMPS			;; Computed clm*SCD4 sin/cos values
	mov	r8, pass2blkdst			;; blkdst
	lea	r10, [2*blkdstreg+blkdstreg]	;; 3*blkdst
	lea	r13, [rbp+4*blkdstreg]		;; Dest+4*blkdst pointer
pfing	mov	rcx, PREMULT_PREFETCH		;; Load premult/sin/cos prefetch pointer

	zr8_rsc_complex_only_preload		;; Preload constants before calling zloop_init
;;	zloop_init 12*6*clm			;; 12 iters of 6*clm
	zloop_init 12*6*clm  *2			;; 12 iters of 6*clm iterations	(work around loops "add al, 1" bug)
	zloop_set_prefetch ZDOUBLE		;; Prefetch 144*clm cache lines
	zloop_set_alternate_prefetch_increment 86*2+1, 144*2  ;; Prefetch approximately 86.5*clm cache lines in those 144*clm prefetches
b9a:	zr8_rsc_wpn_eight_complex_calc_sincos r9, rdi, rbx, clm ;; Generate clm*SCD8 sin/cos values from weights, delayed and complex multipliers
	bump	rdi, clm*ZMM_SCD1		;; Next set of delayed multipliers
b9b:	zr8_rsc_wpn_sgreg_eight_complex_fft8 rsi, 1024, 128, 256, 512, rbp, 128, blkdstreg, blkdst3reg, r13, rdx, ZMM_SCD8, clm, L1PREFETCH_DEST_ALL
	zloop	clm, b9b, -clm*1024+clmblkdst, rbp, -clm*128+8*blkdstreg, r13, -clm*128+8*blkdstreg, rdx, -clm*ZMM_SCD8 ;; Loop, next source/dest pointers, restore premult/sine/cosine pointer
	zloop	6, b9b				;; Test loop counter
	zloop	12, b9a				;; Test loop counter
	end_timer 26

;; Work on next 8*clm sets of 1152 values

endlp:	zpass1_get_next_block pass2, c0b, b0b ;; Get next block and jump
done:	zfft_footer
	ENDM



;; Perform a shared pass 1 FFT of length 1280 doubles.
;; Memory examples are for a 1280K FFT with 10 levels done in pass 2.
;; FFT data needed:
;;		8*clm*1280 doubles					Total: 1280*clm 64-byte cache lines
;; Fixed s/c data needed:
;;		10*ZMM_SCD7 (fixed s/c data),				Total: 140 64-byte cache lines
;; Group multipliers needed:
;;		10*16*64 (weights and fudged weights)
;;		10*16*64 (inverse weights and fudged inv. weights),	Total: 320 64-byte cache lines
;; Variable data needed:
;;		clm*64	(column weights)
;;		clm*64	(column inverse weights)
;;		clm*ZMM_SCD4 (complex reduced s/c multipliers)
;;		clm*ZMM_SCD4 (real s/c multipliers)
;;		7*clm*ZMM_SCD1 (delayed s/c multipliers)
;;		8*clm*ZMM_SCD10 (10-complex 20-real sin/cos data)
;;		80*clm*2 (group multiplier fudge flags)
;;		80*clm*2 (big/lit flags),				Total: 196*clm 64-byte cache lines
;; Total memory consumed (add for scratch area, prefetched data, carries):
;;		(1280*3+196=4036)*clm + (140+320+1280/8=620) cache lines
;;		clm=1,2,4:  291KB, 544KB, 1048KB

zr4dwpn_pass1sc1280 MACRO
	LOCAL	b0b, b1b, b2a, b2b, b2c, b8a, b8b, b9aa, b9a, b9b, c0b, c1b, c2a, c2b, c2c, c8b, c9aa, c9a, c9b
	LOCAL	pass1, pass2, endlp, done, yes_fft

	zfft_header pass1

;; Do pass 2 (the last 10 FFT levels if this is a 1280K FFT)

pass2:	start_timer 0
	mov	rax, ZMM_PASS2_ROUTINE
	call	rax
	end_timer 0
	cmp	ffttype, 1		;; We're done if FFTing only
	je	done

;; Do the inverse FFT

	pass1_inverse_fft_setup c0b

;; Do inverse FFT levels 8,9,10
;; On input the 128-byte double cache lines hold these data values:
;;	0K	+1	+1	+1	+1	+1	+1	+1	1K	+1	...
;;	2K
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0K	+1	+1	+1	+1	+1	+1	+1	8K	+1	...
;;	1K	...
;;	...
;; The output cache lines are swizzled to hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	8K	+1K	...
;;	16K
;;	...

;; Do 1 sixteen_reals_unfft operations
;;	distance between fft data elements is 1K
;;	do 0.125 macros each processing 128 data values
;; Do 79 eight_complex_unfft operations
;;	distance between fft data elements is 1K
;;	do 9.875 macros each processing 128 data values

c0b:	start_timer 16
	mov	rbp, DATA_ADDR			;; Get source address
	mov	rsi, scratch_area		;; Get address of scratch area
	set_data_prefetch_ptrs			;; Init prefetching pointers
	mov	r9, PREMULT_ADDR		;; Variable data
	bump	r9, clm*64			;; Inverse column weights
	lea	rbx, [r9+clm*64]		;; Complex data sin/cos ptr
	lea	rdi, [rbx+clm*ZMM_SCD4]		;; Real data sin/cos pointer, will become delayed sin/cos multipliers
	lea	rdx, ZMM_TMPS			;; Computed clm*SCD8 sin/cos values
	mov	r8, pass2blkdst			;; blkdst
	lea	r10, [2*blkdstreg+blkdstreg]	;; 3*blkdst
	lea	r13, [rbp+4*blkdstreg]		;; Source+4*blkdst pointer
	lea	r12, [rbp+8*blkdstreg]		;; L1 prefetch pointer part 1
	bump2	r12, (clm MOD 2), 8*blkdstreg	;; L1 prefetch pointer part 2
	lea	r14, [r12+4*blkdstreg]		;; L1 prefetch pointer + 4*blkdst

	zr8_rsc_complex_and_real_preload	;; Preload constants before calling zloop_init
	zloop_init 10*8*clm			;; 10 iters of 8*clm iterations
	zr8_rsc_wpn_eight_complex_calc_sincos_simple r9, rbx, clm ;; Generate clm*SCD8 sin/cos values from delayed and complex multipliers
	zloop_set_clm_prefetch ZDOUBLE		;; Prefetch 80 of 640 blocks of clm*128 bytes
c8b:	zr8_rsc_wpn_sgreg_2sc_sixteen_reals_unfft8 rbp, 128, blkdstreg, blkdst3reg, r13, rsi, 1024, 128, 256, 512, rdx, ZMM_SCD8, r9, 64, rdi, ZMM_SCD4, clm, L1PREFETCH_ALL, r12, r14 ;; Prefetch short dist ahead
	zloop	clm, c8b, -clm*1024+clmblkdst, rbp, -clm*128+8*blkdstreg, r13, -clm*128+8*blkdstreg, rdx, -clm*ZMM_SCD8, r9, -clm*64, r12, -clm*128+8*blkdstreg, r14, -clm*128+8*blkdstreg ;; Loop, next src/dest ptrs, restore sin/cos ptr
	jmp	c9aa

	loops_reset
c9a:	zr8_rsc_wpn_eight_complex_calc_sincos r9, rdi, rbx, clm ;; Generate clm*SCD8 sin/cos values from delayed and complex multipliers
	bump	rdi, clm*ZMM_SCD1		;; Next set of delayed sin/cos multipliers
c9aa:	zloop_set_clm_prefetch ZDOUBLE		;; Prefetch 80 of 640 blocks of clm*128 bytes
c9b:	zr8_rsc_wpn_sgreg_eight_complex_unfft8 rbp, 128, blkdstreg, blkdst3reg, r13, rsi, 1024, 128, 256, 512, rdx, ZMM_SCD8, clm, L1PREFETCH_ALL, r12, r14 ;; Prefetch short dist ahead
	zloop	clm, c9b, -clm*1024+clmblkdst, rbp, -clm*128+8*blkdstreg, r13, -clm*128+8*blkdstreg, rdx, -clm*ZMM_SCD8, r12, -clm*128+8*blkdstreg, r14, -clm*128+8*blkdstreg ;; Loop, next src/dest ptrs, restore sin/cos ptr
	zloop	10, c9b				;; Test loop counter
	zloop	8, c9a, -8*10*clmblkdst		;; Test loop counter, restore source pointer
	end_timer 16

;; Do inverse FFT levels 5,6,7
;; On input the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	8K	+1K	...
;;	16K
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	80K	+1K	...
;;	8K
;;	...
;;	72K
;;	160K
;;	...

;; Do 8 twenty_reals_unfft operations
;; Do 56 ten_complex_unfft operations
;;	distance between fft data elements is 8K
;;	do 8 macros each processing 160 data values

	start_timer 18
	zloop_init  8*8*clm			;; 8 iterations of 8*clm
	zloop_set_clm_prefetch ZQUAD		;; Prefetch 128 of 640 blocks of clm*128 bytes
IFDEF OLDDD
	zr10_csc_twenty_reals_unfft_preload
c2a:	zr10_csc_twenty_reals_unfft rsi, 128, clmblkdst, rdi, ZMM_SCD10, 8*clm, L1PREFETCH_ALL, 128
	zloop	8*clm, c2a, -8*clm*128+10*clmblkdst, rdi, -8*clm*ZMM_SCD10 ;; Test loop counter, next source pointer, restore sin/cos pointer
	loops_reset				;; 7 more iters of 8*clm
	zr10_ten_complex_djbunfft_preload
c2b:	zr10_ten_complex_djbunfft rsi, 128, clmblkdst, rdi, ZMM_SCD10, 8*clm, L1PREFETCH_ALL, 128
	zloop	8*clm, c2b, -8*clm*128+10*clmblkdst, rdi, -8*clm*ZMM_SCD10 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	8, c2b, -8*10*clmblkdst, rdi, 8*clm*ZMM_SCD10 ;; Test loop counter, restore source pointer, next sin/cos pointer
ELSE
c2a:	zr10_csc_twenty_reals_unfft_preload
	zr10_csc_twenty_reals_unfft rsi, 10*clmblkdst, clmblkdst, rdi, 0, 1, L1PREFETCH_ALL, 10*clmblkdst
	zr10_ten_complex_djbunfft_preload
	jmp	c2c				;; 7 more iters
c2b:	zr10_ten_complex_djbunfft rsi, 10*clmblkdst, clmblkdst, rdi, 0, 1, L1PREFETCH_ALL, 10*clmblkdst
c2c:	zloop	8, c2b, -8*10*clmblkdst+128, rdi, ZMM_SCD10 ;; Test loop counter, next source pointer, next sin/cos pointer
	zloop	8*clm, c2a, -8*clm*128		;; Test loop counter, restore source pointer
ENDIF
	end_timer 18

;; Do inverse FFT levels 1,2,3,4
;; On input the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	80K	+1K	...
;;	8K
;;	...
;;	72K
;;	160K
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	640K	+1K	...
;;	8K
;;	...
;;	632K

;; Do 80 sixteen_reals_unfft operations
;;	distance between fft data elements is 80K
;;	do 10 macros each processing 128 data values

	start_timer 20
	mov	r15, sincos2			;; Load fixed premultiplier/sin/cos ptr
	sub	rbx, rbx			;; Clear register for loading fudge flags index
	mov	r12, compressed_fudges		;; Address of the compressed fudge flags
	mov	r13, r12			;; The XOR masks for fudge flags come from the compressed fudges
	zloop_init 10*8*clm			;; 10 iters of 8*clm
	zloop_set_clm_prefetch ZDOUBLE		;; Prefetch 80 of 640 blocks of clm*128 bytes
	zr8_wpn_sixteen_reals_last_unfft_preload
c1b:	zr8_wpn_sixteen_reals_last_unfft rsi, 128, 10*clmblkdst, 20*clmblkdst, 40*clmblkdst, r15, 0, rdi, 2, 8*clm, L1PREFETCH_ALL, 128
	zloop	8*clm, c1b, -8*clm*128+clmblkdst, r15, ZMM_SCD7, rdx, 16*64, r13, 16 ;; 8*clm iterations, next fixed s/c, next grp multipliers, next XOR masks
	zloop	10, c1b				;; Test loop counter
	end_timer 20

;; Normalize these values

	pass1_normalize yes_fft			;; Normalize while prefetching 80 of 640 blocks of clm*128 bytes

;; Copy data from the scratch area back to the FFT data area

	copy_scratch_data_to_fft
	jmp	endlp				;; Skip FFT code if flag not set

;; Fire up auxiliary threads to do pass 1 forward FFT

pass1:	pass1_forward_fft_setup b0b

;; Copy data from the FFT data area back to the scratch area

b0b:	copy_fft_data_to_scratch

;; Do FFT levels 1,2,3,4
;; On input the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	640K	+1K	...
;;	8K	...
;;	...
;;	632K
;; On output the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	80K	+1K	...
;;	8K
;;	...
;;	72K
;;	160K
;;	...

;; Do 80 sixteen_reals_first_fft operations
;;	distance between fft data elements is 80K
;;	do 10 macros each processing 128 data values

yes_fft:start_timer 22
	mov	rsi, scratch_area		;; Get address of scratch area
	mov	r15, sincos2			;; Load fixed premultiplier/sin/cos ptr
	mov	rdi, PREMULT_ADDR		;; Calculate pointer to fudge flags
	bump	rdi, 8*clm*ZMM_SCD10+7*clm*ZMM_SCD1+clm*ZMM_SCD4+clm*ZMM_SCD4+clm*64+clm*64 ;; Addr of the group multiplier fudge flags
	mov	rdx, norm_grp_mults		;; Addr of the group multipliers
	sub	rbx, rbx			;; Clear register for loading fudge flags index
	mov	r12, compressed_fudges		;; Address of the compressed fudge flags
	mov	r13, r12			;; The XOR masks for fudge flags come from the compressed fudges
	set_data_prefetch_ptrs			;; Init prefetching pointers
pfing	mov	r8, pass2blkdst			;; blkdst
pfing	imul	rax, blkdstreg, (368-16)	;; We've already prefetched 368 of 640 blocks (16 too many)
pfing	add	rcx, rax
	zloop_init 10*8*clm			;; 10 iters of 8*clm
	zloop_set_clm_prefetch ZQUAD		;; Prefetch 160 of 640 blocks of clm*128 bytes
	zr8_wpn_sixteen_reals_first_fft_preload
b1b:	zr8_wpn_sixteen_reals_first_fft rsi, 128, 10*clmblkdst, 20*clmblkdst, 40*clmblkdst, r15, 0, rdi, 2, rdx, 0, 8*clm, L1PREFETCH_ALL, 128
	zloop	8*clm, b1b, -8*clm*128+clmblkdst, r15, ZMM_SCD7, rdx, 16*64, r13, 16 ;; 8*clm iters prefetching 8*clm cache lines, next source, fixed s/c, grp multipliers pointer, next XOR masks
	zloop	10, b1b, -10*clmblkdst		;; Test loop counter, restore source pointer
	end_timer 22

;; Do FFT levels 5,6,7
;; On input the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	80K	+1K	...
;;	8K
;;	...
;;	72K
;;	160K
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	8K	+1K	...
;;	16K
;;	...

;; Do 8 twenty_reals_fft operations
;; Do 56 ten_complex_fft operations
;;	distance between fft data elements is 8K
;;	do 8 macros each processing 160 data values

	start_timer 24
	bump	rdi, -80*clm*2-8*clm*ZMM_SCD10	;; Sin/cos pointer
	zloop_init  8*8*clm			;; 8 iterations of 8*clm
	zloop_set_clm_prefetch ZQUAD		;; Prefetch 128 of 640 blocks of clm*128 bytes
IFDEF OLDDD
	zr10_csc_twenty_reals_fft_preload
b2a:	zr10_csc_twenty_reals_fft rsi, 128, clmblkdst, rdi, ZMM_SCD10, 8*clm, L1PREFETCH_ALL, 128
	zloop	8*clm, b2a, -8*clm*128+10*clmblkdst, rdi, -8*clm*ZMM_SCD10 ;; Test loop counter, next source pointer, restore sin/cos pointer
	loops_reset				;; 7 more iters of 8*clm
	zr10_ten_complex_djbfft_preload
b2b:	zr10_ten_complex_djbfft rsi, 128, clmblkdst, rdi, ZMM_SCD10, 8*clm, L1PREFETCH_ALL, 128
	zloop	8*clm, b2b, -8*clm*128+10*clmblkdst, rdi, -8*clm*ZMM_SCD10 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	8, b2b, -8*10*clmblkdst		;; Test loop counter, restore source pointer
ELSE
b2a:	zr10_csc_twenty_reals_fft_preload
	zr10_csc_twenty_reals_fft rsi, 10*clmblkdst, clmblkdst, rdi, 0, 1, L1PREFETCH_ALL, 10*clmblkdst
	zr10_ten_complex_djbfft_preload
	jmp	b2c				;; 7 more iters
b2b:	zr10_ten_complex_djbfft rsi, 10*clmblkdst, clmblkdst, rdi, 0, 1, L1PREFETCH_ALL, 10*clmblkdst
b2c:	zloop	8, b2b, -8*10*clmblkdst+128, rdi, ZMM_SCD10 ;; Test loop counter, next source pointer, next sin/cos pointer
	zloop	8*clm, b2a, -8*clm*128		;; Test loop counter, restore source pointer
ENDIF
	end_timer 24

;; Do FFT levels 8,9,10
;; On input the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	8K	+1K	...
;;	16K
;;	...
;; The input is swizzled to hold these data values:
;;	0K	+1	+1	+1	+1	+1	+1	+1	8K	+1	...
;;	1K	...
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0K	+1	+1	+1	+1	+1	+1	+1	1K	+1	...
;;	2K
;;	...

;; Do 1 sixteen_reals_fft operations
;;	distance between fft data elements is 1K
;;	do 0.125 macros each processing 128 data values
;; Do 79 eight_complex_fft operations
;;	distance between fft data elements is 1K
;;	do 9.875 macros each processing 128 data values

	start_timer 26
IFDEF OLDDD
	bump	rdi, -7*clm*ZMM_SCD1-clm*ZMM_SCD4 ;; Real data sin/cos pointer, will become delayed sin/cos multipliers
ELSE
	bump	rdi, -8*clm*ZMM_SCD10-7*clm*ZMM_SCD1-clm*ZMM_SCD4 ;; Real data sin/cos pointer, will become delayed sin/cos multipliers
ENDIF
	lea	rbx, [rdi-clm*ZMM_SCD4]		;; Complex data sin/cos ptr
	lea	r9, [rbx-clm*64-clm*64]		;; Column weights
	lea	rdx, ZMM_TMPS			;; Computed clm*SCD4 sin/cos values
	mov	rbp, DATA_ADDR			;; Destination pointer
	mov	r8, pass2blkdst			;; blkdst
	lea	r10, [2*blkdstreg+blkdstreg]	;; 3*blkdst
	lea	r13, [rbp+4*blkdstreg]		;; Dest+4*blkdst pointer

pfing	mov	rcx, PREMULT_PREFETCH		;; Load premult/sin/cos prefetch pointer

	zr8_rsc_complex_and_real_preload	;; Preload constants before calling zloop_init
	zloop_init 10*8*clm			;; 10 iters of 8*clm
	zloop_set_prefetch ZTRIPLE		;; Prefetch 240*clm times
	zloop_set_alternate_prefetch_increment 196, 240  ;; Prefetch approximately 196*clm cache lines in those 240*clm prefetches
	zr8_rsc_wpn_eight_complex_calc_sincos_simple r9, rbx, clm ;; Generate clm*SCD8 sin/cos values from delayed and complex multipliers
b8b:	zr8_rsc_wpn_sgreg_2sc_sixteen_reals_fft8 rsi, 1024, 128, 256, 512, rbp, 128, blkdstreg, blkdst3reg, r13, rdx, ZMM_SCD8, r9, 64, rdi, ZMM_SCD4, clm, L1PREFETCH_DEST_ALL
	zloop	clm, b8b, -clm*1024+clmblkdst, rbp, -clm*128+8*blkdstreg, r13, -clm*128+8*blkdstreg, rdx, -clm*ZMM_SCD8, r9, -clm*64 ;; Loop, next source/dest pointers, restore premult/sine/cosine pointer
	jmp	b9aa

	loops_reset
b9a:	zr8_rsc_wpn_eight_complex_calc_sincos r9, rdi, rbx, clm ;; Generate clm*SCD8 sin/cos values from delayed and complex multipliers
	bump	rdi, clm*ZMM_SCD1		;; Next set of delayed multipliers
b9aa:
b9b:	zr8_rsc_wpn_sgreg_eight_complex_fft8 rsi, 1024, 128, 256, 512, rbp, 128, blkdstreg, blkdst3reg, r13, rdx, ZMM_SCD8, clm, L1PREFETCH_DEST_ALL
	zloop	clm, b9b, -clm*1024+clmblkdst, rbp, -clm*128+8*blkdstreg, r13, -clm*128+8*blkdstreg, rdx, -clm*ZMM_SCD8 ;; Loop, next source/dest pointers, restore premult/sine/cosine pointer
	zloop	10, b9b				;; Test loop counter
	zloop	8, b9a				;; Test loop counter
	end_timer 26

;; Work on next 8*clm sets of 1280 values

endlp:	zpass1_get_next_block pass2, c0b, b0b ;; Get next block and jump
done:	zfft_footer
	ENDM


;; Perform a shared pass 1 negacyclic FFT of length 1280 doubles.
;; Memory examples are for a 1280K FFT with 10 levels done in pass 2.
;; FFT data needed:
;;		8*clm*1280 doubles					Total: 1280*clm 64-byte cache lines
;; Fixed s/c data needed:
;;		10*ZMM_CD8SCD4 (complex premultipliers and fixed s/c data), Total: 160 64-byte cache lines
;; Group multipliers needed:
;;		10*16*64 (weights)
;;		10*16*64 (inverse weights),				Total: 320 64-byte cache lines
;; Variable data needed:
;;		clm*64	(column weights)
;;		clm*64	(column inverse weights)
;;		clm*ZMM_SCD4 (complex reduced s/c multipliers)
;;		8*clm*ZMM_SCD1 (delayed s/c multipliers)
;;		8*clm*ZMM_SCD5 (10-complex sin/cos data)
;;		80*clm*2 (group multiplier fudge flags)
;;		80*clm*2 (big/lit flags),				Total: 111*clm 64-byte cache lines
;; Total memory consumed (add for scratch area, prefetched data, carries):
;;		(1280*3+111=3951)*clm + (160+320+1280/8=640) cache lines
;;		clm=1,2,4:  287KB, 534KB, 1028KB

zr4dwpn_pass1sc1280ac MACRO
	LOCAL	b0b, b1b, b2b, b9a, b9b, c0b, c1b, c2b, c9a, c9b
	LOCAL	pass1, pass2, endlp, done, yes_fft

	zfft_header pass1

;; Do pass 2 (the last 10 FFT levels if this is a 1280K FFT)

pass2:	start_timer 0
	mov	rax, ZMM_PASS2_ROUTINE
	call	rax
	end_timer 0
	cmp	ffttype, 1		;; We're done if FFTing only
	je	done

;; Do the inverse FFT

	pass1_inverse_fft_setup c0b

;; Do inverse FFT levels 8.9.10
;; On input the 128-byte double cache lines hold these data values:
;;	0K	+1	+1	+1	+1	+1	+1	+1	1K	+1	...
;;	2K
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0K	+1	+1	+1	+1	+1	+1	+1	8K	+1	...
;;	1K	...
;;	...
;; The output cache lines are swizzled to hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	8K	+1K	...
;;	16K
;;	...

;; Do 80 eight_complex_unfft operations
;;	distance between fft data elements is 1K
;;	do 10 macros each processing 128 data values

c0b:	start_timer 16
	mov	rbp, DATA_ADDR			;; Get source address
	mov	rsi, scratch_area		;; Get address of scratch area
	set_data_prefetch_ptrs			;; Init prefetching pointers
	mov	r9, PREMULT_ADDR		;; Variable data
	bump	r9, clm*64			;; Inverse column weights
	lea	rbx, [r9+clm*64]		;; Complex data sin/cos ptr
	lea	rdi, [rbx+clm*ZMM_SCD4]		;; Delayed sin/cos multipliers
	lea	rdx, ZMM_TMPS			;; Computed clm*SCD8 sin/cos values
	mov	r8, pass2blkdst			;; blkdst
	lea	r10, [2*blkdstreg+blkdstreg]	;; 3*blkdst
	lea	r13, [rbp+4*blkdstreg]		;; Source+4*blkdst pointer
	lea	r12, [rbp+8*blkdstreg]		;; L1 prefetch pointer part 1
	bump2	r12, (clm MOD 2), 8*blkdstreg	;; L1 prefetch pointer part 2
	lea	r14, [r12+4*blkdstreg]		;; L1 prefetch pointer + 4*blkdst

	zr8_rsc_complex_only_preload		;; Preload constants before calling zloop_init
	zloop_init 10*8*clm			;; 10 iters of 8*clm iterations
	zloop_set_clm_prefetch ZDOUBLE		;; Prefetch 80 of 640 blocks of clm*128 bytes
c9a:	zr8_rsc_wpn_eight_complex_calc_sincos r9, rdi, rbx, clm ;; Generate clm*SCD8 sin/cos values from weights, delayed, and complex multipliers
	bump	rdi, clm*ZMM_SCD1		;; Next set of delayed sin/cos multipliers
c9b:	zr8_rsc_wpn_sgreg_eight_complex_unfft8 rbp, 128, blkdstreg, blkdst3reg, r13, rsi, 1024, 128, 256, 512, rdx, ZMM_SCD8, clm, L1PREFETCH_ALL, r12, r14 ;; Prefetch short dist ahead
	zloop	clm, c9b, -clm*1024+clmblkdst, rbp, -clm*128+8*blkdstreg, r13, -clm*128+8*blkdstreg, rdx, -clm*ZMM_SCD8, r12, -clm*128+8*blkdstreg, r14, -clm*128+8*blkdstreg ;; Loop, next src/dest ptrs, restore sin/cos ptr
	zloop	10, c9b				;; Test loop counter
	zloop	8, c9a, -8*10*clmblkdst		;; Test loop counter, restore source pointer
	end_timer 16

;; Do inverse FFT levels 5,6,7
;; On input the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	8K	+1K	...
;;	16K
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	80K	+1K	...
;;	8K
;;	...
;;	72K
;;	160K
;;	...

;; Do 64 ten_complex_unfft operations
;;	distance between fft data elements is 8K
;;	do 8 macros each processing 160 data values

	start_timer 18
	zloop_init  8*8*clm			;; 8 iterations of 8*clm
	zloop_set_clm_prefetch ZQUAD		;; Prefetch 128 of 640 blocks of clm*128 bytes
	zr10_ten_complex_djbunfft_preload
IFDEF OLDDD
c2b:	zr10_ten_complex_djbunfft rsi, 128, clmblkdst, rdi, ZMM_SCD5, 8*clm, L1PREFETCH_ALL, 128
	zloop	8*clm, c2b, -8*clm*128+10*clmblkdst, rdi, -8*clm*ZMM_SCD5 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	8, c2b, -8*10*clmblkdst, rdi, 8*clm*ZMM_SCD5 ;; Test loop counter, restore source pointer, next sin/cos pointer
ELSE
c2b:	zr10_ten_complex_djbunfft rsi, 10*clmblkdst, clmblkdst, rdi, 0, 8, L1PREFETCH_ALL, 10*clmblkdst
	zloop	8, c2b, -8*10*clmblkdst+128, rdi, ZMM_SCD5 ;; Test loop counter, next source pointer, next sin/cos pointer
	zloop	8*clm, c2b, -8*clm*128		;; Test loop counter, restore source pointer
ENDIF
	end_timer 18

;; Do inverse FFT levels 1,2,3,4
;; On input the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	80K	+1K	...
;;	8K
;;	...
;;	72K
;;	160K
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	640K	+1K	...
;;	8K	...
;;	...
;;	632K

;; Do 80 eight_complex_unfft operations
;;	distance between fft data elements is 80K
;;	do 10 macros each processing 128 data values

	start_timer 20
	mov	r15, sincos2			;; Load fixed premultiplier/sin/cos ptr
	sub	rbx, rbx			;; Clear register for loading fudge flags index
	mov	r12, compressed_fudges		;; Address of the compressed fudge flags
	mov	r13, r12			;; The XOR masks for fudge flags come from the compressed fudges
	zloop_init 10*8*clm			;; 10 iters of 8*clm
	zloop_set_clm_prefetch ZDOUBLE		;; Prefetch 80 of 640 blocks of clm*128 bytes
	zr8_csc_wpn_eight_complex_last_djbunfft_preload
c1b:	zr8_csc_wpn_eight_complex_last_djbunfft rsi, 128, 10*clmblkdst, 20*clmblkdst, 40*clmblkdst, r15, 0, rdi, 2, 8*clm, L1PREFETCH_ALL, 128
	zloop	8*clm, c1b, -8*clm*128+clmblkdst, r15, ZMM_CD8SCD4, rdx, 16*64, r13, 16 ;; 8*clm iterations, next fixed s/c, next grp multipliers, next XOR masks
	zloop	10, c1b				;; 10 iterations
	end_timer 20


;; Normalize these values

	pass1_normalize yes_fft			;; Normalize while prefetching 80 of 640 blocks of clm*128 bytes

;; Copy data from the scratch area back to the FFT data area

	copy_scratch_data_to_fft
	jmp	endlp				;; Skip FFT code if flag not set

;; Fire up auxiliary threads to do pass 1 forward FFT

pass1:	pass1_forward_fft_setup b0b

;; Copy data from the FFT data area back to the scratch area

b0b:	copy_fft_data_to_scratch

;; Do FFT levels 1,2,3,4
;; On input the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	640K	+1K	...
;;	8K	...
;;	...
;;	632K
;; On output the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	80K	+1K	...
;;	8K
;;	...
;;	72K
;;	160K
;;	...

;; Do 80 eight_complex_first_fft operations
;;	distance between fft data elements is 80K
;;	do 10 macros each processing 128 data values

yes_fft:start_timer 22
	mov	rsi, scratch_area		;; Get address of scratch area
	mov	r15, sincos2			;; Load fixed premultiplier/sin/cos ptr
	mov	rdi, PREMULT_ADDR		;; Calculate pointer to fudge flags
	bump	rdi, 8*clm*ZMM_SCD5+8*clm*ZMM_SCD1+clm*ZMM_SCD4+clm*64+clm*64 ;; Addr of the group multiplier fudge flags
	mov	rdx, norm_grp_mults		;; Addr of the group multipliers
	sub	rbx, rbx			;; Clear register for loading fudge flags index
	mov	r12, compressed_fudges		;; Address of the compressed fudge flags
	mov	r13, r12			;; The XOR masks for fudge flags come from the compressed fudges
	set_data_prefetch_ptrs			;; Init prefetching pointers
pfing	mov	r8, pass2blkdst			;; blkdst
pfing	imul	rax, blkdstreg, (368-16)	;; We've already prefetched 368 of 640 blocks (16 too many)
pfing	add	rcx, rax
	zloop_init 10*8*clm			;; 10 iters of 8*clm
	zloop_set_clm_prefetch ZQUAD		;; Prefetch 160 of 640 blocks of clm*128 bytes
	zr8_csc_wpn_eight_complex_first_djbfft_preload
b1b:	zr8_csc_wpn_eight_complex_first_djbfft rsi, 128, 10*clmblkdst, 20*clmblkdst, 40*clmblkdst, r15, 0, rdi, 2, rdx, 0, 8*clm, L1PREFETCH_ALL, 128
	zloop	8*clm, b1b, -8*clm*128+clmblkdst, r15, ZMM_CD8SCD4, rdx, 16*64, r13, 16 ;; 8*clm iters, next source, fixed s/c, grp multipliers pointer, next xor mask
	zloop	10, b1b, -10*clmblkdst		;; Test loop counter, restore source pointer
	end_timer 22

;; Do FFT levels 5,6,7
;; On input the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	80K	+1K	...
;;	8K
;;	...
;;	72K
;;	160K
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	8K	+1K	...
;;	16K
;;	...

;; Do 64 ten_complex_fft operations
;;	distance between fft data elements is 8K
;;	do 8 macros each processing 160 data values

	start_timer 24
	bump	rdi, -80*clm*2-8*clm*ZMM_SCD5	;; Sin/cos pointer
	zloop_init  8*8*clm			;; 8 iterations of 8*clm
	zloop_set_clm_prefetch ZQUAD		;; Prefetch 128 of 640 blocks of clm*128 bytes
	zr10_ten_complex_djbfft_preload
IFDEF OLDDD
b2b:	zr10_ten_complex_djbfft rsi, 128, clmblkdst, rdi, ZMM_SCD5, 8*clm, L1PREFETCH_ALL, 128
	zloop	8*clm, b2b, -8*clm*128+10*clmblkdst, rdi, -8*clm*ZMM_SCD5 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	8, b2b, -8*10*clmblkdst		;; Test loop counter, restore source pointer
ELSE
b2b:	zr10_ten_complex_djbfft rsi, 10*clmblkdst, clmblkdst, rdi, 0, 8, L1PREFETCH_ALL, 10*clmblkdst
	zloop	8, b2b, -8*10*clmblkdst+128, rdi, ZMM_SCD5 ;; Test loop counter, next source pointer, next sin/cos pointer
	zloop	8*clm, b2b, -8*clm*128		;; Test loop counter, restore source pointer
ENDIF
	end_timer 24

;; Do FFT levels 8,9,10
;; On input the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	8K	+1K	...
;;	16K
;;	...
;; The input is swizzled to hold these data values:
;;	0K	+1	+1	+1	+1	+1	+1	+1	8K	+1	...
;;	1K	...
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0K	+1	+1	+1	+1	+1	+1	+1	1K	+1	...
;;	2K
;;	...

;; Do 80 eight_complex_fft operations
;;	distance between fft data elements is 1K
;;	do 10 macros each processing 128 data values

	start_timer 26
	mov	rbp, DATA_ADDR			;; Destination pointer
IFDEF OLDDD
	bump	rdi, -8*clm*ZMM_SCD1		;; Delayed sin/cos multipliers
ELSE
	bump	rdi, -8*clm*ZMM_SCD5-8*clm*ZMM_SCD1 ;; Delayed sin/cos multipliers
ENDIF
	lea	rbx, [rdi-clm*ZMM_SCD4]		;; Complex data sin/cos ptr
	lea	r9, [rbx-clm*64-clm*64]		;; Column weights
	lea	rdx, ZMM_TMPS			;; Computed clm*SCD4 sin/cos values
	mov	r8, pass2blkdst			;; blkdst
	lea	r10, [2*blkdstreg+blkdstreg]	;; 3*blkdst
	lea	r13, [rbp+4*blkdstreg]		;; Dest+4*blkdst pointer
pfing	mov	rcx, PREMULT_PREFETCH		;; Load premult/sin/cos prefetch pointer

	zr8_rsc_complex_only_preload		;; Preload constants before calling zloop_init
	zloop_init 10*8*clm			;; 10 iters of 8*clm
	zloop_set_prefetch ZDOUBLE		;; Prefetch 160*clm times
	zloop_set_alternate_prefetch_increment 111, 160  ;; Prefetch approximately 111*clm cache lines in those 160*clm prefetches
b9a:	zr8_rsc_wpn_eight_complex_calc_sincos r9, rdi, rbx, clm ;; Generate clm*SCD8 sin/cos values from weights, delayed and complex multipliers
	bump	rdi, clm*ZMM_SCD1		;; Next set of delayed multipliers
b9b:	zr8_rsc_wpn_sgreg_eight_complex_fft8 rsi, 1024, 128, 256, 512, rbp, 128, blkdstreg, blkdst3reg, r13, rdx, ZMM_SCD8, clm, L1PREFETCH_DEST_ALL
	zloop	clm, b9b, -clm*1024+clmblkdst, rbp, -clm*128+8*blkdstreg, r13, -clm*128+8*blkdstreg, rdx, -clm*ZMM_SCD8 ;; Loop, next source/dest pointers, restore premult/sine/cosine pointer
	zloop	10, b9b				;; Test loop counter
	zloop	8, b9a				;; Test loop counter
	end_timer 26

;; Work on next 8*clm sets of 1280 values

endlp:	zpass1_get_next_block pass2, c0b, b0b ;; Get next block and jump
done:	zfft_footer
	ENDM


;; Perform a shared pass 1 FFT of length 1344 doubles.
;; Memory examples are for a 1344K FFT with 10 levels done in pass 2.
;; FFT data needed:
;;		8*clm*1344 doubles					Total: 1344*clm 64-byte cache lines
;; Fixed s/c data needed:
;;		7*ZMM_SCD11 (fixed s/c data),				Total: 154 64-byte cache lines
;; Group multipliers needed:
;;		7*24*64 (weights)
;;		7*24*64 (inverse weights),				Total: 336 64-byte cache lines
;; Variable data needed:
;;		clm*64	(column weights)
;;		clm*64	(column inverse weights)
;;		clm*ZMM_SCD4 (complex reduced s/c multipliers)
;;		clm*ZMM_SCD4 (real s/c multipliers)
;;		11*clm*ZMM_SCD1 (delayed s/c multipliers)
;;		8*clm*ZMM_SCD6 (7-complex 14-real sin/cos data)
;;		56*clm*3 (group multiplier fudge flags)
;;		56*clm*3 (biglit flags),				Total: 141.25*clm 64-byte cache lines
;; Total memory consumed (add for scratch area, prefetched data, carries):
;;		(1344*3+141.25=4173.25)*clm + (154+336+1344/8=658) cache lines
;;		clm=1,2,4:  302KB, 563KB, 1085KB

zr4dwpn_pass1sc1344 MACRO
	LOCAL	b0b, b1b, b2a, b2b, b2c, b8a, b8b, b9aa, b9a, b9b, c0b, c1b, c2a, c2b, c2c, c8b, c9aa, c9a, c9b
	LOCAL	pass1, pass2, endlp, done, yes_fft

	zfft_header pass1

;; Do pass 2 (the last 10 FFT levels if this is a 1344K FFT)

pass2:	start_timer 0
	mov	rax, ZMM_PASS2_ROUTINE
	call	rax
	end_timer 0
	cmp	ffttype, 1		;; We're done if FFTing only
	je	done

;; Do the inverse FFT

	pass1_inverse_fft_setup c0b

;; Do inverse FFT levels 8,9,10
;; On input the 128-byte double cache lines hold these data values:
;;	0K	+1	+1	+1	+1	+1	+1	+1	1K	+1	...
;;	2K
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0K	+1	+1	+1	+1	+1	+1	+1	8K	+1	...
;;	1K	...
;;	...
;; The output cache lines are swizzled to hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	8K	+1K	...
;;	16K
;;	...

;; Do 1 sixteen_reals_unfft operations
;;	distance between fft data elements is 1K
;;	do 0.125 macros each processing 128 data values
;; Do 83 eight_complex_unfft operations
;;	distance between fft data elements is 1K
;;	do 10.375 macros each processing 128 data values

c0b:	start_timer 16
	mov	rbp, DATA_ADDR			;; Get source address
	mov	rsi, scratch_area		;; Get address of scratch area
	set_data_prefetch_ptrs			;; Init prefetching pointers
	mov	r9, PREMULT_ADDR		;; Variable data
	bump	r9, clm*64			;; Inverse column weights
	lea	rbx, [r9+clm*64]		;; Complex data sin/cos ptr
	lea	rdi, [rbx+clm*ZMM_SCD4]		;; Real data sin/cos pointer, will become delayed sin/cos multipliers
	lea	rdx, ZMM_TMPS			;; Computed clm*SCD8 sin/cos values
	mov	r8, pass2blkdst			;; blkdst
	lea	r10, [2*blkdstreg+blkdstreg]	;; 3*blkdst
	lea	r13, [rbp+4*blkdstreg]		;; Source+4*blkdst pointer
	lea	r12, [rbp+8*blkdstreg]		;; L1 prefetch pointer part 1
	bump2	r12, (clm MOD 2), 8*blkdstreg	;; L1 prefetch pointer part 2
	lea	r14, [r12+4*blkdstreg]		;; L1 prefetch pointer + 4*blkdst

	zr8_rsc_complex_and_real_preload	;; Preload constants before calling zloop_init
;;	zloop_init 12*7*clm			;; 12 iters of 7*clm iterations
	zloop_init 12*7*clm  *2			;; 12 iters of 7*clm iterations	(work around loops "add al, 1" bug)
	zloop_set_clm_prefetch ZDOUBLE		;; Prefetch 84 of 672 blocks of clm*128 bytes
	zr8_rsc_wpn_eight_complex_calc_sincos_simple r9, rbx, clm ;; Generate clm*SCD8 sin/cos values from delayed and complex multipliers
c8b:	zr8_rsc_wpn_sgreg_2sc_sixteen_reals_unfft8 rbp, 128, blkdstreg, blkdst3reg, r13, rsi, 1024, 128, 256, 512, rdx, ZMM_SCD8, r9, 64, rdi, ZMM_SCD4, clm, L1PREFETCH_ALL, r12, r14 ;; Prefetch short dist ahead
	zloop	clm, c8b, -clm*1024+clmblkdst, rbp, -clm*128+8*blkdstreg, r13, -clm*128+8*blkdstreg, rdx, -clm*ZMM_SCD8, r9, -clm*64, r12, -clm*128+8*blkdstreg, r14, -clm*128+8*blkdstreg ;; Loop, next src/dest ptrs, restore sin/cos ptr
	jmp	c9aa

	loops_reset
c9a:	zr8_rsc_wpn_eight_complex_calc_sincos r9, rdi, rbx, clm ;; Generate clm*SCD8 sin/cos values from delayed and complex multipliers
	bump	rdi, clm*ZMM_SCD1		;; Next set of delayed sin/cos multipliers
c9aa:
c9b:	zr8_rsc_wpn_sgreg_eight_complex_unfft8 rbp, 128, blkdstreg, blkdst3reg, r13, rsi, 1024, 128, 256, 512, rdx, ZMM_SCD8, clm, L1PREFETCH_ALL, r12, r14 ;; Prefetch short dist ahead
	zloop	clm, c9b, -clm*1024+clmblkdst, rbp, -clm*128+8*blkdstreg, r13, -clm*128+8*blkdstreg, rdx, -clm*ZMM_SCD8, r12, -clm*128+8*blkdstreg, r14, -clm*128+8*blkdstreg ;; Loop, next src/dest ptrs, restore sin/cos ptr
	zloop	7, c9b				;; Test loop counter
	zloop	12, c9a, -12*7*clmblkdst	;; Test loop counter, restore source pointer
	end_timer 16

;; Do inverse FFT levels 5,6,7
;; On input the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	8K	+1K	...
;;	16K
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	56K	+1K	...
;;	8K
;;	...
;;	48K
;;	112K
;;	...

;; Do 8 fourteen_reals_unfft operations
;; Do 88 seven_complex_unfft operations
;;	distance between fft data elements is 8K
;;	do 12 macros each processing 112 data values

	start_timer 18
	zloop_init  12*8*clm			;; 12 iterations of 8*clm
	zloop_set_clm_prefetch ZQUAD		;; Prefetch 192 of 672 blocks of clm*128 bytes
IFDEF OLDDD
	zr7_csc_fourteen_reals_unfft_preload
c2a:	zr7_csc_fourteen_reals_unfft rsi, 128, clmblkdst, rdi, ZMM_SCD6, 8*clm, L1PREFETCH_ALL, 128
	zloop	8*clm, c2a, -8*clm*128+7*clmblkdst, rdi, -8*clm*ZMM_SCD6 ;; Test loop counter, next source pointer, restore sin/cos pointer
	loops_reset				;; 11 more iters of 8*clm
	zr7_seven_complex_djbunfft_preload
c2b:	zr7_seven_complex_djbunfft rsi, 128, clmblkdst, rdi, ZMM_SCD6, 8*clm, L1PREFETCH_ALL, 128
	zloop	8*clm, c2b, -8*clm*128+7*clmblkdst, rdi, -8*clm*ZMM_SCD6 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	12, c2b, -12*7*clmblkdst, rdi, 8*clm*ZMM_SCD6 ;; Test loop counter, restore source pointer, next sin/cos pointer
ELSE
c2a:	zr7_csc_fourteen_reals_unfft_preload
	zr7_csc_fourteen_reals_unfft rsi, 7*clmblkdst, clmblkdst, rdi, 0, 1, L1PREFETCH_ALL, 7*clmblkdst
	zr7_seven_complex_djbunfft_preload
	jmp	c2c				;; 11 more iters
c2b:	zr7_seven_complex_djbunfft rsi, 7*clmblkdst, clmblkdst, rdi, 0, 1, L1PREFETCH_ALL, 7*clmblkdst
c2c:	zloop	12, c2b, -12*7*clmblkdst+128, rdi, ZMM_SCD6 ;; Test loop counter, next source pointer, next sin/cos pointer
	zloop	8*clm, c2a, -8*clm*128		;; Test loop counter, restore source pointer
ENDIF
	end_timer 18

;; Do inverse FFT levels 1,2,3,4
;; On input the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	56K	+1K	...
;;	8K
;;	...
;;	48K
;;	112K
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	672K	+1K	...
;;	8K
;;	...
;;	664K

;; Do 56 twentyfour_reals_unfft operations
;;	distance between fft data elements is 56K
;;	do 7 macros each processing 192 data values

	start_timer 20
	mov	r15, sincos2			;; Load fixed premultiplier/sin/cos ptr
	sub	rbx, rbx			;; Clear register for loading fudge flags index
	mov	r12, compressed_fudges		;; Address of the compressed fudge flags
	mov	r13, r12			;; The XOR masks for fudge flags come from the compressed fudges
	zloop_init 7*8*clm			;; 7 iters of 8*clm
	zloop_set_clm_prefetch ZQUAD		;; Prefetch 112 of 672 blocks of clm*128 bytes
	zr12_wpn_twentyfour_reals_last_unfft_preload
c1b:	zr12_wpn_twentyfour_reals_last_unfft rsi, 128, 7*clmblkdst, 14*clmblkdst, 28*clmblkdst, r15, 0, rdi, 3, 8*clm, L1PREFETCH_ALL, 128
	zloop	8*clm, c1b, -8*clm*128+clmblkdst, r15, ZMM_SCD11, rdx, 24*64, r13, 24 ;; 8*clm iterations, next source, fixed s/c, grp multipliers pointer, next xor mask
	zloop	7, c1b				;; Test loop counter
	end_timer 20

;; Normalize these values

	pass1_normalize yes_fft			;; Normalize while prefetching 84 of 672 blocks of clm*128 bytes

;; Copy data from the scratch area back to the FFT data area

	copy_scratch_data_to_fft
	jmp	endlp				;; Skip FFT code if flag not set

;; Fire up auxiliary threads to do pass 1 forward FFT

pass1:	pass1_forward_fft_setup b0b

;; Copy data from the FFT data area back to the scratch area

b0b:	copy_fft_data_to_scratch

;; Do FFT levels 1,2,3,4
;; On input the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	672K	+1K	...
;;	8K	...
;;	...
;;	664K
;; On output the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	56K	+1K	...
;;	8K
;;	...
;;	48K
;;	112K
;;	...

;; Do 56 twentyfour_reals_first_fft operations
;;	distance between fft data elements is 56K
;;	do 7 macros each processing 192 data values

yes_fft:start_timer 22
	mov	rsi, scratch_area		;; Get address of scratch area
	mov	r15, sincos2			;; Load fixed premultiplier/sin/cos ptr
	mov	rdi, PREMULT_ADDR		;; Calculate pointer to fudge flags
	bump	rdi, 8*clm*ZMM_SCD6+11*clm*ZMM_SCD1+clm*ZMM_SCD4+clm*ZMM_SCD4+clm*64+clm*64 ;; Addr of the group multiplier fudge flags
	mov	rdx, norm_grp_mults		;; Addr of the group multipliers
	sub	rbx, rbx			;; Clear register for loading fudge flags index
	mov	r12, compressed_fudges		;; Address of the compressed fudge flags
	mov	r13, r12			;; The XOR masks for fudge flags come from the compressed fudges
	set_data_prefetch_ptrs			;; Init prefetching pointers
pfing	mov	r8, pass2blkdst			;; blkdst
pfing	imul	rax, blkdstreg, (472-8)		;; We've already prefetched 472 of 672 blocks (8 too many)
pfing	add	rcx, rax
	zloop_init 7*8*clm			;; 7 iters of 8*clm
	zloop_set_clm_prefetch ZQUAD		;; Prefetch 112 of 672 blocks of clm*128 bytes
	zr12_wpn_twentyfour_reals_first_fft_preload
b1b:	zr12_wpn_twentyfour_reals_first_fft rsi, 128, 7*clmblkdst, 14*clmblkdst, 28*clmblkdst, r15, 0, rdi, 3, rdx, 0, 8*clm, L1PREFETCH_ALL, 128
	zloop	8*clm, b1b, -8*clm*128+clmblkdst, r15, ZMM_SCD11, rdx, 24*64, r13, 24 ;; 8*clm iters prefetching 8*clm cache lines, next source, fixed s/c, grp multipliers pointer, next xor mask
	zloop	7, b1b, -7*clmblkdst		;; Test loop counter, restore source pointer
	end_timer 22

;; Do FFT levels 5,6,7
;; On input the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	56K	+1K	...
;;	8K
;;	...
;;	48K
;;	112K
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	8K	+1K	...
;;	16K
;;	...

;; Do 8 fourteen_reals_fft operations
;; Do 88 seven_complex_fft operations
;;	distance between fft data elements is 8K
;;	do 12 macros each processing 112 data values

	start_timer 24
	bump	rdi, -56*clm*3-8*clm*ZMM_SCD6	;; Sin/cos pointer
	zloop_init  12*8*clm			;; 12 iterations of 8*clm
	zloop_set_clm_prefetch ZDOUBLE		;; Prefetch 96 of 672 blocks of clm*128 bytes
IFDEF OLDDD
	zr7_csc_fourteen_reals_fft_preload
b2a:	zr7_csc_fourteen_reals_fft rsi, 128, clmblkdst, rdi, ZMM_SCD6, 8*clm, L1PREFETCH_ALL, 128
	zloop	8*clm, b2a, -8*clm*128+7*clmblkdst, rdi, -8*clm*ZMM_SCD6 ;; Test loop counter, next source pointer, restore sin/cos pointer
	loops_reset				;; 11 more iters of 8*clm
	zr7_seven_complex_djbfft_preload
b2b:	zr7_seven_complex_djbfft rsi, 128, clmblkdst, rdi, ZMM_SCD6, 8*clm, L1PREFETCH_ALL, 128
	zloop	8*clm, b2b, -8*clm*128+7*clmblkdst, rdi, -8*clm*ZMM_SCD6 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	12, b2b, -12*7*clmblkdst	;; Test loop counter, restore source pointer
ELSE
b2a:	zr7_csc_fourteen_reals_fft_preload
	zr7_csc_fourteen_reals_fft rsi, 7*clmblkdst, clmblkdst, rdi, 0, 1, L1PREFETCH_ALL, 7*clmblkdst
	zr7_seven_complex_djbfft_preload
	jmp	b2c				;; 11 more iters
b2b:	zr7_seven_complex_djbfft rsi, 7*clmblkdst, clmblkdst, rdi, 0, 1, L1PREFETCH_ALL, 7*clmblkdst
b2c:	zloop	12, b2b, -12*7*clmblkdst+128, rdi, ZMM_SCD6 ;; Test loop counter, next source pointer, next sin/cos pointer
	zloop	8*clm, b2a, -8*clm*128		;; Test loop counter, restore source pointer
ENDIF
	end_timer 24

;; Do FFT levels 8,9,10
;; On input the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	8K	+1K	...
;;	16K
;;	...
;; The input is swizzled to hold these data values:
;;	0K	+1	+1	+1	+1	+1	+1	+1	8K	+1	...
;;	1K	...
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0K	+1	+1	+1	+1	+1	+1	+1	1K	+1	...
;;	2K
;;	...

;; Do 1 sixteen_reals_fft operations
;;	distance between fft data elements is 1K
;;	do 0.125 macros each processing 128 data values
;; Do 83 eight_complex_fft operations
;;	distance between fft data elements is 1K
;;	do 10.375 macros each processing 128 data values

	start_timer 26
IFDEF OLDDD
	bump	rdi, -11*clm*ZMM_SCD1-clm*ZMM_SCD4 ;; Real data sin/cos pointer, will become delayed sin/cos multipliers
ELSE
	bump	rdi, -8*clm*ZMM_SCD6-11*clm*ZMM_SCD1-clm*ZMM_SCD4 ;; Real data sin/cos pointer, will become delayed sin/cos multipliers
ENDIF
	lea	rbx, [rdi-clm*ZMM_SCD4]		;; Complex data sin/cos ptr
	lea	r9, [rbx-clm*64-clm*64]		;; Column weights
	lea	rdx, ZMM_TMPS			;; Computed clm*SCD4 sin/cos values
	mov	rbp, DATA_ADDR			;; Destination pointer
	mov	r8, pass2blkdst			;; blkdst
	lea	r10, [2*blkdstreg+blkdstreg]	;; 3*blkdst
	lea	r13, [rbp+4*blkdstreg]		;; Dest+4*blkdst pointer
pfing	mov	rcx, PREMULT_PREFETCH		;; Load premult/sin/cos prefetch pointer

	zr8_rsc_complex_and_real_preload	;; Preload constants before calling zloop_init
;;	zloop_init 12*7*clm			;; 12 iters of 7*clm
	zloop_init 12*7*clm  *2			;; 12 iters of 7*clm iterations	(work around loops "add al, 1" bug)
	zloop_set_prefetch ZDOUBLE		;; Prefetch 168*clm times
	zloop_set_alternate_prefetch_increment 141*4+3, 168*4  ;; Prefetch approximately 141.75*clm cache lines in those 168*clm prefetches
	zr8_rsc_wpn_eight_complex_calc_sincos_simple r9, rbx, clm ;; Generate clm*SCD8 sin/cos values from delayed and complex multipliers
b8b:	zr8_rsc_wpn_sgreg_2sc_sixteen_reals_fft8 rsi, 1024, 128, 256, 512, rbp, 128, blkdstreg, blkdst3reg, r13, rdx, ZMM_SCD8, r9, 64, rdi, ZMM_SCD4, clm, L1PREFETCH_DEST_ALL
	zloop	clm, b8b, -clm*1024+clmblkdst, rbp, -clm*128+8*blkdstreg, r13, -clm*128+8*blkdstreg, rdx, -clm*ZMM_SCD8, r9, -clm*64 ;; Loop, next source/dest pointers, restore premult/sine/cosine pointer
	jmp	b9aa

	loops_reset
b9a:	zr8_rsc_wpn_eight_complex_calc_sincos r9, rdi, rbx, clm ;; Generate clm*SCD8 sin/cos values from delayed and complex multipliers
	bump	rdi, clm*ZMM_SCD1		;; Next set of delayed multipliers
b9aa:
b9b:	zr8_rsc_wpn_sgreg_eight_complex_fft8 rsi, 1024, 128, 256, 512, rbp, 128, blkdstreg, blkdst3reg, r13, rdx, ZMM_SCD8, clm, L1PREFETCH_DEST_ALL
	zloop	clm, b9b, -clm*1024+clmblkdst, rbp, -clm*128+8*blkdstreg, r13, -clm*128+8*blkdstreg, rdx, -clm*ZMM_SCD8 ;; Loop, next source/dest pointers, restore premult/sine/cosine pointer
	zloop	7, b9b				;; Test loop counter
	zloop	12, b9a				;; Test loop counter
	end_timer 26

;; Work on next 8*clm sets of 1344 values

endlp:	zpass1_get_next_block pass2, c0b, b0b ;; Get next block and jump
done:	zfft_footer
	ENDM


;; Perform a shared pass 1 negacyclic FFT of length 1344 doubles.
;; Memory examples are for a 1344K FFT with 10 levels done in pass 2.
;; FFT data needed:
;;		8*clm*1344 doubles					Total: 1344*clm 64-byte cache lines
;; Fixed s/c data needed:
;;		7*ZMM_CD12SCD6 (complex premultipliers and fixed s/c data), Total: 168 64-byte cache lines
;; Group multipliers needed:
;;		7*24*64 (weights)
;;		7*24*64 (inverse weights),				Total: 336 64-byte cache lines
;; Variable data needed:
;;		clm*64	(column weights)
;;		clm*64	(column inverse weights)
;;		clm*ZMM_SCD4 (complex reduced s/c multipliers)
;;		12*clm*ZMM_SCD1 (delayed s/c multipliers)
;;		8*clm*ZMM_SCD3 (7-complex sin/cos data)
;;		56*clm*3 (group multiplier fudge flags)
;;		56*clm*3 (biglit flags),				Total: 87.25*clm 64-byte cache lines
;; Total memory consumed (add for scratch area, prefetched data, carries):
;;		(1344*3+87.25=4119.25)*clm + (168+336+1344/8=672) cache lines
;;		clm=1,2,4:  300KB, 557KB, 1072KB

zr4dwpn_pass1sc1344ac MACRO
	LOCAL	b0b, b1b, b2b, b9a, b9b, c0b, c1b, c2b, c9a, c9b
	LOCAL	pass1, pass2, endlp, done, yes_fft

	zfft_header pass1

;; Do pass 2 (the last 10 FFT levels if this is a 1344K FFT)

pass2:	start_timer 0
	mov	rax, ZMM_PASS2_ROUTINE
	call	rax
	end_timer 0
	cmp	ffttype, 1		;; We're done if FFTing only
	je	done

;; Do the inverse FFT

	pass1_inverse_fft_setup c0b

;; Do inverse FFT levels 8.9.10
;; On input the 128-byte double cache lines hold these data values:
;;	0K	+1	+1	+1	+1	+1	+1	+1	1K	+1	...
;;	2K
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0K	+1	+1	+1	+1	+1	+1	+1	8K	+1	...
;;	1K	...
;;	...
;; The output cache lines are swizzled to hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	8K	+1K	...
;;	16K
;;	...

;; Do 84 eight_complex_unfft operations
;;	distance between fft data elements is 1K
;;	do 10.5 macros each processing 128 data values

c0b:	start_timer 16
	mov	rbp, DATA_ADDR			;; Get source address
	mov	rsi, scratch_area		;; Get address of scratch area
	set_data_prefetch_ptrs			;; Init prefetching pointers
	mov	r9, PREMULT_ADDR		;; Variable data
	bump	r9, clm*64			;; Inverse column weights
	lea	rbx, [r9+clm*64]		;; Complex data sin/cos ptr
	lea	rdi, [rbx+clm*ZMM_SCD4]		;; Delayed sin/cos multipliers
	lea	rdx, ZMM_TMPS			;; Computed clm*SCD8 sin/cos values
	mov	r8, pass2blkdst			;; blkdst
	lea	r10, [2*blkdstreg+blkdstreg]	;; 3*blkdst
	lea	r13, [rbp+4*blkdstreg]		;; Source+4*blkdst pointer
	lea	r12, [rbp+8*blkdstreg]		;; L1 prefetch pointer part 1
	bump2	r12, (clm MOD 2), 8*blkdstreg	;; L1 prefetch pointer part 2
	lea	r14, [r12+4*blkdstreg]		;; L1 prefetch pointer + 4*blkdst

	zr8_rsc_complex_only_preload		;; Preload constants before calling zloop_init
;;	zloop_init 12*7*clm			;; 12 iters of 7*clm iterations
	zloop_init 12*7*clm  *2			;; 12 iters of 7*clm iterations	(work around loops "add al, 1" bug)
	zloop_set_clm_prefetch ZDOUBLE		;; Prefetch 84 of 672 blocks of clm*128 bytes
c9a:	zr8_rsc_wpn_eight_complex_calc_sincos r9, rdi, rbx, clm ;; Generate clm*SCD8 sin/cos values from weights, delayed, and complex multipliers
	bump	rdi, clm*ZMM_SCD1		;; Next set of delayed sin/cos multipliers
c9b:	zr8_rsc_wpn_sgreg_eight_complex_unfft8 rbp, 128, blkdstreg, blkdst3reg, r13, rsi, 1024, 128, 256, 512, rdx, ZMM_SCD8, clm, L1PREFETCH_ALL, r12, r14 ;; Prefetch short dist ahead
	zloop	clm, c9b, -clm*1024+clmblkdst, rbp, -clm*128+8*blkdstreg, r13, -clm*128+8*blkdstreg, rdx, -clm*ZMM_SCD8, r12, -clm*128+8*blkdstreg, r14, -clm*128+8*blkdstreg ;; Loop, next src/dest ptrs, restore sin/cos ptr
	zloop	7, c9b				;; Test loop counter
	zloop	12, c9a, -12*7*clmblkdst	;; Test loop counter, restore source pointer
	end_timer 16

;; Do inverse FFT levels 5,6,7
;; On input the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	8K	+1K	...
;;	16K
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	56K	+1K	...
;;	8K
;;	...
;;	48K
;;	112K
;;	...

;; Do 96 seven_complex_unfft operations
;;	distance between fft data elements is 8K
;;	do 12 macros each processing 112 data values

	start_timer 18
	zloop_init  12*8*clm			;; 12 iterations of 8*clm
	zloop_set_clm_prefetch ZQUAD		;; Prefetch 192 of 672 blocks of clm*128 bytes
	zr7_seven_complex_djbunfft_preload
IFDEF OLDDD
c2b:	zr7_seven_complex_djbunfft rsi, 128, clmblkdst, rdi, ZMM_SCD3, 8*clm, L1PREFETCH_ALL, 128
	zloop	8*clm, c2b, -8*clm*128+7*clmblkdst, rdi, -8*clm*ZMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	12, c2b, -12*7*clmblkdst, rdi, 8*clm*ZMM_SCD3 ;; Test loop counter, restore source pointer, next sin/cos pointer
ELSE
c2b:	zr7_seven_complex_djbunfft rsi, 7*clmblkdst, clmblkdst, rdi, 0, 8, L1PREFETCH_ALL, 7*clmblkdst
	zloop	12, c2b, -12*7*clmblkdst+128, rdi, ZMM_SCD3 ;; Test loop counter, next source pointer, next sin/cos pointer
	zloop	8*clm, c2b, -8*clm*128		;; Test loop counter, restore source pointer
ENDIF
	end_timer 18

;; Do inverse FFT levels 1,2,3,4
;; On input the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	56K	+1K	...
;;	8K
;;	...
;;	48K
;;	112K
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	672K	+1K	...
;;	8K	...
;;	...
;;	664K

;; Do 56 twelve_complex_unfft operations
;;	distance between fft data elements is 56K
;;	do 7 macros each processing 192 data values

	start_timer 20
	mov	r15, sincos2			;; Load fixed premultiplier/sin/cos ptr
	sub	rbx, rbx			;; Clear register for loading fudge flags index
	mov	r12, compressed_fudges		;; Address of the compressed fudge flags
	mov	r13, r12			;; The XOR masks for fudge flags come from the compressed fudges
	zloop_init 7*8*clm			;; 7 iters of 8*clm
	zloop_set_clm_prefetch ZQUAD		;; Prefetch 112 of 672 blocks of clm*128 bytes
	zr12_csc_wpn_twelve_complex_last_djbunfft_preload
c1b:	zr12_csc_wpn_twelve_complex_last_djbunfft rsi, 128, 7*clmblkdst, 14*clmblkdst, 28*clmblkdst, r15, 0, rdi, 3, 8*clm, L1PREFETCH_ALL, 128
	zloop	8*clm, c1b, -8*clm*128+clmblkdst, r15, ZMM_CD12SCD6, rdx, 24*64, r13, 24 ;; 8*clm iterations, next fixed s/c, next grp multipliers, next xor mask
	zloop	7, c1b				;; 7 iterations
	end_timer 20

;; Normalize these values

	pass1_normalize yes_fft			;; Normalize while prefetching 84 of 672 blocks of clm*128 bytes

;; Copy data from the scratch area back to the FFT data area

	copy_scratch_data_to_fft
	jmp	endlp				;; Skip FFT code if flag not set

;; Fire up auxiliary threads to do pass 1 forward FFT

pass1:	pass1_forward_fft_setup b0b

;; Copy data from the FFT data area back to the scratch area

b0b:	copy_fft_data_to_scratch

;; Do FFT levels 1,2,3,4
;; On input the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	672K	+1K	...
;;	8K	...
;;	...
;;	664K
;; On output the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	56K	+1K	...
;;	8K
;;	...
;;	48K
;;	112K
;;	...

;; Do 56 twelve_complex_first_fft operations
;;	distance between fft data elements is 56K
;;	do 7 macros each processing 128 data values

yes_fft:start_timer 22
	mov	rsi, scratch_area		;; Get address of scratch area
	mov	r15, sincos2			;; Load fixed premultiplier/sin/cos ptr
	mov	rdi, PREMULT_ADDR		;; Calculate pointer to fudge flags
	bump	rdi, 8*clm*ZMM_SCD3+12*clm*ZMM_SCD1+clm*ZMM_SCD4+clm*64+clm*64 ;; Addr of the group multiplier fudge flags
	mov	rdx, norm_grp_mults		;; Addr of the group multipliers
	sub	rbx, rbx			;; Clear register for loading fudge flags index
	mov	r12, compressed_fudges		;; Address of the compressed fudge flags
	mov	r13, r12			;; The XOR masks for fudge flags come from the compressed fudges
	set_data_prefetch_ptrs			;; Init prefetching pointers
pfing	mov	r8, pass2blkdst			;; blkdst
pfing	imul	rax, blkdstreg, (472-8)		;; We've already prefetched 472 of 672 blocks (8 too many)
pfing	add	rcx, rax
	zloop_init 7*8*clm			;; 7 iters of 8*clm
	zloop_set_clm_prefetch ZQUAD		;; Prefetch 112 of 672 blocks of clm*128 bytes
	zr12_csc_wpn_twelve_complex_first_djbfft_preload
b1b:	zr12_csc_wpn_twelve_complex_first_djbfft rsi, 128, 7*clmblkdst, 14*clmblkdst, 28*clmblkdst, r15, 0, rdi, 3, rdx, 0, 8*clm, L1PREFETCH_ALL, 128
	zloop	8*clm, b1b, -8*clm*128+clmblkdst, r15, ZMM_CD12SCD6, rdx, 24*64, r13, 24 ;; 8*clm iters, next source, fixed s/c, grp multipliers pointer, next xor mask
	zloop	7, b1b, -7*clmblkdst		;; Test loop counter, restore source pointer
	end_timer 22

;; Do FFT levels 5,6,7
;; On input the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	56K	+1K	...
;;	8K
;;	...
;;	48K
;;	112K
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	8K	+1K	...
;;	16K
;;	...

;; Do 96 seven_complex_fft operations
;;	distance between fft data elements is 8K
;;	do 12 macros each processing 112 data values

	start_timer 24
	bump	rdi, -56*clm*3-8*clm*ZMM_SCD3	;; Sin/cos pointer
	zloop_init  12*8*clm			;; 12 iterations of 8*clm
	zloop_set_clm_prefetch ZDOUBLE		;; Prefetch 96 of 672 blocks of clm*128 bytes
	zr7_seven_complex_djbfft_preload
IFDEF OLDDD
b2b:	zr7_seven_complex_djbfft rsi, 128, clmblkdst, rdi, ZMM_SCD3, 8*clm, L1PREFETCH_ALL, 128
	zloop	8*clm, b2b, -8*clm*128+7*clmblkdst, rdi, -8*clm*ZMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	12, b2b, -12*7*clmblkdst	;; Test loop counter, restore source pointer
ELSE
b2b:	zr7_seven_complex_djbfft rsi, 7*clmblkdst, clmblkdst, rdi, 0, 8, L1PREFETCH_ALL, 7*clmblkdst
	zloop	12, b2b, -12*7*clmblkdst+128, rdi, ZMM_SCD3 ;; Test loop counter, next source pointer, next sin/cos pointer
	zloop	8*clm, b2b, -8*clm*128		;; Test loop counter, restore source pointer
ENDIF
	end_timer 24

;; Do FFT levels 8,9,10
;; On input the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	8K	+1K	...
;;	16K
;;	...
;; The input is swizzled to hold these data values:
;;	0K	+1	+1	+1	+1	+1	+1	+1	8K	+1	...
;;	1K	...
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0K	+1	+1	+1	+1	+1	+1	+1	1K	+1	...
;;	2K
;;	...

;; Do 84 eight_complex_fft operations
;;	distance between fft data elements is 1K
;;	do 10.5 macros each processing 128 data values

	start_timer 26
	mov	rbp, DATA_ADDR			;; Destination pointer
IFDEF OLDDD
	bump	rdi, -12*clm*ZMM_SCD1		;; Delayed sin/cos multipliers
ELSE
	bump	rdi, -8*clm*ZMM_SCD3-12*clm*ZMM_SCD1 ;; Delayed sin/cos multipliers
ENDIF
	lea	rbx, [rdi-clm*ZMM_SCD4]		;; Complex data sin/cos ptr
	lea	r9, [rbx-clm*64-clm*64]		;; Column weights
	lea	rdx, ZMM_TMPS			;; Computed clm*SCD4 sin/cos values
	mov	r8, pass2blkdst			;; blkdst
	lea	r10, [2*blkdstreg+blkdstreg]	;; 3*blkdst
	lea	r13, [rbp+4*blkdstreg]		;; Dest+4*blkdst pointer
pfing	mov	rcx, PREMULT_PREFETCH		;; Load premult/sin/cos prefetch pointer

	zr8_rsc_complex_only_preload		;; Preload constants before calling zloop_init
;;	zloop_init 12*7*clm			;; 12 iters of 7*clm
	zloop_init 12*7*clm  *2			;; 12 iters of 7*clm iterations	(work around loops "add al, 1" bug)
	zloop_set_prefetch ZDOUBLE		;; Prefetch 168*clm cache lines
	zloop_set_alternate_prefetch_increment 87*4+3, 168*4  ;; Prefetch approximately 87.25*clm cache lines in those 168*clm prefetches
b9a:	zr8_rsc_wpn_eight_complex_calc_sincos r9, rdi, rbx, clm ;; Generate clm*SCD8 sin/cos values from weights, delayed and complex multipliers
	bump	rdi, clm*ZMM_SCD1		;; Next set of delayed multipliers
b9b:	zr8_rsc_wpn_sgreg_eight_complex_fft8 rsi, 1024, 128, 256, 512, rbp, 128, blkdstreg, blkdst3reg, r13, rdx, ZMM_SCD8, clm, L1PREFETCH_DEST_ALL
	zloop	clm, b9b, -clm*1024+clmblkdst, rbp, -clm*128+8*blkdstreg, r13, -clm*128+8*blkdstreg, rdx, -clm*ZMM_SCD8 ;; Loop, next source/dest pointers, restore premult/sine/cosine pointer
	zloop	7, b9b				;; Test loop counter
	zloop	12, b9a				;; Test loop counter
	end_timer 26

;; Work on next 8*clm sets of 1344 values

endlp:	zpass1_get_next_block pass2, c0b, b0b ;; Get next block and jump
done:	zfft_footer
	ENDM



;; Perform a shared pass 1 FFT of length 1536 doubles.
;; Memory examples are for a 1536K FFT with 10 levels done in pass 2.
;; FFT data needed:
;;		8*clm*1536 doubles					Total: 1536*clm 64-byte cache lines
;; Fixed s/c data needed:
;;		12*ZMM_SCD7 (fixed s/c data),				Total: 168 64-byte cache lines
;; Group multipliers needed:
;;		12*16*64 (weights)
;;		12*16*64 (inverse weights),				Total: 384 64-byte cache lines
;; Variable data needed:
;;		clm*64	(column weights)
;;		clm*64	(column inverse weights)
;;		clm*ZMM_SCD4 (complex reduced s/c multipliers)
;;		clm*ZMM_SCD4 (real s/c multipliers)
;;		7*clm*ZMM_SCD1 (delayed s/c multipliers)
;;		8*clm*ZMM_SCD12 (12-complex 24-real sin/cos data)
;;		96*clm*2 (group multiplier fudge flags)
;;		96*clm*2 (biglit flags),				Total: 230*clm 64-byte cache lines
;; Total memory consumed (add for scratch area, prefetched data, carries):
;;		(1536*3+230=4838)*clm + (168+384+1536/8=744) cache lines
;;		clm=1,2,4:  349KB, 652KB, 1256KB

zr4dwpn_pass1sc1536 MACRO
	LOCAL	b0b, b1b, b2a, b2b, b2c, b8a, b8b, b9aa, b9a, b9b, c0b, c1b, c2a, c2b, c2c, c8b, c9aa, c9a, c9b
	LOCAL	pass1, pass2, endlp, done, yes_fft

	zfft_header pass1

;; Do pass 2 (the last 10 FFT levels if this is a 1536K FFT)

pass2:	start_timer 0
	mov	rax, ZMM_PASS2_ROUTINE
	call	rax
	end_timer 0
	cmp	ffttype, 1		;; We're done if FFTing only
	je	done

;; Do the inverse FFT

	pass1_inverse_fft_setup c0b

;; Do inverse FFT levels 8,9,10
;; On input the 128-byte double cache lines hold these data values:
;;	0K	+1	+1	+1	+1	+1	+1	+1	1K	+1	...
;;	2K
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0K	+1	+1	+1	+1	+1	+1	+1	8K	+1	...
;;	1K	...
;;	...
;; The output cache lines are swizzled to hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	8K	+1K	...
;;	16K
;;	...

;; Do 1 sixteen_reals_unfft operations
;;	distance between fft data elements is 1K
;;	do 0.125 macros each processing 128 data values
;; Do 95 eight_complex_unfft operations
;;	distance between fft data elements is 1K
;;	do 11.875 macros each processing 128 data values

c0b:	start_timer 16
	mov	rbp, DATA_ADDR			;; Get source address
	mov	rsi, scratch_area		;; Get address of scratch area
	set_data_prefetch_ptrs			;; Init prefetching pointers
	mov	r9, PREMULT_ADDR		;; Variable data
	bump	r9, clm*64			;; Inverse column weights
	lea	rbx, [r9+clm*64]		;; Complex data sin/cos ptr
	lea	rdi, [rbx+clm*ZMM_SCD4]		;; Real data sin/cos pointer, will become delayed sin/cos multipliers
	lea	rdx, ZMM_TMPS			;; Computed clm*SCD8 sin/cos values
	mov	r8, pass2blkdst			;; blkdst
	lea	r10, [2*blkdstreg+blkdstreg]	;; 3*blkdst
	lea	r13, [rbp+4*blkdstreg]		;; Source+4*blkdst pointer
	lea	r12, [rbp+8*blkdstreg]		;; L1 prefetch pointer part 1
	bump2	r12, (clm MOD 2), 8*blkdstreg	;; L1 prefetch pointer part 2
	lea	r14, [r12+4*blkdstreg]		;; L1 prefetch pointer + 4*blkdst

	zr8_rsc_complex_and_real_preload	;; Preload constants before calling zloop_init
	zloop_init 12*8*clm			;; 12 iters of 8*clm iterations
	zloop_set_clm_prefetch ZDOUBLE		;; Prefetch 96 of 768 blocks of clm*128 bytes
	zr8_rsc_wpn_eight_complex_calc_sincos_simple r9, rbx, clm ;; Generate clm*SCD8 sin/cos values from delayed and complex multipliers
c8b:	zr8_rsc_wpn_sgreg_2sc_sixteen_reals_unfft8 rbp, 128, blkdstreg, blkdst3reg, r13, rsi, 1024, 128, 256, 512, rdx, ZMM_SCD8, r9, 64, rdi, ZMM_SCD4, clm, L1PREFETCH_ALL, r12, r14 ;; Prefetch short dist ahead
	zloop	clm, c8b, -clm*1024+clmblkdst, rbp, -clm*128+8*blkdstreg, r13, -clm*128+8*blkdstreg, rdx, -clm*ZMM_SCD8, r9, -clm*64, r12, -clm*128+8*blkdstreg, r14, -clm*128+8*blkdstreg ;; Loop, next src/dest ptrs, restore sin/cos ptr
	jmp	c9aa

	loops_reset
c9a:	zr8_rsc_wpn_eight_complex_calc_sincos r9, rdi, rbx, clm ;; Generate clm*SCD8 sin/cos values from delayed and complex multipliers
	bump	rdi, clm*ZMM_SCD1		;; Next set of delayed sin/cos multipliers
c9aa:
c9b:	zr8_rsc_wpn_sgreg_eight_complex_unfft8 rbp, 128, blkdstreg, blkdst3reg, r13, rsi, 1024, 128, 256, 512, rdx, ZMM_SCD8, clm, L1PREFETCH_ALL, r12, r14 ;; Prefetch short dist ahead
	zloop	clm, c9b, -clm*1024+clmblkdst, rbp, -clm*128+8*blkdstreg, r13, -clm*128+8*blkdstreg, rdx, -clm*ZMM_SCD8, r12, -clm*128+8*blkdstreg, r14, -clm*128+8*blkdstreg ;; Loop, next src/dest ptrs, restore sin/cos ptr
	zloop	12, c9b				;; Test loop counter
	zloop	8, c9a, -8*12*clmblkdst		;; Test loop counter, restore source pointer
	end_timer 16

;; Do inverse FFT levels 5,6,7
;; On input the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	8K	+1K	...
;;	16K
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	96K	+1K	...
;;	8K
;;	...
;;	88K
;;	192K
;;	...

;; Do 8 twentyfour_reals_unfft operations
;; Do 56 twelve_complex_unfft operations
;;	distance between fft data elements is 8K
;;	do 8 macros each processing 192 data values

	start_timer 18
	zloop_init  8*8*clm			;; 8 iterations of 8*clm
	zloop_set_clm_prefetch ZQUAD		;; Prefetch 128 of 768 blocks of clm*128 bytes
IFDEF OLDDD
	zr12_csc_twentyfour_reals_unfft_preload
c2a:	zr12_csc_twentyfour_reals_unfft rsi, 128, clmblkdst, 2*clmblkdst, 4*clmblkdst, rdi, ZMM_SCD12, 8*clm, L1PREFETCH_ALL, 128
	zloop	8*clm, c2a, -8*clm*128+12*clmblkdst, rdi, -8*clm*ZMM_SCD12 ;; Test loop counter, next source pointer, restore sin/cos pointer
	loops_reset				;; 7 more iters of 8*clm
	zr12_twelve_complex_djbunfft_preload
c2b:	zr12_twelve_complex_djbunfft rsi, 128, clmblkdst, 2*clmblkdst, 4*clmblkdst, rdi, ZMM_SCD12, 8*clm, L1PREFETCH_ALL, 128
	zloop	8*clm, c2b, -8*clm*128+12*clmblkdst, rdi, -8*clm*ZMM_SCD12 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	8, c2b, -8*12*clmblkdst, rdi, 8*clm*ZMM_SCD12 ;; Test loop counter, restore source pointer, next sin/cos pointer
ELSE
c2a:	zr12_csc_twentyfour_reals_unfft_preload
	zr12_csc_twentyfour_reals_unfft rsi, 12*clmblkdst, clmblkdst, 2*clmblkdst, 4*clmblkdst, rdi, 0, 1, L1PREFETCH_ALL, 12*clmblkdst
	zr12_twelve_complex_djbunfft_preload
	jmp	c2c				;; 7 more iters
c2b:	zr12_twelve_complex_djbunfft rsi, 12*clmblkdst, clmblkdst, 2*clmblkdst, 4*clmblkdst, rdi, 0, 1, L1PREFETCH_ALL, 12*clmblkdst
c2c:	zloop	8, c2b, -8*12*clmblkdst+128, rdi, ZMM_SCD12 ;; Test loop counter, next source pointer, next sin/cos pointer
	zloop	8*clm, c2a, -8*clm*128		;; Test loop counter, restore source pointer
ENDIF

;; Do inverse FFT levels 1,2,3,4
;; On input the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	96K	+1K	...
;;	8K
;;	...
;;	88K
;;	192K
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	768K	+1K	...
;;	8K
;;	...
;;	760K

;; Do 96 sixteen_reals_unfft operations
;;	distance between fft data elements is 96K
;;	do 12 macros each processing 128 data values

	start_timer 20
	mov	r15, sincos2			;; Load fixed premultiplier/sin/cos ptr
	sub	rbx, rbx			;; Clear register for loading fudge flags index
	mov	r12, compressed_fudges		;; Address of the compressed fudge flags
	mov	r13, r12			;; The XOR masks for fudge flags come from the compressed fudges
	zloop_init 12*8*clm			;; 12 iters of 8*clm
	zloop_set_clm_prefetch ZDOUBLE		;; Prefetch 96 of 768 blocks of clm*128 bytes
	zr8_wpn_sixteen_reals_last_unfft_preload
c1b:	zr8_wpn_sixteen_reals_last_unfft rsi, 128, 12*clmblkdst, 24*clmblkdst, 48*clmblkdst, r15, 0, rdi, 2, 8*clm, L1PREFETCH_ALL, 128
	zloop	8*clm, c1b, -8*clm*128+clmblkdst, r15, ZMM_SCD7, rdx, 16*64, r13, 16 ;; 8*clm iters, next source, fixed s/c, grp multipliers pointer, next xor mask
	zloop	12, c1b				;; Test loop counter
	end_timer 20

;; Normalize these values

	pass1_normalize yes_fft			;; Normalize while prefetching 96 of 768 blocks of clm*128 bytes

;; Copy data from the scratch area back to the FFT data area

	copy_scratch_data_to_fft
	jmp	endlp				;; Skip FFT code if flag not set

;; Fire up auxiliary threads to do pass 1 forward FFT

pass1:	pass1_forward_fft_setup b0b

;; Copy data from the FFT data area back to the scratch area

b0b:	copy_fft_data_to_scratch

;; Do FFT levels 1,2,3,4
;; On input the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	768K	+1K	...
;;	8K	...
;;	...
;;	760K
;; On output the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	96K	+1K	...
;;	8K
;;	...
;;	88K
;;	192K
;;	...

;; Do 96 sixteen_reals_first_fft operations
;;	distance between fft data elements is 96K
;;	do 12 macros each processing 128 data values

yes_fft:start_timer 22
	mov	rsi, scratch_area		;; Get address of scratch area
	mov	r15, sincos2			;; Load fixed premultiplier/sin/cos ptr
	mov	rdi, PREMULT_ADDR		;; Calculate pointer to fudge flags
	bump	rdi, 8*clm*ZMM_SCD12+7*clm*ZMM_SCD1+clm*ZMM_SCD4+clm*ZMM_SCD4+clm*64+clm*64 ;; Addr of the group multiplier fudge flags
	mov	rdx, norm_grp_mults		;; Addr of the group multipliers
	sub	rbx, rbx			;; Clear register for loading fudge flags index
	mov	r12, compressed_fudges		;; Address of the compressed fudge flags
	mov	r13, r12			;; The XOR masks for fudge flags come from the compressed fudges
	set_data_prefetch_ptrs			;; Init prefetching pointers
pfing	mov	r8, pass2blkdst			;; blkdst
pfing	imul	rax, blkdstreg, (424-0)		;; We've already prefetched 424 of 768 blocks (4 too few)
pfing	add	rcx, rax
	zloop_init 12*8*clm			;; 12 iters of 8*clm
	zloop_set_clm_prefetch ZQUAD		;; Prefetch 192 of 768 blocks of clm*128 bytes
	zr8_wpn_sixteen_reals_first_fft_preload
b1b:	zr8_wpn_sixteen_reals_first_fft rsi, 128, 12*clmblkdst, 24*clmblkdst, 48*clmblkdst, r15, 0, rdi, 2, rdx, 0, 8*clm, L1PREFETCH_ALL, 128
	zloop	8*clm, b1b, -8*clm*128+clmblkdst, r15, ZMM_SCD7, rdx, 16*64, r13, 16 ;; 8*clm iters prefetching 8*clm cache lines, next source, fixed s/c, grp multipliers pointer, next xor mask
	zloop	12, b1b, -12*clmblkdst		;; Test loop counter, restore source pointer
	end_timer 22

;; Do FFT levels 5,6,7
;; On input the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	96K	+1K	...
;;	8K
;;	...
;;	88K
;;	192K
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	8K	+1K	...
;;	16K
;;	...

;; Do 8 twentyfour_reals_fft operations
;; Do 56 twelve_complex_fft operations
;;	distance between fft data elements is 8K
;;	do 8 macros each processing 192 data values

	start_timer 24
	bump	rdi, -96*clm*2-8*clm*ZMM_SCD12	;; Sin/cos pointer
	zloop_init  8*8*clm			;; 8 iterations of 8*clm
	zloop_set_clm_prefetch ZQUAD		;; Prefetch 128 of 768 blocks of clm*128 bytes
IFDEF OLDDD
	zr12_csc_twentyfour_reals_fft_preload
b2a:	zr12_csc_twentyfour_reals_fft rsi, 128, clmblkdst, 2*clmblkdst, 4*clmblkdst, rdi, ZMM_SCD12, 8*clm, L1PREFETCH_ALL, 128
	zloop	8*clm, b2a, -8*clm*128+12*clmblkdst, rdi, -8*clm*ZMM_SCD12 ;; Test loop counter, next source pointer, restore sin/cos pointer
	loops_reset				;; 7 more iters of 8*clm
	zr12_twelve_complex_djbfft_preload
b2b:	zr12_twelve_complex_djbfft rsi, 128, clmblkdst, 2*clmblkdst, 4*clmblkdst, rdi, ZMM_SCD12, 8*clm, L1PREFETCH_ALL, 128
	zloop	8*clm, b2b, -8*clm*128+12*clmblkdst, rdi, -8*clm*ZMM_SCD12 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	8, b2b, -8*12*clmblkdst		;; Test loop counter, restore source pointer
ELSE
b2a:	zr12_csc_twentyfour_reals_fft_preload
	zr12_csc_twentyfour_reals_fft rsi, 12*clmblkdst, clmblkdst, 2*clmblkdst, 4*clmblkdst, rdi, 0, 1, L1PREFETCH_ALL, 12*clmblkdst
	zr12_twelve_complex_djbfft_preload
	jmp	b2c				;; 7 more iters
b2b:	zr12_twelve_complex_djbfft rsi, 12*clmblkdst, clmblkdst, 2*clmblkdst, 4*clmblkdst, rdi, 0, 1, L1PREFETCH_ALL, 12*clmblkdst
b2c:	zloop	8, b2b, -8*12*clmblkdst+128, rdi, ZMM_SCD12 ;; Test loop counter, next source pointer, next sin/cos ptr
	zloop	8*clm, b2a, -8*clm*128		;; Test loop counter, restore source pointer
ENDIF
	end_timer 24

;; Do FFT levels 8,9,10
;; On input the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	8K	+1K	...
;;	16K
;;	...
;; The input is swizzled to hold these data values:
;;	0K	+1	+1	+1	+1	+1	+1	+1	8K	+1	...
;;	1K	...
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0K	+1	+1	+1	+1	+1	+1	+1	1K	+1	...
;;	2K
;;	...

;; Do 1 sixteen_reals_fft operations
;;	distance between fft data elements is 1K
;;	do 0.125 macros each processing 128 data values
;; Do 95 eight_complex_fft operations
;;	distance between fft data elements is 1K
;;	do 11.875 macros each processing 128 data values

	start_timer 26
IFDEF OLDDD
	bump	rdi, -7*clm*ZMM_SCD1-clm*ZMM_SCD4 ;; Real data sin/cos pointer, will become delayed sin/cos multipliers
ELSE
	bump	rdi, -8*clm*ZMM_SCD12-7*clm*ZMM_SCD1-clm*ZMM_SCD4 ;; Real data sin/cos pointer, will become delayed sin/cos multipliers
ENDIF
	lea	rbx, [rdi-clm*ZMM_SCD4]		;; Complex data sin/cos ptr
	lea	r9, [rbx-clm*64-clm*64]		;; Column weights
	lea	rdx, ZMM_TMPS			;; Computed clm*SCD4 sin/cos values
	mov	rbp, DATA_ADDR			;; Destination pointer
	mov	r8, pass2blkdst			;; blkdst
	lea	r10, [2*blkdstreg+blkdstreg]	;; 3*blkdst
	lea	r13, [rbp+4*blkdstreg]		;; Dest+4*blkdst pointer
pfing	mov	rcx, PREMULT_PREFETCH		;; Load premult/sin/cos prefetch pointer

	zr8_rsc_complex_and_real_preload	;; Preload constants before calling zloop_init
	zloop_init 12*8*clm			;; 12 iters of 8*clm
	zloop_set_prefetch ZTRIPLE		;; Prefetch 288*clm times
	zloop_set_alternate_prefetch_increment 230, 288  ;; Prefetch approximately 230*clm cache lines in those 288*clm prefetches
	zr8_rsc_wpn_eight_complex_calc_sincos_simple r9, rbx, clm ;; Generate clm*SCD8 sin/cos values from delayed and complex multipliers
b8b:	zr8_rsc_wpn_sgreg_2sc_sixteen_reals_fft8 rsi, 1024, 128, 256, 512, rbp, 128, blkdstreg, blkdst3reg, r13, rdx, ZMM_SCD8, r9, 64, rdi, ZMM_SCD4, clm, L1PREFETCH_DEST_ALL
	zloop	clm, b8b, -clm*1024+clmblkdst, rbp, -clm*128+8*blkdstreg, r13, -clm*128+8*blkdstreg, rdx, -clm*ZMM_SCD8, r9, -clm*64 ;; Loop, next source/dest pointers, restore premult/sine/cosine pointer
	jmp	b9aa

	loops_reset
b9a:	zr8_rsc_wpn_eight_complex_calc_sincos r9, rdi, rbx, clm ;; Generate clm*SCD8 sin/cos values from delayed and complex multipliers
	bump	rdi, clm*ZMM_SCD1		;; Next set of delayed multipliers
b9aa:
b9b:	zr8_rsc_wpn_sgreg_eight_complex_fft8 rsi, 1024, 128, 256, 512, rbp, 128, blkdstreg, blkdst3reg, r13, rdx, ZMM_SCD8, clm, L1PREFETCH_DEST_ALL
	zloop	clm, b9b, -clm*1024+clmblkdst, rbp, -clm*128+8*blkdstreg, r13, -clm*128+8*blkdstreg, rdx, -clm*ZMM_SCD8 ;; Loop, next source/dest pointers, restore premult/sine/cosine pointer
	zloop	12, b9b				;; Test loop counter
	zloop	8, b9a				;; Test loop counter
	end_timer 26

;; Work on next 8*clm sets of 1536 values

endlp:	zpass1_get_next_block pass2, c0b, b0b ;; Get next block and jump
done:	zfft_footer
	ENDM


;; Perform a shared pass 1 negacyclic FFT of length 1536 doubles.
;; Memory examples are for a 1536K FFT with 10 levels done in pass 2.
;; FFT data needed:
;;		8*clm*1536 doubles					Total: 1536*clm 64-byte cache lines
;; Fixed s/c data needed:
;;		12*ZMM_CD8SCD4 (complex premultipliers and fixed s/c data), Total: 192 64-byte cache lines
;; Group multipliers needed:
;;		12*16*64 (weights)
;;		12*16*64 (inverse weights),				Total: 384 64-byte cache lines
;; Variable data needed:
;;		clm*64	(column weights)
;;		clm*64	(column inverse weights)
;;		clm*ZMM_SCD4 (complex reduced s/c multipliers)
;;		8*clm*ZMM_SCD1 (delayed s/c multipliers)
;;		8*clm*ZMM_SCD6 (12-complex sin/cos data)
;;		96*clm*2 (group multiplier fudge flags)
;;		96*clm*2 (biglit flags),				Total: 128*clm 64-byte cache lines
;; Total memory consumed (add for scratch area, prefetched data, carries):
;;		(1536*3+128=4736)*clm + (192+384+1536/8=768) cache lines
;;		clm=1,2,4:  344KB, 640KB, 1232KB

zr4dwpn_pass1sc1536ac MACRO
	LOCAL	b0b, b1b, b2b, b9a, b9b, c0b, c1b, c2b, c9a, c9b
	LOCAL	pass1, pass2, endlp, done, yes_fft

	zfft_header pass1

;; Do pass 2 (the last 10 FFT levels if this is a 1536K FFT)

pass2:	start_timer 0
	mov	rax, ZMM_PASS2_ROUTINE
	call	rax
	end_timer 0
	cmp	ffttype, 1		;; We're done if FFTing only
	je	done

;; Do the inverse FFT

	pass1_inverse_fft_setup c0b

;; Do inverse FFT levels 8.9.10
;; On input the 128-byte double cache lines hold these data values:
;;	0K	+1	+1	+1	+1	+1	+1	+1	1K	+1	...
;;	2K
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0K	+1	+1	+1	+1	+1	+1	+1	8K	+1	...
;;	1K	...
;;	...
;; The output cache lines are swizzled to hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	8K	+1K	...
;;	16K
;;	...

;; Do 96 eight_complex_unfft operations
;;	distance between fft data elements is 1K
;;	do 12 macros each processing 128 data values

c0b:	start_timer 16
	mov	rbp, DATA_ADDR			;; Get source address
	mov	rsi, scratch_area		;; Get address of scratch area
	set_data_prefetch_ptrs			;; Init prefetching pointers
	mov	r9, PREMULT_ADDR		;; Variable data
	bump	r9, clm*64			;; Inverse column weights
	lea	rbx, [r9+clm*64]		;; Complex data sin/cos ptr
	lea	rdi, [rbx+clm*ZMM_SCD4]		;; Delayed sin/cos multipliers
	lea	rdx, ZMM_TMPS			;; Computed clm*SCD8 sin/cos values
	mov	r8, pass2blkdst			;; blkdst
	lea	r10, [2*blkdstreg+blkdstreg]	;; 3*blkdst
	lea	r13, [rbp+4*blkdstreg]		;; Source+4*blkdst pointer
	lea	r12, [rbp+8*blkdstreg]		;; L1 prefetch pointer part 1
	bump2	r12, (clm MOD 2), 8*blkdstreg	;; L1 prefetch pointer part 2
	lea	r14, [r12+4*blkdstreg]		;; L1 prefetch pointer + 4*blkdst

	zr8_rsc_complex_only_preload		;; Preload constants before calling zloop_init
	zloop_init 12*8*clm			;; 12 iters of 8*clm iterations
	zloop_set_clm_prefetch ZDOUBLE		;; Prefetch 96 of 768 blocks of clm*128 bytes
c9a:	zr8_rsc_wpn_eight_complex_calc_sincos r9, rdi, rbx, clm ;; Generate clm*SCD8 sin/cos values from weights, delayed, and complex multipliers
	bump	rdi, clm*ZMM_SCD1		;; Next set of delayed sin/cos multipliers
c9b:	zr8_rsc_wpn_sgreg_eight_complex_unfft8 rbp, 128, blkdstreg, blkdst3reg, r13, rsi, 1024, 128, 256, 512, rdx, ZMM_SCD8, clm, L1PREFETCH_ALL, r12, r14 ;; Prefetch short dist ahead
	zloop	clm, c9b, -clm*1024+clmblkdst, rbp, -clm*128+8*blkdstreg, r13, -clm*128+8*blkdstreg, rdx, -clm*ZMM_SCD8, r12, -clm*128+8*blkdstreg, r14, -clm*128+8*blkdstreg ;; Loop, next src/dest ptrs, restore sin/cos ptr
	zloop	12, c9b				;; Test loop counter
	zloop	8, c9a, -8*12*clmblkdst		;; Test loop counter, restore source pointer
	end_timer 16

;; Do inverse FFT levels 5,6,7
;; On input the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	8K	+1K	...
;;	16K
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	96K	+1K	...
;;	8K
;;	...
;;	88K
;;	192K
;;	...

;; Do 64 twelve_complex_unfft operations
;;	distance between fft data elements is 8K
;;	do 8 macros each processing 192 data values

	start_timer 18
	zloop_init  8*8*clm			;; 8 iterations of 8*clm
	zloop_set_clm_prefetch ZQUAD		;; Prefetch 128 of 768 blocks of clm*128 bytes
	zr12_twelve_complex_djbunfft_preload
IFDEF OLDDD
c2b:	zr12_twelve_complex_djbunfft rsi, 128, clmblkdst, 2*clmblkdst, 4*clmblkdst, rdi, ZMM_SCD6, 8*clm, L1PREFETCH_ALL, 128
	zloop	8*clm, c2b, -8*clm*128+12*clmblkdst, rdi, -8*clm*ZMM_SCD6 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	8, c2b, -8*12*clmblkdst, rdi, 8*clm*ZMM_SCD6 ;; Test loop counter, restore source pointer, next sin/cos pointer
ELSE
c2b:	zr12_twelve_complex_djbunfft rsi, 12*clmblkdst, clmblkdst, 2*clmblkdst, 4*clmblkdst, rdi, 0, 8, L1PREFETCH_ALL, 12*clmblkdst
	zloop	8, c2b, -8*12*clmblkdst+128, rdi, ZMM_SCD6 ;; Test loop counter, next source pointer, next sin/cos pointer
	zloop	8*clm, c2b, -8*clm*128		;; Test loop counter, restore source pointer
ENDIF
	end_timer 18

;; Do inverse FFT levels 1,2,3,4
;; On input the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	96K	+1K	...
;;	8K
;;	...
;;	88K
;;	192K
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	768K	+1K	...
;;	8K	...
;;	...
;;	760K

;; Do 96 eight_complex_unfft operations
;;	distance between fft data elements is 96K
;;	do 12 macros each processing 128 data values

	start_timer 20
	mov	r15, sincos2			;; Load fixed premultiplier/sin/cos ptr
	sub	rbx, rbx			;; Clear register for loading fudge flags index
	mov	r12, compressed_fudges		;; Address of the compressed fudge flags
	mov	r13, r12			;; The XOR masks for fudge flags come from the compressed fudges
	zloop_init 12*8*clm			;; 12 iters of 8*clm
	zloop_set_clm_prefetch ZDOUBLE		;; Prefetch 96 of 768 blocks of clm*128 bytes
	zr8_csc_wpn_eight_complex_last_djbunfft_preload
c1b:	zr8_csc_wpn_eight_complex_last_djbunfft rsi, 128, 12*clmblkdst, 24*clmblkdst, 48*clmblkdst, r15, 0, rdi, 2, 8*clm, L1PREFETCH_ALL, 128
	zloop	8*clm, c1b, -8*clm*128+clmblkdst, r15, ZMM_CD8SCD4, rdx, 16*64, r13, 16 ;; 8*clm iterations, next fixed s/c, next grp multipliers, next xor mask
	zloop	12, c1b				;; 12 iterations
	end_timer 20

;; Normalize these values

	pass1_normalize yes_fft			;; Normalize while prefetching 96 of 768 blocks of clm*128 bytes

;; Copy data from the scratch area back to the FFT data area

	copy_scratch_data_to_fft
	jmp	endlp				;; Skip FFT code if flag not set

;; Fire up auxiliary threads to do pass 1 forward FFT

pass1:	pass1_forward_fft_setup b0b

;; Copy data from the FFT data area back to the scratch area

b0b:	copy_fft_data_to_scratch

;; Do FFT levels 1,2,3,4
;; On input the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	768K	+1K	...
;;	8K	...
;;	...
;;	760K
;; On output the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	96K	+1K	...
;;	8K
;;	...
;;	88K
;;	192K
;;	...

;; Do 96 eight_complex_first_fft operations
;;	distance between fft data elements is 96K
;;	do 12 macros each processing 128 data values

yes_fft:start_timer 22
	mov	rsi, scratch_area		;; Get address of scratch area
	mov	r15, sincos2			;; Load fixed premultiplier/sin/cos ptr
	mov	rdi, PREMULT_ADDR		;; Calculate pointer to fudge flags
	bump	rdi, 8*clm*ZMM_SCD6+8*clm*ZMM_SCD1+clm*ZMM_SCD4+clm*64+clm*64 ;; Addr of the group multiplier fudge flags
	mov	rdx, norm_grp_mults		;; Addr of the group multipliers
	sub	rbx, rbx			;; Clear register for loading fudge flags index
	mov	r12, compressed_fudges		;; Address of the compressed fudge flags
	mov	r13, r12			;; The XOR masks for fudge flags come from the compressed fudges
	set_data_prefetch_ptrs			;; Init prefetching pointers
pfing	mov	r8, pass2blkdst			;; blkdst
pfing	imul	rax, blkdstreg, (424-0)		;; We've already prefetched 424 of 768 blocks (4 too few)
pfing	add	rcx, rax
	zloop_init 12*8*clm			;; 12 iters of 8*clm
	zloop_set_clm_prefetch ZQUAD		;; Prefetch 192 of 768 blocks of clm*128 bytes
	zr8_csc_wpn_eight_complex_first_djbfft_preload
b1b:	zr8_csc_wpn_eight_complex_first_djbfft rsi, 128, 12*clmblkdst, 24*clmblkdst, 48*clmblkdst, r15, 0, rdi, 2, rdx, 0, 8*clm, L1PREFETCH_ALL, 128
	zloop	8*clm, b1b, -8*clm*128+clmblkdst, r15, ZMM_CD8SCD4, rdx, 16*64, r13, 16 ;; 8*clm iters, next source, fixed s/c, grp multipliers pointer, next xor mask
	zloop	12, b1b, -12*clmblkdst		;; Test loop counter, restore source pointer
	end_timer 22

;; Do FFT levels 5,6,7
;; On input the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	96K	+1K	...
;;	8K
;;	...
;;	88K
;;	192K
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	8K	+1K	...
;;	16K
;;	...

;; Do 64 twelve_complex_fft operations
;;	distance between fft data elements is 8K
;;	do 8 macros each processing 192 data values

	start_timer 24
	bump	rdi, -96*clm*2-8*clm*ZMM_SCD6	;; Sin/cos pointer
	zloop_init  8*8*clm			;; 8 iterations of 8*clm
	zloop_set_clm_prefetch ZQUAD		;; Prefetch 128 of 768 blocks of clm*128 bytes
	zr12_twelve_complex_djbfft_preload
IFDEF OLDDD
b2b:	zr12_twelve_complex_djbfft rsi, 128, clmblkdst, 2*clmblkdst, 4*clmblkdst, rdi, ZMM_SCD6, 8*clm, L1PREFETCH_ALL, 128
	zloop	8*clm, b2b, -8*clm*128+12*clmblkdst, rdi, -8*clm*ZMM_SCD6 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	8, b2b, -8*12*clmblkdst		;; Test loop counter, restore source pointer
ELSE
b2b:	zr12_twelve_complex_djbfft rsi, 12*clmblkdst, clmblkdst, 2*clmblkdst, 4*clmblkdst, rdi, 0, 8, L1PREFETCH_ALL, 12*clmblkdst
	zloop	8, b2b, -8*12*clmblkdst+128, rdi, ZMM_SCD6 ;; Test loop counter, next source pointer, next sin/cos pointer
	zloop	8*clm, b2b, -8*clm*128		;; Test loop counter, restore source pointer
ENDIF
	end_timer 24

;; Do FFT levels 8,9,10
;; On input the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	8K	+1K	...
;;	16K
;;	...
;; The input is swizzled to hold these data values:
;;	0K	+1	+1	+1	+1	+1	+1	+1	8K	+1	...
;;	1K	...
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0K	+1	+1	+1	+1	+1	+1	+1	1K	+1	...
;;	2K
;;	...

;; Do 96 eight_complex_fft operations
;;	distance between fft data elements is 1K
;;	do 12 macros each processing 128 data values

	start_timer 26
	mov	rbp, DATA_ADDR			;; Destination pointer
IFDEF OLDDD
	bump	rdi, -8*clm*ZMM_SCD1		;; Delayed sin/cos multipliers
ELSE
	bump	rdi, -8*clm*ZMM_SCD6-8*clm*ZMM_SCD1 ;; Delayed sin/cos multipliers
ENDIF
	lea	rbx, [rdi-clm*ZMM_SCD4]		;; Complex data sin/cos ptr
	lea	r9, [rbx-clm*64-clm*64]		;; Column weights
	lea	rdx, ZMM_TMPS			;; Computed clm*SCD4 sin/cos values
	mov	r8, pass2blkdst			;; blkdst
	lea	r10, [2*blkdstreg+blkdstreg]	;; 3*blkdst
	lea	r13, [rbp+4*blkdstreg]		;; Dest+4*blkdst pointer
pfing	mov	rcx, PREMULT_PREFETCH		;; Load premult/sin/cos prefetch pointer

	zr8_rsc_complex_only_preload		;; Preload constants before calling zloop_init
	zloop_init 12*8*clm			;; 12 iters of 8*clm
	zloop_set_prefetch ZDOUBLE		;; Prefetch 192*clm cache lines
	zloop_set_alternate_prefetch_increment 128, 192  ;; Prefetch approximately 111*clm cache lines in those 192*clm prefetches
b9a:	zr8_rsc_wpn_eight_complex_calc_sincos r9, rdi, rbx, clm ;; Generate clm*SCD8 sin/cos values from weights, delayed and complex multipliers
	bump	rdi, clm*ZMM_SCD1		;; Next set of delayed multipliers
b9b:	zr8_rsc_wpn_sgreg_eight_complex_fft8 rsi, 1024, 128, 256, 512, rbp, 128, blkdstreg, blkdst3reg, r13, rdx, ZMM_SCD8, clm, L1PREFETCH_DEST_ALL
	zloop	clm, b9b, -clm*1024+clmblkdst, rbp, -clm*128+8*blkdstreg, r13, -clm*128+8*blkdstreg, rdx, -clm*ZMM_SCD8 ;; Loop, next source/dest pointers, restore premult/sine/cosine pointer
	zloop	12, b9b				;; Test loop counter
	zloop	8, b9a				;; Test loop counter
	end_timer 26

;; Work on next 8*clm sets of 1536 values

endlp:	zpass1_get_next_block pass2, c0b, b0b ;; Get next block and jump
done:	zfft_footer
	ENDM



;; Perform a shared pass 1 FFT of length 1536 doubles.
;; Memory examples are for a 1536K FFT with 10 levels done in pass 2.
;; FFT data needed:
;;		8*clm*1536 doubles					Total: 1536*clm 64-byte cache lines
;; Fixed s/c data needed:
;;		8*ZMM_SCD11 (fixed s/c data),				Total: 176 64-byte cache lines
;; Group multipliers needed:
;;		8*24*64 (weights)
;;		8*24*64 (inverse weights),				Total: 384 64-byte cache lines
;; Variable data needed:
;;		clm*64	(column weights)
;;		clm*64	(column inverse weights)
;;		clm*ZMM_SCD4 (complex reduced s/c multipliers)
;;		clm*ZMM_SCD4 (real s/c multipliers)
;;		11*clm*ZMM_SCD1 (delayed s/c multipliers)
;;		8*clm*ZMM_SCD8 (8-complex 16-real sin/cos data)
;;		64*clm*3 (group multiplier fudge flags)
;;		64*clm*3 (biglit flags),				Total: 174*clm 64-byte cache lines
;; Total memory consumed (add for scratch area, prefetched data, carries):
;;		(1536*3+174=4782)*clm + (176+384+1536/8=752) cache lines
;;		clm=1,2,4:  346KB, 645KB, 1243KB

ALTERNATEzr4dwpn_pass1sc1536 MACRO
	LOCAL	b0b, b1b, b2a, b2b, b2c, b8a, b8b, b9aa, b9a, b9b, c0b, c1b, c2a, c2b, c2c, c8b, c9aa, c9a, c9b
	LOCAL	pass1, pass2, endlp, done, yes_fft

	zfft_header pass1

;; Do pass 2 (the last 10 FFT levels if this is a 1536K FFT)

pass2:	start_timer 0
	mov	rax, ZMM_PASS2_ROUTINE
	call	rax
	end_timer 0
	cmp	ffttype, 1		;; We're done if FFTing only
	je	done

;; Do the inverse FFT

	pass1_inverse_fft_setup c0b

;; Do inverse FFT levels 8,9,10
;; On input the 128-byte double cache lines hold these data values:
;;	0K	+1	+1	+1	+1	+1	+1	+1	1K	+1	...
;;	2K
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0K	+1	+1	+1	+1	+1	+1	+1	8K	+1	...
;;	1K	...
;;	...
;; The output cache lines are swizzled to hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	8K	+1K	...
;;	16K
;;	...

;; Do 1 sixteen_reals_unfft operations
;;	distance between fft data elements is 1K
;;	do 0.125 macros each processing 128 data values
;; Do 95 eight_complex_unfft operations
;;	distance between fft data elements is 1K
;;	do 11.875 macros each processing 128 data values

c0b:	start_timer 16
	mov	rbp, DATA_ADDR			;; Get source address
	mov	rsi, scratch_area		;; Get address of scratch area
	set_data_prefetch_ptrs			;; Init prefetching pointers
	mov	r9, PREMULT_ADDR		;; Variable data
	bump	r9, clm*64			;; Inverse column weights
	lea	rbx, [r9+clm*64]		;; Complex data sin/cos ptr
	lea	rdi, [rbx+clm*ZMM_SCD4]		;; Real data sin/cos pointer, will become delayed sin/cos multipliers
	lea	rdx, ZMM_TMPS			;; Computed clm*SCD8 sin/cos values
	mov	r8, pass2blkdst			;; blkdst
	lea	r10, [2*blkdstreg+blkdstreg]	;; 3*blkdst
	lea	r13, [rbp+4*blkdstreg]		;; Source+4*blkdst pointer
	lea	r12, [rbp+8*blkdstreg]		;; L1 prefetch pointer part 1
	bump2	r12, (clm MOD 2), 8*blkdstreg	;; L1 prefetch pointer part 2
	lea	r14, [r12+4*blkdstreg]		;; L1 prefetch pointer + 4*blkdst

	zr8_rsc_complex_and_real_preload	;; Preload constants before calling zloop_init
	zloop_init 12*8*clm			;; 12 iters of 8*clm iterations
	zloop_set_clm_prefetch ZDOUBLE		;; Prefetch 96 of 768 blocks of clm*128 bytes
	zr8_rsc_wpn_eight_complex_calc_sincos_simple r9, rbx, clm ;; Generate clm*SCD8 sin/cos values from delayed and complex multipliers
c8b:	zr8_rsc_wpn_sgreg_2sc_sixteen_reals_unfft8 rbp, 128, blkdstreg, blkdst3reg, r13, rsi, 1024, 128, 256, 512, rdx, ZMM_SCD8, r9, 64, rdi, ZMM_SCD4, clm, L1PREFETCH_ALL, r12, r14 ;; Prefetch short dist ahead
	zloop	clm, c8b, -clm*1024+clmblkdst, rbp, -clm*128+8*blkdstreg, r13, -clm*128+8*blkdstreg, rdx, -clm*ZMM_SCD8, r9, -clm*64, r12, -clm*128+8*blkdstreg, r14, -clm*128+8*blkdstreg ;; Loop, next src/dest ptrs, restore sin/cos ptr
	jmp	c9aa

	loops_reset
c9a:	zr8_rsc_wpn_eight_complex_calc_sincos r9, rdi, rbx, clm ;; Generate clm*SCD8 sin/cos values from delayed and complex multipliers
	bump	rdi, clm*ZMM_SCD1		;; Next set of delayed sin/cos multipliers
c9aa:
c9b:	zr8_rsc_wpn_sgreg_eight_complex_unfft8 rbp, 128, blkdstreg, blkdst3reg, r13, rsi, 1024, 128, 256, 512, rdx, ZMM_SCD8, clm, L1PREFETCH_ALL, r12, r14 ;; Prefetch short dist ahead
	zloop	clm, c9b, -clm*1024+clmblkdst, rbp, -clm*128+8*blkdstreg, r13, -clm*128+8*blkdstreg, rdx, -clm*ZMM_SCD8, r12, -clm*128+8*blkdstreg, r14, -clm*128+8*blkdstreg ;; Loop, next src/dest ptrs, restore sin/cos ptr
	zloop	8, c9b, -8*clmblkdst+clmblkdst8	;; Test loop counter, next source pointer
	zloop	12, c9a, -12*clmblkdst8		;; Test loop counter, restore source pointer
	end_timer 16

;; Do inverse FFT levels 5,6,7
;; On input the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	8K	+1K	...
;;	16K
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	64K	+1K	...
;;	8K
;;	...
;;	56K
;;	128K
;;	...

;; Do 8 sixteen_reals_unfft operations
;; Do 88 eight_complex_unfft operations
;;	distance between fft data elements is 8K
;;	do 12 macros each processing 128 data values

	start_timer 18
	zloop_init  12*8*clm			;; 12 iterations of 8*clm
	zloop_set_clm_prefetch ZQUAD		;; Prefetch 192 of 786 blocks of clm*128 bytes
IFDEF OLDDD
	zr8_csc_sixteen_reals_unfft_preload
c2a:	zr8_csc_sixteen_reals_unfft rsi, 128, clmblkdst, 2*clmblkdst, 4*clmblkdst, rdi, ZMM_SCD8, 8*clm, L1PREFETCH_ALL, 128
	zloop	8*clm, c2a, -8*clm*128+clmblkdst8, rdi, -8*clm*ZMM_SCD8 ;; Test loop counter, next source pointer, restore sin/cos pointer
	loops_reset				;; 11 more iters of 8*clm
	zr8_eight_complex_djbunfft_preload
c2b:	zr8_eight_complex_djbunfft rsi, 128, clmblkdst, 2*clmblkdst, 4*clmblkdst, rdi, ZMM_SCD8, 8*clm, L1PREFETCH_ALL, 128
	zloop	8*clm, c2b, -8*clm*128+clmblkdst8, rdi, -8*clm*ZMM_SCD8 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	12, c2b, -12*clmblkdst8, rdi, 8*clm*ZMM_SCD8 ;; Test loop counter, restore source pointer, next sin/cos pointer
ELSE
c2a:	zr8_csc_sixteen_reals_unfft_preload
	zr8_csc_sixteen_reals_unfft rsi, clmblkdst8, clmblkdst, 2*clmblkdst, 4*clmblkdst, rdi, 0, 1, L1PREFETCH_ALL, clmblkdst8
	zr8_eight_complex_djbunfft_preload
	jmp	c2c				;; 11 more iters
c2b:	zr8_eight_complex_djbunfft rsi, clmblkdst8, clmblkdst, 2*clmblkdst, 4*clmblkdst, rdi, 0, 1, L1PREFETCH_ALL, clmblkdst8
c2c:	zloop	12, c2b, -12*clmblkdst8+128, rdi, ZMM_SCD8 ;; Test loop counter, next source pointer, next sin/cos pointer
	zloop	8*clm, c2a, -8*clm*128		;; Test loop counter, restore source pointer
ENDIF
	end_timer 18

;; Do inverse FFT levels 1,2,3,4
;; On input the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	64K	+1K	...
;;	8K
;;	...
;;	56K
;;	128K
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	768K	+1K	...
;;	8K
;;	...
;;	760K

;; Do 64 twentyfour_reals_unfft operations
;;	distance between fft data elements is 64K
;;	do 8 macros each processing 192 data values

	start_timer 20
	mov	r15, sincos2			;; Load fixed premultiplier/sin/cos ptr
	sub	rbx, rbx			;; Clear register for loading fudge flags index
	mov	r12, compressed_fudges		;; Address of the compressed fudge flags
	mov	r13, r12			;; The XOR masks for fudge flags come from the compressed fudges
	zloop_init 8*8*clm			;; 8 iters of 8*clm
	zloop_set_clm_prefetch ZDOUBLE		;; Prefetch 64 of 768 blocks of clm*128 bytes
	zr12_wpn_twentyfour_reals_last_unfft_preload
c1b:	zr12_wpn_twentyfour_reals_last_unfft rsi, 128, clmblkdst8, 2*clmblkdst8, 4*clmblkdst8, r15, 0, rdi, 3, 8*clm, L1PREFETCH_ALL, 128
	zloop	8*clm, c1b, -8*clm*128+clmblkdst, r15, ZMM_SCD11, rdx, 24*64, r13, 24 ;; 8*clm iters, next source, fixed s/c, grp multipliers pointer, next xor mask
	zloop	8, c1b				;; Test loop counter
	end_timer 20

;; Normalize these values

	pass1_normalize yes_fft			;; Normalize while prefetching 96 of 768 blocks of clm*128 bytes

;; Copy data from the scratch area back to the FFT data area

	copy_scratch_data_to_fft
	jmp	endlp				;; Skip FFT code if flag not set

;; Fire up auxiliary threads to do pass 1 forward FFT

pass1:	pass1_forward_fft_setup b0b

;; Copy data from the FFT data area back to the scratch area

b0b:	copy_fft_data_to_scratch

;; Do FFT levels 1,2,3,4
;; On input the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	768K	+1K	...
;;	8K	...
;;	...
;;	760K
;; On output the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	64K	+1K	...
;;	8K
;;	...
;;	56K
;;	128K
;;	...

;; Do 64 twentyfour_reals_first_fft operations
;;	distance between fft data elements is 64K
;;	do 8 macros each processing 192 data values

yes_fft:start_timer 22
	mov	rsi, scratch_area		;; Get address of scratch area
	mov	r15, sincos2			;; Load fixed premultiplier/sin/cos ptr
	mov	rdi, PREMULT_ADDR		;; Calculate pointer to fudge flags
	bump	rdi, 8*clm*ZMM_SCD8+11*clm*ZMM_SCD1+clm*ZMM_SCD4+clm*ZMM_SCD4+clm*64+clm*64 ;; Addr of the group multiplier fudge flags
	mov	rdx, norm_grp_mults		;; Addr of the group multipliers
	sub	rbx, rbx			;; Clear register for loading fudge flags index
	mov	r12, compressed_fudges		;; Address of the compressed fudge flags
	mov	r13, r12			;; The XOR masks for fudge flags come from the compressed fudges
	set_data_prefetch_ptrs			;; Init prefetching pointers
pfing	mov	r8, pass2blkdst			;; blkdst
pfing	imul	rax, blkdstreg, (448-0)		;; We've already prefetched 448 of 768 blocks (zero too many)
pfing	add	rcx, rax
	zloop_init 8*8*clm			;; 8 iters of 8*clm
	zloop_set_clm_prefetch ZQUAD		;; Prefetch 128 of 512 blocks of clm*128 bytes
	zr12_wpn_twentyfour_reals_first_fft_preload
b1b:	zr12_wpn_twentyfour_reals_first_fft rsi, 128, clmblkdst8, 2*clmblkdst8, 4*clmblkdst8, r15, 0, rdi, 3, rdx, 0, 8*clm, L1PREFETCH_ALL, 128
	zloop	8*clm, b1b, -8*clm*128+clmblkdst, r15, ZMM_SCD11, rdx, 24*64, r13, 24 ;; 8*clm iters prefetching 8*clm cache lines, next source, fixed s/c, grp multipliers pointer, next xor mask
	zloop	8, b1b, -8*clmblkdst		;; Test loop counter, restore source pointer
	end_timer 22

;; Do FFT levels 5,6,7
;; On input the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	64K	+1K	...
;;	8K
;;	...
;;	56K
;;	128K
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	8K	+1K	...
;;	16K
;;	...

;; Do 8 sixteen_reals_fft operations
;; Do 88 eight_complex_fft operations
;;	distance between fft data elements is 8K
;;	do 12 macros each processing 128 data values

	start_timer 24
	bump	rdi, -64*clm*3-8*clm*ZMM_SCD8	;; Sin/cos pointer
	zloop_init  12*8*clm			;; 12 iterations of 8*clm
	zloop_set_clm_prefetch ZQUAD		;; Prefetch 192 of 768 blocks of clm*128 bytes
IFDEF OLDDD
	zr8_csc_sixteen_reals_fft_preload
b2a:	zr8_csc_sixteen_reals_fft rsi, 128, clmblkdst, 2*clmblkdst, 4*clmblkdst, rdi, ZMM_SCD8, 8*clm, L1PREFETCH_ALL, 128
	zloop	8*clm, b2a, -8*clm*128+clmblkdst8, rdi, -8*clm*ZMM_SCD8 ;; Test loop counter, next source pointer, restore sin/cos pointer
	loops_reset				;; 11 more iters of 8*clm
	zr8_eight_complex_djbfft_preload
b2b:	zr8_eight_complex_djbfft rsi, 128, clmblkdst, 2*clmblkdst, 4*clmblkdst, rdi, ZMM_SCD8, 8*clm, L1PREFETCH_ALL, 128
	zloop	8*clm, b2b, -8*clm*128+clmblkdst8, rdi, -8*clm*ZMM_SCD8 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	12, b2b, -12*clmblkdst8		;; Test loop counter, restore source pointer
ELSE
b2a:	zr8_csc_sixteen_reals_fft_preload
	zr8_csc_sixteen_reals_fft rsi, clmblkdst8, clmblkdst, 2*clmblkdst, 4*clmblkdst, rdi, 0, 1, L1PREFETCH_ALL, clmblkdst8
	zr8_eight_complex_djbfft_preload
	jmp	b2c				;; 11 more iters
b2b:	zr8_eight_complex_djbfft rsi, clmblkdst8, clmblkdst, 2*clmblkdst, 4*clmblkdst, rdi, 0, 1, L1PREFETCH_ALL, clmblkdst8
b2c:	zloop	12, b2b, -12*clmblkdst8+128, rdi, ZMM_SCD8 ;; Test loop counter,  source pointer, next sin/cos pointer
	zloop	8*clm, b2a, -8*clm*128		;; Test loop counter, restore source pointer
ENDIF
	end_timer 24

;; Do FFT levels 8,9,10
;; On input the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	8K	+1K	...
;;	16K
;;	...
;; The input is swizzled to hold these data values:
;;	0K	+1	+1	+1	+1	+1	+1	+1	8K	+1	...
;;	1K	...
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0K	+1	+1	+1	+1	+1	+1	+1	1K	+1	...
;;	2K
;;	...

;; Do 1 sixteen_reals_fft operations
;;	distance between fft data elements is 1K
;;	do 0.125 macros each processing 128 data values
;; Do 95 eight_complex_fft operations
;;	distance between fft data elements is 1K
;;	do 11.875 macros each processing 128 data values

	start_timer 26
IFDEF OLDDD
	bump	rdi, -11*clm*ZMM_SCD1-clm*ZMM_SCD4 ;; Real data sin/cos pointer, will become delayed sin/cos multipliers
ELSE
	bump	rdi, -8*clm*ZMM_SCD8-11*clm*ZMM_SCD1-clm*ZMM_SCD4 ;; Real data sin/cos pointer, will become delayed sin/cos multipliers
ENDIF
	lea	rbx, [rdi-clm*ZMM_SCD4]		;; Complex data sin/cos ptr
	lea	r9, [rbx-clm*64-clm*64]		;; Column weights
	lea	rdx, ZMM_TMPS			;; Computed clm*SCD4 sin/cos values
	mov	rbp, DATA_ADDR			;; Destination pointer
	mov	r8, pass2blkdst			;; blkdst
	lea	r10, [2*blkdstreg+blkdstreg]	;; 3*blkdst
	lea	r13, [rbp+4*blkdstreg]		;; Dest+4*blkdst pointer
pfing	mov	rcx, PREMULT_PREFETCH		;; Load premult/sin/cos prefetch pointer

	zr8_rsc_complex_and_real_preload	;; Preload constants before calling zloop_init
	zloop_init 12*8*clm			;; 12 iters of 8*clm
	zloop_set_prefetch ZDOUBLE		;; Prefetch 192*clm times
	zloop_set_alternate_prefetch_increment 174, 192  ;; Prefetch approximately 174*clm cache lines in those 192*clm prefetches
	zr8_rsc_wpn_eight_complex_calc_sincos_simple r9, rbx, clm ;; Generate clm*SCD8 sin/cos values from delayed and complex multipliers
b8b:	zr8_rsc_wpn_sgreg_2sc_sixteen_reals_fft8 rsi, 1024, 128, 256, 512, rbp, 128, blkdstreg, blkdst3reg, r13, rdx, ZMM_SCD8, r9, 64, rdi, ZMM_SCD4, clm, L1PREFETCH_DEST_ALL
	zloop	clm, b8b, -clm*1024+clmblkdst, rbp, -clm*128+8*blkdstreg, r13, -clm*128+8*blkdstreg, rdx, -clm*ZMM_SCD8, r9, -clm*64 ;; Loop, next source/dest pointers, restore premult/sine/cosine pointer
	jmp	b9aa

	loops_reset
b9a:	zr8_rsc_wpn_eight_complex_calc_sincos r9, rdi, rbx, clm ;; Generate clm*SCD8 sin/cos values from delayed and complex multipliers
	bump	rdi, clm*ZMM_SCD1		;; Next set of delayed multipliers
b9aa:
b9b:	zr8_rsc_wpn_sgreg_eight_complex_fft8 rsi, 1024, 128, 256, 512, rbp, 128, blkdstreg, blkdst3reg, r13, rdx, ZMM_SCD8, clm, L1PREFETCH_DEST_ALL
	zloop	clm, b9b, -clm*1024+clmblkdst, rbp, -clm*128+8*blkdstreg, r13, -clm*128+8*blkdstreg, rdx, -clm*ZMM_SCD8 ;; Loop, next source/dest pointers, restore premult/sine/cosine pointer
	zloop	8, b9b, -8*clmblkdst+clmblkdst8	;; Test loop counter, next source pointer
	zloop	12, b9a				;; Test loop counter
	end_timer 26

;; Work on next 8*clm sets of 1536 values

endlp:	zpass1_get_next_block pass2, c0b, b0b ;; Get next block and jump
done:	zfft_footer
	ENDM


;; Perform a shared pass 1 negacyclic FFT of length 1536 doubles.
;; Memory examples are for a 1536K FFT with 10 levels done in pass 2.
;; FFT data needed:
;;		8*clm*1536 doubles					Total: 1536*clm 64-byte cache lines
;; Fixed s/c data needed:
;;		8*ZMM_CD12SCD6 (complex premultipliers and fixed s/c data), Total: 192 64-byte cache lines
;; Group multipliers needed:
;;		8*24*64 (weights)
;;		8*24*64 (inverse weights),				Total: 384 64-byte cache lines
;; Variable data needed:
;;		clm*64	(column weights)
;;		clm*64	(column inverse weights)
;;		clm*ZMM_SCD4 (complex reduced s/c multipliers)
;;		12*clm*ZMM_SCD1 (delayed s/c multipliers)
;;		8*clm*ZMM_SCD4 (8-complex sin/cos data)
;;		64*clm*3 (group multiplier fudge flags)
;;		64*clm*3 (biglit flags),				Total: 104*clm 64-byte cache lines
;; Total memory consumed (add for scratch area, prefetched data, carries):
;;		(1536*3+104=4712)*clm + (192+384+1536/8=768) cache lines
;;		clm=1,2,4:  343KB, 637KB, 1226KB

ALTERNATEzr4dwpn_pass1sc1536ac MACRO
	LOCAL	b0b, b1b, b2b, b9a, b9b, c0b, c1b, c2b, c9a, c9b
	LOCAL	pass1, pass2, endlp, done, yes_fft

	zfft_header pass1

;; Do pass 2 (the last 10 FFT levels if this is a 1536K FFT)

pass2:	start_timer 0
	mov	rax, ZMM_PASS2_ROUTINE
	call	rax
	end_timer 0
	cmp	ffttype, 1		;; We're done if FFTing only
	je	done

;; Do the inverse FFT

	pass1_inverse_fft_setup c0b

;; Do inverse FFT levels 8.9.10
;; On input the 128-byte double cache lines hold these data values:
;;	0K	+1	+1	+1	+1	+1	+1	+1	1K	+1	...
;;	2K
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0K	+1	+1	+1	+1	+1	+1	+1	8K	+1	...
;;	1K	...
;;	...
;; The output cache lines are swizzled to hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	8K	+1K	...
;;	16K
;;	...

;; Do 96 eight_complex_unfft operations
;;	distance between fft data elements is 1K
;;	do 12 macros each processing 128 data values

c0b:	start_timer 16
	mov	rbp, DATA_ADDR			;; Get source address
	mov	rsi, scratch_area		;; Get address of scratch area
	set_data_prefetch_ptrs			;; Init prefetching pointers
	mov	r9, PREMULT_ADDR		;; Variable data
	bump	r9, clm*64			;; Inverse column weights
	lea	rbx, [r9+clm*64]		;; Complex data sin/cos ptr
	lea	rdi, [rbx+clm*ZMM_SCD4]		;; Delayed sin/cos multipliers
	lea	rdx, ZMM_TMPS			;; Computed clm*SCD8 sin/cos values
	mov	r8, pass2blkdst			;; blkdst
	lea	r10, [2*blkdstreg+blkdstreg]	;; 3*blkdst
	lea	r13, [rbp+4*blkdstreg]		;; Source+4*blkdst pointer
	lea	r12, [rbp+8*blkdstreg]		;; L1 prefetch pointer part 1
	bump2	r12, (clm MOD 2), 8*blkdstreg	;; L1 prefetch pointer part 2
	lea	r14, [r12+4*blkdstreg]		;; L1 prefetch pointer + 4*blkdst

	zr8_rsc_complex_only_preload		;; Preload constants before calling zloop_init
	zloop_init 12*8*clm			;; 12 iters of 8*clm iterations
	zloop_set_clm_prefetch ZDOUBLE		;; Prefetch 96 of 768 blocks of clm*128 bytes
c9a:	zr8_rsc_wpn_eight_complex_calc_sincos r9, rdi, rbx, clm ;; Generate clm*SCD8 sin/cos values from weights, delayed, and complex multipliers
	bump	rdi, clm*ZMM_SCD1		;; Next set of delayed sin/cos multipliers
c9b:	zr8_rsc_wpn_sgreg_eight_complex_unfft8 rbp, 128, blkdstreg, blkdst3reg, r13, rsi, 1024, 128, 256, 512, rdx, ZMM_SCD8, clm, L1PREFETCH_ALL, r12, r14 ;; Prefetch short dist ahead
	zloop	clm, c9b, -clm*1024+clmblkdst, rbp, -clm*128+8*blkdstreg, r13, -clm*128+8*blkdstreg, rdx, -clm*ZMM_SCD8, r12, -clm*128+8*blkdstreg, r14, -clm*128+8*blkdstreg ;; Loop, next src/dest ptrs, restore sin/cos ptr
	zloop	8, c9b, -8*clmblkdst+clmblkdst8	;; Test loop counter, next source pointer
	zloop	12, c9a, -12*clmblkdst8		;; Test loop counter, restore source pointer
	end_timer 16

;; Do inverse FFT levels 5,6,7
;; On input the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	8K	+1K	...
;;	16K
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	64K	+1K	...
;;	8K
;;	...
;;	56K
;;	128K
;;	...

;; Do 96 eight_complex_unfft operations
;;	distance between fft data elements is 8K
;;	do 12 macros each processing 128 data values

	start_timer 18
	zloop_init  12*8*clm			;; 12 iterations of 8*clm
	zloop_set_clm_prefetch ZQUAD		;; Prefetch 192 of 768 blocks of clm*128 bytes
	zr8_eight_complex_djbunfft_preload
IFDEF OLDDD
c2b:	zr8_eight_complex_djbunfft rsi, 128, clmblkdst, 2*clmblkdst, 4*clmblkdst, rdi, ZMM_SCD4, 8*clm, L1PREFETCH_ALL, 128
	zloop	8*clm, c2b, -8*clm*128+clmblkdst8, rdi, -8*clm*ZMM_SCD4 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	12, c2b, -12*clmblkdst8, rdi, 8*clm*ZMM_SCD4 ;; Test loop counter, restore source pointer, next sin/cos pointer
ELSE
c2b:	zr8_eight_complex_djbunfft rsi, clmblkdst8, clmblkdst, 2*clmblkdst, 4*clmblkdst, rdi, 0, 8, L1PREFETCH_ALL, clmblkdst8
	zloop	12, c2b, -12*clmblkdst8+128, rdi, ZMM_SCD4 ;; Test loop counter, next source pointer, next sin/cos pointer
	zloop	8*clm, c2b, -8*clm*128		;; Test loop counter, restore source pointer
ENDIF
	end_timer 18

;; Do inverse FFT levels 1,2,3,4
;; On input the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	64K	+1K	...
;;	8K
;;	...
;;	56K
;;	128K
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	768K	+1K	...
;;	8K	...
;;	...
;;	760K

;; Do 64 twelve_complex_unfft operations
;;	distance between fft data elements is 64K
;;	do 8 macros each processing 192 data values

	start_timer 20
	mov	r15, sincos2			;; Load fixed premultiplier/sin/cos ptr
	sub	rbx, rbx			;; Clear register for loading fudge flags index
	mov	r12, compressed_fudges		;; Address of the compressed fudge flags
	mov	r13, r12			;; The XOR masks for fudge flags come from the compressed fudges
	zloop_init 8*8*clm			;; 8 iters of 8*clm
	zloop_set_clm_prefetch ZDOUBLE		;; Prefetch 64 of 768 blocks of clm*128 bytes
	zr12_csc_wpn_twelve_complex_last_djbunfft_preload
c1b:	zr12_csc_wpn_twelve_complex_last_djbunfft rsi, 128, clmblkdst8, 2*clmblkdst8, 4*clmblkdst8, r15, 0, rdi, 3, 8*clm, L1PREFETCH_ALL, 128
	zloop	8*clm, c1b, -8*clm*128+clmblkdst, r15, ZMM_CD12SCD6, rdx, 24*64, r13, 24 ;; 8*clm iterations, next fixed s/c, next grp multipliers, next xor mask
	zloop	8, c1b				;; 8 iterations
	end_timer 20

;; Normalize these values

	pass1_normalize yes_fft			;; Normalize while prefetching 96 of 768 blocks of clm*128 bytes

;; Copy data from the scratch area back to the FFT data area

	copy_scratch_data_to_fft
	jmp	endlp				;; Skip FFT code if flag not set

;; Fire up auxiliary threads to do pass 1 forward FFT

pass1:	pass1_forward_fft_setup b0b

;; Copy data from the FFT data area back to the scratch area

b0b:	copy_fft_data_to_scratch

;; Do FFT levels 1,2,3,4
;; On input the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	768K	+1K	...
;;	8K	...
;;	...
;;	760K
;; On output the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	64K	+1K	...
;;	8K
;;	...
;;	56K
;;	128K
;;	...

;; Do 64 twelve_complex_first_fft operations
;;	distance between fft data elements is 64K
;;	do 8 macros each processing 192 data values

yes_fft:start_timer 22
	mov	rsi, scratch_area		;; Get address of scratch area
	mov	r15, sincos2			;; Load fixed premultiplier/sin/cos ptr
	mov	rdi, PREMULT_ADDR		;; Calculate pointer to fudge flags
	bump	rdi, 8*clm*ZMM_SCD4+12*clm*ZMM_SCD1+clm*ZMM_SCD4+clm*64+clm*64 ;; Addr of the group multiplier fudge flags
	mov	rdx, norm_grp_mults		;; Addr of the group multipliers
	sub	rbx, rbx			;; Clear register for loading fudge flags index
	mov	r12, compressed_fudges		;; Address of the compressed fudge flags
	mov	r13, r12			;; The XOR masks for fudge flags come from the compressed fudges
	set_data_prefetch_ptrs			;; Init prefetching pointers
pfing	mov	r8, pass2blkdst			;; blkdst
pfing	imul	rax, blkdstreg, (448-0)		;; We've already prefetched 448 of 768 blocks (zero too many)
pfing	add	rcx, rax
	zloop_init 8*8*clm			;; 8 iters of 8*clm
	zloop_set_clm_prefetch ZQUAD		;; Prefetch 128 of 768 blocks of clm*128 bytes
	zr12_csc_wpn_twelve_complex_first_djbfft_preload
b1b:	zr12_csc_wpn_twelve_complex_first_djbfft rsi, 128, clmblkdst8, 2*clmblkdst8, 4*clmblkdst8, r15, 0, rdi, 3, rdx, 0, 8*clm, L1PREFETCH_ALL, 128
	zloop	8*clm, b1b, -8*clm*128+clmblkdst, r15, ZMM_CD12SCD6, rdx, 24*64, r13, 24 ;; 8*clm iters, next source, fixed s/c, grp multipliers pointer, next xor mask
	zloop	8, b1b, -8*clmblkdst		;; Test loop counter, restore source pointer
	end_timer 22

;; Do FFT levels 5,6,7
;; On input the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	64K	+1K	...
;;	8K
;;	...
;;	56K
;;	128K
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	8K	+1K	...
;;	16K
;;	...

;; Do 96 eight_complex_fft operations
;;	distance between fft data elements is 8K
;;	do 12 macros each processing 128 data values

	start_timer 24
	bump	rdi, -64*clm*3-8*clm*ZMM_SCD4	;; Sin/cos pointer
	zloop_init  12*8*clm			;; 12 iterations of 8*clm
	zloop_set_clm_prefetch ZQUAD		;; Prefetch 192 of 768 blocks of clm*128 bytes
	zr8_eight_complex_djbfft_preload
IFDEF OLDDD
b2b:	zr8_eight_complex_djbfft rsi, 128, clmblkdst, 2*clmblkdst, 4*clmblkdst, rdi, ZMM_SCD4, 8*clm, L1PREFETCH_ALL, 128
	zloop	8*clm, b2b, -8*clm*128+clmblkdst8, rdi, -8*clm*ZMM_SCD4 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	12, b2b, -12*clmblkdst8		;; Test loop counter, restore source pointer
ELSE
b2b:	zr8_eight_complex_djbfft rsi, clmblkdst8, clmblkdst, 2*clmblkdst, 4*clmblkdst, rdi, 0, 8, L1PREFETCH_ALL, clmblkdst8
	zloop	12, b2b, -12*clmblkdst8+128, rdi, ZMM_SCD4 ;; Test loop counter, next source pointer, next sin/cos pointer
	zloop	8*clm, b2b, -8*clm*128		;; Test loop counter, restore source pointer
ENDIF
	end_timer 24

;; Do FFT levels 8,9,10
;; On input the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	8K	+1K	...
;;	16K
;;	...
;; The input is swizzled to hold these data values:
;;	0K	+1	+1	+1	+1	+1	+1	+1	8K	+1	...
;;	1K	...
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0K	+1	+1	+1	+1	+1	+1	+1	1K	+1	...
;;	2K
;;	...

;; Do 96 eight_complex_fft operations
;;	distance between fft data elements is 1K
;;	do 8 macros each processing 128 data values

	start_timer 26
	mov	rbp, DATA_ADDR			;; Destination pointer
IFDEF OLDDD
	bump	rdi, -12*clm*ZMM_SCD1		;; Delayed sin/cos multipliers
ELSE
	bump	rdi, -8*clm*ZMM_SCD4-12*clm*ZMM_SCD1 ;; Delayed sin/cos multipliers
ENDIF
	lea	rbx, [rdi-clm*ZMM_SCD4]		;; Complex data sin/cos ptr
	lea	r9, [rbx-clm*64-clm*64]		;; Column weights
	lea	rdx, ZMM_TMPS			;; Computed clm*SCD4 sin/cos values
	mov	r8, pass2blkdst			;; blkdst
	lea	r10, [2*blkdstreg+blkdstreg]	;; 3*blkdst
	lea	r13, [rbp+4*blkdstreg]		;; Dest+4*blkdst pointer
pfing	mov	rcx, PREMULT_PREFETCH		;; Load premult/sin/cos prefetch pointer

	zr8_rsc_complex_only_preload		;; Preload constants before calling zloop_init
	zloop_init 12*8*clm			;; 12 iters of 8*clm
	zloop_set_prefetch ZDOUBLE		;; Prefetch 192*clm cache lines
	zloop_set_alternate_prefetch_increment 104, 192  ;; Prefetch approximately 104*clm cache lines in those 192*clm prefetches
b9a:	zr8_rsc_wpn_eight_complex_calc_sincos r9, rdi, rbx, clm ;; Generate clm*SCD8 sin/cos values from weights, delayed and complex multipliers
	bump	rdi, clm*ZMM_SCD1		;; Next set of delayed multipliers
b9b:	zr8_rsc_wpn_sgreg_eight_complex_fft8 rsi, 1024, 128, 256, 512, rbp, 128, blkdstreg, blkdst3reg, r13, rdx, ZMM_SCD8, clm, L1PREFETCH_DEST_ALL
	zloop	clm, b9b, -clm*1024+clmblkdst, rbp, -clm*128+8*blkdstreg, r13, -clm*128+8*blkdstreg, rdx, -clm*ZMM_SCD8 ;; Loop, next source/dest pointers, restore premult/sine/cosine pointer
	zloop	8, b9b, -8*clmblkdst+clmblkdst8	;; Test loop counter, next source pointer
	zloop	12, b9a				;; Test loop counter
	end_timer 26

;; Work on next 8*clm sets of 1536 values

endlp:	zpass1_get_next_block pass2, c0b, b0b ;; Get next block and jump
done:	zfft_footer
	ENDM



;; Perform a shared pass 1 FFT of length 1920 doubles.
;; Memory examples are for a 1920K FFT with 10 levels done in pass 2.
;; FFT data needed:
;;		8*clm*1920 doubles					Total: 1920*clm 64-byte cache lines
;; Fixed s/c data needed:
;;		10*ZMM_SCD11 (fixed s/c data),				Total: 220 64-byte cache lines
;; Group multipliers needed:
;;		10*24*64 (weights and fudged weights)
;;		10*24*64 (inverse weights and fudged inv. weights),	Total: 480 64-byte cache lines
;; Variable data needed:
;;		clm*64	(column weights)
;;		clm*64	(column inverse weights)
;;		clm*ZMM_SCD4 (complex reduced s/c multipliers)
;;		clm*ZMM_SCD4 (real s/c multipliers)
;;		11*clm*ZMM_SCD1 (delayed s/c multipliers)
;;		8*clm*ZMM_SCD10 (10-complex 20-real sin/cos data)
;;		80*clm*3 (group multiplier fudge flags)
;;		80*clm*3 (big/lit flags),				Total: 206.5*clm 64-byte cache lines
;; Total memory consumed (add for scratch area, prefetched data, carries):
;;		(1920*3+206.5=5966.5)*clm + (220+480+1920/8=940) cache lines
;;		clm=1,2,4:  432KB, 805KB, 1551KB

zr4dwpn_pass1sc1920 MACRO
	LOCAL	b0b, b1b, b2a, b2b, b2c, b8a, b8b, b9aa, b9a, b9b, c0b, c1b, c2a, c2b, c2c, c8b, c9aa, c9a, c9b
	LOCAL	pass1, pass2, endlp, done, yes_fft

	zfft_header pass1

;; Do pass 2 (the last 10 FFT levels if this is a 1920K FFT)

pass2:	start_timer 0
	mov	rax, ZMM_PASS2_ROUTINE
	call	rax
	end_timer 0
	cmp	ffttype, 1		;; We're done if FFTing only
	je	done

;; Do the inverse FFT

	pass1_inverse_fft_setup c0b

;; Do inverse FFT levels 8,9,10
;; On input the 128-byte double cache lines hold these data values:
;;	0K	+1	+1	+1	+1	+1	+1	+1	1K	+1	...
;;	2K
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0K	+1	+1	+1	+1	+1	+1	+1	8K	+1	...
;;	1K	...
;;	...
;; The output cache lines are swizzled to hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	8K	+1K	...
;;	16K
;;	...

;; Do 1 sixteen_reals_unfft operations
;;	distance between fft data elements is 1K
;;	do 0.125 macros each processing 128 data values
;; Do 119 eight_complex_unfft operations
;;	distance between fft data elements is 1K
;;	do 14.875 macros each processing 128 data values

c0b:	start_timer 16
	mov	rbp, DATA_ADDR			;; Get source address
	mov	rsi, scratch_area		;; Get address of scratch area
	set_data_prefetch_ptrs			;; Init prefetching pointers
	mov	r9, PREMULT_ADDR		;; Variable data
	bump	r9, clm*64			;; Inverse column weights
	lea	rbx, [r9+clm*64]		;; Complex data sin/cos ptr
	lea	rdi, [rbx+clm*ZMM_SCD4]		;; Real data sin/cos pointer, will become delayed sin/cos multipliers
	lea	rdx, ZMM_TMPS			;; Computed clm*SCD8 sin/cos values
	mov	r8, pass2blkdst			;; blkdst
	lea	r10, [2*blkdstreg+blkdstreg]	;; 3*blkdst
	lea	r13, [rbp+4*blkdstreg]		;; Source+4*blkdst pointer
	lea	r12, [rbp+8*blkdstreg]		;; L1 prefetch pointer part 1
	bump2	r12, (clm MOD 2), 8*blkdstreg	;; L1 prefetch pointer part 2
	lea	r14, [r12+4*blkdstreg]		;; L1 prefetch pointer + 4*blkdst

	zr8_rsc_complex_and_real_preload	;; Preload constants before calling zloop_init
;;BUG	zloop_init 12*10*clm			;; 12 iters of 10*clm iterations
	zloop_init 12*10*clm  *2		;; 12 iters of 10*clm iterations	(work around loops "add al, 1" bug)
	zr8_rsc_wpn_eight_complex_calc_sincos_simple r9, rbx, clm ;; Generate clm*SCD8 sin/cos values from delayed and complex multipliers
	zloop_set_clm_prefetch ZSINGLE		;; Prefetch 60 of 960 blocks of clm*128 bytes
c8b:	zr8_rsc_wpn_sgreg_2sc_sixteen_reals_unfft8 rbp, 128, blkdstreg, blkdst3reg, r13, rsi, 1024, 128, 256, 512, rdx, ZMM_SCD8, r9, 64, rdi, ZMM_SCD4, clm, L1PREFETCH_ALL, r12, r14 ;; Prefetch short dist ahead
	zloop	clm, c8b, -clm*1024+clmblkdst, rbp, -clm*128+8*blkdstreg, r13, -clm*128+8*blkdstreg, rdx, -clm*ZMM_SCD8, r9, -clm*64, r12, -clm*128+8*blkdstreg, r14, -clm*128+8*blkdstreg ;; Loop, next src/dest ptrs, restore sin/cos ptr
	jmp	c9aa

	loops_reset
c9a:	zr8_rsc_wpn_eight_complex_calc_sincos r9, rdi, rbx, clm ;; Generate clm*SCD8 sin/cos values from delayed and complex multipliers
	bump	rdi, clm*ZMM_SCD1		;; Next set of delayed sin/cos multipliers
c9aa:	zloop_set_clm_prefetch ZSINGLE		;; Prefetch 60 of 960 blocks of clm*128 bytes
c9b:	zr8_rsc_wpn_sgreg_eight_complex_unfft8 rbp, 128, blkdstreg, blkdst3reg, r13, rsi, 1024, 128, 256, 512, rdx, ZMM_SCD8, clm, L1PREFETCH_ALL, r12, r14 ;; Prefetch short dist ahead
	zloop	clm, c9b, -clm*1024+clmblkdst, rbp, -clm*128+8*blkdstreg, r13, -clm*128+8*blkdstreg, rdx, -clm*ZMM_SCD8, r12, -clm*128+8*blkdstreg, r14, -clm*128+8*blkdstreg ;; Loop, next src/dest ptrs, restore sin/cos ptr
	zloop	10, c9b				;; Test loop counter
	zloop	12, c9a, -12*10*clmblkdst	;; Test loop counter, restore source pointer
	end_timer 16

;; Do inverse FFT levels 5,6,7
;; On input the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	8K	+1K	...
;;	16K
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	80K	+1K	...
;;	8K
;;	...
;;	72K
;;	160K
;;	...

;; Do 8 twenty_reals_unfft operations
;; Do 88 ten_complex_unfft operations
;;	distance between fft data elements is 8K
;;	do 12 macros each processing 160 data values

	start_timer 18
	zloop_init  12*8*clm			;; 12 iterations of 8*clm
	zloop_set_clm_prefetch ZQUAD		;; Prefetch 192 of 960 blocks of clm*128 bytes
IFDEF OLDDD
	zr10_csc_twenty_reals_unfft_preload
c2a:	zr10_csc_twenty_reals_unfft rsi, 128, clmblkdst, rdi, ZMM_SCD10, 8*clm, L1PREFETCH_ALL, 128
	zloop	8*clm, c2a, -8*clm*128+10*clmblkdst, rdi, -8*clm*ZMM_SCD10 ;; Test loop counter, next source pointer, restore sin/cos pointer
	loops_reset				;; 11 more iters of 8*clm
	zr10_ten_complex_djbunfft_preload
c2b:	zr10_ten_complex_djbunfft rsi, 128, clmblkdst, rdi, ZMM_SCD10, 8*clm, L1PREFETCH_ALL, 128
	zloop	8*clm, c2b, -8*clm*128+10*clmblkdst, rdi, -8*clm*ZMM_SCD10 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	12, c2b, -12*10*clmblkdst, rdi, 8*clm*ZMM_SCD10 ;; Test loop counter, restore source pointer, next sin/cos pointer
ELSE
c2a:	zr10_csc_twenty_reals_unfft_preload
	zr10_csc_twenty_reals_unfft rsi, 10*clmblkdst, clmblkdst, rdi, 0, 1, L1PREFETCH_ALL, 10*clmblkdst
	zr10_ten_complex_djbunfft_preload
	jmp	c2c				;; 11 more iters
c2b:	zr10_ten_complex_djbunfft rsi, 10*clmblkdst, clmblkdst, rdi, 0, 1, L1PREFETCH_ALL, 10*clmblkdst
c2c:	zloop	12, c2b, -12*10*clmblkdst+128, rdi, ZMM_SCD10 ;; Test loop counter, next source pointer, next sin/cos pointer
	zloop	8*clm, c2a, -8*clm*128		;; Test loop counter, restore source pointer
ENDIF
	end_timer 18

;; Do inverse FFT levels 1,2,3,4
;; On input the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	80K	+1K	...
;;	8K
;;	...
;;	72K
;;	160K
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	960K	+1K	...
;;	8K
;;	...
;;	952K

;; Do 80 twentyfour_reals_unfft operations
;;	distance between fft data elements is 80K
;;	do 10 macros each processing 192 data values

	start_timer 20
	mov	r15, sincos2			;; Load fixed premultiplier/sin/cos ptr
	sub	rbx, rbx			;; Clear register for loading fudge flags index
	mov	r12, compressed_fudges		;; Address of the compressed fudge flags
	mov	r13, r12			;; The XOR masks for fudge flags come from the compressed fudges
	zloop_init 10*8*clm			;; 10 iters of 8*clm
	zloop_set_clm_prefetch ZDOUBLE		;; Prefetch 80 of 960 blocks of clm*128 bytes
	zr12_wpn_twentyfour_reals_last_unfft_preload
c1b:	zr12_wpn_twentyfour_reals_last_unfft rsi, 128, 10*clmblkdst, 20*clmblkdst, 40*clmblkdst, r15, 0, rdi, 3, 8*clm, L1PREFETCH_ALL, 128
	zloop	8*clm, c1b, -8*clm*128+clmblkdst, r15, ZMM_SCD11, rdx, 24*64, r13, 24 ;; 8*clm iterations, next fixed s/c, next grp multipliers, next XOR masks
	zloop	10, c1b				;; Test loop counter
	end_timer 20

;; Normalize these values

	pass1_normalize yes_fft			;; Normalize while prefetching 120 of 960 blocks of clm*128 bytes

;; Copy data from the scratch area back to the FFT data area

	copy_scratch_data_to_fft
	jmp	endlp				;; Skip FFT code if flag not set

;; Fire up auxiliary threads to do pass 1 forward FFT

pass1:	pass1_forward_fft_setup b0b

;; Copy data from the FFT data area back to the scratch area

b0b:	copy_fft_data_to_scratch

;; Do FFT levels 1,2,3,4
;; On input the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	960K	+1K	...
;;	8K	...
;;	...
;;	952K
;; On output the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	80K	+1K	...
;;	8K
;;	...
;;	72K
;;	160K
;;	...

;; Do 80 twentyfour_reals_first_fft operations
;;	distance between fft data elements is 80K
;;	do 10 macros each processing 192 data values

yes_fft:start_timer 22
	mov	rsi, scratch_area		;; Get address of scratch area
	mov	r15, sincos2			;; Load fixed premultiplier/sin/cos ptr
	mov	rdi, PREMULT_ADDR		;; Calculate pointer to fudge flags
	bump	rdi, 8*clm*ZMM_SCD10+11*clm*ZMM_SCD1+clm*ZMM_SCD4+clm*ZMM_SCD4+clm*64+clm*64 ;; Addr of the group multiplier fudge flags
	mov	rdx, norm_grp_mults		;; Addr of the group multipliers
	sub	rbx, rbx			;; Clear register for loading fudge flags index
	mov	r12, compressed_fudges		;; Address of the compressed fudge flags
	mov	r13, r12			;; The XOR masks for fudge flags come from the compressed fudges
	set_data_prefetch_ptrs			;; Init prefetching pointers
pfing	mov	r8, pass2blkdst			;; blkdst
pfing	imul	rax, blkdstreg, (452-4)		;; We've already prefetched 452 of 960 blocks (4 too many)
pfing	add	rcx, rax
	zloop_init 10*8*clm			;; 10 iters of 8*clm
	zloop_set_clm_prefetch ZOCT		;; Prefetch 320 of 960 blocks of clm*128 bytes
	zr12_wpn_twentyfour_reals_first_fft_preload
b1b:	zr12_wpn_twentyfour_reals_first_fft rsi, 128, 10*clmblkdst, 20*clmblkdst, 40*clmblkdst, r15, 0, rdi, 3, rdx, 0, 8*clm, L1PREFETCH_ALL, 128
	zloop	8*clm, b1b, -8*clm*128+clmblkdst, r15, ZMM_SCD11, rdx, 24*64, r13, 24 ;; 8*clm iters prefetching 8*clm cache lines, next source, fixed s/c, grp multipliers pointer, next XOR masks
	zloop	10, b1b, -10*clmblkdst		;; Test loop counter, restore source pointer
	end_timer 22

;; Do FFT levels 5,6,7
;; On input the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	80K	+1K	...
;;	8K
;;	...
;;	72K
;;	160K
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	8K	+1K	...
;;	16K
;;	...

;; Do 8 twenty_reals_fft operations
;; Do 88 ten_complex_fft operations
;;	distance between fft data elements is 8K
;;	do 12 macros each processing 160 data values

	start_timer 24
	bump	rdi, -80*clm*3-8*clm*ZMM_SCD10	;; Sin/cos pointer
	zloop_init  12*8*clm			;; 12 iterations of 8*clm
	zloop_set_clm_prefetch ZQUAD		;; Prefetch 192 of 960 blocks of clm*128 bytes
IFDEF OLDDD
	zr10_csc_twenty_reals_fft_preload
b2a:	zr10_csc_twenty_reals_fft rsi, 128, clmblkdst, rdi, ZMM_SCD10, 8*clm, L1PREFETCH_ALL, 128
	zloop	8*clm, b2a, -8*clm*128+10*clmblkdst, rdi, -8*clm*ZMM_SCD10 ;; Test loop counter, next source pointer, restore sin/cos pointer
	loops_reset				;; 11 more iters of 8*clm
	zr10_ten_complex_djbfft_preload
b2b:	zr10_ten_complex_djbfft rsi, 128, clmblkdst, rdi, ZMM_SCD10, 8*clm, L1PREFETCH_ALL, 128
	zloop	8*clm, b2b, -8*clm*128+10*clmblkdst, rdi, -8*clm*ZMM_SCD10 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	12, b2b, -12*10*clmblkdst	;; Test loop counter, restore source pointer
ELSE
b2a:	zr10_csc_twenty_reals_fft_preload
	zr10_csc_twenty_reals_fft rsi, 10*clmblkdst, clmblkdst, rdi, 0, 1, L1PREFETCH_ALL, 10*clmblkdst
	zr10_ten_complex_djbfft_preload
	jmp	b2c				;; 11 more iters
b2b:	zr10_ten_complex_djbfft rsi, 10*clmblkdst, clmblkdst, rdi, 0, 1, L1PREFETCH_ALL, 10*clmblkdst
b2c:	zloop	12, b2b, -12*10*clmblkdst+128, rdi, ZMM_SCD10 ;; Test loop counter, next source pointer, next sin/cos pointer
	zloop	8*clm, b2a, -8*clm*128		;; Test loop counter, restore source pointer
ENDIF
	end_timer 24

;; Do FFT levels 8,9,10
;; On input the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	8K	+1K	...
;;	16K
;;	...
;; The input is swizzled to hold these data values:
;;	0K	+1	+1	+1	+1	+1	+1	+1	8K	+1	...
;;	1K	...
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0K	+1	+1	+1	+1	+1	+1	+1	1K	+1	...
;;	2K
;;	...

;; Do 1 sixteen_reals_fft operations
;;	distance between fft data elements is 1K
;;	do 0.125 macros each processing 128 data values
;; Do 119 eight_complex_fft operations
;;	distance between fft data elements is 1K
;;	do 14.875 macros each processing 128 data values

	start_timer 26
IFDEF OLDDD
	bump	rdi, -11*clm*ZMM_SCD1-clm*ZMM_SCD4 ;; Real data sin/cos pointer, will become delayed sin/cos multipliers
ELSE
	bump	rdi, -8*clm*ZMM_SCD10-11*clm*ZMM_SCD1-clm*ZMM_SCD4 ;; Real data sin/cos pointer, will become delayed sin/cos multipliers
ENDIF
	lea	rbx, [rdi-clm*ZMM_SCD4]		;; Complex data sin/cos ptr
	lea	r9, [rbx-clm*64-clm*64]		;; Column weights
	lea	rdx, ZMM_TMPS			;; Computed clm*SCD4 sin/cos values
	mov	rbp, DATA_ADDR			;; Destination pointer
	mov	r8, pass2blkdst			;; blkdst
	lea	r10, [2*blkdstreg+blkdstreg]	;; 3*blkdst
	lea	r13, [rbp+4*blkdstreg]		;; Dest+4*blkdst pointer

pfing	mov	rcx, PREMULT_PREFETCH		;; Load premult/sin/cos prefetch pointer

	zr8_rsc_complex_and_real_preload	;; Preload constants before calling zloop_init
;;BUG	zloop_init 12*10*clm			;; 12 iters of 10*clm iterations
	zloop_init 12*10*clm  *2		;; 12 iters of 10*clm iterations	(work around loops "add al, 1" bug)
	zloop_set_prefetch ZDOUBLE		;; Prefetch 240*clm times
	zloop_set_alternate_prefetch_increment 206*2+1, 240*2  ;; Prefetch approximately 206.5*clm cache lines in those 240*clm prefetches
	zr8_rsc_wpn_eight_complex_calc_sincos_simple r9, rbx, clm ;; Generate clm*SCD8 sin/cos values from delayed and complex multipliers
b8b:	zr8_rsc_wpn_sgreg_2sc_sixteen_reals_fft8 rsi, 1024, 128, 256, 512, rbp, 128, blkdstreg, blkdst3reg, r13, rdx, ZMM_SCD8, r9, 64, rdi, ZMM_SCD4, clm, L1PREFETCH_DEST_ALL
	zloop	clm, b8b, -clm*1024+clmblkdst, rbp, -clm*128+8*blkdstreg, r13, -clm*128+8*blkdstreg, rdx, -clm*ZMM_SCD8, r9, -clm*64 ;; Loop, next source/dest pointers, restore premult/sine/cosine pointer
	jmp	b9aa

	loops_reset
b9a:	zr8_rsc_wpn_eight_complex_calc_sincos r9, rdi, rbx, clm ;; Generate clm*SCD8 sin/cos values from delayed and complex multipliers
	bump	rdi, clm*ZMM_SCD1		;; Next set of delayed multipliers
b9aa:
b9b:	zr8_rsc_wpn_sgreg_eight_complex_fft8 rsi, 1024, 128, 256, 512, rbp, 128, blkdstreg, blkdst3reg, r13, rdx, ZMM_SCD8, clm, L1PREFETCH_DEST_ALL
	zloop	clm, b9b, -clm*1024+clmblkdst, rbp, -clm*128+8*blkdstreg, r13, -clm*128+8*blkdstreg, rdx, -clm*ZMM_SCD8 ;; Loop, next source/dest pointers, restore premult/sine/cosine pointer
	zloop	10, b9b				;; Test loop counter
	zloop	12, b9a				;; Test loop counter
	end_timer 26

;; Work on next 8*clm sets of 1920 values

endlp:	zpass1_get_next_block pass2, c0b, b0b ;; Get next block and jump
done:	zfft_footer
	ENDM


;; Perform a shared pass 1 negacyclic FFT of length 1920 doubles.
;; Memory examples are for a 1920K FFT with 10 levels done in pass 2.
;; FFT data needed:
;;		8*clm*1920 doubles					Total: 1920*clm 64-byte cache lines
;; Fixed s/c data needed:
;;		10*ZMM_CD12SCD9 (complex premultipliers and fixed s/c data), Total: 240 64-byte cache lines
;; Group multipliers needed:
;;		10*24*64 (weights)
;;		10*24*64 (inverse weights),				Total: 480 64-byte cache lines
;; Variable data needed:
;;		clm*64	(column weights)
;;		clm*64	(column inverse weights)
;;		clm*ZMM_SCD4 (complex reduced s/c multipliers)
;;		12*clm*ZMM_SCD1 (delayed s/c multipliers)
;;		8*clm*ZMM_SCD5 (10-complex sin/cos data)
;;		80*clm*3 (group multiplier fudge flags)
;;		80*clm*3 (big/lit flags),				Total: 121.5*clm 64-byte cache lines
;; Total memory consumed (add for scratch area, prefetched data, carries):
;;		(1920*3+121.5=5881.5)*clm + (240+480+1920/8=960) cache lines
;;		clm=1,2,4:  428KB, 796KB, 1531KB

zr4dwpn_pass1sc1920ac MACRO
	LOCAL	b0b, b1b, b2b, b9a, b9b, c0b, c1b, c2b, c9a, c9b
	LOCAL	pass1, pass2, endlp, done, yes_fft

	zfft_header pass1

;; Do pass 2 (the last 10 FFT levels if this is a 1920K FFT)

pass2:	start_timer 0
	mov	rax, ZMM_PASS2_ROUTINE
	call	rax
	end_timer 0
	cmp	ffttype, 1		;; We're done if FFTing only
	je	done

;; Do the inverse FFT

	pass1_inverse_fft_setup c0b

;; Do inverse FFT levels 8.9.10
;; On input the 128-byte double cache lines hold these data values:
;;	0K	+1	+1	+1	+1	+1	+1	+1	1K	+1	...
;;	2K
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0K	+1	+1	+1	+1	+1	+1	+1	8K	+1	...
;;	1K	...
;;	...
;; The output cache lines are swizzled to hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	8K	+1K	...
;;	16K
;;	...

;; Do 120 eight_complex_unfft operations
;;	distance between fft data elements is 1K
;;	do 15 macros each processing 128 data values

c0b:	start_timer 16
	mov	rbp, DATA_ADDR			;; Get source address
	mov	rsi, scratch_area		;; Get address of scratch area
	set_data_prefetch_ptrs			;; Init prefetching pointers
	mov	r9, PREMULT_ADDR		;; Variable data
	bump	r9, clm*64			;; Inverse column weights
	lea	rbx, [r9+clm*64]		;; Complex data sin/cos ptr
	lea	rdi, [rbx+clm*ZMM_SCD4]		;; Delayed sin/cos multipliers
	lea	rdx, ZMM_TMPS			;; Computed clm*SCD8 sin/cos values
	mov	r8, pass2blkdst			;; blkdst
	lea	r10, [2*blkdstreg+blkdstreg]	;; 3*blkdst
	lea	r13, [rbp+4*blkdstreg]		;; Source+4*blkdst pointer
	lea	r12, [rbp+8*blkdstreg]		;; L1 prefetch pointer part 1
	bump2	r12, (clm MOD 2), 8*blkdstreg	;; L1 prefetch pointer part 2
	lea	r14, [r12+4*blkdstreg]		;; L1 prefetch pointer + 4*blkdst

	zr8_rsc_complex_only_preload		;; Preload constants before calling zloop_init
;;BUG	zloop_init 12*10*clm			;; 12 iters of 10*clm iterations
	zloop_init 12*10*clm  *2		;; 12 iters of 10*clm iterations	(work around loops "add al, 1" bug)
	zloop_set_clm_prefetch ZSINGLE		;; Prefetch 60 of 960 blocks of clm*128 bytes
c9a:	zr8_rsc_wpn_eight_complex_calc_sincos r9, rdi, rbx, clm ;; Generate clm*SCD8 sin/cos values from weights, delayed, and complex multipliers
	bump	rdi, clm*ZMM_SCD1		;; Next set of delayed sin/cos multipliers
c9b:	zr8_rsc_wpn_sgreg_eight_complex_unfft8 rbp, 128, blkdstreg, blkdst3reg, r13, rsi, 1024, 128, 256, 512, rdx, ZMM_SCD8, clm, L1PREFETCH_ALL, r12, r14 ;; Prefetch short dist ahead
	zloop	clm, c9b, -clm*1024+clmblkdst, rbp, -clm*128+8*blkdstreg, r13, -clm*128+8*blkdstreg, rdx, -clm*ZMM_SCD8, r12, -clm*128+8*blkdstreg, r14, -clm*128+8*blkdstreg ;; Loop, next src/dest ptrs, restore sin/cos ptr
	zloop	10, c9b				;; Test loop counter
	zloop	12, c9a, -12*10*clmblkdst	;; Test loop counter, restore source pointer
	end_timer 16

;; Do inverse FFT levels 5,6,7
;; On input the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	8K	+1K	...
;;	16K
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	80K	+1K	...
;;	8K
;;	...
;;	72K
;;	160K
;;	...

;; Do 96 ten_complex_unfft operations
;;	distance between fft data elements is 8K
;;	do 12 macros each processing 160 data values

	start_timer 18
	zloop_init  12*8*clm			;; 12 iterations of 8*clm
	zloop_set_clm_prefetch ZQUAD		;; Prefetch 192 of 960 blocks of clm*128 bytes
	zr10_ten_complex_djbunfft_preload
IFDEF OLDDD
c2b:	zr10_ten_complex_djbunfft rsi, 128, clmblkdst, rdi, ZMM_SCD5, 8*clm, L1PREFETCH_ALL, 128
	zloop	8*clm, c2b, -8*clm*128+10*clmblkdst, rdi, -8*clm*ZMM_SCD5 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	12, c2b, -12*10*clmblkdst, rdi, 8*clm*ZMM_SCD5 ;; Test loop counter, restore source pointer, next sin/cos pointer
ELSE
c2b:	zr10_ten_complex_djbunfft rsi, 10*clmblkdst, clmblkdst, rdi, 0, 8, L1PREFETCH_ALL, 10*clmblkdst
	zloop	12, c2b, -12*10*clmblkdst+128, rdi, ZMM_SCD5 ;; Test loop counter, next source pointer, next sin/cos pointer
	zloop	8*clm, c2b, -8*clm*128		;; Test loop counter, restore source pointer
ENDIF
	end_timer 18

;; Do inverse FFT levels 1,2,3,4
;; On input the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	80K	+1K	...
;;	8K
;;	...
;;	72K
;;	160K
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	960K	+1K	...
;;	8K	...
;;	...
;;	952K

;; Do 80 twelve_complex_unfft operations
;;	distance between fft data elements is 80K
;;	do 10 macros each processing 192 data values

	start_timer 20
	mov	r15, sincos2			;; Load fixed premultiplier/sin/cos ptr
	sub	rbx, rbx			;; Clear register for loading fudge flags index
	mov	r12, compressed_fudges		;; Address of the compressed fudge flags
	mov	r13, r12			;; The XOR masks for fudge flags come from the compressed fudges
	zloop_init 10*8*clm			;; 10 iters of 8*clm
	zloop_set_clm_prefetch ZDOUBLE		;; Prefetch 80 of 960 blocks of clm*128 bytes
	zr12_csc_wpn_twelve_complex_last_djbunfft_preload
c1b:	zr12_csc_wpn_twelve_complex_last_djbunfft rsi, 128, 10*clmblkdst, 20*clmblkdst, 40*clmblkdst, r15, 0, rdi, 3, 8*clm, L1PREFETCH_ALL, 128
	zloop	8*clm, c1b, -8*clm*128+clmblkdst, r15, ZMM_CD12SCD6, rdx, 24*64, r13, 24 ;; 8*clm iterations, next fixed s/c, next grp multipliers, next XOR masks
	zloop	10, c1b				;; 10 iterations
	end_timer 20


;; Normalize these values

	pass1_normalize yes_fft			;; Normalize while prefetching 120 of 960 blocks of clm*128 bytes

;; Copy data from the scratch area back to the FFT data area

	copy_scratch_data_to_fft
	jmp	endlp				;; Skip FFT code if flag not set

;; Fire up auxiliary threads to do pass 1 forward FFT

pass1:	pass1_forward_fft_setup b0b

;; Copy data from the FFT data area back to the scratch area

b0b:	copy_fft_data_to_scratch

;; Do FFT levels 1,2,3,4
;; On input the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	960K	+1K	...
;;	8K	...
;;	...
;;	952K
;; On output the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	80K	+1K	...
;;	8K
;;	...
;;	72K
;;	160K
;;	...

;; Do 80 twelve_complex_first_fft operations
;;	distance between fft data elements is 80K
;;	do 10 macros each processing 192 data values

yes_fft:start_timer 22
	mov	rsi, scratch_area		;; Get address of scratch area
	mov	r15, sincos2			;; Load fixed premultiplier/sin/cos ptr
	mov	rdi, PREMULT_ADDR		;; Calculate pointer to fudge flags
	bump	rdi, 8*clm*ZMM_SCD5+12*clm*ZMM_SCD1+clm*ZMM_SCD4+clm*64+clm*64 ;; Addr of the group multiplier fudge flags
	mov	rdx, norm_grp_mults		;; Addr of the group multipliers
	sub	rbx, rbx			;; Clear register for loading fudge flags index
	mov	r12, compressed_fudges		;; Address of the compressed fudge flags
	mov	r13, r12			;; The XOR masks for fudge flags come from the compressed fudges
	set_data_prefetch_ptrs			;; Init prefetching pointers
pfing	mov	r8, pass2blkdst			;; blkdst
pfing	imul	rax, blkdstreg, (452-4)		;; We've already prefetched 452 of 960 blocks (4 too many)
pfing	add	rcx, rax
	zloop_init 10*8*clm			;; 10 iters of 8*clm
	zloop_set_clm_prefetch ZOCT		;; Prefetch 320 of 960 blocks of clm*128 bytes
	zr12_csc_wpn_twelve_complex_first_djbfft_preload
b1b:	zr12_csc_wpn_twelve_complex_first_djbfft rsi, 128, 10*clmblkdst, 20*clmblkdst, 40*clmblkdst, r15, 0, rdi, 3, rdx, 0, 8*clm, L1PREFETCH_ALL, 128
	zloop	8*clm, b1b, -8*clm*128+clmblkdst, r15, ZMM_CD12SCD6, rdx, 24*64, r13, 24 ;; 8*clm iters, next source, fixed s/c, grp multipliers pointer, next xor mask
	zloop	10, b1b, -10*clmblkdst		;; Test loop counter, restore source pointer
	end_timer 22

;; Do FFT levels 5,6,7
;; On input the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	80K	+1K	...
;;	8K
;;	...
;;	72K
;;	160K
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	8K	+1K	...
;;	16K
;;	...

;; Do 96 ten_complex_fft operations
;;	distance between fft data elements is 8K
;;	do 12 macros each processing 160 data values

	start_timer 24
	bump	rdi, -80*clm*3-8*clm*ZMM_SCD5	;; Sin/cos pointer
	zloop_init  12*8*clm			;; 12 iterations of 8*clm
	zloop_set_clm_prefetch ZQUAD		;; Prefetch 192 of 960 blocks of clm*128 bytes
	zr10_ten_complex_djbfft_preload
IFDEF OLDDD
b2b:	zr10_ten_complex_djbfft rsi, 128, clmblkdst, rdi, ZMM_SCD5, 8*clm, L1PREFETCH_ALL, 128
	zloop	8*clm, b2b, -8*clm*128+10*clmblkdst, rdi, -8*clm*ZMM_SCD5 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	12, b2b, -12*10*clmblkdst	;; Test loop counter, restore source pointer
ELSE
b2b:	zr10_ten_complex_djbfft rsi, 10*clmblkdst, clmblkdst, rdi, 0, 8, L1PREFETCH_ALL, 10*clmblkdst
	zloop	12, b2b, -12*10*clmblkdst+128, rdi, ZMM_SCD5 ;; Test loop counter, next source pointer, next sin/cos pointer
	zloop	8*clm, b2b, -8*clm*128		;; Test loop counter, restore source pointer
ENDIF
	end_timer 24

;; Do FFT levels 8,9,10
;; On input the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	8K	+1K	...
;;	16K
;;	...
;; The input is swizzled to hold these data values:
;;	0K	+1	+1	+1	+1	+1	+1	+1	8K	+1	...
;;	1K	...
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0K	+1	+1	+1	+1	+1	+1	+1	1K	+1	...
;;	2K
;;	...

;; Do 120 eight_complex_fft operations
;;	distance between fft data elements is 1K
;;	do 15 macros each processing 128 data values

	start_timer 26
	mov	rbp, DATA_ADDR			;; Destination pointer
IFDEF OLDDD
	bump	rdi, -12*clm*ZMM_SCD1		;; Delayed sin/cos multipliers
ELSE
	bump	rdi, -8*clm*ZMM_SCD5-12*clm*ZMM_SCD1 ;; Delayed sin/cos multipliers
ENDIF
	lea	rbx, [rdi-clm*ZMM_SCD4]		;; Complex data sin/cos ptr
	lea	r9, [rbx-clm*64-clm*64]		;; Column weights
	lea	rdx, ZMM_TMPS			;; Computed clm*SCD4 sin/cos values
	mov	r8, pass2blkdst			;; blkdst
	lea	r10, [2*blkdstreg+blkdstreg]	;; 3*blkdst
	lea	r13, [rbp+4*blkdstreg]		;; Dest+4*blkdst pointer
pfing	mov	rcx, PREMULT_PREFETCH		;; Load premult/sin/cos prefetch pointer

	zr8_rsc_complex_only_preload		;; Preload constants before calling zloop_init
;;BUG	zloop_init 12*10*clm			;; 12 iters of 10*clm iterations
	zloop_init 12*10*clm  *2		;; 12 iters of 10*clm iterations	(work around loops "add al, 1" bug)
	zloop_set_prefetch ZDOUBLE		;; Prefetch 240*clm times
	zloop_set_alternate_prefetch_increment 121*2+1, 240*2  ;; Prefetch approximately 121*clm cache lines in those 240*clm prefetches
b9a:	zr8_rsc_wpn_eight_complex_calc_sincos r9, rdi, rbx, clm ;; Generate clm*SCD8 sin/cos values from weights, delayed and complex multipliers
	bump	rdi, clm*ZMM_SCD1		;; Next set of delayed multipliers
b9b:	zr8_rsc_wpn_sgreg_eight_complex_fft8 rsi, 1024, 128, 256, 512, rbp, 128, blkdstreg, blkdst3reg, r13, rdx, ZMM_SCD8, clm, L1PREFETCH_DEST_ALL
	zloop	clm, b9b, -clm*1024+clmblkdst, rbp, -clm*128+8*blkdstreg, r13, -clm*128+8*blkdstreg, rdx, -clm*ZMM_SCD8 ;; Loop, next source/dest pointers, restore premult/sine/cosine pointer
	zloop	10, b9b				;; Test loop counter
	zloop	12, b9a				;; Test loop counter
	end_timer 26

;; Work on next 8*clm sets of 1920 values

endlp:	zpass1_get_next_block pass2, c0b, b0b ;; Get next block and jump
done:	zfft_footer
	ENDM



;; Perform a shared pass 1 FFT of length 2048 doubles.
;; Memory examples are for a 2M FFT with 10 levels done in pass 2.
;; FFT data needed:
;;		8*clm*2048 doubles					Total: 2048*clm 64-byte cache lines
;; Fixed s/c data needed:
;;		16*ZMM_SCD7 (fixed s/c data),				Total: 224 64-byte cache lines
;; Group multipliers needed:
;;		16*16*64 (weights)
;;		16*16*64 (inverse weights),				Total: 512 64-byte cache lines
;; Variable data needed:
;;		clm*64	(column weights)
;;		clm*64	(column inverse weights)
;;		clm*ZMM_SCD4 (complex reduced s/c multipliers)
;;		clm*ZMM_SCD4 (real s/c multipliers)
;;		7*clm*ZMM_SCD1 (delayed s/c multipliers)
;;		8*clm*ZMM_SCD16 (16-complex 32-real sin/cos data)
;;		16*8*clm*2 (group multiplier fudge flags)
;;		16*8*clm*2 (biglit flags),				Total: 296*clm 64-byte cache lines
;; Total memory consumed (add for scratch area, prefetched data, carries):
;;		(2048*3+296=6440)*clm + (224+512+2048/8=992) cache lines
;;		clm=1,2,4:  465KB, 867KB, 1672KB

zr4dwpn_pass1sc2048 MACRO
	LOCAL	b0b, b1b, b2a, b2b, b2c, b8a, b8b, b9aa, b9a, b9b, c0b, c1b, c2a, c2b, c2c, c8b, c9aa, c9a, c9b
	LOCAL	pass1, pass2, endlp, done, yes_fft

	zfft_header pass1

;; Do pass 2 (the last 10 FFT levels if this is a 2M FFT)

pass2:	start_timer 0
	mov	rax, ZMM_PASS2_ROUTINE
	call	rax
	end_timer 0
	cmp	ffttype, 1		;; We're done if FFTing only
	je	done

;; Do the inverse FFT

	pass1_inverse_fft_setup c0b

;; Do inverse FFT levels 9,10,11
;; On input the 128-byte double cache lines hold these data values:
;;	0K	+1	+1	+1	+1	+1	+1	+1	1K	+1	...
;;	2K
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0K	+1	+1	+1	+1	+1	+1	+1	8K	+1	...
;;	1K	...
;;	...
;; The output cache lines are swizzled to hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	8K	+1K	...
;;	16K
;;	...

;; Do 1 sixteen_reals_unfft operations
;;	distance between fft data elements is 1K
;;	do 0.125 macros each processing 128 data values
;; Do 127 eight_complex_unfft operations
;;	distance between fft data elements is 1K
;;	do 15.875 macros each processing 128 data values

c0b:	start_timer 16
	mov	rbp, DATA_ADDR			;; Get source address
	mov	rsi, scratch_area		;; Get address of scratch area
	set_data_prefetch_ptrs			;; Init prefetching pointers
	mov	r9, PREMULT_ADDR		;; Variable data
	bump	r9, clm*64			;; Inverse column weights
	lea	rbx, [r9+clm*64]		;; Complex data sin/cos ptr
	lea	rdi, [rbx+clm*ZMM_SCD4]		;; Real data sin/cos pointer, will become delayed sin/cos multipliers
	lea	rdx, ZMM_TMPS			;; Computed clm*SCD8 sin/cos values
	mov	r8, pass2blkdst			;; blkdst
	lea	r10, [2*blkdstreg+blkdstreg]	;; 3*blkdst
	lea	r13, [rbp+4*blkdstreg]		;; Source+4*blkdst pointer
	lea	r12, [rbp+8*blkdstreg]		;; L1 prefetch pointer part 1
	bump2	r12, (clm MOD 2), 8*blkdstreg	;; L1 prefetch pointer part 2
	lea	r14, [r12+4*blkdstreg]		;; L1 prefetch pointer + 4*blkdst

	zr8_rsc_complex_and_real_preload	;; Preload constants before calling zloop_init
	zloop_init 16*8*clm			;; 16 iters of 8*clm iterations
	zloop_set_clm_prefetch ZDOUBLE		;; Prefetch 128 of 1024 blocks of clm*128 bytes
	zr8_rsc_wpn_eight_complex_calc_sincos_simple r9, rbx, clm ;; Generate clm*SCD8 sin/cos values from delayed and complex multipliers
c8b:	zr8_rsc_wpn_sgreg_2sc_sixteen_reals_unfft8 rbp, 128, blkdstreg, blkdst3reg, r13, rsi, 1024, 128, 256, 512, rdx, ZMM_SCD8, r9, 64, rdi, ZMM_SCD4, clm, L1PREFETCH_ALL, r12, r14 ;; Prefetch short dist ahead
	zloop	clm, c8b, -clm*1024+clmblkdst, rbp, -clm*128+8*blkdstreg, r13, -clm*128+8*blkdstreg, rdx, -clm*ZMM_SCD8, r9, -clm*64, r12, -clm*128+8*blkdstreg, r14, -clm*128+8*blkdstreg ;; Loop, next src/dest ptrs, restore sin/cos ptr
	jmp	c9aa

	loops_reset
c9a:	zr8_rsc_wpn_eight_complex_calc_sincos r9, rdi, rbx, clm ;; Generate clm*SCD8 sin/cos values from delayed and complex multipliers
	bump	rdi, clm*ZMM_SCD1		;; Next set of delayed sin/cos multipliers
c9aa:
c9b:	zr8_rsc_wpn_sgreg_eight_complex_unfft8 rbp, 128, blkdstreg, blkdst3reg, r13, rsi, 1024, 128, 256, 512, rdx, ZMM_SCD8, clm, L1PREFETCH_ALL, r12, r14 ;; Prefetch short dist ahead
	zloop	clm, c9b, -clm*1024+clmblkdst, rbp, -clm*128+8*blkdstreg, r13, -clm*128+8*blkdstreg, rdx, -clm*ZMM_SCD8, r12, -clm*128+8*blkdstreg, r14, -clm*128+8*blkdstreg ;; Loop, next src/dest ptrs, restore sin/cos ptr
	zloop	8, c9b, -8*clmblkdst+clmblkdst8	;; Test loop counter, next source pointer
	zloop	2, c9b				;; Test loop counter
	zloop	8, c9a, -16*clmblkdst8		;; Test loop counter, restore source pointer
	end_timer 16

;; Do inverse FFT levels 5,6,7,8
;; On input the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	8K	+1K	...
;;	16K
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	128K	+1K	...
;;	8K
;;	...
;;	120K
;;	256K
;;	...

;; Do 8 thirtytwo_reals_unfft operations
;; Do 56 sixteen_complex_unfft operations
;;	distance between fft data elements is 8K
;;	do 8 macros each processing 256 data values

	start_timer 18
	zloop_init  8*8*clm			;; 8 iterations of 8*clm
	zloop_set_clm_prefetch ZQUAD		;; Prefetch 128 of 1024 blocks of clm*128 bytes
IFDEF OLDDD
	zr16_csc_thirtytwo_reals_unfft_preload
c2a:	zr16_csc_thirtytwo_reals_unfft rsi, 128, clmblkdst, 2*clmblkdst, 4*clmblkdst, clmblkdst8, rdi, ZMM_SCD16, 8*clm, L1PREFETCH_ALL, 128
	zloop	8*clm, c2a, -8*clm*128+2*clmblkdst8, rdi, -8*clm*ZMM_SCD16 ;; Test loop counter, next source pointer, restore sin/cos pointer
	loops_reset				;; 7 more iters of 8*clm
	zr16_sixteen_complex_djbunfft_preload
c2b:	zr16_sixteen_complex_djbunfft rsi, 128, clmblkdst, 2*clmblkdst, 4*clmblkdst, clmblkdst8, rdi, ZMM_SCD16, 8*clm, L1PREFETCH_ALL, 128
	zloop	8*clm, c2b, -8*clm*128+2*clmblkdst8, rdi, -8*clm*ZMM_SCD16 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	8, c2b, -8*2*clmblkdst8, rdi, 8*clm*ZMM_SCD16 ;; Test loop counter, restore source pointer, next sin/cos pointer
ELSE
c2a:	zr16_csc_thirtytwo_reals_unfft_preload
	zr16_csc_thirtytwo_reals_unfft rsi, 2*clmblkdst8, clmblkdst, 2*clmblkdst, 4*clmblkdst, clmblkdst8, rdi, 0, 1, L1PREFETCH_ALL, 2*clmblkdst8
	zr16_sixteen_complex_djbunfft_preload
	jmp	c2c				;; 7 more iters
c2b:	zr16_sixteen_complex_djbunfft rsi, 2*clmblkdst8, clmblkdst, 2*clmblkdst, 4*clmblkdst, clmblkdst8, rdi, 0, 1, L1PREFETCH_ALL, 2*clmblkdst8
c2c:	zloop	8, c2b, -8*2*clmblkdst8+128, rdi, ZMM_SCD16 ;; Test loop counter, next source pointer, next sin/cos pointer
	zloop	8*clm, c2a, -8*clm*128		;; Test loop counter, restore source pointer
ENDIF
	end_timer 18

;; Do inverse FFT levels 1,2,3,4
;; On input the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	128K	+1K	...
;;	8K
;;	...
;;	120K
;;	256K
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	1024K	+1K	...
;;	8K
;;	...
;;	1016K

;; Do 128 sixteen_reals_unfft operations
;;	distance between fft data elements is 128K
;;	do 16 macros each processing 128 data values

	start_timer 20
	mov	r15, sincos2			;; Load fixed premultiplier/sin/cos ptr
	sub	rbx, rbx			;; Clear register for loading fudge flags index
	mov	r12, compressed_fudges		;; Address of the compressed fudge flags
	mov	r13, r12			;; The XOR masks for fudge flags come from the compressed fudges
	zloop_init 16*8*clm			;; 16 iters of 8*clm
	zloop_set_clm_prefetch ZQUAD		;; Prefetch 256 of 1024 blocks of clm*128 bytes
	zr8_wpn_sixteen_reals_last_unfft_preload
c1b:	zr8_wpn_sixteen_reals_last_unfft rsi, 128, 2*clmblkdst8, 4*clmblkdst8, 8*clmblkdst8, r15, 0, rdi, 2, 8*clm, L1PREFETCH_ALL, 128
	zloop	8*clm, c1b, -8*clm*128+clmblkdst, r15, ZMM_SCD7, rdx, 16*64, r13, 16 ;; 8*clm iters, next source, fixed s/c, grp multipliers pointer, next xor mask
	zloop	8, c1b, -8*clmblkdst+clmblkdst8	;; Test loop counter, next source pointer
	zloop	2, c1b				;; Test loop counter
	end_timer 20

;; Normalize these values

	pass1_normalize yes_fft			;; Normalize while prefetching 128 of 1024 blocks of clm*128 bytes

;; Copy data from the scratch area back to the FFT data area

	copy_scratch_data_to_fft
	jmp	endlp				;; Skip FFT code if flag not set

;; Fire up auxiliary threads to do pass 1 forward FFT

pass1:	pass1_forward_fft_setup b0b

;; Copy data from the FFT data area back to the scratch area

b0b:	copy_fft_data_to_scratch

;; Do FFT levels 1,2,3,4
;; On input the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	1024K	+1K	...
;;	8K	...
;;	...
;;	1016K
;; On output the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	128K	+1K	...
;;	8K
;;	...
;;	120K
;;	256K
;;	...

;; Do 128 sixteen_reals_first_fft operations
;;	distance between fft data elements is 64K
;;	do 16 macros each processing 128 data values

yes_fft:start_timer 22
	mov	rsi, scratch_area		;; Get address of scratch area
	mov	r15, sincos2			;; Load fixed premultiplier/sin/cos ptr
	mov	rdi, PREMULT_ADDR		;; Calculate pointer to fudge flags
	bump	rdi, 8*clm*ZMM_SCD16+7*clm*ZMM_SCD1+clm*ZMM_SCD4+clm*ZMM_SCD4+clm*64+clm*64 ;; Addr of the group multiplier fudge flags
	mov	rdx, norm_grp_mults		;; Addr of the group multipliers
	sub	rbx, rbx			;; Clear register for loading fudge flags index
	mov	r12, compressed_fudges		;; Address of the compressed fudge flags
	mov	r13, r12			;; The XOR masks for fudge flags come from the compressed fudges
	set_data_prefetch_ptrs			;; Init prefetching pointers
pfing	mov	r8, pass2blkdst			;; blkdst
pfing	imul	rax, blkdstreg, (640-0)		;; We've already prefetched 640 of 1024 blocks (zero too many)
pfing	add	rcx, rax
	zloop_init 16*8*clm			;; 16 iters of 8*clm
	zloop_set_clm_prefetch ZQUAD		;; Prefetch 256 of 1024 blocks of clm*128 bytes
	zr8_wpn_sixteen_reals_first_fft_preload
b1b:	zr8_wpn_sixteen_reals_first_fft rsi, 128, 2*clmblkdst8, 4*clmblkdst8, 8*clmblkdst8, r15, 0, rdi, 2, rdx, 0, 8*clm, L1PREFETCH_ALL, 128
	zloop	8*clm, b1b, -8*clm*128+clmblkdst, r15, ZMM_SCD7, rdx, 16*64, r13, 16 ;; 8*clm iters prefetching 8*clm cache lines, next source, fixed s/c, grp multipliers pointer, next xor mask
	zloop	8, b1b, -8*clmblkdst+clmblkdst8	;; Test loop counter, next source pointer
	zloop	2, b1b, -2*clmblkdst8		;; Test loop counter, restore source pointer
	end_timer 22

;; Do FFT levels 5,6,7,8
;; On input the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	128K	+1K	...
;;	8K
;;	...
;;	120K
;;	256K
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	8K	+1K	...
;;	16K
;;	...

;; Do 8 thirtytwo_reals_fft operations
;; Do 56 sixteen_complex_fft operations
;;	distance between fft data elements is 8K
;;	do 8 macros each processing 256 data values

	start_timer 24
	bump	rdi, -16*8*clm*2-8*clm*ZMM_SCD16 ;; Sin/cos pointer
	zloop_init  8*8*clm			;; 8 iterations of 8*clm
	zloop_set_clm_prefetch ZQUAD		;; Prefetch 128 of 1024 blocks of clm*128 bytes
IFDEF OLDDD
	zr16_csc_thirtytwo_reals_fft_preload
b2a:	zr16_csc_thirtytwo_reals_fft rsi, 128, clmblkdst, 2*clmblkdst, 4*clmblkdst, clmblkdst8, rdi, ZMM_SCD16, 8*clm, L1PREFETCH_ALL, 128
	zloop	8*clm, b2a, -8*clm*128+2*clmblkdst8, rdi, -8*clm*ZMM_SCD16 ;; Test loop counter, next source pointer, restore sin/cos pointer
	loops_reset				;; 7 more iters of 8*clm
	zr16_sixteen_complex_djbfft_preload
b2b:	zr16_sixteen_complex_djbfft rsi, 128, clmblkdst, 2*clmblkdst, 4*clmblkdst, clmblkdst8, rdi, ZMM_SCD16, 8*clm, L1PREFETCH_ALL, 128
	zloop	8*clm, b2b, -8*clm*128+2*clmblkdst8, rdi, -8*clm*ZMM_SCD16 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	8, b2b, -8*2*clmblkdst8		;; Test loop counter, restore source pointer
ELSE
b2a:	zr16_csc_thirtytwo_reals_fft_preload
	zr16_csc_thirtytwo_reals_fft rsi, 2*clmblkdst8, clmblkdst, 2*clmblkdst, 4*clmblkdst, clmblkdst8, rdi, 0, 1, L1PREFETCH_ALL, 2*clmblkdst8
	zr16_sixteen_complex_djbfft_preload
	jmp	b2c				;; 7 more iters
b2b:	zr16_sixteen_complex_djbfft rsi, 2*clmblkdst8, clmblkdst, 2*clmblkdst, 4*clmblkdst, clmblkdst8, rdi, 0, 1, L1PREFETCH_ALL, 2*clmblkdst8
b2c:	zloop	8, b2b, -8*2*clmblkdst8+128, rdi, ZMM_SCD16 ;; Test loop counter, next source pointer, next sin/cos pointer
	zloop	8*clm, b2a, -8*clm*128		;; Test loop counter, restore source pointer
ENDIF
	end_timer 24

;; Do FFT levels 9,10,11
;; On input the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	8K	+1K	...
;;	16K
;;	...
;; The input is swizzled to hold these data values:
;;	0K	+1	+1	+1	+1	+1	+1	+1	8K	+1	...
;;	1K	...
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0K	+1	+1	+1	+1	+1	+1	+1	1K	+1	...
;;	2K
;;	...

;; Do 1 sixteen_reals_fft operations
;;	distance between fft data elements is 1K
;;	do 0.125 macros each processing 128 data values
;; Do 127 eight_complex_fft operations
;;	distance between fft data elements is 1K
;;	do 15.875 macros each processing 128 data values

	start_timer 26
IFDEF OLDDD
	bump	rdi, -7*clm*ZMM_SCD1-clm*ZMM_SCD4 ;; Real data sin/cos pointer, will become delayed sin/cos multipliers
ELSE
	bump	rdi, -8*clm*ZMM_SCD16-7*clm*ZMM_SCD1-clm*ZMM_SCD4 ;; Real data sin/cos pointer, will become delayed sin/cos multipliers
ENDIF
	lea	rbx, [rdi-clm*ZMM_SCD4]		;; Complex data sin/cos ptr
	lea	r9, [rbx-clm*64-clm*64]		;; Column weights
	lea	rdx, ZMM_TMPS			;; Computed clm*SCD4 sin/cos values
	mov	rbp, DATA_ADDR			;; Destination pointer
	mov	r8, pass2blkdst			;; blkdst
	lea	r10, [2*blkdstreg+blkdstreg]	;; 3*blkdst
	lea	r13, [rbp+4*blkdstreg]		;; Dest+4*blkdst pointer
pfing	mov	rcx, PREMULT_PREFETCH		;; Load premult/sin/cos prefetch pointer

	zr8_rsc_complex_and_real_preload	;; Preload constants before calling zloop_init
	zloop_init 16*8*clm			;; 16 iters of 8*clm
	zloop_set_prefetch ZTRIPLE		;; Prefetch 384*clm times
	zloop_set_alternate_prefetch_increment 296, 384  ;; Prefetch approximately 296*clm cache lines in those 384*clm prefetches
	zr8_rsc_wpn_eight_complex_calc_sincos_simple r9, rbx, clm ;; Generate clm*SCD8 sin/cos values from delayed and complex multipliers
b8b:	zr8_rsc_wpn_sgreg_2sc_sixteen_reals_fft8 rsi, 1024, 128, 256, 512, rbp, 128, blkdstreg, blkdst3reg, r13, rdx, ZMM_SCD8, r9, 64, rdi, ZMM_SCD4, clm, L1PREFETCH_DEST_ALL
	zloop	clm, b8b, -clm*1024+clmblkdst, rbp, -clm*128+8*blkdstreg, r13, -clm*128+8*blkdstreg, rdx, -clm*ZMM_SCD8, r9, -clm*64 ;; Loop, next source/dest pointers, restore premult/sine/cosine pointer
	jmp	b9aa

	loops_reset
b9a:	zr8_rsc_wpn_eight_complex_calc_sincos r9, rdi, rbx, clm ;; Generate clm*SCD8 sin/cos values from delayed and complex multipliers
	bump	rdi, clm*ZMM_SCD1		;; Next set of delayed multipliers
b9aa:
b9b:	zr8_rsc_wpn_sgreg_eight_complex_fft8 rsi, 1024, 128, 256, 512, rbp, 128, blkdstreg, blkdst3reg, r13, rdx, ZMM_SCD8, clm, L1PREFETCH_DEST_ALL
	zloop	clm, b9b, -clm*1024+clmblkdst, rbp, -clm*128+8*blkdstreg, r13, -clm*128+8*blkdstreg, rdx, -clm*ZMM_SCD8 ;; Loop, next source/dest pointers, restore premult/sine/cosine pointer
	zloop	8, b9b, -8*clmblkdst+clmblkdst8	;; Test loop counter, next source pointer
	zloop	2, b9b				;; Test loop counter
	zloop	8, b9a				;; Test loop counter
	end_timer 26

;; Work on next 8*clm sets of 2048 values

endlp:	zpass1_get_next_block pass2, c0b, b0b ;; Get next block and jump
done:	zfft_footer
	ENDM


;; Perform a shared pass 1 negacyclic FFT of length 2048 doubles.
;; Memory examples are for a 2M FFT with 10 levels done in pass 2.
;; FFT data needed:
;;		8*clm*2048 doubles					Total: 2048*clm 64-byte cache lines
;; Fixed s/c data needed:
;;		16*ZMM_CD8SCD4 (complex premultipliers and fixed s/c data), Total: 256 64-byte cache lines
;; Group multipliers needed:
;;		16*16*64 (weights)
;;		16*16*64 (inverse weights),				Total: 512 64-byte cache lines
;; Variable data needed:
;;		clm*64	(column weights)
;;		clm*64	(column inverse weights)
;;		clm*ZMM_SCD4 (complex reduced s/c multipliers)
;;		8*clm*ZMM_SCD1 (delayed s/c multipliers)
;;		8*clm*ZMM_SCD8 (16-complex sin/cos data)
;;		16*8*clm*2 (group multiplier fudge flags)
;;		16*8*clm*2 (biglit flags),				Total: 162*clm 64-byte cache lines
;; Total memory consumed (add for scratch area, prefetched data, carries):
;;		(2048*3+162=6306)*clm + (256+512+2048/8=1024) cache lines
;;		clm=1,2,4:  459KB, 853KB, 1641KB

zr4dwpn_pass1sc2048ac MACRO
	LOCAL	b0b, b1b, b2b, b9a, b9b, c0b, c1b, c2b, c9a, c9b
	LOCAL	pass1, pass2, endlp, done, yes_fft

	zfft_header pass1

;; Do pass 2 (the last 10 FFT levels if this is a 2M FFT)

pass2:	start_timer 0
	mov	rax, ZMM_PASS2_ROUTINE
	call	rax
	end_timer 0
	cmp	ffttype, 1		;; We're done if FFTing only
	je	done

;; Do the inverse FFT

	pass1_inverse_fft_setup c0b

;; Do inverse FFT levels 9.10,11
;; On input the 128-byte double cache lines hold these data values:
;;	0K	+1	+1	+1	+1	+1	+1	+1	1K	+1	...
;;	2K
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0K	+1	+1	+1	+1	+1	+1	+1	8K	+1	...
;;	1K	...
;;	...
;; The output cache lines are swizzled to hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	8K	+1K	...
;;	16K
;;	...

;; Do 128 eight_complex_unfft operations
;;	distance between fft data elements is 1K
;;	do 16 macros each processing 128 data values

c0b:	start_timer 16
	mov	rbp, DATA_ADDR			;; Get source address
	mov	rsi, scratch_area		;; Get address of scratch area
	set_data_prefetch_ptrs			;; Init prefetching pointers
	mov	r9, PREMULT_ADDR		;; Variable data
	bump	r9, clm*64			;; Inverse column weights
	lea	rbx, [r9+clm*64]		;; Complex data sin/cos ptr
	lea	rdi, [rbx+clm*ZMM_SCD4]		;; Delayed sin/cos multipliers
	lea	rdx, ZMM_TMPS			;; Computed clm*SCD8 sin/cos values
	mov	r8, pass2blkdst			;; blkdst
	lea	r10, [2*blkdstreg+blkdstreg]	;; 3*blkdst
	lea	r13, [rbp+4*blkdstreg]		;; Source+4*blkdst pointer
	lea	r12, [rbp+8*blkdstreg]		;; L1 prefetch pointer part 1
	bump2	r12, (clm MOD 2), 8*blkdstreg	;; L1 prefetch pointer part 2
	lea	r14, [r12+4*blkdstreg]		;; L1 prefetch pointer + 4*blkdst

	zr8_rsc_complex_only_preload		;; Preload constants before calling zloop_init
	zloop_init 16*8*clm			;; 16 iters of 8*clm iterations
	zloop_set_clm_prefetch ZDOUBLE		;; Prefetch 128 of 1024 blocks of clm*128 bytes
c9a:	zr8_rsc_wpn_eight_complex_calc_sincos r9, rdi, rbx, clm ;; Generate clm*SCD8 sin/cos values from weights, delayed, and complex multipliers
	bump	rdi, clm*ZMM_SCD1		;; Next set of delayed sin/cos multipliers
c9b:	zr8_rsc_wpn_sgreg_eight_complex_unfft8 rbp, 128, blkdstreg, blkdst3reg, r13, rsi, 1024, 128, 256, 512, rdx, ZMM_SCD8, clm, L1PREFETCH_ALL, r12, r14 ;; Prefetch short dist ahead
	zloop	clm, c9b, -clm*1024+clmblkdst, rbp, -clm*128+8*blkdstreg, r13, -clm*128+8*blkdstreg, rdx, -clm*ZMM_SCD8, r12, -clm*128+8*blkdstreg, r14, -clm*128+8*blkdstreg ;; Loop, next src/dest ptrs, restore sin/cos ptr
	zloop	8, c9b, -8*clmblkdst+clmblkdst8	;; Test loop counter, next source pointer
	zloop	2, c9b				;; Test loop counter
	zloop	8, c9a, -16*clmblkdst8		;; Test loop counter, restore source pointer
	end_timer 16

;; Do inverse FFT levels 5,6,7,8
;; On input the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	8K	+1K	...
;;	16K
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	128K	+1K	...
;;	8K
;;	...
;;	120K
;;	256K
;;	...

;; Do 64 sixteen_complex_unfft operations
;;	distance between fft data elements is 8K
;;	do 8 macros each processing 256 data values

	start_timer 18
	zloop_init  8*8*clm			;; 8 iterations of 8*clm
	zloop_set_clm_prefetch ZQUAD		;; Prefetch 128 of 1024 blocks of clm*128 bytes
	zr16_sixteen_complex_djbunfft_preload
IFDEF OLDDD
c2b:	zr16_sixteen_complex_djbunfft rsi, 128, clmblkdst, 2*clmblkdst, 4*clmblkdst, clmblkdst8, rdi, ZMM_SCD8, 8*clm, L1PREFETCH_ALL, 128
	zloop	8*clm, c2b, -8*clm*128+2*clmblkdst8, rdi, -8*clm*ZMM_SCD8 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	8, c2b, -8*2*clmblkdst8, rdi, 8*clm*ZMM_SCD8 ;; Test loop counter, restore source pointer, next sin/cos pointer
ELSE
c2b:	zr16_sixteen_complex_djbunfft rsi, 2*clmblkdst8, clmblkdst, 2*clmblkdst, 4*clmblkdst, clmblkdst8, rdi, 0, 8, L1PREFETCH_ALL, 2*clmblkdst8
	zloop	8, c2b, -8*2*clmblkdst8+128, rdi, ZMM_SCD8 ;; Test loop counter, next source pointer, next sin/cos pointer
	zloop	8*clm, c2b, -8*clm*128		;; Test loop counter, restore source pointer
ENDIF
	end_timer 18

;; Do inverse FFT levels 1,2,3,4
;; On input the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	128K	+1K	...
;;	8K
;;	...
;;	120K
;;	256K
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	1024K	+1K	...
;;	8K	...
;;	...
;;	1016K

;; Do 128 eight_complex_unfft operations
;;	distance between fft data elements is 128K
;;	do 16 macros each processing 128 data values

	start_timer 20
	mov	r15, sincos2			;; Load fixed premultiplier/sin/cos ptr
	sub	rbx, rbx			;; Clear register for loading fudge flags index
	mov	r12, compressed_fudges		;; Address of the compressed fudge flags
	mov	r13, r12			;; The XOR masks for fudge flags come from the compressed fudges
	zloop_init 16*8*clm			;; 16 iters of 8*clm
	zloop_set_clm_prefetch ZQUAD		;; Prefetch 256 of 1024 blocks of clm*128 bytes
	zr8_csc_wpn_eight_complex_last_djbunfft_preload
c1b:	zr8_csc_wpn_eight_complex_last_djbunfft rsi, 128, 2*clmblkdst8, 4*clmblkdst8, 8*clmblkdst8, r15, 0, rdi, 2, 8*clm, L1PREFETCH_ALL, 128
	zloop	8*clm, c1b, -8*clm*128+clmblkdst, r15, ZMM_CD8SCD4, rdx, 16*64, r13, 16 ;; 8*clm iterations, next fixed s/c, next grp multipliers, next xor mask
	zloop	8, c1b, -8*clmblkdst+clmblkdst8	;; Test loop counter, next source pointer
	zloop	2, c1b				;; Test loop counter
	end_timer 20

;; Normalize these values

	pass1_normalize yes_fft			;; Normalize while prefetching 128 of 1024 blocks of clm*128 bytes

;; Copy data from the scratch area back to the FFT data area

	copy_scratch_data_to_fft
	jmp	endlp				;; Skip FFT code if flag not set

;; Fire up auxiliary threads to do pass 1 forward FFT

pass1:	pass1_forward_fft_setup b0b

;; Copy data from the FFT data area back to the scratch area

b0b:	copy_fft_data_to_scratch

;; Do FFT levels 1,2,3,4
;; On input the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	1024K	+1K	...
;;	8K	...
;;	...
;;	1016K
;; On output the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	128K	+1K	...
;;	8K
;;	...
;;	120K
;;	256K
;;	...

;; Do 128 eight_complex_first_fft operations
;;	distance between fft data elements is 128K
;;	do 16 macros each processing 128 data values

yes_fft:start_timer 22
	mov	rsi, scratch_area		;; Get address of scratch area
	mov	r15, sincos2			;; Load fixed premultiplier/sin/cos ptr
	mov	rdi, PREMULT_ADDR		;; Calculate pointer to fudge flags
	bump	rdi, 8*clm*ZMM_SCD8+8*clm*ZMM_SCD1+clm*ZMM_SCD4+clm*64+clm*64 ;; Addr of the group multiplier fudge flags
	mov	rdx, norm_grp_mults		;; Addr of the group multipliers
	sub	rbx, rbx			;; Clear register for loading fudge flags index
	mov	r12, compressed_fudges		;; Address of the compressed fudge flags
	mov	r13, r12			;; The XOR masks for fudge flags come from the compressed fudges
	set_data_prefetch_ptrs			;; Init prefetching pointers
pfing	mov	r8, pass2blkdst			;; blkdst
pfing	imul	rax, blkdstreg, (640-0)		;; We've already prefetched 640 of 1024 blocks (zero too many)
pfing	add	rcx, rax
	zloop_init 16*8*clm			;; 16 iters of 8*clm
	zloop_set_clm_prefetch ZQUAD		;; Prefetch 256 of 1024 blocks of clm*128 bytes
	zr8_csc_wpn_eight_complex_first_djbfft_preload
b1b:	zr8_csc_wpn_eight_complex_first_djbfft rsi, 128, 2*clmblkdst8, 4*clmblkdst8, 8*clmblkdst8, r15, 0, rdi, 2, rdx, 0, 8*clm, L1PREFETCH_ALL, 128
	zloop	8*clm, b1b, -8*clm*128+clmblkdst, r15, ZMM_CD8SCD4, rdx, 16*64, r13, 16 ;; 8*clm iters, next source, fixed s/c, grp multipliers pointer, next xor mask
	zloop	8, b1b, -8*clmblkdst+clmblkdst8	;; Test loop counter, next source pointer
	zloop	2, b1b, -2*clmblkdst8		;; Test loop counter, restore source pointer
	end_timer 22

;; Do FFT levels 5,6,7,8
;; On input the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	128K	+1K	...
;;	8K
;;	...
;;	120K
;;	256K
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	8K	+1K	...
;;	16K
;;	...

;; Do 64 sixteen_complex_fft operations
;;	distance between fft data elements is 8K
;;	do 8 macros each processing 256 data values

	start_timer 24
	bump	rdi, -16*8*clm*2-8*clm*ZMM_SCD8 ;; Sin/cos pointer
	zloop_init  8*8*clm			;; 8 iterations of 8*clm
	zloop_set_clm_prefetch ZQUAD		;; Prefetch 128 of 1024 blocks of clm*128 bytes
	zr16_sixteen_complex_djbfft_preload
IFDEF OLDDD
b2b:	zr16_sixteen_complex_djbfft rsi, 128, clmblkdst, 2*clmblkdst, 4*clmblkdst, clmblkdst8, rdi, ZMM_SCD8, 8*clm, L1PREFETCH_ALL, 128
	zloop	8*clm, b2b, -8*clm*128+2*clmblkdst8, rdi, -8*clm*ZMM_SCD8 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	8, b2b, -8*2*clmblkdst8		;; Test loop counter, restore source pointer
ELSE
b2b:	zr16_sixteen_complex_djbfft rsi, 2*clmblkdst8, clmblkdst, 2*clmblkdst, 4*clmblkdst, clmblkdst8, rdi, 0, 8, L1PREFETCH_ALL, 2*clmblkdst8
	zloop	8, b2b, -8*2*clmblkdst8+128, rdi, ZMM_SCD8 ;; Test loop counter, next source pointer, next sin/cos pointer
	zloop	8*clm, b2b, -8*clm*128		;; Test loop counter, restore source pointer
ENDIF
	end_timer 24

;; Do FFT levels 9,10,11
;; On input the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	8K	+1K	...
;;	16K
;;	...
;; The input is swizzled to hold these data values:
;;	0K	+1	+1	+1	+1	+1	+1	+1	8K	+1	...
;;	1K	...
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0K	+1	+1	+1	+1	+1	+1	+1	1K	+1	...
;;	2K
;;	...

;; Do 128 eight_complex_fft operations
;;	distance between fft data elements is 1K
;;	do 16 macros each processing 128 data values

	start_timer 26
	mov	rbp, DATA_ADDR			;; Destination pointer
IFDEF OLDDD
	bump	rdi, -8*clm*ZMM_SCD1		;; Delayed sin/cos multipliers
ELSE
	bump	rdi, -8*clm*ZMM_SCD8-8*clm*ZMM_SCD1 ;; Delayed sin/cos multipliers
ENDIF
	lea	rbx, [rdi-clm*ZMM_SCD4]		;; Complex data sin/cos ptr
	lea	r9, [rbx-clm*64-clm*64]		;; Column weights
	lea	rdx, ZMM_TMPS			;; Computed clm*SCD4 sin/cos values
	mov	r8, pass2blkdst			;; blkdst
	lea	r10, [2*blkdstreg+blkdstreg]	;; 3*blkdst
	lea	r13, [rbp+4*blkdstreg]		;; Dest+4*blkdst pointer
pfing	mov	rcx, PREMULT_PREFETCH		;; Load premult/sin/cos prefetch pointer

	zr8_rsc_complex_only_preload		;; Preload constants before calling zloop_init
	zloop_init 16*8*clm			;; 16 iters of 8*clm
	zloop_set_prefetch ZDOUBLE		;; Prefetch 256*clm cache lines
	zloop_set_alternate_prefetch_increment 162, 256  ;; Prefetch approximately 162*clm cache lines in those 256*clm prefetches
b9a:	zr8_rsc_wpn_eight_complex_calc_sincos r9, rdi, rbx, clm ;; Generate clm*SCD8 sin/cos values from weights, delayed and complex multipliers
	bump	rdi, clm*ZMM_SCD1		;; Next set of delayed multipliers
b9b:	zr8_rsc_wpn_sgreg_eight_complex_fft8 rsi, 1024, 128, 256, 512, rbp, 128, blkdstreg, blkdst3reg, r13, rdx, ZMM_SCD8, clm, L1PREFETCH_DEST_ALL
	zloop	clm, b9b, -clm*1024+clmblkdst, rbp, -clm*128+8*blkdstreg, r13, -clm*128+8*blkdstreg, rdx, -clm*ZMM_SCD8 ;; Loop, next source/dest pointers, restore premult/sine/cosine pointer
	zloop	8, b9b, -8*clmblkdst+clmblkdst8	;; Test loop counter, next source pointer
	zloop	2, b9b				;; Test loop counter
	zloop	8, b9a				;; Test loop counter
	end_timer 26

;; Work on next 8*clm sets of 2048 values

endlp:	zpass1_get_next_block pass2, c0b, b0b ;; Get next block and jump
done:	zfft_footer
	ENDM



;; Perform a shared pass 1 FFT of length 2304 doubles.
;; Memory examples are for a 2304K FFT with 10 levels done in pass 2.
;; FFT data needed:
;;		8*clm*2304 doubles					Total: 2304*clm 64-byte cache lines
;; Fixed s/c data needed:
;;		12*ZMM_SCD11 (fixed s/c data),				Total: 252 64-byte cache lines
;; Group multipliers needed:
;;		12*24*64 (weights)
;;		12*24*64 (inverse weights),				Total: 576 64-byte cache lines
;; Variable data needed:
;;		clm*64	(column weights)
;;		clm*64	(column inverse weights)
;;		clm*ZMM_SCD4 (complex reduced s/c multipliers)
;;		clm*ZMM_SCD4 (real s/c multipliers)
;;		11*clm*ZMM_SCD1 (delayed s/c multipliers)
;;		8*clm*ZMM_SCD12 (12-complex 24-real sin/cos data)
;;		96*clm*3 (group multiplier fudge flags)
;;		96*clm*3 (biglit flags),				Total: 241*clm 64-byte cache lines
;; Total memory consumed (add for scratch area, prefetched data, carries):
;;		(2304*3+241=7153)*clm + (252+576+2304/8=1116) cache lines
;;		clm=1,2,4:  517KB, 964KB, 1858KB

zr4dwpn_pass1sc2304 MACRO
	LOCAL	b0b, b1b, b2a, b2b, b2c, b8a, b8b, b9aa, b9a, b9b, c0b, c1b, c2a, c2b, c2c, c8b, c9aa, c9a, c9b
	LOCAL	pass1, pass2, endlp, done, yes_fft

	zfft_header pass1

;; Do pass 2 (the last 10 FFT levels if this is a 2304K FFT)

pass2:	start_timer 0
	mov	rax, ZMM_PASS2_ROUTINE
	call	rax
	end_timer 0
	cmp	ffttype, 1		;; We're done if FFTing only
	je	done

;; Do the inverse FFT

	pass1_inverse_fft_setup c0b

;; Do inverse FFT levels 8,9,10
;; On input the 128-byte double cache lines hold these data values:
;;	0K	+1	+1	+1	+1	+1	+1	+1	1K	+1	...
;;	2K
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0K	+1	+1	+1	+1	+1	+1	+1	8K	+1	...
;;	1K	...
;;	...
;; The output cache lines are swizzled to hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	8K	+1K	...
;;	16K
;;	...

;; Do 1 sixteen_reals_unfft operations
;;	distance between fft data elements is 1K
;;	do 0.125 macros each processing 128 data values
;; Do 143 eight_complex_unfft operations
;;	distance between fft data elements is 1K
;;	do 17.875 macros each processing 128 data values

c0b:	start_timer 16
	mov	rbp, DATA_ADDR			;; Get source address
	mov	rsi, scratch_area		;; Get address of scratch area
	set_data_prefetch_ptrs			;; Init prefetching pointers
	mov	r9, PREMULT_ADDR		;; Variable data
	bump	r9, clm*64			;; Inverse column weights
	lea	rbx, [r9+clm*64]		;; Complex data sin/cos ptr
	lea	rdi, [rbx+clm*ZMM_SCD4]		;; Real data sin/cos pointer, will become delayed sin/cos multipliers
	lea	rdx, ZMM_TMPS			;; Computed clm*SCD8 sin/cos values
	mov	r8, pass2blkdst			;; blkdst
	lea	r10, [2*blkdstreg+blkdstreg]	;; 3*blkdst
	lea	r13, [rbp+4*blkdstreg]		;; Source+4*blkdst pointer
	lea	r12, [rbp+8*blkdstreg]		;; L1 prefetch pointer part 1
	bump2	r12, (clm MOD 2), 8*blkdstreg	;; L1 prefetch pointer part 2
	lea	r14, [r12+4*blkdstreg]		;; L1 prefetch pointer + 4*blkdst

	zr8_rsc_complex_and_real_preload	;; Preload constants before calling zloop_init
;;	zloop_init 12*12*clm			;; 12 iters of 12*clm iterations
	zloop_init 12*12*clm  *2		;; 12 iters of 12*clm iterations (work around loops "add al, 1" bug)
	zloop_set_clm_prefetch ZQUAD		;; Prefetch 288 of 1152 blocks of clm*128 bytes
	zr8_rsc_wpn_eight_complex_calc_sincos_simple r9, rbx, clm ;; Generate clm*SCD8 sin/cos values from delayed and complex multipliers
c8b:	zr8_rsc_wpn_sgreg_2sc_sixteen_reals_unfft8 rbp, 128, blkdstreg, blkdst3reg, r13, rsi, 1024, 128, 256, 512, rdx, ZMM_SCD8, r9, 64, rdi, ZMM_SCD4, clm, L1PREFETCH_ALL, r12, r14 ;; Prefetch short dist ahead
	zloop	clm, c8b, -clm*1024+clmblkdst, rbp, -clm*128+8*blkdstreg, r13, -clm*128+8*blkdstreg, rdx, -clm*ZMM_SCD8, r9, -clm*64, r12, -clm*128+8*blkdstreg, r14, -clm*128+8*blkdstreg ;; Loop, next src/dest ptrs, restore sin/cos ptr
	jmp	c9aa

	loops_reset
c9a:	zr8_rsc_wpn_eight_complex_calc_sincos r9, rdi, rbx, clm ;; Generate clm*SCD8 sin/cos values from delayed and complex multipliers
	bump	rdi, clm*ZMM_SCD1		;; Next set of delayed sin/cos multipliers
c9aa:
c9b:	zr8_rsc_wpn_sgreg_eight_complex_unfft8 rbp, 128, blkdstreg, blkdst3reg, r13, rsi, 1024, 128, 256, 512, rdx, ZMM_SCD8, clm, L1PREFETCH_ALL, r12, r14 ;; Prefetch short dist ahead
	zloop	clm, c9b, -clm*1024+clmblkdst, rbp, -clm*128+8*blkdstreg, r13, -clm*128+8*blkdstreg, rdx, -clm*ZMM_SCD8, r12, -clm*128+8*blkdstreg, r14, -clm*128+8*blkdstreg ;; Loop, next src/dest ptrs, restore sin/cos ptr
	zloop	12, c9b				;; Test loop counter
	zloop	12, c9a, -12*12*clmblkdst	;; Test loop counter, restore source pointer
	end_timer 16

;; Do inverse FFT levels 5,6,7
;; On input the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	8K	+1K	...
;;	16K
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	96K	+1K	...
;;	8K
;;	...
;;	88K
;;	192K
;;	...

;; Do 8 twentyfour_reals_unfft operations
;; Do 88 twelve_complex_unfft operations
;;	distance between fft data elements is 8K
;;	do 12 macros each processing 192 data values

	start_timer 18
	zloop_init  12*8*clm			;; 8 iterations of 8*clm
	zloop_set_clm_prefetch ZQUAD		;; Prefetch 192 of 1152 blocks of clm*128 bytes
IFDEF OLDDD
	zr12_csc_twentyfour_reals_unfft_preload
c2a:	zr12_csc_twentyfour_reals_unfft rsi, 128, clmblkdst, 2*clmblkdst, 4*clmblkdst, rdi, ZMM_SCD12, 8*clm, L1PREFETCH_ALL, 128
	zloop	8*clm, c2a, -8*clm*128+12*clmblkdst, rdi, -8*clm*ZMM_SCD12 ;; Test loop counter, next source pointer, restore sin/cos pointer
	loops_reset				;; 11 more iters of 8*clm
	zr12_twelve_complex_djbunfft_preload
c2b:	zr12_twelve_complex_djbunfft rsi, 128, clmblkdst, 2*clmblkdst, 4*clmblkdst, rdi, ZMM_SCD12, 8*clm, L1PREFETCH_ALL, 128
	zloop	8*clm, c2b, -8*clm*128+12*clmblkdst, rdi, -8*clm*ZMM_SCD12 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	12, c2b, -12*12*clmblkdst, rdi, 8*clm*ZMM_SCD12 ;; Test loop counter, restore source pointer, next sin/cos pointer
ELSE
c2a:	zr12_csc_twentyfour_reals_unfft_preload
	zr12_csc_twentyfour_reals_unfft rsi, 12*clmblkdst, clmblkdst, 2*clmblkdst, 4*clmblkdst, rdi, 0, 1, L1PREFETCH_ALL, 12*clmblkdst
	zr12_twelve_complex_djbunfft_preload
	jmp	c2c				;; 11 more iters
c2b:	zr12_twelve_complex_djbunfft rsi, 12*clmblkdst, clmblkdst, 2*clmblkdst, 4*clmblkdst, rdi, 0, 1, L1PREFETCH_ALL, 12*clmblkdst
c2c:	zloop	12, c2b, -12*12*clmblkdst+128, rdi, ZMM_SCD12 ;; Test loop counter, next source pointer, next sin/cos pointer
	zloop	8*clm, c2a, -8*clm*128		;; Test loop counter, restore source pointer
ENDIF

;; Do inverse FFT levels 1,2,3,4
;; On input the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	96K	+1K	...
;;	8K
;;	...
;;	88K
;;	192K
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	1152K	+1K	...
;;	8K
;;	...
;;	1144K

;; Do 96 twentyfour_reals_unfft operations
;;	distance between fft data elements is 96K
;;	do 12 macros each processing 192 data values

	start_timer 20
	mov	r15, sincos2			;; Load fixed premultiplier/sin/cos ptr
	sub	rbx, rbx			;; Clear register for loading fudge flags index
	mov	r12, compressed_fudges		;; Address of the compressed fudge flags
	mov	r13, r12			;; The XOR masks for fudge flags come from the compressed fudges
	zloop_init 12*8*clm			;; 12 iters of 8*clm
	zloop_set_clm_prefetch ZQUAD		;; Prefetch 192 of 1152 blocks of clm*128 bytes
	zr12_wpn_twentyfour_reals_last_unfft_preload
c1b:	zr12_wpn_twentyfour_reals_last_unfft rsi, 128, 12*clmblkdst, 24*clmblkdst, 48*clmblkdst, r15, 0, rdi, 3, 8*clm, L1PREFETCH_ALL, 128
	zloop	8*clm, c1b, -8*clm*128+clmblkdst, r15, ZMM_SCD11, rdx, 24*64, r13, 24 ;; 8*clm iters, next source, fixed s/c, grp multipliers pointer, next xor mask
	zloop	12, c1b				;; Test loop counter
	end_timer 20

;; Normalize these values

	pass1_normalize yes_fft			;; Normalize while prefetching 144 of 768 blocks of clm*128 bytes

;; Copy data from the scratch area back to the FFT data area

	copy_scratch_data_to_fft
	jmp	endlp				;; Skip FFT code if flag not set

;; Fire up auxiliary threads to do pass 1 forward FFT

pass1:	pass1_forward_fft_setup b0b

;; Copy data from the FFT data area back to the scratch area

b0b:	copy_fft_data_to_scratch

;; Do FFT levels 1,2,3,4
;; On input the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	1152K	+1K	...
;;	8K	...
;;	...
;;	1144K
;; On output the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	96K	+1K	...
;;	8K
;;	...
;;	88K
;;	192K
;;	...

;; Do 96 twentyfour_reals_first_fft operations
;;	distance between fft data elements is 96K
;;	do 12 macros each processing 192 data values

yes_fft:start_timer 22
	mov	rsi, scratch_area		;; Get address of scratch area
	mov	r15, sincos2			;; Load fixed premultiplier/sin/cos ptr
	mov	rdi, PREMULT_ADDR		;; Calculate pointer to fudge flags
	bump	rdi, 8*clm*ZMM_SCD12+11*clm*ZMM_SCD1+clm*ZMM_SCD4+clm*ZMM_SCD4+clm*64+clm*64 ;; Addr of the group multiplier fudge flags
	mov	rdx, norm_grp_mults		;; Addr of the group multipliers
	sub	rbx, rbx			;; Clear register for loading fudge flags index
	mov	r12, compressed_fudges		;; Address of the compressed fudge flags
	mov	r13, r12			;; The XOR masks for fudge flags come from the compressed fudges
	set_data_prefetch_ptrs			;; Init prefetching pointers
pfing	mov	r8, pass2blkdst			;; blkdst
pfing	imul	rax, blkdstreg, (816-48)	;; We've already prefetched 816 of 1152 blocks (48 too many)
pfing	add	rcx, rax
	zloop_init 12*8*clm			;; 12 iters of 8*clm
	zloop_set_clm_prefetch ZQUAD		;; Prefetch 192 of 1152 blocks of clm*128 bytes
	zr12_wpn_twentyfour_reals_first_fft_preload
b1b:	zr12_wpn_twentyfour_reals_first_fft rsi, 128, 12*clmblkdst, 24*clmblkdst, 48*clmblkdst, r15, 0, rdi, 3, rdx, 0, 8*clm, L1PREFETCH_ALL, 128
	zloop	8*clm, b1b, -8*clm*128+clmblkdst, r15, ZMM_SCD11, rdx, 24*64, r13, 24 ;; 8*clm iters prefetching 8*clm cache lines, next source, fixed s/c, grp multipliers pointer, next xor mask
	zloop	12, b1b, -12*clmblkdst		;; Test loop counter, restore source pointer
	end_timer 22

;; Do FFT levels 5,6,7
;; On input the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	96K	+1K	...
;;	8K
;;	...
;;	88K
;;	192K
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	8K	+1K	...
;;	16K
;;	...

;; Do 8 twentyfour_reals_fft operations
;; Do 88 twelve_complex_fft operations
;;	distance between fft data elements is 8K
;;	do 12 macros each processing 192 data values

	start_timer 24
	bump	rdi, -96*clm*3-8*clm*ZMM_SCD12	;; Sin/cos pointer
	zloop_init  12*8*clm			;; 12 iterations of 8*clm
	zloop_set_clm_prefetch ZQUAD		;; Prefetch 192 of 768 blocks of clm*128 bytes
IFDEF OLDDD
	zr12_csc_twentyfour_reals_fft_preload
b2a:	zr12_csc_twentyfour_reals_fft rsi, 128, clmblkdst, 2*clmblkdst, 4*clmblkdst, rdi, ZMM_SCD12, 8*clm, L1PREFETCH_ALL, 128
	zloop	8*clm, b2a, -8*clm*128+12*clmblkdst, rdi, -8*clm*ZMM_SCD12 ;; Test loop counter, next source pointer, restore sin/cos pointer
	loops_reset				;; 11 more iters of 8*clm
	zr12_twelve_complex_djbfft_preload
b2b:	zr12_twelve_complex_djbfft rsi, 128, clmblkdst, 2*clmblkdst, 4*clmblkdst, rdi, ZMM_SCD12, 8*clm, L1PREFETCH_ALL, 128
	zloop	8*clm, b2b, -8*clm*128+12*clmblkdst, rdi, -8*clm*ZMM_SCD12 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	12, b2b, -12*12*clmblkdst	;; Test loop counter, restore source pointer
ELSE
b2a:	zr12_csc_twentyfour_reals_fft_preload
	zr12_csc_twentyfour_reals_fft rsi, 12*clmblkdst, clmblkdst, 2*clmblkdst, 4*clmblkdst, rdi, 0, 1, L1PREFETCH_ALL, 12*clmblkdst
	zr12_twelve_complex_djbfft_preload
	jmp	b2c				;; 11 more iters
b2b:	zr12_twelve_complex_djbfft rsi, 12*clmblkdst, clmblkdst, 2*clmblkdst, 4*clmblkdst, rdi, 0, 1, L1PREFETCH_ALL, 12*clmblkdst
b2c:	zloop	12, b2b, -12*12*clmblkdst+128, rdi, ZMM_SCD12 ;; Test loop counter, next source pointer, next sin/cos ptr
	zloop	8*clm, b2a, -8*clm*128		;; Test loop counter, restore source pointer
ENDIF
	end_timer 24

;; Do FFT levels 8,9,10
;; On input the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	8K	+1K	...
;;	16K
;;	...
;; The input is swizzled to hold these data values:
;;	0K	+1	+1	+1	+1	+1	+1	+1	8K	+1	...
;;	1K	...
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0K	+1	+1	+1	+1	+1	+1	+1	1K	+1	...
;;	2K
;;	...

;; Do 1 sixteen_reals_fft operations
;;	distance between fft data elements is 1K
;;	do 0.125 macros each processing 128 data values
;; Do 143 eight_complex_fft operations
;;	distance between fft data elements is 1K
;;	do 17.875 macros each processing 128 data values

	start_timer 26
IFDEF OLDDD
	bump	rdi, -11*clm*ZMM_SCD1-clm*ZMM_SCD4 ;; Real data sin/cos pointer, will become delayed sin/cos multipliers
ELSE
	bump	rdi, -8*clm*ZMM_SCD12-11*clm*ZMM_SCD1-clm*ZMM_SCD4 ;; Real data sin/cos pointer, will become delayed sin/cos multipliers
ENDIF
	lea	rbx, [rdi-clm*ZMM_SCD4]		;; Complex data sin/cos ptr
	lea	r9, [rbx-clm*64-clm*64]		;; Column weights
	lea	rdx, ZMM_TMPS			;; Computed clm*SCD4 sin/cos values
	mov	rbp, DATA_ADDR			;; Destination pointer
	mov	r8, pass2blkdst			;; blkdst
	lea	r10, [2*blkdstreg+blkdstreg]	;; 3*blkdst
	lea	r13, [rbp+4*blkdstreg]		;; Dest+4*blkdst pointer
pfing	mov	rcx, PREMULT_PREFETCH		;; Load premult/sin/cos prefetch pointer

	zr8_rsc_complex_and_real_preload	;; Preload constants before calling zloop_init
;;	zloop_init 12*12*clm			;; 12 iters of 12*clm
	zloop_init 12*12*clm  *2		;; 12 iters of 12*clm iterations (work around loops "add al, 1" bug)
	zloop_set_prefetch ZDOUBLE		;; Prefetch 288*clm times
	zloop_set_alternate_prefetch_increment 241, 288  ;; Prefetch approximately 241*clm cache lines in those 288*clm prefetches
	zr8_rsc_wpn_eight_complex_calc_sincos_simple r9, rbx, clm ;; Generate clm*SCD8 sin/cos values from delayed and complex multipliers
b8b:	zr8_rsc_wpn_sgreg_2sc_sixteen_reals_fft8 rsi, 1024, 128, 256, 512, rbp, 128, blkdstreg, blkdst3reg, r13, rdx, ZMM_SCD8, r9, 64, rdi, ZMM_SCD4, clm, L1PREFETCH_DEST_ALL
	zloop	clm, b8b, -clm*1024+clmblkdst, rbp, -clm*128+8*blkdstreg, r13, -clm*128+8*blkdstreg, rdx, -clm*ZMM_SCD8, r9, -clm*64 ;; Loop, next source/dest pointers, restore premult/sine/cosine pointer
	jmp	b9aa

	loops_reset
b9a:	zr8_rsc_wpn_eight_complex_calc_sincos r9, rdi, rbx, clm ;; Generate clm*SCD8 sin/cos values from delayed and complex multipliers
	bump	rdi, clm*ZMM_SCD1		;; Next set of delayed multipliers
b9aa:
b9b:	zr8_rsc_wpn_sgreg_eight_complex_fft8 rsi, 1024, 128, 256, 512, rbp, 128, blkdstreg, blkdst3reg, r13, rdx, ZMM_SCD8, clm, L1PREFETCH_DEST_ALL
	zloop	clm, b9b, -clm*1024+clmblkdst, rbp, -clm*128+8*blkdstreg, r13, -clm*128+8*blkdstreg, rdx, -clm*ZMM_SCD8 ;; Loop, next source/dest pointers, restore premult/sine/cosine pointer
	zloop	12, b9b				;; Test loop counter
	zloop	12, b9a				;; Test loop counter
	end_timer 26

;; Work on next 8*clm sets of 2304 values

endlp:	zpass1_get_next_block pass2, c0b, b0b ;; Get next block and jump
done:	zfft_footer
	ENDM


;; Perform a shared pass 1 negacyclic FFT of length 2304 doubles.
;; Memory examples are for a 2304K FFT with 10 levels done in pass 2.
;; FFT data needed:
;;		8*clm*2304 doubles					Total: 2304*clm 64-byte cache lines
;; Fixed s/c data needed:
;;		12*ZMM_CD12SCD6 (complex premultipliers and fixed s/c data), Total: 288 64-byte cache lines
;; Group multipliers needed:
;;		12*24*64 (weights)
;;		12*24*64 (inverse weights),				Total: 576 64-byte cache lines
;; Variable data needed:
;;		clm*64	(column weights)
;;		clm*64	(column inverse weights)
;;		clm*ZMM_SCD4 (complex reduced s/c multipliers)
;;		12*clm*ZMM_SCD1 (delayed s/c multipliers)
;;		8*clm*ZMM_SCD6 (12-complex sin/cos data)
;;		96*clm*3 (group multiplier fudge flags)
;;		96*clm*3 (biglit flags),				Total: 139*clm 64-byte cache lines
;; Total memory consumed (add for scratch area, prefetched data, carries):
;;		(2304*3+139=7051)*clm + (288+576+2304/8=1152) cache lines
;;		clm=1,2,4:  513KB, 954KB, 1835KB

zr4dwpn_pass1sc2304ac MACRO
	LOCAL	b0b, b1b, b2b, b9a, b9b, c0b, c1b, c2b, c9a, c9b
	LOCAL	pass1, pass2, endlp, done, yes_fft

	zfft_header pass1

;; Do pass 2 (the last 10 FFT levels if this is a 2304K FFT)

pass2:	start_timer 0
	mov	rax, ZMM_PASS2_ROUTINE
	call	rax
	end_timer 0
	cmp	ffttype, 1		;; We're done if FFTing only
	je	done

;; Do the inverse FFT

	pass1_inverse_fft_setup c0b

;; Do inverse FFT levels 8.9.10
;; On input the 128-byte double cache lines hold these data values:
;;	0K	+1	+1	+1	+1	+1	+1	+1	1K	+1	...
;;	2K
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0K	+1	+1	+1	+1	+1	+1	+1	8K	+1	...
;;	1K	...
;;	...
;; The output cache lines are swizzled to hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	8K	+1K	...
;;	16K
;;	...

;; Do 144 eight_complex_unfft operations
;;	distance between fft data elements is 1K
;;	do 18 macros each processing 128 data values

c0b:	start_timer 16
	mov	rbp, DATA_ADDR			;; Get source address
	mov	rsi, scratch_area		;; Get address of scratch area
	set_data_prefetch_ptrs			;; Init prefetching pointers
	mov	r9, PREMULT_ADDR		;; Variable data
	bump	r9, clm*64			;; Inverse column weights
	lea	rbx, [r9+clm*64]		;; Complex data sin/cos ptr
	lea	rdi, [rbx+clm*ZMM_SCD4]		;; Delayed sin/cos multipliers
	lea	rdx, ZMM_TMPS			;; Computed clm*SCD8 sin/cos values
	mov	r8, pass2blkdst			;; blkdst
	lea	r10, [2*blkdstreg+blkdstreg]	;; 3*blkdst
	lea	r13, [rbp+4*blkdstreg]		;; Source+4*blkdst pointer
	lea	r12, [rbp+8*blkdstreg]		;; L1 prefetch pointer part 1
	bump2	r12, (clm MOD 2), 8*blkdstreg	;; L1 prefetch pointer part 2
	lea	r14, [r12+4*blkdstreg]		;; L1 prefetch pointer + 4*blkdst

	zr8_rsc_complex_only_preload		;; Preload constants before calling zloop_init
;;	zloop_init 12*12*clm			;; 12 iters of 12*clm iterations
	zloop_init 12*12*clm  *2		;; 12 iters of 12*clm iterations (work around loops "add al, 1" bug)
	zloop_set_clm_prefetch ZQUAD		;; Prefetch 288 of 1152 blocks of clm*128 bytes
c9a:	zr8_rsc_wpn_eight_complex_calc_sincos r9, rdi, rbx, clm ;; Generate clm*SCD8 sin/cos values from weights, delayed, and complex multipliers
	bump	rdi, clm*ZMM_SCD1		;; Next set of delayed sin/cos multipliers
c9b:	zr8_rsc_wpn_sgreg_eight_complex_unfft8 rbp, 128, blkdstreg, blkdst3reg, r13, rsi, 1024, 128, 256, 512, rdx, ZMM_SCD8, clm, L1PREFETCH_ALL, r12, r14 ;; Prefetch short dist ahead
	zloop	clm, c9b, -clm*1024+clmblkdst, rbp, -clm*128+8*blkdstreg, r13, -clm*128+8*blkdstreg, rdx, -clm*ZMM_SCD8, r12, -clm*128+8*blkdstreg, r14, -clm*128+8*blkdstreg ;; Loop, next src/dest ptrs, restore sin/cos ptr
	zloop	12, c9b				;; Test loop counter
	zloop	12, c9a, -12*12*clmblkdst	;; Test loop counter, restore source pointer
	end_timer 16

;; Do inverse FFT levels 5,6,7
;; On input the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	8K	+1K	...
;;	16K
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	96K	+1K	...
;;	8K
;;	...
;;	88K
;;	192K
;;	...

;; Do 96 twelve_complex_unfft operations
;;	distance between fft data elements is 8K
;;	do 12 macros each processing 192 data values

	start_timer 18
	zloop_init  12*8*clm			;; 12 iterations of 8*clm
	zloop_set_clm_prefetch ZQUAD		;; Prefetch 192 of 1152 blocks of clm*128 bytes
	zr12_twelve_complex_djbunfft_preload
IFDEF OLDDD
c2b:	zr12_twelve_complex_djbunfft rsi, 128, clmblkdst, 2*clmblkdst, 4*clmblkdst, rdi, ZMM_SCD6, 8*clm, L1PREFETCH_ALL, 128
	zloop	8*clm, c2b, -8*clm*128+12*clmblkdst, rdi, -8*clm*ZMM_SCD6 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	12, c2b, -12*12*clmblkdst, rdi, 8*clm*ZMM_SCD6 ;; Test loop counter, restore source pointer, next sin/cos pointer
ELSE
c2b:	zr12_twelve_complex_djbunfft rsi, 12*clmblkdst, clmblkdst, 2*clmblkdst, 4*clmblkdst, rdi, 0, 8, L1PREFETCH_ALL, 12*clmblkdst
	zloop	12, c2b, -12*12*clmblkdst+128, rdi, ZMM_SCD6 ;; Test loop counter, next source pointer, next sin/cos pointer
	zloop	8*clm, c2b, -8*clm*128		;; Test loop counter, restore source pointer
ENDIF
	end_timer 18

;; Do inverse FFT levels 1,2,3,4
;; On input the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	96K	+1K	...
;;	8K
;;	...
;;	88K
;;	192K
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	1152K	+1K	...
;;	8K	...
;;	...
;;	1144K

;; Do 96 twelve_complex_unfft operations
;;	distance between fft data elements is 96K
;;	do 12 macros each processing 192 data values

	start_timer 20
	mov	r15, sincos2			;; Load fixed premultiplier/sin/cos ptr
	sub	rbx, rbx			;; Clear register for loading fudge flags index
	mov	r12, compressed_fudges		;; Address of the compressed fudge flags
	mov	r13, r12			;; The XOR masks for fudge flags come from the compressed fudges
	zloop_init 12*8*clm			;; 12 iters of 8*clm
	zloop_set_clm_prefetch ZQUAD		;; Prefetch 192 of 1152 blocks of clm*128 bytes
	zr12_csc_wpn_twelve_complex_last_djbunfft_preload
c1b:	zr12_csc_wpn_twelve_complex_last_djbunfft rsi, 128, 12*clmblkdst, 24*clmblkdst, 48*clmblkdst, r15, 0, rdi, 3, 8*clm, L1PREFETCH_ALL, 128
	zloop	8*clm, c1b, -8*clm*128+clmblkdst, r15, ZMM_CD12SCD6, rdx, 24*64, r13, 24 ;; 8*clm iterations, next fixed s/c, next grp multipliers, next xor mask
	zloop	12, c1b				;; 12 iterations
	end_timer 20

;; Normalize these values

	pass1_normalize yes_fft			;; Normalize while prefetching 144 of 1152 blocks of clm*128 bytes

;; Copy data from the scratch area back to the FFT data area

	copy_scratch_data_to_fft
	jmp	endlp				;; Skip FFT code if flag not set

;; Fire up auxiliary threads to do pass 1 forward FFT

pass1:	pass1_forward_fft_setup b0b

;; Copy data from the FFT data area back to the scratch area

b0b:	copy_fft_data_to_scratch

;; Do FFT levels 1,2,3,4
;; On input the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	1152K	+1K	...
;;	8K	...
;;	...
;;	1144K
;; On output the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	96K	+1K	...
;;	8K
;;	...
;;	88K
;;	192K
;;	...

;; Do 96 twelve_complex_first_fft operations
;;	distance between fft data elements is 96K
;;	do 12 macros each processing 192 data values

yes_fft:start_timer 22
	mov	rsi, scratch_area		;; Get address of scratch area
	mov	r15, sincos2			;; Load fixed premultiplier/sin/cos ptr
	mov	rdi, PREMULT_ADDR		;; Calculate pointer to fudge flags
	bump	rdi, 8*clm*ZMM_SCD6+12*clm*ZMM_SCD1+clm*ZMM_SCD4+clm*64+clm*64 ;; Addr of the group multiplier fudge flags
	mov	rdx, norm_grp_mults		;; Addr of the group multipliers
	sub	rbx, rbx			;; Clear register for loading fudge flags index
	mov	r12, compressed_fudges		;; Address of the compressed fudge flags
	mov	r13, r12			;; The XOR masks for fudge flags come from the compressed fudges
	set_data_prefetch_ptrs			;; Init prefetching pointers
pfing	mov	r8, pass2blkdst			;; blkdst
pfing	imul	rax, blkdstreg, (816-48)	;; We've already prefetched 816 of 1152 blocks (48 too many)
pfing	add	rcx, rax
	zloop_init 12*8*clm			;; 12 iters of 8*clm
	zloop_set_clm_prefetch ZQUAD		;; Prefetch 192 of 1152 blocks of clm*128 bytes
	zr12_csc_wpn_twelve_complex_first_djbfft_preload
b1b:	zr12_csc_wpn_twelve_complex_first_djbfft rsi, 128, 12*clmblkdst, 24*clmblkdst, 48*clmblkdst, r15, 0, rdi, 3, rdx, 0, 8*clm, L1PREFETCH_ALL, 128
	zloop	8*clm, b1b, -8*clm*128+clmblkdst, r15, ZMM_CD12SCD6, rdx, 24*64, r13, 24 ;; 8*clm iters, next source, fixed s/c, grp multipliers pointer, next xor mask
	zloop	12, b1b, -12*clmblkdst		;; Test loop counter, restore source pointer
	end_timer 22

;; Do FFT levels 5,6,7
;; On input the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	96K	+1K	...
;;	8K
;;	...
;;	88K
;;	192K
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	8K	+1K	...
;;	16K
;;	...

;; Do 96 twelve_complex_fft operations
;;	distance between fft data elements is 8K
;;	do 12 macros each processing 192 data values

	start_timer 24
	bump	rdi, -96*clm*3-8*clm*ZMM_SCD6	;; Sin/cos pointer
	zloop_init  12*8*clm			;; 12 iterations of 8*clm
	zloop_set_clm_prefetch ZQUAD		;; Prefetch 192 of 1152 blocks of clm*128 bytes
	zr12_twelve_complex_djbfft_preload
IFDEF OLDDD
b2b:	zr12_twelve_complex_djbfft rsi, 128, clmblkdst, 2*clmblkdst, 4*clmblkdst, rdi, ZMM_SCD6, 8*clm, L1PREFETCH_ALL, 128
	zloop	8*clm, b2b, -8*clm*128+12*clmblkdst, rdi, -8*clm*ZMM_SCD6 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	12, b2b, -12*12*clmblkdst	;; Test loop counter, restore source pointer
ELSE
b2b:	zr12_twelve_complex_djbfft rsi, 12*clmblkdst, clmblkdst, 2*clmblkdst, 4*clmblkdst, rdi, 0, 8, L1PREFETCH_ALL, 12*clmblkdst
	zloop	12, b2b, -12*12*clmblkdst+128, rdi, ZMM_SCD6 ;; Test loop counter, next source pointer, next sin/cos pointer
	zloop	8*clm, b2b, -8*clm*128		;; Test loop counter, restore source pointer
ENDIF
	end_timer 24

;; Do FFT levels 8,9,10
;; On input the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	8K	+1K	...
;;	16K
;;	...
;; The input is swizzled to hold these data values:
;;	0K	+1	+1	+1	+1	+1	+1	+1	8K	+1	...
;;	1K	...
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0K	+1	+1	+1	+1	+1	+1	+1	1K	+1	...
;;	2K
;;	...

;; Do 144 eight_complex_fft operations
;;	distance between fft data elements is 1K
;;	do 18 macros each processing 128 data values

	start_timer 26
	mov	rbp, DATA_ADDR			;; Destination pointer
IFDEF OLDDD
	bump	rdi, -12*clm*ZMM_SCD1		;; Delayed sin/cos multipliers
ELSE
	bump	rdi, -8*clm*ZMM_SCD6-12*clm*ZMM_SCD1 ;; Delayed sin/cos multipliers
ENDIF
	lea	rbx, [rdi-clm*ZMM_SCD4]		;; Complex data sin/cos ptr
	lea	r9, [rbx-clm*64-clm*64]		;; Column weights
	lea	rdx, ZMM_TMPS			;; Computed clm*SCD4 sin/cos values
	mov	r8, pass2blkdst			;; blkdst
	lea	r10, [2*blkdstreg+blkdstreg]	;; 3*blkdst
	lea	r13, [rbp+4*blkdstreg]		;; Dest+4*blkdst pointer
pfing	mov	rcx, PREMULT_PREFETCH		;; Load premult/sin/cos prefetch pointer

	zr8_rsc_complex_only_preload		;; Preload constants before calling zloop_init
;;	zloop_init 12*12*clm			;; 12 iters of 12*clm
	zloop_init 12*12*clm  *2		;; 12 iters of 12*clm iterations (work around loops "add al, 1" bug)
	zloop_set_prefetch ZSINGLE		;; Prefetch 144*clm cache lines
	zloop_set_alternate_prefetch_increment 139, 144  ;; Prefetch approximately 139*clm cache lines in those 144*clm prefetches
b9a:	zr8_rsc_wpn_eight_complex_calc_sincos r9, rdi, rbx, clm ;; Generate clm*SCD8 sin/cos values from weights, delayed and complex multipliers
	bump	rdi, clm*ZMM_SCD1		;; Next set of delayed multipliers
b9b:	zr8_rsc_wpn_sgreg_eight_complex_fft8 rsi, 1024, 128, 256, 512, rbp, 128, blkdstreg, blkdst3reg, r13, rdx, ZMM_SCD8, clm, L1PREFETCH_DEST_ALL
	zloop	clm, b9b, -clm*1024+clmblkdst, rbp, -clm*128+8*blkdstreg, r13, -clm*128+8*blkdstreg, rdx, -clm*ZMM_SCD8 ;; Loop, next source/dest pointers, restore premult/sine/cosine pointer
	zloop	12, b9b				;; Test loop counter
	zloop	12, b9a				;; Test loop counter
	end_timer 26

;; Work on next 8*clm sets of 2304 values

endlp:	zpass1_get_next_block pass2, c0b, b0b ;; Get next block and jump
done:	zfft_footer
	ENDM



;; Perform a shared pass 1 FFT of length 3072 doubles.
;; Memory examples are for a 3M FFT with 10 levels done in pass 2.
;; FFT data needed:
;;		8*clm*3072 doubles					Total: 3072*clm 64-byte cache lines
;; Fixed s/c data needed:
;;		16*ZMM_SCD11 (fixed s/c data),				Total: 352 64-byte cache lines
;; Group multipliers needed:
;;		16*24*64 (weights)
;;		16*24*64 (inverse weights),				Total: 768 64-byte cache lines
;; Variable data needed:
;;		clm*64	(column weights)
;;		clm*64	(column inverse weights)
;;		clm*ZMM_SCD4 (complex reduced s/c multipliers)
;;		clm*ZMM_SCD4 (real s/c multipliers)
;;		11*clm*ZMM_SCD1 (delayed s/c multipliers)
;;		8*clm*ZMM_SCD16 (16-complex 32-real sin/cos data)
;;		16*8*clm*3 (group multiplier fudge flags)
;;		16*8*clm*3 (biglit flags),				Total: 308*clm 64-byte cache lines
;; Total memory consumed (add for scratch area, prefetched data, carries):
;;		(3072*3+308=9524)*clm + (352+768+3072/8=1504) cache lines
;;		clm=1,2,4:  690KB, 1285KB, 2475KB

zr4dwpn_pass1sc3072 MACRO
	LOCAL	b0b, b1b, b2a, b2b, b2c, b8a, b8b, b9aa, b9a, b9b, c0b, c1b, c2a, c2b, c2c, c8b, c9aa, c9a, c9b
	LOCAL	pass1, pass2, endlp, done, yes_fft

	zfft_header pass1

;; Do pass 2 (the last 10 FFT levels if this is a 3M FFT)

pass2:	start_timer 0
	mov	rax, ZMM_PASS2_ROUTINE
	call	rax
	end_timer 0
	cmp	ffttype, 1		;; We're done if FFTing only
	je	done

;; Do the inverse FFT

	pass1_inverse_fft_setup c0b

;; Do inverse FFT levels 9,10,11
;; On input the 128-byte double cache lines hold these data values:
;;	0K	+1	+1	+1	+1	+1	+1	+1	1K	+1	...
;;	2K
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0K	+1	+1	+1	+1	+1	+1	+1	8K	+1	...
;;	1K	...
;;	...
;; The output cache lines are swizzled to hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	8K	+1K	...
;;	16K
;;	...

;; Do 1 sixteen_reals_unfft operations
;;	distance between fft data elements is 1K
;;	do 0.125 macros each processing 128 data values
;; Do 191 eight_complex_unfft operations
;;	distance between fft data elements is 1K
;;	do 23.875 macros each processing 128 data values

c0b:	start_timer 16
	mov	rbp, DATA_ADDR			;; Get source address
	mov	rsi, scratch_area		;; Get address of scratch area
	set_data_prefetch_ptrs			;; Init prefetching pointers
	mov	r9, PREMULT_ADDR		;; Variable data
	bump	r9, clm*64			;; Inverse column weights
	lea	rbx, [r9+clm*64]		;; Complex data sin/cos ptr
	lea	rdi, [rbx+clm*ZMM_SCD4]		;; Real data sin/cos pointer, will become delayed sin/cos multipliers
	lea	rdx, ZMM_TMPS			;; Computed clm*SCD8 sin/cos values
	mov	r8, pass2blkdst			;; blkdst
	lea	r10, [2*blkdstreg+blkdstreg]	;; 3*blkdst
	lea	r13, [rbp+4*blkdstreg]		;; Source+4*blkdst pointer
	lea	r12, [rbp+8*blkdstreg]		;; L1 prefetch pointer part 1
	bump2	r12, (clm MOD 2), 8*blkdstreg	;; L1 prefetch pointer part 2
	lea	r14, [r12+4*blkdstreg]		;; L1 prefetch pointer + 4*blkdst

	zr8_rsc_complex_and_real_preload	;; Preload constants before calling zloop_init
	zloop_init 12*16*clm			;; 12 iters of 16*clm iterations
	zloop_set_clm_prefetch ZDOUBLE		;; Prefetch 192 of 1536 blocks of clm*128 bytes
	zr8_rsc_wpn_eight_complex_calc_sincos_simple r9, rbx, clm ;; Generate clm*SCD8 sin/cos values from delayed and complex multipliers
c8b:	zr8_rsc_wpn_sgreg_2sc_sixteen_reals_unfft8 rbp, 128, blkdstreg, blkdst3reg, r13, rsi, 1024, 128, 256, 512, rdx, ZMM_SCD8, r9, 64, rdi, ZMM_SCD4, clm, L1PREFETCH_ALL, r12, r14 ;; Prefetch short dist ahead
	zloop	clm, c8b, -clm*1024+clmblkdst, rbp, -clm*128+8*blkdstreg, r13, -clm*128+8*blkdstreg, rdx, -clm*ZMM_SCD8, r9, -clm*64, r12, -clm*128+8*blkdstreg, r14, -clm*128+8*blkdstreg ;; Loop, next src/dest ptrs, restore sin/cos ptr
	jmp	c9aa

	loops_reset
c9a:	zr8_rsc_wpn_eight_complex_calc_sincos r9, rdi, rbx, clm ;; Generate clm*SCD8 sin/cos values from delayed and complex multipliers
	bump	rdi, clm*ZMM_SCD1		;; Next set of delayed sin/cos multipliers
c9aa:
c9b:	zr8_rsc_wpn_sgreg_eight_complex_unfft8 rbp, 128, blkdstreg, blkdst3reg, r13, rsi, 1024, 128, 256, 512, rdx, ZMM_SCD8, clm, L1PREFETCH_ALL, r12, r14 ;; Prefetch short dist ahead
	zloop	clm, c9b, -clm*1024+clmblkdst, rbp, -clm*128+8*blkdstreg, r13, -clm*128+8*blkdstreg, rdx, -clm*ZMM_SCD8, r12, -clm*128+8*blkdstreg, r14, -clm*128+8*blkdstreg ;; Loop, next src/dest ptrs, restore sin/cos ptr
	zloop	8, c9b, -8*clmblkdst+clmblkdst8	;; Test loop counter, next source pointer
	zloop	2, c9b				;; Test loop counter
	zloop	12, c9a, -24*clmblkdst8		;; Test loop counter, restore source pointer
	end_timer 16

;; Do inverse FFT levels 5,6,7,8
;; On input the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	8K	+1K	...
;;	16K
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	128K	+1K	...
;;	8K
;;	...
;;	120K
;;	256K
;;	...

;; Do 8 thirtytwo_reals_unfft operations
;; Do 88 sixteen_complex_unfft operations
;;	distance between fft data elements is 8K
;;	do 12 macros each processing 256 data values

	start_timer 18
	zloop_init  12*8*clm			;; 12 iterations of 8*clm
	zloop_set_clm_prefetch ZOCT		;; Prefetch 384 of 1536 blocks of clm*128 bytes
IFDEF OLDDD
	zr16_csc_thirtytwo_reals_unfft_preload
c2a:	zr16_csc_thirtytwo_reals_unfft rsi, 128, clmblkdst, 2*clmblkdst, 4*clmblkdst, clmblkdst8, rdi, ZMM_SCD16, 8*clm, L1PREFETCH_ALL, 128
	zloop	8*clm, c2a, -8*clm*128+2*clmblkdst8, rdi, -8*clm*ZMM_SCD16 ;; Test loop counter, next source pointer, restore sin/cos pointer
	loops_reset				;; 11 more iters of 8*clm
	zr16_sixteen_complex_djbunfft_preload
c2b:	zr16_sixteen_complex_djbunfft rsi, 128, clmblkdst, 2*clmblkdst, 4*clmblkdst, clmblkdst8, rdi, XMM_SCD16, 8*clm, L1PREFETCH_ALL, 128
	zloop	8*clm, c2b, -8*clm*128+2*clmblkdst8, rdi, -8*clm*ZMM_SCD16 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	12, c2b, -12*2*clmblkdst8, rdi, 8*clm*ZMM_SCD16 ;; Test loop counter, restore source pointer, next sin/cos pointer
ELSE
c2a:	zr16_csc_thirtytwo_reals_unfft_preload
	zr16_csc_thirtytwo_reals_unfft rsi, 2*clmblkdst8, clmblkdst, 2*clmblkdst, 4*clmblkdst, clmblkdst8, rdi, 0, 1, L1PREFETCH_ALL, 2*clmblkdst8
	zr16_sixteen_complex_djbunfft_preload
	jmp	c2c				;; 11 more iters
c2b:	zr16_sixteen_complex_djbunfft rsi, 2*clmblkdst8, clmblkdst, 2*clmblkdst, 4*clmblkdst, clmblkdst8, rdi, 0, 1, L1PREFETCH_ALL, 2*clmblkdst8
c2c:	zloop	12, c2b, -12*2*clmblkdst8+128, rdi, ZMM_SCD16 ;; Test loop counter, next source pointer, next sin/cos pointer
	zloop	8*clm, c2a, -8*clm*128		;; Test loop counter, restore source pointer
ENDIF
	end_timer 18

;; Do inverse FFT levels 1,2,3,4
;; On input the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	128K	+1K	...
;;	8K
;;	...
;;	120K
;;	256K
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	1536K	+1K	...
;;	8K
;;	...
;;	1528K

;; Do 128 twentyfour_reals_unfft operations
;;	distance between fft data elements is 128K
;;	do 16 macros each processing 192 data values

	start_timer 20
	mov	r15, sincos2			;; Load fixed premultiplier/sin/cos ptr
	sub	rbx, rbx			;; Clear register for loading fudge flags index
	mov	r12, compressed_fudges		;; Address of the compressed fudge flags
	mov	r13, r12			;; The XOR masks for fudge flags come from the compressed fudges
	zloop_init 16*8*clm			;; 16 iters of 8*clm
	zloop_set_clm_prefetch ZDOUBLE		;; Prefetch 128 of 1536 blocks of clm*128 bytes
	zr12_wpn_twentyfour_reals_last_unfft_preload
c1b:	zr12_wpn_twentyfour_reals_last_unfft rsi, 128, 2*clmblkdst8, 4*clmblkdst8, 8*clmblkdst8, r15, 0, rdi, 3, 8*clm, L1PREFETCH_ALL, 128
	zloop	8*clm, c1b, -8*clm*128+clmblkdst, r15, ZMM_SCD11, rdx, 24*64, r13, 24 ;; 8*clm iters, next source, fixed s/c, grp multipliers pointer, next xor mask
	zloop	8, c1b, -8*clmblkdst+clmblkdst8	;; Test loop counter, next source pointer
	zloop	2, c1b				;; Test loop counter
	end_timer 20

;; Normalize these values

	pass1_normalize yes_fft			;; Normalize while prefetching 192 of 1536 blocks of clm*128 bytes

;; Copy data from the scratch area back to the FFT data area

	copy_scratch_data_to_fft
	jmp	endlp				;; Skip FFT code if flag not set

;; Fire up auxiliary threads to do pass 1 forward FFT

pass1:	pass1_forward_fft_setup b0b

;; Copy data from the FFT data area back to the scratch area

b0b:	copy_fft_data_to_scratch

;; Do FFT levels 1,2,3,4
;; On input the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	1536K	+1K	...
;;	8K	...
;;	...
;;	1528K
;; On output the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	128K	+1K	...
;;	8K
;;	...
;;	120K
;;	256K
;;	...

;; Do 128 twentyfour_reals_first_fft operations
;;	distance between fft data elements is 64K
;;	do 16 macros each processing 192 data values

yes_fft:start_timer 22
	mov	rsi, scratch_area		;; Get address of scratch area
	mov	r15, sincos2			;; Load fixed premultiplier/sin/cos ptr
	mov	rdi, PREMULT_ADDR		;; Calculate pointer to fudge flags
	bump	rdi, 8*clm*ZMM_SCD16+11*clm*ZMM_SCD1+clm*ZMM_SCD4+clm*ZMM_SCD4+clm*64+clm*64 ;; Addr of the group multiplier fudge flags
	mov	rdx, norm_grp_mults		;; Addr of the group multipliers
	sub	rbx, rbx			;; Clear register for loading fudge flags index
	mov	r12, compressed_fudges		;; Address of the compressed fudge flags
	mov	r13, r12			;; The XOR masks for fudge flags come from the compressed fudges
	set_data_prefetch_ptrs			;; Init prefetching pointers
pfing	mov	r8, pass2blkdst			;; blkdst
pfing	imul	rax, blkdstreg, (896-0)		;; We've already prefetched 896 of 1536 blocks (zero too many)
pfing	add	rcx, rax
	zloop_init 16*8*clm			;; 16 iters of 8*clm
	zloop_set_clm_prefetch ZQUAD		;; Prefetch 256 of 1536 blocks of clm*128 bytes
	zr12_wpn_twentyfour_reals_first_fft_preload
b1b:	zr12_wpn_twentyfour_reals_first_fft rsi, 128, 2*clmblkdst8, 4*clmblkdst8, 8*clmblkdst8, r15, 0, rdi, 3, rdx, 0, 8*clm, L1PREFETCH_ALL, 128
	zloop	8*clm, b1b, -8*clm*128+clmblkdst, r15, ZMM_SCD11, rdx, 24*64, r13, 24 ;; 8*clm iters prefetching 8*clm cache lines, next source, fixed s/c, grp multipliers pointer, next xor mask
	zloop	8, b1b, -8*clmblkdst+clmblkdst8	;; Test loop counter, next source pointer
	zloop	2, b1b, -2*clmblkdst8		;; Test loop counter, restore source pointer
	end_timer 22

;; Do FFT levels 5,6,7,8
;; On input the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	128K	+1K	...
;;	8K
;;	...
;;	120K
;;	256K
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	8K	+1K	...
;;	16K
;;	...

;; Do 8 thirtytwo_reals_fft operations
;; Do 88 sixteen_complex_fft operations
;;	distance between fft data elements is 8K
;;	do 12 macros each processing 256 data values

	start_timer 24
	bump	rdi, -16*8*clm*3-8*clm*ZMM_SCD16 ;; Sin/cos pointer
	zloop_init  12*8*clm			;; 12 iterations of 8*clm
	zloop_set_clm_prefetch ZOCT		;; Prefetch 384 of 1536 blocks of clm*128 bytes
IFDEF OLDDD
	zr16_csc_thirtytwo_reals_fft_preload
b2a:	zr16_csc_thirtytwo_reals_fft rsi, 128, clmblkdst, 2*clmblkdst, 4*clmblkdst, clmblkdst8, rdi, ZMM_SCD16, 8*clm, L1PREFETCH_ALL, 128
	zloop	8*clm, b2a, -8*clm*128+2*clmblkdst8, rdi, -8*clm*ZMM_SCD16 ;; Test loop counter, next source pointer, restore sin/cos pointer
	loops_reset				;; 11 more iters of 8*clm
	zr16_sixteen_complex_djbfft_preload
b2b:	zr16_sixteen_complex_djbfft rsi, 128, clmblkdst, 2*clmblkdst, 4*clmblkdst, clmblkdst8, rdi, ZMM_SCD16, 8*clm, L1PREFETCH_ALL, 128
	zloop	8*clm, b2b, -8*clm*128+2*clmblkdst8, rdi, -8*clm*ZMM_SCD16 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	12, b2b, -12*2*clmblkdst8	 ;; Test loop counter, restore source pointer
ELSE
b2a:	zr16_csc_thirtytwo_reals_fft_preload
	zr16_csc_thirtytwo_reals_fft rsi, 2*clmblkdst8, clmblkdst, 2*clmblkdst, 4*clmblkdst, clmblkdst8, rdi, 0, 1, L1PREFETCH_ALL, 2*clmblkdst8
	zr16_sixteen_complex_djbfft_preload
	jmp	b2c				;; 11 more iters
b2b:	zr16_sixteen_complex_djbfft rsi, 2*clmblkdst8, clmblkdst, 2*clmblkdst, 4*clmblkdst, clmblkdst8, rdi, 0, 1, L1PREFETCH_ALL, 2*clmblkdst8
b2c:	zloop	12, b2b, -12*2*clmblkdst8+128, rdi, ZMM_SCD16 ;; Test loop counter, next source pointer, next sin/cos pointer
	zloop	8*clm, b2a, -8*clm*128		;; Test loop counter, restore source pointer
ENDIF
	end_timer 24

;; Do FFT levels 9,10,11
;; On input the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	8K	+1K	...
;;	16K
;;	...
;; The input is swizzled to hold these data values:
;;	0K	+1	+1	+1	+1	+1	+1	+1	8K	+1	...
;;	1K	...
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0K	+1	+1	+1	+1	+1	+1	+1	1K	+1	...
;;	2K
;;	...

;; Do 1 sixteen_reals_fft operations
;;	distance between fft data elements is 1K
;;	do 0.125 macros each processing 128 data values
;; Do 191 eight_complex_fft operations
;;	distance between fft data elements is 1K
;;	do 23.875 macros each processing 128 data values

	start_timer 26
IFDEF OLDDD
	bump	rdi, -11*clm*ZMM_SCD1-clm*ZMM_SCD4 ;; Real data sin/cos pointer, will become delayed sin/cos multipliers
ELSE
	bump	rdi, -8*clm*ZMM_SCD16-11*clm*ZMM_SCD1-clm*ZMM_SCD4 ;; Real data sin/cos pointer, will become delayed sin/cos multipliers
ENDIF
	lea	rbx, [rdi-clm*ZMM_SCD4]		;; Complex data sin/cos ptr
	lea	r9, [rbx-clm*64-clm*64]		;; Column weights
	lea	rdx, ZMM_TMPS			;; Computed clm*SCD4 sin/cos values
	mov	rbp, DATA_ADDR			;; Destination pointer
	mov	r8, pass2blkdst			;; blkdst
	lea	r10, [2*blkdstreg+blkdstreg]	;; 3*blkdst
	lea	r13, [rbp+4*blkdstreg]		;; Dest+4*blkdst pointer
pfing	mov	rcx, PREMULT_PREFETCH		;; Load premult/sin/cos prefetch pointer

	zr8_rsc_complex_and_real_preload	;; Preload constants before calling zloop_init
	zloop_init 12*16*clm			;; 12 iters of 16*clm
	zloop_set_prefetch ZDOUBLE		;; Prefetch 384*clm times
	zloop_set_alternate_prefetch_increment 308, 384  ;; Prefetch approximately 308*clm cache lines in those 384*clm prefetches
	zr8_rsc_wpn_eight_complex_calc_sincos_simple r9, rbx, clm ;; Generate clm*SCD8 sin/cos values from delayed and complex multipliers
b8b:	zr8_rsc_wpn_sgreg_2sc_sixteen_reals_fft8 rsi, 1024, 128, 256, 512, rbp, 128, blkdstreg, blkdst3reg, r13, rdx, ZMM_SCD8, r9, 64, rdi, ZMM_SCD4, clm, L1PREFETCH_DEST_ALL
	zloop	clm, b8b, -clm*1024+clmblkdst, rbp, -clm*128+8*blkdstreg, r13, -clm*128+8*blkdstreg, rdx, -clm*ZMM_SCD8, r9, -clm*64 ;; Loop, next source/dest pointers, restore premult/sine/cosine pointer
	jmp	b9aa

	loops_reset
b9a:	zr8_rsc_wpn_eight_complex_calc_sincos r9, rdi, rbx, clm ;; Generate clm*SCD8 sin/cos values from delayed and complex multipliers
	bump	rdi, clm*ZMM_SCD1		;; Next set of delayed multipliers
b9aa:
b9b:	zr8_rsc_wpn_sgreg_eight_complex_fft8 rsi, 1024, 128, 256, 512, rbp, 128, blkdstreg, blkdst3reg, r13, rdx, ZMM_SCD8, clm, L1PREFETCH_DEST_ALL
	zloop	clm, b9b, -clm*1024+clmblkdst, rbp, -clm*128+8*blkdstreg, r13, -clm*128+8*blkdstreg, rdx, -clm*ZMM_SCD8 ;; Loop, next source/dest pointers, restore premult/sine/cosine pointer
	zloop	8, b9b, -8*clmblkdst+clmblkdst8	;; Test loop counter, next source pointer
	zloop	2, b9b				;; Test loop counter
	zloop	12, b9a				;; Test loop counter
	end_timer 26

;; Work on next 8*clm sets of 3072 values

endlp:	zpass1_get_next_block pass2, c0b, b0b ;; Get next block and jump
done:	zfft_footer
	ENDM


;; Perform a shared pass 1 negacyclic FFT of length 3072 doubles.
;; Memory examples are for a 3M FFT with 10 levels done in pass 2.
;; FFT data needed:
;;		8*clm*3072 doubles					Total: 3072*clm 64-byte cache lines
;; Fixed s/c data needed:
;;		16*ZMM_CD12SCD6 (complex premultipliers and fixed s/c data), Total: 384 64-byte cache lines
;; Group multipliers needed:
;;		16*24*64 (weights)
;;		16*24*64 (inverse weights),				Total: 768 64-byte cache lines
;; Variable data needed:
;;		clm*64	(column weights)
;;		clm*64	(column inverse weights)
;;		clm*ZMM_SCD4 (complex reduced s/c multipliers)
;;		12*clm*ZMM_SCD1 (delayed s/c multipliers)
;;		8*clm*ZMM_SCD8 (16-complex sin/cos data)
;;		16*8*clm*3 (group multiplier fudge flags)
;;		16*8*clm*3 (biglit flags),				Total: 174*clm 64-byte cache lines
;; Total memory consumed (add for scratch area, prefetched data, carries):
;;		(3072*3+174=9390)*clm + (384+768+3072/8=1536) cache lines
;;		clm=1,2,4:  683KB, 1270KB, 2444KB

zr4dwpn_pass1sc3072ac MACRO
	LOCAL	b0b, b1b, b2b, b9a, b9b, c0b, c1b, c2b, c9a, c9b
	LOCAL	pass1, pass2, endlp, done, yes_fft

	zfft_header pass1

;; Do pass 2 (the last 10 FFT levels if this is a 3M FFT)

pass2:	start_timer 0
	mov	rax, ZMM_PASS2_ROUTINE
	call	rax
	end_timer 0
	cmp	ffttype, 1		;; We're done if FFTing only
	je	done

;; Do the inverse FFT

	pass1_inverse_fft_setup c0b

;; Do inverse FFT levels 9.10,11
;; On input the 128-byte double cache lines hold these data values:
;;	0K	+1	+1	+1	+1	+1	+1	+1	1K	+1	...
;;	2K
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0K	+1	+1	+1	+1	+1	+1	+1	8K	+1	...
;;	1K	...
;;	...
;; The output cache lines are swizzled to hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	8K	+1K	...
;;	16K
;;	...

;; Do 192 eight_complex_unfft operations
;;	distance between fft data elements is 1K
;;	do 24 macros each processing 128 data values

c0b:	start_timer 16
	mov	rbp, DATA_ADDR			;; Get source address
	mov	rsi, scratch_area		;; Get address of scratch area
	set_data_prefetch_ptrs			;; Init prefetching pointers
	mov	r9, PREMULT_ADDR		;; Variable data
	bump	r9, clm*64			;; Inverse column weights
	lea	rbx, [r9+clm*64]		;; Complex data sin/cos ptr
	lea	rdi, [rbx+clm*ZMM_SCD4]		;; Delayed sin/cos multipliers
	lea	rdx, ZMM_TMPS			;; Computed clm*SCD8 sin/cos values
	mov	r8, pass2blkdst			;; blkdst
	lea	r10, [2*blkdstreg+blkdstreg]	;; 3*blkdst
	lea	r13, [rbp+4*blkdstreg]		;; Source+4*blkdst pointer
	lea	r12, [rbp+8*blkdstreg]		;; L1 prefetch pointer part 1
	bump2	r12, (clm MOD 2), 8*blkdstreg	;; L1 prefetch pointer part 2
	lea	r14, [r12+4*blkdstreg]		;; L1 prefetch pointer + 4*blkdst

	zr8_rsc_complex_only_preload		;; Preload constants before calling zloop_init
	zloop_init 12*16*clm			;; 12 iters of 16*clm iterations
	zloop_set_clm_prefetch ZDOUBLE		;; Prefetch 192 of 1536 blocks of clm*128 bytes
c9a:	zr8_rsc_wpn_eight_complex_calc_sincos r9, rdi, rbx, clm ;; Generate clm*SCD8 sin/cos values from weights, delayed, and complex multipliers
	bump	rdi, clm*ZMM_SCD1		;; Next set of delayed sin/cos multipliers
c9b:	zr8_rsc_wpn_sgreg_eight_complex_unfft8 rbp, 128, blkdstreg, blkdst3reg, r13, rsi, 1024, 128, 256, 512, rdx, ZMM_SCD8, clm, L1PREFETCH_ALL, r12, r14 ;; Prefetch short dist ahead
	zloop	clm, c9b, -clm*1024+clmblkdst, rbp, -clm*128+8*blkdstreg, r13, -clm*128+8*blkdstreg, rdx, -clm*ZMM_SCD8, r12, -clm*128+8*blkdstreg, r14, -clm*128+8*blkdstreg ;; Loop, next src/dest ptrs, restore sin/cos ptr
	zloop	8, c9b, -8*clmblkdst+clmblkdst8	;; Test loop counter, next source pointer
	zloop	2, c9b				;; Test loop counter
	zloop	12, c9a, -24*clmblkdst8		;; Test loop counter, restore source pointer
	end_timer 16

;; Do inverse FFT levels 5,6,7,8
;; On input the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	8K	+1K	...
;;	16K
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	128K	+1K	...
;;	8K
;;	...
;;	120K
;;	256K
;;	...

;; Do 96 sixteen_complex_unfft operations
;;	distance between fft data elements is 8K
;;	do 12 macros each processing 256 data values

	start_timer 18
	zloop_init  12*8*clm			;; 12 iterations of 8*clm
	zloop_set_clm_prefetch ZOCT		;; Prefetch 384 of 1536 blocks of clm*128 bytes
	zr16_sixteen_complex_djbunfft_preload
IFDEF OLDDD
c2b:	zr16_sixteen_complex_djbunfft rsi, 128, clmblkdst, 2*clmblkdst, 4*clmblkdst, clmblkdst8, rdi, ZMM_SCD8, 8*clm, L1PREFETCH_ALL, 128
	zloop	8*clm, c2b, -8*clm*128+2*clmblkdst8, rdi, -8*clm*ZMM_SCD8 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	12, c2b, -12*2*clmblkdst8, rdi, 8*clm*ZMM_SCD8 ;; Test loop counter, restore source pointer, next sin/cos pointer
ELSE
c2b:	zr16_sixteen_complex_djbunfft rsi, 2*clmblkdst8, clmblkdst, 2*clmblkdst, 4*clmblkdst, clmblkdst8, rdi, 0, 8, L1PREFETCH_ALL, 2*clmblkdst8
	zloop	12, c2b, -12*2*clmblkdst8+128, rdi, ZMM_SCD8 ;; Test loop counter, next source pointer, next sin/cos pointer
	zloop	8*clm, c2b, -8*clm*128		;; Test loop counter, restore source pointer
ENDIF
	end_timer 18

;; Do inverse FFT levels 1,2,3,4
;; On input the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	128K	+1K	...
;;	8K
;;	...
;;	120K
;;	256K
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	1536K	+1K	...
;;	8K	...
;;	...
;;	1528K

;; Do 128 twelve_complex_unfft operations
;;	distance between fft data elements is 128K
;;	do 16 macros each processing 192 data values

	start_timer 20
	mov	r15, sincos2			;; Load fixed premultiplier/sin/cos ptr
	sub	rbx, rbx			;; Clear register for loading fudge flags index
	mov	r12, compressed_fudges		;; Address of the compressed fudge flags
	mov	r13, r12			;; The XOR masks for fudge flags come from the compressed fudges
	zloop_init 16*8*clm			;; 16 iters of 8*clm
	zloop_set_clm_prefetch ZDOUBLE		;; Prefetch 128 of 1536 blocks of clm*128 bytes
	zr12_csc_wpn_twelve_complex_last_djbunfft_preload
c1b:	zr12_csc_wpn_twelve_complex_last_djbunfft rsi, 128, 2*clmblkdst8, 4*clmblkdst8, 8*clmblkdst8, r15, 0, rdi, 3, 8*clm, L1PREFETCH_ALL, 128
	zloop	8*clm, c1b, -8*clm*128+clmblkdst, r15, ZMM_CD12SCD6, rdx, 24*64, r13, 24 ;; 8*clm iterations, next fixed s/c, next grp multipliers, next xor mask
	zloop	8, c1b, -8*clmblkdst+clmblkdst8	;; Test loop counter, next source pointer
	zloop	2, c1b				;; Test loop counter
	end_timer 20

;; Normalize these values

	pass1_normalize yes_fft			;; Normalize while prefetching 192 of 1536 blocks of clm*128 bytes

;; Copy data from the scratch area back to the FFT data area

	copy_scratch_data_to_fft
	jmp	endlp				;; Skip FFT code if flag not set

;; Fire up auxiliary threads to do pass 1 forward FFT

pass1:	pass1_forward_fft_setup b0b

;; Copy data from the FFT data area back to the scratch area

b0b:	copy_fft_data_to_scratch

;; Do FFT levels 1,2,3,4
;; On input the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	1536K	+1K	...
;;	8K	...
;;	...
;;	1528K
;; On output the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	128K	+1K	...
;;	8K
;;	...
;;	120K
;;	256K
;;	...

;; Do 128 twelve_complex_first_fft operations
;;	distance between fft data elements is 128K
;;	do 16 macros each processing 192 data values

yes_fft:start_timer 22
	mov	rsi, scratch_area		;; Get address of scratch area
	mov	r15, sincos2			;; Load fixed premultiplier/sin/cos ptr
	mov	rdi, PREMULT_ADDR		;; Calculate pointer to fudge flags
	bump	rdi, 8*clm*ZMM_SCD8+12*clm*ZMM_SCD1+clm*ZMM_SCD4+clm*64+clm*64 ;; Addr of the group multiplier fudge flags
	mov	rdx, norm_grp_mults		;; Addr of the group multipliers
	sub	rbx, rbx			;; Clear register for loading fudge flags index
	mov	r12, compressed_fudges		;; Address of the compressed fudge flags
	mov	r13, r12			;; The XOR masks for fudge flags come from the compressed fudges
	set_data_prefetch_ptrs			;; Init prefetching pointers
pfing	mov	r8, pass2blkdst			;; blkdst
pfing	imul	rax, blkdstreg, (896-0)		;; We've already prefetched 896 of 1536 blocks (zero too many)
pfing	add	rcx, rax
	zloop_init 16*8*clm			;; 16 iters of 8*clm
	zloop_set_clm_prefetch ZQUAD		;; Prefetch 256 of 1536 blocks of clm*128 bytes
	zr12_csc_wpn_twelve_complex_first_djbfft_preload
b1b:	zr12_csc_wpn_twelve_complex_first_djbfft rsi, 128, 2*clmblkdst8, 4*clmblkdst8, 8*clmblkdst8, r15, 0, rdi, 3, rdx, 0, 8*clm, L1PREFETCH_ALL, 128
	zloop	8*clm, b1b, -8*clm*128+clmblkdst, r15, ZMM_CD12SCD6, rdx, 24*64, r13, 24 ;; 8*clm iters, next source, fixed s/c, grp multipliers pointer, next xor mask
	zloop	8, b1b, -8*clmblkdst+clmblkdst8	;; Test loop counter, next source pointer
	zloop	2, b1b, -2*clmblkdst8		;; Test loop counter, restore source pointer
	end_timer 22

;; Do FFT levels 5,6,7,8
;; On input the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	128K	+1K	...
;;	8K
;;	...
;;	120K
;;	256K
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	8K	+1K	...
;;	16K
;;	...

;; Do 96 sixteen_complex_fft operations
;;	distance between fft data elements is 8K
;;	do 12 macros each processing 256 data values

	start_timer 24
	bump	rdi, -16*8*clm*3-8*clm*ZMM_SCD8 ;; Sin/cos pointer
	zloop_init  12*8*clm			;; 12 iterations of 8*clm
	zloop_set_clm_prefetch ZOCT		;; Prefetch 384 of 1536 blocks of clm*128 bytes
	zr16_sixteen_complex_djbfft_preload
IFDEF OLDDD
b2b:	zr16_sixteen_complex_djbfft rsi, 128, clmblkdst, 2*clmblkdst, 4*clmblkdst, clmblkdst8, rdi, ZMM_SCD8, 8*clm, L1PREFETCH_ALL, 128
	zloop	8*clm, b2b, -8*clm*128+2*clmblkdst8, rdi, -8*clm*ZMM_SCD8 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	12, b2b, -12*2*clmblkdst8	;; Test loop counter, restore source pointer
ELSE
b2b:	zr16_sixteen_complex_djbfft rsi, 2*clmblkdst8, clmblkdst, 2*clmblkdst, 4*clmblkdst, clmblkdst8, rdi, 0, 8, L1PREFETCH_ALL, 2*clmblkdst8
	zloop	12, b2b, -12*2*clmblkdst8+128, rdi, ZMM_SCD8 ;; Test loop counter, next source pointer, next sin/cos pointer
	zloop	8*clm, b2b, -8*clm*128		;; Test loop counter, restore source pointer
ENDIF
	end_timer 24

;; Do FFT levels 9,10,11
;; On input the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	8K	+1K	...
;;	16K
;;	...
;; The input is swizzled to hold these data values:
;;	0K	+1	+1	+1	+1	+1	+1	+1	8K	+1	...
;;	1K	...
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0K	+1	+1	+1	+1	+1	+1	+1	1K	+1	...
;;	2K
;;	...

;; Do 192 eight_complex_fft operations
;;	distance between fft data elements is 1K
;;	do 24 macros each processing 128 data values

	start_timer 26
	mov	rbp, DATA_ADDR			;; Destination pointer
IFDEF OLDDD
	bump	rdi, -12*clm*ZMM_SCD1		;; Delayed sin/cos multipliers
ELSE
	bump	rdi, -8*clm*ZMM_SCD8-12*clm*ZMM_SCD1 ;; Delayed sin/cos multipliers
ENDIF
	lea	rbx, [rdi-clm*ZMM_SCD4]		;; Complex data sin/cos ptr
	lea	r9, [rbx-clm*64-clm*64]		;; Column weights
	lea	rdx, ZMM_TMPS			;; Computed clm*SCD4 sin/cos values
	mov	r8, pass2blkdst			;; blkdst
	lea	r10, [2*blkdstreg+blkdstreg]	;; 3*blkdst
	lea	r13, [rbp+4*blkdstreg]		;; Dest+4*blkdst pointer
pfing	mov	rcx, PREMULT_PREFETCH		;; Load premult/sin/cos prefetch pointer

	zr8_rsc_complex_only_preload		;; Preload constants before calling zloop_init
	zloop_init 12*16*clm			;; 12 iters of 16*clm
	zloop_set_prefetch ZSINGLE		;; Prefetch 192*clm times
	zloop_set_alternate_prefetch_increment 174, 192  ;; Prefetch approximately 174*clm cache lines in those 192*clm prefetches
b9a:	zr8_rsc_wpn_eight_complex_calc_sincos r9, rdi, rbx, clm ;; Generate clm*SCD8 sin/cos values from weights, delayed and complex multipliers
	bump	rdi, clm*ZMM_SCD1		;; Next set of delayed multipliers
b9b:	zr8_rsc_wpn_sgreg_eight_complex_fft8 rsi, 1024, 128, 256, 512, rbp, 128, blkdstreg, blkdst3reg, r13, rdx, ZMM_SCD8, clm, L1PREFETCH_DEST_ALL
	zloop	clm, b9b, -clm*1024+clmblkdst, rbp, -clm*128+8*blkdstreg, r13, -clm*128+8*blkdstreg, rdx, -clm*ZMM_SCD8 ;; Loop, next source/dest pointers, restore premult/sine/cosine pointer
	zloop	8, b9b, -8*clmblkdst+clmblkdst8	;; Test loop counter, next source pointer
	zloop	2, b9b				;; Test loop counter
	zloop	12, b9a				;; Test loop counter
	end_timer 26

;; Work on next 8*clm sets of 3072 values

endlp:	zpass1_get_next_block pass2, c0b, b0b ;; Get next block and jump
done:	zfft_footer
	ENDM





;********************************************************************
;  Testing different loop structure for better 32KB L1 cache usage
;********************************************************************

;; Perform a shared pass 1 FFT of length 1280 doubles.
;; Memory examples are for a 1280K FFT with 10 levels done in pass 2.
;; FFT data needed:
;;		8*clm*1280 doubles					Total: 1280*clm 64-byte cache lines
;; Fixed s/c data needed:
;;		10*ZMM_SCD7 (fixed s/c data),				Total: 140 64-byte cache lines
;; Group multipliers needed:
;;		10*16*64 (weights)
;;		10*16*64 (inverse weights),				Total: 320 64-byte cache lines
;; Variable data needed:
;;		clm*64	(column weights)
;;		clm*64	(column inverse weights)
;;		clm*ZMM_SCD4 (complex reduced s/c multipliers)
;;		clm*ZMM_SCD4 (real s/c multipliers)
;;		7*clm*ZMM_SCD1 (delayed s/c multipliers)
;;		8*clm*ZMM_SCD10 (10-complex 20-real sin/cos data)
;;		80*clm*16 (group multiplier fudge flags),		Total: 212*clm 64-byte cache lines

TEST_SOON_zr4dwpn_pass1sc1280 MACRO
	LOCAL	b0b, b1b, b2a, b2b, b8a, b8b, b9aa, b9a, b9b, c0b, c1b, c2a, c2b, c8b, c9aa, c9a, c9b
	LOCAL	pass1, pass2, endlp, done, yes_fft

	zfft_header pass1

;; Do pass 2 (the last 10 FFT levels if this is a 1280K FFT)

pass2:	start_timer 0
	mov	rax, ZMM_PASS2_ROUTINE
	call	rax
	end_timer 0
	cmp	ffttype, 1		;; We're done if FFTing only
	je	done

;; Do the inverse FFT

	pass1_inverse_fft_setup c0b

;; Do inverse FFT levels 8,9,10
;; On input the 128-byte double cache lines hold these data values:
;;	0K	+1	+1	+1	+1	+1	+1	+1	1K	+1	...
;;	2K
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0K	+1	+1	+1	+1	+1	+1	+1	8K	+1	...
;;	1K	...
;;	...
;; The output cache lines are swizzled to hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	8K	+1K	...
;;	16K
;;	...

;; Do 1 sixteen_reals_unfft operations
;;	distance between fft data elements is 1K
;;	do 0.125 macros each processing 128 data values
;; Do 79 eight_complex_unfft operations
;;	distance between fft data elements is 1K
;;	do 9.875 macros each processing 128 data values

c0b:	start_timer 16
	mov	rbp, DATA_ADDR			;; Get source address
	mov	rsi, scratch_area		;; Get address of scratch area
	set_data_prefetch_ptrs			;; Init prefetching pointers
	mov	r9, PREMULT_ADDR		;; Variable data
	bump	r9, clm*64			;; Inverse column weights
	lea	rbx, [r9+clm*64]		;; Complex data sin/cos ptr
	lea	rdi, [rbx+clm*ZMM_SCD4]		;; Real data sin/cos pointer, will become delayed sin/cos multipliers
	lea	rdx, ZMM_TMPS			;; Computed clm*SCD8 sin/cos values
	mov	r8, pass2blkdst			;; blkdst
	lea	r10, [2*blkdstreg+blkdstreg]	;; 3*blkdst
	lea	r13, [rbp+4*blkdstreg]		;; Source+4*blkdst pointer
	lea	r12, [rbp+8*blkdstreg]		;; L1 prefetch pointer part 1
	bump2	r12, (clm MOD 2), 8*blkdstreg	;; L1 prefetch pointer part 2
	lea	r14, [r12+4*blkdstreg]		;; L1 prefetch pointer + 4*blkdst

	zr8_rsc_complex_and_real_preload	;; Preload constants before calling zloop_init
	zloop_init 10*8*clm			;; 10 iters of 8*clm iterations
	zr8_rsc_wpn_eight_complex_calc_sincos_simple r9, rbx, clm ;; Generate clm*SCD8 sin/cos values from delayed and complex multipliers
	zloop_set_clm_prefetch ZQUAD		;; Prefetch 20 blocks of 8*clm*128 bytes
c8b:	zr8_rsc_wpn_sgreg_2sc_sixteen_reals_unfft8 rbp, 128, blkdstreg, blkdst3reg, r13, rsi, 1024, 128, 256, 512, rdx, ZMM_SCD8, r9, 64, rdi, ZMM_SCD4, clm, L1PREFETCH_ALL, r12, r14 ;; Prefetch short dist ahead
	zloop	clm, c8b, -clm*1024+clmblkdst, rbp, -clm*128+8*blkdstreg, r13, -clm*128+8*blkdstreg, rdx, -clm*ZMM_SCD8, r9, -clm*64, r12, -clm*128+8*blkdstreg, r14, -clm*128+8*blkdstreg ;; Loop, next src/dest ptrs, restore sin/cos ptr
	jmp	c9aa

	loops_reset
c9a:	zr8_rsc_wpn_eight_complex_calc_sincos r9, rdi, rbx, clm ;; Generate clm*SCD8 sin/cos values from delayed and complex multipliers
	bump	rdi, clm*ZMM_SCD1		;; Next set of delayed sin/cos multipliers
c9aa:	zloop_set_clm_prefetch ZQUAD		;; Prefetch 20 blocks of 8*clm*128 bytes
c9b:	zr8_rsc_wpn_sgreg_eight_complex_unfft8 rbp, 128, blkdstreg, blkdst3reg, r13, rsi, 1024, 128, 256, 512, rdx, ZMM_SCD8, clm, L1PREFETCH_ALL, r12, r14 ;; Prefetch short dist ahead
	zloop	clm, c9b, -clm*1024+clmblkdst, rbp, -clm*128+8*blkdstreg, r13, -clm*128+8*blkdstreg, rdx, -clm*ZMM_SCD8, r12, -clm*128+8*blkdstreg, r14, -clm*128+8*blkdstreg ;; Loop, next src/dest ptrs, restore sin/cos ptr
	zloop	10, c9b				;; Test loop counter
	zloop	8, c9a, -8*10*clmblkdst		;; Test loop counter, restore source pointer
	end_timer 16

;; Do inverse FFT levels 5,6,7
;; On input the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	8K	+1K	...
;;	16K
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	80K	+1K	...
;;	8K
;;	...
;;	72K
;;	160K
;;	...

;; Do 8 twenty_reals_unfft operations
;; Do 56 ten_complex_unfft operations
;;	distance between fft data elements is 8K
;;	do 8 macros each processing 160 data values

	start_timer 18
	zloop_init  8*8*clm			;; 8 iterations of 8*clm
	zloop_set_clm_prefetch ZDOUBLE		;; Prefetch 8 blocks of 8*clm*128 bytes
	zr10_csc_twenty_reals_unfft_preload
c2a:	zr10_csc_twenty_reals_unfft rsi, 128, clmblkdst, rdi, ZMM_SCD10, 8*clm, L1PREFETCH_ALL, 128
	zloop	8*clm, c2a, -8*clm*128+10*clmblkdst, rdi, -8*clm*ZMM_SCD10 ;; Test loop counter, next source pointer, restore sin/cos pointer
	loops_reset				;; 7 more iters of 8*clm
	zr10_ten_complex_djbunfft_preload
c2b:	zr10_ten_complex_djbunfft rsi, 128, clmblkdst, rdi, ZMM_SCD10, 8*clm, L1PREFETCH_ALL, 128
	zloop	8*clm, c2b, -8*clm*128+10*clmblkdst, rdi, -8*clm*ZMM_SCD10 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	8, c2b, -8*10*clmblkdst, rdi, 8*clm*ZMM_SCD10 ;; Test loop counter, restore source pointer, next sin/cos pointer
	end_timer 18

;; Do inverse FFT levels 1,2,3,4
;; On input the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	80K	+1K	...
;;	8K
;;	...
;;	72K
;;	160K
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	640K	+1K	...
;;	8K
;;	...
;;	632K

;; Do 80 sixteen_reals_unfft operations
;;	distance between fft data elements is 80K
;;	do 10 macros each processing 128 data values

	start_timer 20
	mov	rbx, sincos2			;; Load fixed premultiplier/sin/cos ptr
	zloop_init 10*8*clm			;; 10 iters of 8*clm
	zloop_set_clm_prefetch ZDOUBLE		;; Prefetch 10 blocks of 8*clm*128 bytes
	zr8_wpn_sixteen_reals_last_unfft_preload
c1b:	zr8_wpn_sixteen_reals_last_unfft rsi, 128, 10*clmblkdst, 20*clmblkdst, 40*clmblkdst, rbx, 0, rdi, 16, 8*clm, L1PREFETCH_ALL, 128
	zloop	8*clm, c1b, -8*clm*128+clmblkdst, rbx, ZMM_SCD7, rdx, 16*64 ;; 8*clm iterations
	zloop	10, c1b				;; Test loop counter
	end_timer 20

;; Normalize these values

	pass1_normalize yes_fft			;; Normalize while prefetching 10 blocks of 8*clm*128 bytes

;; Copy data from the scratch area back to the FFT data area

	copy_scratch_data_to_fft
	jmp	endlp				;; Skip FFT code if flag not set

;; Fire up auxiliary threads to do pass 1 forward FFT

pass1:	pass1_forward_fft_setup b0b

;; Copy data from the FFT data area back to the scratch area

b0b:	copy_fft_data_to_scratch

;; Do FFT levels 1,2,3,4
;; On input the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	640K	+1K	...
;;	8K	...
;;	...
;;	632K
;; On output the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	80K	+1K	...
;;	8K
;;	...
;;	72K
;;	160K
;;	...

;; Do 80 sixteen_reals_first_fft operations
;;	distance between fft data elements is 80K
;;	do 10 macros each processing 128 data values

yes_fft:start_timer 22
	mov	rsi, scratch_area		;; Get address of scratch area
	mov	rbx, sincos2			;; Load fixed premultiplier/sin/cos ptr
	mov	rdi, PREMULT_ADDR		;; Load variable premultiplier/sin/cos ptr
	bump	rdi, 7*clm*ZMM_SCD1+clm*ZMM_SCD4+clm*ZMM_SCD4+clm*64+clm*64 ;; Addr of the sin/cos data for FFT levels 5,6,7
	lea	r9, [rdi+8*clm*ZMM_SCD10]	;; Addr of the group multiplier fudge flags
	mov	rdx, norm_grp_mults		;; Addr of the group multipliers
	set_data_prefetch_ptrs			;; Init prefetching pointers
pfing	mov	r8, pass2blkdst			;; blkdst
pfing	imul	rax, blkdstreg, (48-4)*8	;; We've already prefetched 48 blocks (4 too many)
pfing	add	rcx, rax

	zloop_init 10*8*clm			;; 10 iters of 8*clm OR 8 iters of 8*clm
	zloop_set_clm_prefetch ZQUAD		;; Prefetch 20 blocks of 8*clm*128 bytes
	zr8_wpn_sixteen_reals_first_fft_preload
b1b:	zr8_wpn_sixteen_reals_first_fft rsi, clmblkdst, 10*clmblkdst, 20*clmblkdst, 40*clmblkdst, rbx, ZMM_SCD7, r9, 8*clm*16, rdx, 16*64, 10, L1PREFETCH_ALL, clmblkdst
	zloop	10, b1b, -10*clmblkdst+128, rbx, -10*ZMM_SCD7, r9, -10*8*clm*16+16, rdx, -10*16*64 ;; next source, restore fixed s/c, grp multipliers pointer

one loop has 10*8*128 data (10K), 10*SCD7 fixed s/c (8960), 10*64 fudgies (640) (could reduce to 10*16), 10*16*64 grp mults (10K).
Total of ~30KB

	zloop	8*clm, b1b, -8*clm*128		;; Test loop counter, restore source pointer
	end_timer 22

;; Do FFT levels 5,6,7
;; On input the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	80K	+1K	...
;;	8K
;;	...
;;	72K
;;	160K
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	8K	+1K	...
;;	16K
;;	...

;; Do 8 twenty_reals_fft operations
;; Do 56 ten_complex_fft operations
;;	distance between fft data elements is 8K
;;	do 8 macros each processing 160 data values

	bump	rdi, -80*clm*16-8*clm*ZMM_SCD10	;; Sin/cos pointer
	zloop_init  8*8*clm			;; 8 iterations of 8*clm
	zloop_set_clm_prefetch ZQUAD		;; Prefetch 16 blocks of 8*clm*128 bytes
	zr10_csc_twenty_reals_fft_preload
b2a:	zr10_csc_twenty_reals_fft rsi, 128, clmblkdst, rdi, ZMM_SCD10, 8*clm, L1PREFETCH_ALL, 128
	zloop	8*clm, b2a, -8*clm*128+10*clmblkdst, rdi, -8*clm*ZMM_SCD10 ;; Test loop counter, next source pointer, restore sin/cos pointer
	loops_reset				;; 7 more iters of 8*clm
	zr10_ten_complex_djbfft_preload
b2b:	zr10_ten_complex_djbfft rsi, 128, clmblkdst, rdi, ZMM_SCD10, 8*clm, L1PREFETCH_ALL, 128

	zloop	8*clm, b2b, -8*clm*128+10*clmblkdst, rdi, -8*clm*ZMM_SCD10 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	8, b2b, -8*10*clmblkdst		;; Test loop counter, restore source pointer

;; Do FFT levels 8,9,10
;; On input the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	8K	+1K	...
;;	16K
;;	...
;; The input is swizzled to hold these data values:
;;	0K	+1	+1	+1	+1	+1	+1	+1	8K	+1	...
;;	1K	...
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0K	+1	+1	+1	+1	+1	+1	+1	1K	+1	...
;;	2K
;;	...

;; Do 1 sixteen_reals_fft operations
;;	distance between fft data elements is 1K
;;	do 0.125 macros each processing 128 data values
;; Do 79 eight_complex_fft operations
;;	distance between fft data elements is 1K
;;	do 9.875 macros each processing 128 data values

	start_timer 26
	bump	rdi, -7*clm*ZMM_SCD1-clm*ZMM_SCD4 ;; Real data sin/cos pointer, will become delayed sin/cos multipliers
	lea	rbx, [rdi-clm*ZMM_SCD4]		;; Complex data sin/cos ptr
	lea	r9, [rbx-clm*64-clm*64]		;; Column weights
	lea	rdx, ZMM_TMPS			;; Computed clm*SCD4 sin/cos values
	mov	rbp, DATA_ADDR			;; Destination pointer
	mov	r8, pass2blkdst			;; blkdst
	lea	r10, [2*blkdstreg+blkdstreg]	;; 3*blkdst
	lea	r13, [rbp+4*blkdstreg]		;; Dest+4*blkdst pointer
pfing	mov	rcx, PREMULT_PREFETCH		;; Load premult/sin/cos prefetch pointer

	zr8_rsc_complex_and_real_preload	;; Preload constants before calling zloop_init
	zloop_init 10*8*clm			;; 10 iters of 8*clm
	zloop_set_prefetch ZTRIPLE		;; Prefetch 240*clm times
	zloop_set_alternate_prefetch_increment 212, 240  ;; Prefetch approximately 212*clm cache lines in those 240*clm prefetches
	zr8_rsc_wpn_eight_complex_calc_sincos_simple r9, rbx, clm ;; Generate clm*SCD8 sin/cos values from delayed and complex multipliers
b8b:	zr8_rsc_wpn_sgreg_2sc_sixteen_reals_fft8 rsi, 1024, 128, 256, 512, rbp, 128, blkdstreg, blkdst3reg, r13, rdx, ZMM_SCD8, r9, 64, rdi, ZMM_SCD4, clm, L1PREFETCH_DEST_ALL
	zloop	clm, b8b, -clm*1024+clmblkdst, rbp, -clm*128+8*blkdstreg, r13, -clm*128+8*blkdstreg, rdx, -clm*ZMM_SCD8, r9, -clm*64 ;; Loop, next source/dest pointers, restore premult/sine/cosine pointer
	jmp	b9aa

	loops_reset
b9a:	zr8_rsc_wpn_eight_complex_calc_sincos r9, rdi, rbx, clm ;; Generate clm*SCD8 sin/cos values from delayed and complex multipliers
	bump	rdi, clm*ZMM_SCD1		;; Next set of delayed multipliers
b9aa:
b9b:	zr8_rsc_wpn_sgreg_eight_complex_fft8 rsi, 1024, 128, 256, 512, rbp, 128, blkdstreg, blkdst3reg, r13, rdx, ZMM_SCD8, clm, L1PREFETCH_DEST_ALL
	zloop	clm, b9b, -clm*1024+clmblkdst, rbp, -clm*128+8*blkdstreg, r13, -clm*128+8*blkdstreg, rdx, -clm*ZMM_SCD8 ;; Loop, next source/dest pointers, restore premult/sine/cosine pointer
	zloop	10, b9b				;; Test loop counter
	zloop	8, b9a				;; Test loop counter
	end_timer 26

;; Work on next 8*clm sets of 1280 values

endlp:	zpass1_get_next_block pass2, c0b, b0b ;; Get next block and jump
done:	zfft_footer
	ENDM









;; Perform a shared pass 1 FFT of length 1280 doubles.
;; Memory examples are for a 1280K FFT with 10 levels done in pass 2.
;; FFT data needed:
;;		8*clm*1280 doubles					Total: 1280*clm 64-byte cache lines
;; Fixed s/c data needed:
;;		10*ZMM_SCD7 (fixed s/c data),				Total: 140 64-byte cache lines
;; Group multipliers needed:
;;		10*16*64 (weights and fudged weights)
;;		10*16*64 (inverse weights and fudged inv. weights),	Total: 320 64-byte cache lines
;; Variable data needed:
;;		clm*64	(column weights)
;;		clm*64	(column inverse weights)
;;		clm*ZMM_SCD4 (complex reduced s/c multipliers)
;;		clm*ZMM_SCD4 (real s/c multipliers)
;;		7*clm*ZMM_SCD1 (delayed s/c multipliers)
;;		8*clm*ZMM_SCD10 (10-complex 20-real sin/cos data)
;;		80*clm*16 (group multiplier fudge flags),		Total: 212*clm 64-byte cache lines

TSTzr4dwpn_pass1sc1280 MACRO
	LOCAL	b0b, b1b, b2a, b2b, b8a, b8b, b9aa, b9a, b9b, c0b, c1b, c2a, c2b, c8b, c9aa, c9a, c9b
	LOCAL	pass1, pass2, endlp, done, yes_fft
LOCAL xyz,abc,xyz2

	zfft_header pass1

;; Do pass 2 (the last 10 FFT levels if this is a 1280K FFT)

pass2:	start_timer 0
	mov	rax, ZMM_PASS2_ROUTINE
	call	rax
	end_timer 0
	cmp	ffttype, 1		;; We're done if FFTing only
	je	done

;; Do the inverse FFT

	pass1_inverse_fft_setup c0b

;; Do inverse FFT levels 8,9,10
;; On input the 128-byte double cache lines hold these data values:
;;	0K	+1	+1	+1	+1	+1	+1	+1	1K	+1	...
;;	2K
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0K	+1	+1	+1	+1	+1	+1	+1	8K	+1	...
;;	1K	...
;;	...
;; The output cache lines are swizzled to hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	8K	+1K	...
;;	16K
;;	...

;; Do 1 sixteen_reals_unfft operations
;;	distance between fft data elements is 1K
;;	do 0.125 macros each processing 128 data values
;; Do 79 eight_complex_unfft operations
;;	distance between fft data elements is 1K
;;	do 9.875 macros each processing 128 data values

c0b:
	start_timer 15
	zloop_init 80*clm			;; 10 iters of 8*clm iterations
	mov	rbp, DATA_ADDR			;; Get source address
	mov	r8, pass2blkdst			;; blkdst
	lea	r10, [2*blkdstreg+blkdstreg]	;; 3*blkdst
	lea	r13, [rbp+4*blkdstreg]		;; Source+4*blkdst pointer
xyz:	vmovapd	zmm0, [rbp]
	vmovapd	zmm0, [rbp+64]
	vmovapd	zmm0, [rbp+r8]
	vmovapd	zmm0, [rbp+r8+64]
	vmovapd	zmm0, [rbp+2*r8]
	vmovapd	zmm0, [rbp+2*r8+64]
	vmovapd	zmm0, [rbp+r10]
	vmovapd	zmm0, [rbp+r10+64]
	vmovapd	zmm0, [r13]
	vmovapd	zmm0, [r13+64]
	vmovapd	zmm0, [r13+r8]
	vmovapd	zmm0, [r13+r8+64]
	vmovapd	zmm0, [r13+2*r8]
	vmovapd	zmm0, [r13+2*r8+64]
	vmovapd	zmm0, [r13+r10]
	vmovapd	zmm0, [r13+r10+64]
	bump	rbp, 128
	bump	r13, 128
	zloop	clm, xyz, 0, rbp, -clm*128+8*blkdstreg, r13, -clm*128+8*blkdstreg
	zloop	80, xyz
	end_timer 15

	start_timer 17
	zloop_init 26*clm			;; 10 iters of 8*clm iterations
	mov	r9, PREMULT_ADDR		;; Variable data
abc:	vmovapd	zmm0, [r9]
	bump	r9, 64
	zloop	26*clm, abc
	end_timer 17

	start_timer 19
	zloop_init 80*clm			;; 10 iters of 8*clm iterations
	mov	rbp, DATA_ADDR			;; Get source address
	mov	r8, pass2blkdst			;; blkdst
	lea	r10, [2*blkdstreg+blkdstreg]	;; 3*blkdst
	lea	r13, [rbp+4*blkdstreg]		;; Source+4*blkdst pointer
xyz2:	vmovapd	zmm0, [rbp]
	vmovapd	zmm0, [rbp+64]
	vmovapd	zmm0, [rbp+r8]
	vmovapd	zmm0, [rbp+r8+64]
	vmovapd	zmm0, [rbp+2*r8]
	vmovapd	zmm0, [rbp+2*r8+64]
	vmovapd	zmm0, [rbp+r10]
	vmovapd	zmm0, [rbp+r10+64]
	vmovapd	zmm0, [r13]
	vmovapd	zmm0, [r13+64]
	vmovapd	zmm0, [r13+r8]
	vmovapd	zmm0, [r13+r8+64]
	vmovapd	zmm0, [r13+2*r8]
	vmovapd	zmm0, [r13+2*r8+64]
	vmovapd	zmm0, [r13+r10]
	vmovapd	zmm0, [r13+r10+64]
	bump	rbp, 128
	bump	r13, 128
	zloop	clm, xyz2, 0, rbp, -clm*128+8*blkdstreg, r13, -clm*128+8*blkdstreg
	zloop	80, xyz2
	end_timer 19


	start_timer 16
	mov	rbp, DATA_ADDR			;; Get source address
	mov	rsi, scratch_area		;; Get address of scratch area
	set_data_prefetch_ptrs			;; Init prefetching pointers
	mov	r9, PREMULT_ADDR		;; Variable data
	bump	r9, clm*64			;; Inverse column weights
	lea	rbx, [r9+clm*64]		;; Complex data sin/cos ptr
	lea	rdi, [rbx+clm*ZMM_SCD4]		;; Real data sin/cos pointer, will become delayed sin/cos multipliers
	lea	rdx, ZMM_TMPS			;; Computed clm*SCD8 sin/cos values
	mov	r8, pass2blkdst			;; blkdst
	lea	r10, [2*blkdstreg+blkdstreg]	;; 3*blkdst
	lea	r13, [rbp+4*blkdstreg]		;; Source+4*blkdst pointer
	lea	r12, [rbp+8*blkdstreg]		;; L1 prefetch pointer part 1
	bump2	r12, (clm MOD 2), 8*blkdstreg	;; L1 prefetch pointer part 2
	lea	r14, [r12+4*blkdstreg]		;; L1 prefetch pointer + 4*blkdst

	zr8_rsc_complex_and_real_preload	;; Preload constants before calling zloop_init
	zloop_init 10*8*clm			;; 10 iters of 8*clm iterations
	zr8_rsc_wpn_eight_complex_calc_sincos_simple r9, rbx, clm ;; Generate clm*SCD8 sin/cos values from delayed and complex multipliers
	zloop_set_clm_prefetch ZDOUBLE		;; Prefetch 20 blocks of 8*clm*128 bytes
c8b:	zr8_rsc_wpn_sgreg_2sc_sixteen_reals_unfft8 rbp, 128, blkdstreg, blkdst3reg, r13, rsi, 1024, 128, 256, 512, rdx, 0*ZMM_SCD8, r9, 64, rdi, ZMM_SCD4, clm, L1PREFETCH_ALL, r12, r14 ;; Prefetch short dist ahead
	zloop	clm, c8b, -clm*1024+clmblkdst, rbp, -clm*128+8*blkdstreg, r13, -clm*128+8*blkdstreg, r9, -clm*64, r12, -clm*128+8*blkdstreg, r14, -clm*128+8*blkdstreg ;; Loop, next src/dest ptrs, restore sin/cos ptr
	jmp	c9aa
data= 16cl = 1KB, dest = 1kb
s/c=16cl = 1kb
read-ahead 2 is fine.    

	loops_reset
c9a:	;zr8_rsc_wpn_eight_complex_calc_sincos r9, rdi, rbx, clm ;; Generate clm*SCD8 sin/cos values from delayed and complex multipliers
	bump	rdi, clm*ZMM_SCD1		;; Next set of delayed sin/cos multipliers
c9aa:	zloop_set_clm_prefetch ZDOUBLE		;; Prefetch 20 blocks of 8*clm*128 bytes
c9b:	zr8_rsc_wpn_sgreg_eight_complex_unfft8 rbp, 128, blkdstreg, blkdst3reg, r13, rsi, 1024, 128, 256, 512, rdx, 0*ZMM_SCD8, clm, L1PREFETCH_ALL, r12, r14 ;; Prefetch short dist ahead
	zloop	clm, c9b, -clm*1024+clmblkdst, rbp, -clm*128+8*blkdstreg, r13, -clm*128+8*blkdstreg, r12, -clm*128+8*blkdstreg, r14, -clm*128+8*blkdstreg ;; Loop, next src/dest ptrs, restore sin/cos ptr
	zloop	10, c9b				;; Test loop counter
	zloop	8, c9a, -8*10*clmblkdst		;; Test loop counter, restore source pointer
	end_timer 16

;; Do inverse FFT levels 5,6,7
;; On input the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	8K	+1K	...
;;	16K
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	80K	+1K	...
;;	8K
;;	...
;;	72K
;;	160K
;;	...

;; Do 8 twenty_reals_unfft operations
;; Do 56 ten_complex_unfft operations
;;	distance between fft data elements is 8K
;;	do 8 macros each processing 160 data values

	start_timer 18
	zloop_init  8*8*clm			;; 8 iterations of 8*clm
	zloop_set_clm_prefetch ZDOUBLE		;; Prefetch 8 blocks of 8*clm*128 bytes
	zr10_csc_twenty_reals_unfft_preload
c2a:	zr10_csc_twenty_reals_unfft rsi, 128, clmblkdst, rdi, ZMM_SCD10, 8*clm, L1PREFETCH_ALL, 128
	zloop	8*clm, c2a, -8*clm*128+10*clmblkdst, rdi, -8*clm*ZMM_SCD10 ;; Test loop counter, next source pointer, restore sin/cos pointer
	loops_reset				;; 7 more iters of 8*clm
	zr10_ten_complex_djbunfft_preload
c2b:	zr10_ten_complex_djbunfft rsi, 128, clmblkdst, rdi, ZMM_SCD10, 8*clm, L1PREFETCH_ALL, 128
	zloop	8*clm, c2b, -8*clm*128+10*clmblkdst, rdi, -8*clm*ZMM_SCD10 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	8, c2b, -8*10*clmblkdst, rdi, 8*clm*ZMM_SCD10 ;; Test loop counter, restore source pointer, next sin/cos pointer
	end_timer 18

;; Do inverse FFT levels 1,2,3,4
;; On input the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	80K	+1K	...
;;	8K
;;	...
;;	72K
;;	160K
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	640K	+1K	...
;;	8K
;;	...
;;	632K

;; Do 80 sixteen_reals_unfft operations
;;	distance between fft data elements is 80K
;;	do 10 macros each processing 128 data values

	start_timer 20
	mov	rbx, sincos2			;; Load fixed premultiplier/sin/cos ptr
	zloop_init 10*8*clm			;; 10 iters of 8*clm
	zloop_set_clm_prefetch ZDOUBLE		;; Prefetch 10 blocks of 8*clm*128 bytes
	zr8_wpn_sixteen_reals_last_unfft_preload
c1b:	zr8_wpn_sixteen_reals_last_unfft rsi, 128, 10*clmblkdst, 20*clmblkdst, 40*clmblkdst, rbx, 0, rdi, 16, 8*clm, L1PREFETCH_ALL, 128
	zloop	8*clm, c1b, -8*clm*128+clmblkdst, rbx, ZMM_SCD7, rdx, 16*64 ;; 8*clm iterations
	zloop	10, c1b				;; Test loop counter
	end_timer 20

;; Normalize these values

	pass1_normalize yes_fft			;; Normalize while prefetching 10 blocks of 8*clm*128 bytes

;; Copy data from the scratch area back to the FFT data area

	copy_scratch_data_to_fft
	jmp	endlp				;; Skip FFT code if flag not set

;; Fire up auxiliary threads to do pass 1 forward FFT

pass1:	pass1_forward_fft_setup b0b

;; Copy data from the FFT data area back to the scratch area

b0b:	copy_fft_data_to_scratch

;; Do FFT levels 1,2,3,4
;; On input the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	640K	+1K	...
;;	8K	...
;;	...
;;	632K
;; On output the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	80K	+1K	...
;;	8K
;;	...
;;	72K
;;	160K
;;	...

;; Do 80 sixteen_reals_first_fft operations
;;	distance between fft data elements is 80K
;;	do 10 macros each processing 128 data values

yes_fft:start_timer 22
	mov	rsi, scratch_area		;; Get address of scratch area
	mov	rbx, sincos2			;; Load fixed premultiplier/sin/cos ptr
	mov	rdi, PREMULT_ADDR		;; Calculate pointer to fudge flags
	bump	rdi, 8*clm*ZMM_SCD10+7*clm*ZMM_SCD1+clm*ZMM_SCD4+clm*ZMM_SCD4+clm*64+clm*64 ;; Addr of the group multiplier fudge flags
	mov	rdx, norm_grp_mults		;; Addr of the group multipliers
	set_data_prefetch_ptrs			;; Init prefetching pointers
pfing	mov	r8, pass2blkdst			;; blkdst
pfing	imul	rax, blkdstreg, (48-4)*8	;; We've already prefetched 48 blocks (4 too many)
pfing	add	rcx, rax
	zloop_init 10*8*clm			;; 10 iters of 8*clm
	zloop_set_clm_prefetch ZQUAD		;; Prefetch 20 blocks of 8*clm*128 bytes
	zr8_wpn_sixteen_reals_first_fft_preload
b1b:	zr8_wpn_sixteen_reals_first_fft rsi, 128, 10*clmblkdst, 20*clmblkdst, 40*clmblkdst, rbx, 0, rdi, 16, rdx, 0, 8*clm, L1PREFETCH_ALL, 128
	zloop	8*clm, b1b, -8*clm*128+clmblkdst, rbx, ZMM_SCD7, rdx, 16*64 ;; 8*clm iters prefetching 8*clm cache lines, next source, fixed s/c, grp multipliers pointer
	zloop	10, b1b, -10*clmblkdst		;; Test loop counter, restore source pointer
	end_timer 22

;; Do FFT levels 5,6,7
;; On input the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	80K	+1K	...
;;	8K
;;	...
;;	72K
;;	160K
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	8K	+1K	...
;;	16K
;;	...

;; Do 8 twenty_reals_fft operations
;; Do 56 ten_complex_fft operations
;;	distance between fft data elements is 8K
;;	do 8 macros each processing 160 data values

	start_timer 24
	bump	rdi, -80*clm*16-8*clm*ZMM_SCD10	;; Sin/cos pointer
	zloop_init  8*8*clm			;; 8 iterations of 8*clm
	zloop_set_clm_prefetch ZQUAD		;; Prefetch 16 blocks of 8*clm*128 bytes
	zr10_csc_twenty_reals_fft_preload
b2a:	zr10_csc_twenty_reals_fft rsi, 128, clmblkdst, rdi, ZMM_SCD10, 8*clm, L1PREFETCH_ALL, 128
	zloop	8*clm, b2a, -8*clm*128+10*clmblkdst, rdi, -8*clm*ZMM_SCD10 ;; Test loop counter, next source pointer, restore sin/cos pointer
	loops_reset				;; 7 more iters of 8*clm
	zr10_ten_complex_djbfft_preload
b2b:	zr10_ten_complex_djbfft rsi, 128, clmblkdst, rdi, ZMM_SCD10, 8*clm, L1PREFETCH_ALL, 128
	zloop	8*clm, b2b, -8*clm*128+10*clmblkdst, rdi, -8*clm*ZMM_SCD10 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	8, b2b, -8*10*clmblkdst		;; Test loop counter, restore source pointer
	end_timer 24

;; Do FFT levels 8,9,10
;; On input the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	8K	+1K	...
;;	16K
;;	...
;; The input is swizzled to hold these data values:
;;	0K	+1	+1	+1	+1	+1	+1	+1	8K	+1	...
;;	1K	...
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0K	+1	+1	+1	+1	+1	+1	+1	1K	+1	...
;;	2K
;;	...

;; Do 1 sixteen_reals_fft operations
;;	distance between fft data elements is 1K
;;	do 0.125 macros each processing 128 data values
;; Do 79 eight_complex_fft operations
;;	distance between fft data elements is 1K
;;	do 9.875 macros each processing 128 data values

	start_timer 26
	bump	rdi, -7*clm*ZMM_SCD1-clm*ZMM_SCD4 ;; Real data sin/cos pointer, will become delayed sin/cos multipliers
	lea	rbx, [rdi-clm*ZMM_SCD4]		;; Complex data sin/cos ptr
	lea	r9, [rbx-clm*64-clm*64]		;; Column weights
	lea	rdx, ZMM_TMPS			;; Computed clm*SCD4 sin/cos values
	mov	rbp, DATA_ADDR			;; Destination pointer
	mov	r8, pass2blkdst			;; blkdst
	lea	r10, [2*blkdstreg+blkdstreg]	;; 3*blkdst
	lea	r13, [rbp+4*blkdstreg]		;; Dest+4*blkdst pointer

pfing	mov	rcx, PREMULT_PREFETCH		;; Load premult/sin/cos prefetch pointer

	zr8_rsc_complex_and_real_preload	;; Preload constants before calling zloop_init
	zloop_init 10*8*clm			;; 10 iters of 8*clm
	zloop_set_prefetch ZTRIPLE		;; Prefetch 240*clm times
	zloop_set_alternate_prefetch_increment 212, 240  ;; Prefetch approximately 212*clm cache lines in those 240*clm prefetches
	zr8_rsc_wpn_eight_complex_calc_sincos_simple r9, rbx, clm ;; Generate clm*SCD8 sin/cos values from delayed and complex multipliers
b8b:	zr8_rsc_wpn_sgreg_2sc_sixteen_reals_fft8 rsi, 1024, 128, 256, 512, rbp, 128, blkdstreg, blkdst3reg, r13, rdx, ZMM_SCD8, r9, 64, rdi, ZMM_SCD4, clm, L1PREFETCH_DEST_ALL
	zloop	clm, b8b, -clm*1024+clmblkdst, rbp, -clm*128+8*blkdstreg, r13, -clm*128+8*blkdstreg, rdx, -clm*ZMM_SCD8, r9, -clm*64 ;; Loop, next source/dest pointers, restore premult/sine/cosine pointer
	jmp	b9aa

	loops_reset
b9a:	zr8_rsc_wpn_eight_complex_calc_sincos r9, rdi, rbx, clm ;; Generate clm*SCD8 sin/cos values from delayed and complex multipliers
	bump	rdi, clm*ZMM_SCD1		;; Next set of delayed multipliers
b9aa:
b9b:	zr8_rsc_wpn_sgreg_eight_complex_fft8 rsi, 1024, 128, 256, 512, rbp, 128, blkdstreg, blkdst3reg, r13, rdx, ZMM_SCD8, clm, L1PREFETCH_DEST_ALL
	zloop	clm, b9b, -clm*1024+clmblkdst, rbp, -clm*128+8*blkdstreg, r13, -clm*128+8*blkdstreg, rdx, -clm*ZMM_SCD8 ;; Loop, next source/dest pointers, restore premult/sine/cosine pointer
	zloop	10, b9b				;; Test loop counter
	zloop	8, b9a				;; Test loop counter
	end_timer 26

;; Work on next 8*clm sets of 1280 values

endlp:	zpass1_get_next_block pass2, c0b, b0b ;; Get next block and jump
done:	zfft_footer
	ENDM









;; Perform a shared pass 1 FFT of length 1280 doubles.
;; Memory examples are for a 1280K FFT with 10 levels done in pass 2.
;; FFT data needed:
;;		8*clm*1280 doubles					Total: 1280*clm 64-byte cache lines
;; Fixed s/c data needed:
;;		10*ZMM_SCD7 (fixed s/c data),				Total: 140 64-byte cache lines
;; Group multipliers needed:
;;		10*16*64 (weights and fudged weights)
;;		10*16*64 (inverse weights and fudged inv. weights),	Total: 320 64-byte cache lines
;; Variable data needed:
;;		clm*64	(column weights)
;;		clm*64	(column inverse weights)
;;		clm*ZMM_SCD4 (complex reduced s/c multipliers)
;;		clm*ZMM_SCD4 (real s/c multipliers)
;;		7*clm*ZMM_SCD1 (delayed s/c multipliers)
;;		8*clm*ZMM_SCD10 (10-complex 20-real sin/cos data)
;;		80*clm*16 (group multiplier fudge flags),		Total: 212*clm 64-byte cache lines

;; TIMING alternate timer 16 loop structure.  Maybe a little faster (timer 16 went down 400K)
;; Should also try scrblkdst from non_temporal
MAYBEzr4dwpn_pass1sc1280 MACRO
	LOCAL	b0b, b1b, b2a, b2b, b2c, b8a, b8b, b9aa, b9a, b9b, c0b, c1b, c2a, c2b, c2c, c9, c9a, c9b, c9c
	LOCAL	pass1, pass2, endlp, done, yes_fft

	zfft_header pass1

;; Do pass 2 (the last 10 FFT levels if this is a 1280K FFT)

pass2:	start_timer 0
	mov	rax, ZMM_PASS2_ROUTINE
	call	rax
	end_timer 0
	cmp	ffttype, 1		;; We're done if FFTing only
	je	done

;; Do the inverse FFT

	pass1_inverse_fft_setup c0b

;; Do inverse FFT levels 8,9,10
;; On input the 128-byte double cache lines hold these data values:
;;	0K	+1	+1	+1	+1	+1	+1	+1	1K	+1	...
;;	2K
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0K	+1	+1	+1	+1	+1	+1	+1	8K	+1	...
;;	1K	...
;;	...
;; The output cache lines are swizzled to hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	8K	+1K	...
;;	16K
;;	...

;; Do 1 sixteen_reals_unfft operations
;;	distance between fft data elements is 1K
;;	do 0.125 macros each processing 128 data values
;; Do 79 eight_complex_unfft operations
;;	distance between fft data elements is 1K
;;	do 9.875 macros each processing 128 data values

c0b:	start_timer 16
	mov	rbp, DATA_ADDR			;; Get source address
	mov	rsi, scratch_area		;; Get address of scratch area
	set_data_prefetch_ptrs			;; Init prefetching pointers
	mov	r9, PREMULT_ADDR		;; Variable data
	bump	r9, clm*64			;; Inverse column weights
	lea	rbx, [r9+clm*64]		;; Complex data sin/cos ptr
	lea	rdi, [rbx+clm*ZMM_SCD4]		;; Real data sin/cos pointer, will become delayed sin/cos multipliers
	mov	r8, pass2blkdst			;; blkdst
	lea	r10, [2*blkdstreg+blkdstreg]	;; 3*blkdst

	zr8_rsc_complex_and_real_preload	;; Preload constants before calling zloop_init
	zloop_init clm*8*10			;; clm iters of 8*10 iterations
IF clm EQ 4
	zloop_set_clm_prefetch ZQUAD		;; Prefetch 80 of 640 blocks of clm*128 bytes
ELSE
	zloop_set_clm_prefetch ZDOUBLE		;; Prefetch 80 of 640 blocks of clm*128 bytes
ENDIF
	lea	rdi, [rdi+clm*ZMM_SCD4]		;; delayed sin/cos multipliers
;;BUG - reduce ZMM_TMPS size now that we don't gen clm*SCD8 values
c9:	zr8_rsc_wpn_eight_complex_calc_sincos_simple r9, rbx, 1 ;; Generate SCD8 sin/cos values from wpn weights and complex multipliers
	mov	rdx, rbp			;; Save starting source address
	lea	r13, [rbp+4*blkdstreg]		;; Source+4*blkdst pointer
	lea	r12, [rbp+8*blkdstreg]		;; L1 prefetch pointer
	lea	r14, [r12+4*blkdstreg]		;; L1 prefetch pointer + 4*blkdst
;; BUG re-org real and complex s/c data into one SCD8 -- see non-temporal
	zr8_rsc_wpn_sgreg_2sc_sixteen_reals_unfft8 rbp, 8*blkdstreg, blkdstreg, blkdst3reg, r13, rsi, clmblkdst, 128, 256, 512, ZMM_TMPS, 0, r9, 0, rbx+clm*ZMM_SCD4, 0, 1, L1PREFETCH_ALL, r12, r14 ;; Prefetch short dist ahead
	jmp	c9c
c9a:	zr8_rsc_wpn_eight_complex_calc_sincos r9, rdi, rbx, 1 ;; Generate SCD8 sin/cos values from wpn weights, delayed and complex multipliers
	bump	rdi, clm*ZMM_SCD1			;; Next delayed sin/cos ptr
c9b:	zr8_rsc_wpn_sgreg_eight_complex_unfft8 rbp, 8*blkdstreg, blkdstreg, blkdst3reg, r13, rsi, clmblkdst, 128, 256, 512, ZMM_TMPS, 0, 1, L1PREFETCH_ALL, r12, r14 ;; Prefetch short dist ahead
c9c:	zloop	10, c9b				;; Test loop counter reusing generated s/c values
;; BUG - "10" runs afoul of clm prefetching requirements unless we go to ZQUAD
	zloop	8, c9a, -8*10*clmblkdst+1024, r9, 64, rbx, ZMM_SCD4 ;; Test loop counter, next dest pointer, next weights, next complex/real s/c data ptr
	lea	rbp, [rdx+128]			;; Next source pointer
;; BUG - have C code rearrange rdi ptrs -- see non-temporal code
	bump	rdi, -7*clm*ZMM_SCD1+ZMM_SCD1	;; Next delayed sin/cos ptr
	zloop	clm, c9, -clm*1024		;; Loop, restore scratch area pointer
	bump	rdi, -clm*ZMM_SCD1+7*clm*ZMM_SCD1 ;; Next delayed sin/cos ptr
	end_timer 16

;; Do inverse FFT levels 5,6,7
;; On input the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	8K	+1K	...
;;	16K
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	80K	+1K	...
;;	8K
;;	...
;;	72K
;;	160K
;;	...

;; Do 8 twenty_reals_unfft operations
;; Do 56 ten_complex_unfft operations
;;	distance between fft data elements is 8K
;;	do 8 macros each processing 160 data values

	start_timer 18
	zloop_init  8*8*clm			;; 8 iterations of 8*clm
	zloop_set_clm_prefetch ZQUAD		;; Prefetch 128 of 640 blocks of clm*128 bytes
	zr10_csc_twenty_reals_unfft_preload
c2a:	zr10_csc_twenty_reals_unfft rsi, 10*clmblkdst, clmblkdst, rdi, 0, 1, L1PREFETCH_ALL, 10*clmblkdst
	zr10_ten_complex_djbunfft_preload
	jmp	c2c				;; 7 more iters
c2b:	zr10_ten_complex_djbunfft rsi, 10*clmblkdst, clmblkdst, rdi, 0, 1, L1PREFETCH_ALL, 10*clmblkdst
c2c:	zloop	8, c2b, -8*10*clmblkdst+128, rdi, ZMM_SCD10 ;; Test loop counter, next source pointer, next sin/cos pointer
	zloop	8*clm, c2a, -8*clm*128		;; Test loop counter, restore source pointer
	end_timer 18

;; Do inverse FFT levels 1,2,3,4
;; On input the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	80K	+1K	...
;;	8K
;;	...
;;	72K
;;	160K
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	640K	+1K	...
;;	8K
;;	...
;;	632K

;; Do 80 sixteen_reals_unfft operations
;;	distance between fft data elements is 80K
;;	do 10 macros each processing 128 data values

	start_timer 20
	mov	rbx, sincos2			;; Load fixed premultiplier/sin/cos ptr
	zloop_init 10*8*clm			;; 10 iters of 8*clm
	zloop_set_clm_prefetch ZDOUBLE		;; Prefetch 80 of 640 blocks of clm*128 bytes
	zr8_wpn_sixteen_reals_last_unfft_preload
c1b:	zr8_wpn_sixteen_reals_last_unfft rsi, 128, 10*clmblkdst, 20*clmblkdst, 40*clmblkdst, rbx, 0, rdi, 16, 8*clm, L1PREFETCH_ALL, 128
	zloop	8*clm, c1b, -8*clm*128+clmblkdst, rbx, ZMM_SCD7, rdx, 16*64 ;; 8*clm iterations
	zloop	10, c1b				;; Test loop counter
	end_timer 20

;; Normalize these values

	pass1_normalize yes_fft			;; Normalize while prefetching 80 of 640 blocks of clm*128 bytes

;; Copy data from the scratch area back to the FFT data area

	copy_scratch_data_to_fft
	jmp	endlp				;; Skip FFT code if flag not set

;; Fire up auxiliary threads to do pass 1 forward FFT

pass1:	pass1_forward_fft_setup b0b

;; Copy data from the FFT data area back to the scratch area

b0b:	copy_fft_data_to_scratch

;; Do FFT levels 1,2,3,4
;; On input the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	640K	+1K	...
;;	8K	...
;;	...
;;	632K
;; On output the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	80K	+1K	...
;;	8K
;;	...
;;	72K
;;	160K
;;	...

;; Do 80 sixteen_reals_first_fft operations
;;	distance between fft data elements is 80K
;;	do 10 macros each processing 128 data values

yes_fft:start_timer 22
	mov	rsi, scratch_area		;; Get address of scratch area
	mov	rbx, sincos2			;; Load fixed premultiplier/sin/cos ptr
	mov	rdi, PREMULT_ADDR		;; Calculate pointer to fudge flags
	bump	rdi, 8*clm*ZMM_SCD10+7*clm*ZMM_SCD1+clm*ZMM_SCD4+clm*ZMM_SCD4+clm*64+clm*64 ;; Addr of the group multiplier fudge flags
	mov	rdx, norm_grp_mults		;; Addr of the group multipliers
	set_data_prefetch_ptrs			;; Init prefetching pointers
pfing	mov	r8, pass2blkdst			;; blkdst
pfing	imul	rax, blkdstreg, (368-16)	;; We've already prefetched 368 of 640 blocks (16 too many)
pfing	add	rcx, rax
	zloop_init 10*8*clm			;; 10 iters of 8*clm
	zloop_set_clm_prefetch ZQUAD		;; Prefetch 160 of 640 blocks of clm*128 bytes
	zr8_wpn_sixteen_reals_first_fft_preload
b1b:	zr8_wpn_sixteen_reals_first_fft rsi, 128, 10*clmblkdst, 20*clmblkdst, 40*clmblkdst, rbx, 0, rdi, 16, rdx, 0, 8*clm, L1PREFETCH_ALL, 128
	zloop	8*clm, b1b, -8*clm*128+clmblkdst, rbx, ZMM_SCD7, rdx, 16*64 ;; 8*clm iters prefetching 8*clm cache lines, next source, fixed s/c, grp multipliers pointer
	zloop	10, b1b, -10*clmblkdst		;; Test loop counter, restore source pointer
	end_timer 22

;; Do FFT levels 5,6,7
;; On input the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	80K	+1K	...
;;	8K
;;	...
;;	72K
;;	160K
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	8K	+1K	...
;;	16K
;;	...

;; Do 8 twenty_reals_fft operations
;; Do 56 ten_complex_fft operations
;;	distance between fft data elements is 8K
;;	do 8 macros each processing 160 data values

	start_timer 24
	bump	rdi, -80*clm*16-8*clm*ZMM_SCD10	;; Sin/cos pointer
	zloop_init  8*8*clm			;; 8 iterations of 8*clm
	zloop_set_clm_prefetch ZQUAD		;; Prefetch 128 of 640 blocks of clm*128 bytes
	zr10_csc_twenty_reals_fft_preload
b2a:	zr10_csc_twenty_reals_fft rsi, 10*clmblkdst, clmblkdst, rdi, 0, 1, L1PREFETCH_ALL, 10*clmblkdst
	zr10_ten_complex_djbfft_preload
	jmp	b2c				;; 7 more iters
b2b:	zr10_ten_complex_djbfft rsi, 10*clmblkdst, clmblkdst, rdi, 0, 1, L1PREFETCH_ALL, 10*clmblkdst
b2c:	zloop	8, b2b, -8*10*clmblkdst+128, rdi, ZMM_SCD10 ;; Test loop counter, next source pointer, next sin/cos pointer
	zloop	8*clm, b2a, -8*clm*128		;; Test loop counter, restore source pointer
	end_timer 24

;; Do FFT levels 8,9,10
;; On input the 128-byte double cache lines hold these data values:
;;	0K	+1K	+1K	+1K	+1K	+1K	+1K	+1K	8K	+1K	...
;;	16K
;;	...
;; The input is swizzled to hold these data values:
;;	0K	+1	+1	+1	+1	+1	+1	+1	8K	+1	...
;;	1K	...
;;	...
;; On output the 128-byte double cache lines hold these data values:
;;	0K	+1	+1	+1	+1	+1	+1	+1	1K	+1	...
;;	2K
;;	...

;; Do 1 sixteen_reals_fft operations
;;	distance between fft data elements is 1K
;;	do 0.125 macros each processing 128 data values
;; Do 79 eight_complex_fft operations
;;	distance between fft data elements is 1K
;;	do 9.875 macros each processing 128 data values

	start_timer 26
	bump	rdi, -8*clm*ZMM_SCD10-7*clm*ZMM_SCD1-clm*ZMM_SCD4 ;; Real data sin/cos pointer, will become delayed sin/cos multipliers
	lea	rbx, [rdi-clm*ZMM_SCD4]		;; Complex data sin/cos ptr
	lea	r9, [rbx-clm*64-clm*64]		;; Column weights
	lea	rdx, ZMM_TMPS			;; Computed clm*SCD4 sin/cos values
	mov	rbp, DATA_ADDR			;; Destination pointer
	mov	r8, pass2blkdst			;; blkdst
	lea	r10, [2*blkdstreg+blkdstreg]	;; 3*blkdst
	lea	r13, [rbp+4*blkdstreg]		;; Dest+4*blkdst pointer

pfing	mov	rcx, PREMULT_PREFETCH		;; Load premult/sin/cos prefetch pointer

	zr8_rsc_complex_and_real_preload	;; Preload constants before calling zloop_init
	zloop_init 10*8*clm			;; 10 iters of 8*clm
	zloop_set_prefetch ZTRIPLE		;; Prefetch 240*clm times
	zloop_set_alternate_prefetch_increment 212, 240  ;; Prefetch approximately 212*clm cache lines in those 240*clm prefetches
	zr8_rsc_wpn_eight_complex_calc_sincos_simple r9, rbx, clm ;; Generate clm*SCD8 sin/cos values from delayed and complex multipliers
b8b:	zr8_rsc_wpn_sgreg_2sc_sixteen_reals_fft8 rsi, 1024, 128, 256, 512, rbp, 128, blkdstreg, blkdst3reg, r13, rdx, ZMM_SCD8, r9, 64, rdi, ZMM_SCD4, clm, L1PREFETCH_DEST_ALL
	zloop	clm, b8b, -clm*1024+clmblkdst, rbp, -clm*128+8*blkdstreg, r13, -clm*128+8*blkdstreg, rdx, -clm*ZMM_SCD8, r9, -clm*64 ;; Loop, next source/dest pointers, restore premult/sine/cosine pointer
	jmp	b9aa

	loops_reset
b9a:	zr8_rsc_wpn_eight_complex_calc_sincos r9, rdi, rbx, clm ;; Generate clm*SCD8 sin/cos values from delayed and complex multipliers
	bump	rdi, clm*ZMM_SCD1		;; Next set of delayed multipliers
b9aa:
b9b:	zr8_rsc_wpn_sgreg_eight_complex_fft8 rsi, 1024, 128, 256, 512, rbp, 128, blkdstreg, blkdst3reg, r13, rdx, ZMM_SCD8, clm, L1PREFETCH_DEST_ALL
	zloop	clm, b9b, -clm*1024+clmblkdst, rbp, -clm*128+8*blkdstreg, r13, -clm*128+8*blkdstreg, rdx, -clm*ZMM_SCD8 ;; Loop, next source/dest pointers, restore premult/sine/cosine pointer
	zloop	10, b9b				;; Test loop counter
	zloop	8, b9a				;; Test loop counter
	end_timer 26

;; Work on next 8*clm sets of 1280 values

endlp:	zpass1_get_next_block pass2, c0b, b0b ;; Get next block and jump
done:	zfft_footer
	ENDM

