; Copyright 2001-2023 - Mersenne Research, Inc.  All rights reserved
; Author:  George Woltman
; Email: woltman@alum.mit.edu
;
; These macros do the first pass of 2 pass FFTs using SSE2 instructions.
;

; ********************************************************
; ********************************************************
; ******************  PASS 1 MACROS  *********************
; ********************************************************
; ********************************************************

;;***********************************************************************
;; Pass 1 macros
;;
;; p2cl = number of 128-byte cache lines processed from a pass 2 block
;; clm = cache line multiplier (num cache lines processed each iteration)
;;***********************************************************************

; Note:  All memory layouts are for 64 byte L1 cache lines.  The high
; order XMM words are not shown (they are 1K higher than the low
; order XMM word.  Since the L2 cache reads 128 byte cache lines, two
; 64 byte cache lines are processed at a time.  The second 64 byte
; cache line contains a completely independent set of data.


;; Perform a 320K-element FFT.  This is done in two passes.  Pass 1
;; does 8 levels, pass 2 uses common code to perform the last 11 levels.

hg_pass1sclevels8pfa5 MACRO fftname, pass2_levels, clmarg
	LOCAL	b0b, b1a, b1aa, b1b, b2b, b3b, b4b, b5b, b6b, b7b
	LOCAL	c0b, c1b, c2a, c2b, c3b, c4b, c5b, c6b, c7b
	LOCAL	pass1, pass2, endlp, done, yes_fft

	set_FFT_constants fftname, 0, 0, pass2_levels, clmarg
	xfft_header pass1

;; Do the last 11 FFT levels
;;	do 1 pass2_11_levels_real and 39 pass2_11_levels_complex

pass2:	start_timer 0
	call	xpass2name
	end_timer 0
	cmp	ffttype, 1		;; We're done if FFTing only
 	je	done

;; Do the inverse FFT

	pass1_inverse_fft_setup c0b

;; Do inverse FFT levels 7,8
;; On input the 64-byte cache lines hold these data values:
;;	0K	2K	4K	6K
;;	8K	10K	12K	14K
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0K	4K	8K	12K
;;	2K	...
;;	16K	...
;;	...

;; Do 1 eight_reals_unfft_2 macros
;; Do 1 nop_two_two_complex_unfft_2 macros
;;	distance between fft data elements is 2K
;;	do 1 macros each processing 16 data values

c0b:	start_timer 16
	mov	rsi, DATA_ADDR		;; Get source address
	mov	rcx, scratch_area	;; Get address of scratch area
	sub	rax, rax
	sub	rdx, rdx
pfing	mov	rbp, DATA_PREFETCH	;; Set prefetch distance
pfing	sub	rbp, rsi
c6b:	g2cl_eight_reals_unfft_2 rsi, 64, blkdst, rcx, 64, clmblkdst
	add	al, 256/2/clm		;; Test loop counter
	jnc	c6b			;; Iterate if necessary

;; Do 38 four_complex_unfft macros
;;	distance between fft data elements is 2K
;;	do 19 macros each processing 16 data values

	bump	rsi, -2*clm*64+2*blkdst	;; Next source pointer
	bump	rcx, -2*clm*64+2*clmblkdst ;; Next dest pointer
	mov	rdi, sincos1		;; Load sin/cos pointer
	mov	al, 256/4+5		;; 19 iterations of 2*clm
c7b:	g2cl_four_complex_unfft rsi, 64, blkdst, rcx, 64, clmblkdst
	add	dl, 256/2/clm		;; Test loop counter
	jnc	c7b			;; Iterate if necessary
	bump	rsi, -2*clm*64+2*blkdst	;; Next source pointer
	bump	rcx, -2*clm*64+2*clmblkdst ;; Next dest pointer
	bump	rdi, XMM_SCD		;; Next sine/cosine pointer
	add	al, 256/4		;; Test loop counter
	jnc	c7b			;; Iterate if necessary
	bump	rcx, -8*clmblkdst+clmblkdst8 ;; Next dest pointer
	sub	al, 1			;; Test outer loop counter
	jnz	c7b			;; Iterate if necessary
	lea	rcx, [rsi-20*2*blkdst]	;; Create prefetch pointer
	mov	rsi, scratch_area	;; Restore source pointer
	end_timer 16

;; Do inverse FFT levels 6
;; On input the 64-byte cache lines hold these data values:
;;	0K	4K	8K	12K
;;	2K	...
;;	16K	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0K	8K	16K	24K
;;	2K	...
;;	...
;;	6K
;;	32K	...
;;	...
;;
;; Do 4 eight_reals_unfft_1 macros
;;	distance between fft data elements is 8K
;;	do 2 macros each processing 16 data values
;;
;; Do 36 two_two_complex_unfft macros
;;	distance between fft data elements is 8K
;;	do 18 macros each processing 16 data values

;; Do inverse FFT levels 4,5
;; On input the 64-byte cache lines hold these data values:
;;	0K	8K	16K	24K
;;	2K	...
;;	...
;;	6K
;;	32K	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0K	16K	32K	48K
;;	2K	...
;;	...
;;	14K
;;	64K	96K	128K	160K
;;	...
;;
;; Do 8 eight_reals_unfft_2 macros
;;	distance between fft data elements is 16K
;;	do 4 macros each processing 16 data values
;;
;; Do 32 four_complex_unfft macros
;;	distance between fft data elements is 16K
;;	do 8 macros each processing 32 data values

;; Do inverse FFT levels 1,2,3
;; On input the 64-byte cache lines hold these data values:
;;	0K	16K	32K	48K
;;	2K	...
;;	...
;;	14K
;;	64K	96K	128K	160K
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0K	16K	160K	176K
;;	2K	...
;;	...
;;	14K
;;	32K	96K	192K	256K
;;	34K
;;	...
;;	94K
;;
;; Do 64 five_reals_last_unfft macros
;;	distance between fft data elements is 64K
;;	do 8 macros each processing 40 data values

	start_timer 17
	mov	rdi, sincos2		;; Load sin/cos pointer
	mov	rbx, sincos3		;; Load sin/cos pointer
	mov	eax, 2			;; 2 iterations

c2a:	xtouchlines 10			;; Preload TLBs

c4b:	x2cl_eight_reals_unfft_1 rsi, 64, 2*clmblkdst
	xloop1sp c4b, 4*clmblkdst

	mov	ah, 256/2+5		;; 9 iterations
c5b:	x2cl_two_complex_unfft rsi, 64, 2*clmblkdst
	add	al, 256/2		;; Test loop counter
	jnc	c5b			;; Iterate if necessary
	bump	rsi, -2*64+4*clmblkdst	;; Next source pointer
	bump	rdi, XMM_SCD		;; Next sine/cosine pointer
	add	ah, 256/2		;; Test loop counter
	jnc	c5b			;; Iterate if necessary
	bump	rsi, -2*4*clmblkdst+clmblkdst8 ;; Next source pointer
	sub	ah, 1			;; Test outer loop counter
	jnz	c5b			;; Iterate if necessary
	bump	rdi, -9*XMM_SCD		;; Restore sine/cosine pointer
	L2prefetch128 [rcx][rbp]
pfing	bump	rcx, blkdst		;; Next prefetch pointer

	bump	rsi, -5*clmblkdst8	;; Restore source pointer
c2b:	x2cl_half_eight_reals_unfft_2 rsi, 64, 4*clmblkdst
	add	al, 256/2		;; Test loop counter
	jnc	c2b			;; Iterate if necessary
	bump	rsi, -2*64+2*clmblkdst
	add	ah, 256/2		;; Test loop counter
	jnc	c2b			;; Iterate if necessary

	bump	rsi, -2*2*clmblkdst+clmblkdst8 ;; Next source pointer
c3b:	x4cl_four_complex_unfft rsi, 64, 4*clmblkdst, clmblkdst8, rbx
	xloop1sp c3b, 2*clmblkdst
	add	edx, 80000000h		;; Test loop counter
	jnc	c3b			;; Iterate if necessary
	bump	rsi, -2*2*clmblkdst+2*clmblkdst8
	bump	rbx, XMM_SCD		;; Next sine/cosine pointer
	add	ah, 256/2		;; Test outer loop counter
	jnc	c3b			;; Iterate if necessary
	bump	rbx, -2*XMM_SCD		;; Restore sine/cosine pointer

	bump	rsi, -2*2*clmblkdst8-clmblkdst8 ;; Restore source pointer
	xfive_reals_unfft_preload	;; Preload consts for five real macros
c1b:	x5cl_five_reals_last_unfft rsi, 64, clmblkdst8
	xloop1sp c1b, 2*clmblkdst
	add	ah, 256/4
	jnc	c1b

	IF clm GT 1
	bump	rsi, -4*2*clmblkdst+2*64;; Next source pointer
	bump	rcx, -10*blkdst+128	;; Next prefetch pointer
	add	dl, 256/clm		;; clm iterations
	jnc	c4b
	bump	rsi, -clm*2*64+clmblkdst;; Next source pointer
	bump	rcx, -clm*128+10*blkdst	;; Next prefetch pointer
	ELSE
	bump	rsi, -4*2*clmblkdst+clmblkdst;; Next source pointer
	ENDIF

	sub	al, 1			;; Test loop counter
	jnz	c2a			;; Iterate if necessary
	end_timer 17

;; Normalize these values

	pass1_normalize 1, yes_fft

;; Copy data from the scratch area back to the FFT data area

	copy_scratch_data_to_fft 40
	jmp	endlp			;; Skip FFT code if flag not set

;; Fire up auxiliary threads to do pass 1 forward FFT

pass1:	pass1_forward_fft_setup b0b

;; Copy data from the FFT data area back to the scratch area

b0b:	copy_fft_data_to_scratch 40

;; Do FFT levels 1,2,3
;;
;; On input the 64-byte cache lines hold these data values:
;;	0K	16K	160K	176K
;;	2K	...
;;	...
;;	14K
;;	32K	96K	192K	256K
;;	34K
;;	...
;;	94K
;; On output the 64-byte cache lines hold these data values:
;;	0K	16K	32K	48K
;;	2K	...
;;	...
;;	14K
;;	64K	128K	192K	256K
;;	66K
;;	...
;;	126K
;;
;; Do 64 five_reals_first_fft macros (FFT levels 1,2,3)
;;	distance between fft data elements is 64K
;;	do 8 iterations each processing 40 data values

;; Do FFT levels 4,5
;; Values 0K-63K is real data, 64K-319K is complex data.
;;
;; On input the 64-byte cache lines hold these data values:
;;	0K	16K	32K	48K
;;	2K	...
;;	...
;;	14K
;;	64K	128K	192K	256K
;;	66K
;;	...
;;	126K
;; On output the 64-byte cache lines hold these data values:
;;	0K	16K	32K	48K
;;	2K	...
;;	...
;;	14K	...
;;	64K	...
;;	...
;;
;; Do 8 eight_reals_fft_2 macros
;;	distance between fft data elements is 16K
;;	do 8 macros each processing 8 data values
;;
;; Do 32 four_complex_fft macros
;;	distance between fft data elements is 16K
;;	do 8 macros each processing 32 data values

;; Do FFT levels 6
;; Values 0K-15K is real data, 16K-31K is semi-real data, 32K-319K is
;; complex data.
;;
;; On input the 64-byte cache lines hold these data values:
;;	0K	16K	32K	48K
;;	2K	...
;;	...
;;	14K	...
;;	64K	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0K	8K	16K	24K
;;	2K	...
;;	...
;;	6K	...
;;	32K	...
;;	...
;;
;; Do 4 eight_reals_fft_1 macros
;; Do 4 two_two_complex_fft macros
;;	distance between fft data elements is 8K
;;	do 4 macros each processing 16 data values
;;
;; Do 32 two_two_complex_fft macros
;;	distance between fft data elements is 8K
;;	do 16 macros each processing 16 data values

yes_fft:start_timer 21
	mov	rsi, scratch_area	;; Get address of scratch area
	mov	rcx, DATA_ADDR		;; Load source address for prefetching
pfing	mov	rbp, DATA_PREFETCH	;; Set prefetch distance
pfing	sub	rbp, rcx
pfing	bump	rcx, 20*blkdst		;; Create prefetching pointer
	mov	rdi, sincos3		;; Load sin/cos pointer
	mov	rbx, sincos2		;; Load sin/cos pointer
	mov	eax, 4			;; 4 iterations
	sub	rdx, rdx

b1a:	xtouchlines 5			;; Preload TLBs

b1aa:	xfive_reals_fft_preload		;; Preload consts for five real macros
b1b:	x5cl_five_reals_fft rsi, 64, clmblkdst8
	xloop1sp b1b, 4*clmblkdst
	add	ah, 256/2
	jnc	b1b

	bump	rsi, -2*4*clmblkdst	;; Restore source pointer
b2b:	x1cl_half_eight_reals_fft_2 rsi, 64
	add	al, 256/2		;; Test inner loop counter
	jnc	b2b			;; Iterate if necessary
	bump	rsi, -2*64+4*clmblkdst	;; Next source pointer
	add	ah, 256/2
	jnc	b2b

	bump	rsi, -2*4*clmblkdst+clmblkdst8;; Next source pointer
b3b:	x4cl_four_complex_fft rsi, 64, clmblkdst8, 2*clmblkdst8, rdi
	xloop1sp b3b, 4*clmblkdst
	add	ah, 256/2
	jnc	b3b

	bump	rsi, -2*4*clmblkdst-clmblkdst8;; Restore source pointer
b4b:	x2cl_eight_reals_fft_1 rsi, 64, 4*clmblkdst
	add	al, 256/2		;; Test inner loop counter
	jnc	b4b			;; Iterate if necessary

	bump	rsi, -2*64+clmblkdst8	;; Next source pointer
	bump	rbx, XMM_SCD		;; Next sine/cosine pointer
b5b:	x2cl_two_complex_fft rsi, 64, 4*clmblkdst, rbx
	add	al, 256/2		;; Test inner loop counter
	jnc	b5b			;; Iterate if necessary
	bump	rsi, -2*64+clmblkdst8	;; Next source pointer
	bump	rbx, 2*XMM_SCD		;; Next sine/cosine pointer
	add	ah, 256/4		;; Test outer loop counter
	jnc	b5b			;; Iterate if necessary
	L2prefetch128 [rcx][rbp]
pfing	bump	rcx, blkdst		;; Next prefetch pointer
	bump	rbx, -9*XMM_SCD		;; Restore sine/cosine pointer

	IF clm GT 1
	bump	rsi, -5*clmblkdst8+2*64	;; Next source pointer
	bump	rcx, -5*blkdst+128	;; Next prefetch pointer
	add	dl, 256/clm		;; clm iterations
	jnc	b1aa
	bump	rsi, -clm*2*64+clmblkdst;; Next source pointer
	bump	rcx, -clm*128+5*blkdst	;; Next prefetch pointer
	ELSE
	bump	rsi, -5*clmblkdst8+clmblkdst;; Next source pointer
	ENDIF

	sub	al, 1			;; Test loop counter
	jnz	b1a			;; Iterate if necessary
	bump	rsi, -4*clmblkdst	;; Restore source pointer
pfing	bump	rcx, -40*blkdst		;; Restore prefetch pointer
	end_timer 21

;; Do FFT levels 7,8
;; Values 0K-7K is real data, 8K-15K is semi-real data, 16K-319K is
;; complex data.
;;
;; On input the 64-byte cache lines hold these data values:
;;	0K	8K	16K	24K
;;	2K	...
;;	...
;;	6K	...
;;	32K	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0K	2K	4K	6K
;;	8K	...
;;	...

;; Do 1 eight_reals_fft_2 macros
;; Do 1 nop_two_two_complex_fft_2 macros
;; Do 2 four_complex_fft macros
;;	distance between fft data elements is 2K
;;	do 1 macros each processing 32 data values

	start_timer 24
	mov	rdi, sincos1		;; Load sin/cos pointer
b6b:	g4cl_eight_reals_fft_2 rsi, 64, clmblkdst, 2*clmblkdst, rcx, 64, blkdst, 2*blkdst
	add	al, 256/2/clm		;; Test inner loop counter
	jnc	b6b			;; Iterate if necessary

;; Do 36 four_complex_fft macros
;;	distance between fft data elements is 2K
;;	do 9 macros each processing 32 data values

	bump	rsi, -2*clm*64+4*clmblkdst;; Next source pointer
	bump	rcx, -2*clm*64+4*blkdst	;; Next dest pointer
	bump	rdi, XMM_SCD		;; Next sine/cosine pointer
	mov	eax, 5*256+1		;; 9 iterations of 2*clm
b7b:	g4cl_four_complex_fft rsi, 64, clmblkdst, 2*clmblkdst, rcx, 64, blkdst, 2*blkdst
	L2prefetch128 [rcx-64][rbp]
	L2prefetch128 [rcx-64+blkdst][rbp]
	bump	rbp, -64+2*blkdst
	add	al, 256/2		;; Test inner loop counter
	jnc	b7b			;; Iterate if necessary
	bump	rbp, 128-4*blkdst
	IF clm NE 1
	add	dl, 256/clm
	jnc	b7b
	ENDIF
	bump	rsi, -2*clm*64+4*clmblkdst;; Next source pointer
	bump	rcx, -2*clm*64+4*blkdst	;; Next dest pointer
	bump	rdi, 2*XMM_SCD		;; Next sine/cosine pointer
	sub	al, 1			;; Test loop counter
	jnz	b7b			;; Iterate if necessary
	bump	rsi, -8*clmblkdst+clmblkdst8
	sub	ah, 1			;; Test outer loop counter
	mov	al, 2
	jnz	b7b			;; Iterate if necessary
	end_timer 24

;; Work on next 4 sets of 160 values

endlp:	xpass1_get_next_block pass2, c0b, b0b ;; Get next block and jump
	done:	xfft_footer
	ENDM


;; Perform a 384K-element FFT.  This is done in two passes.  Pass 1
;; does 8 levels, pass 2 uses common code to perform the last 11 levels.

hg_pass1sclevels8pfa6 MACRO fftname, pass2_levels, clmarg
	LOCAL	b0b, b1a, b1b, b2b, b3b, b4b, b5b, b6b, b7b
	LOCAL	c0b, c1b, c2a, c2b, c3b, c4b, c5b, c6b, c7b
	LOCAL	pass1, pass2, endlp, done, yes_fft

	set_FFT_constants fftname, 0, 0, pass2_levels, clmarg
	xfft_header pass1

;; Do the last 11 FFT levels
;;	do 1 pass2_11_levels_real and 47 pass2_11_levels_complex

pass2:	start_timer 0
	call	xpass2name
	end_timer 0
	cmp	ffttype, 1		;; We're done if FFTing only
	je	done

;; Do the inverse FFT

	pass1_inverse_fft_setup c0b

;; Do inverse FFT levels 7,8
;; On input the 64-byte cache lines hold these data values:
;;	0K	2K	4K	6K
;;	8K	10K	12K	14K
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0K	4K	8K	12K
;;	2K	...
;;	16K	...
;;	...

;; Do 1 eight_reals_unfft_2 macros
;; Do 1 nop_two_two_complex_unfft_2 macros
;;	distance between fft data elements is 2K
;;	do 1 macros each processing 16 data values

c0b:	start_timer 16
	mov	rsi, DATA_ADDR		;; Get source address
	mov	rcx, scratch_area	;; Get address of scratch area
	sub	rax, rax
	sub	rdx, rdx
pfing	mov	rbp, DATA_PREFETCH	;; Set prefetch distance
pfing	sub	rbp, rsi
c6b:	g2cl_eight_reals_unfft_2 rsi, 64, blkdst, rcx, 64, clmblkdst
	add	al, 256/2/clm		;; Test loop counter
	jnc	c6b			;; Iterate if necessary

;; Do 46 four_complex_unfft macros
;;	distance between fft data elements is 2K
;;	do 23 macros each processing 16 data values

	bump	rsi, -2*clm*64+2*blkdst	;; Next source pointer
	bump	rcx, -2*clm*64+2*clmblkdst;; Next dest pointer
	mov	rdi, sincos1		;; Load sin/cos pointer
	mov	al, 256/4+6		;; 23 iterations of 2*clm
c7b:	g2cl_four_complex_unfft rsi, 64, blkdst, rcx, 64, clmblkdst
	add	dl, 256/2/clm		;; Test loop counter
	jnc	c7b			;; Iterate if necessary
	bump	rsi, -2*clm*64+2*blkdst	;; Next source pointer
	bump	rcx, -2*clm*64+2*clmblkdst;; Next dest pointer
	bump	rdi, XMM_SCD		;; Next sine/cosine pointer
	add	al, 256/4		;; Test loop counter
	jnc	c7b			;; Iterate if necessary
	bump	rcx, -8*clmblkdst+clmblkdst8
	sub	al, 1			;; Test outer loop counter
	jnz	c7b			;; Iterate if necessary
	lea	rcx, [rsi-24*2*blkdst]	;; Create prefetch pointer
	mov	rsi, scratch_area	;; Restore source pointer
	end_timer 16

;; Do inverse FFT levels 6
;; On input the 64-byte cache lines hold these data values:
;;	0K	4K	8K	12K
;;	2K	...
;;	16K	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0K	8K	16K	24K
;;	2K	...
;;	...
;;	6K
;;	32K	...
;;	...
;;
;; Do 4 eight_reals_unfft_1 macros
;;	distance between fft data elements is 8K
;;	do 2 macros each processing 16 data values
;;
;; Do 44 two_two_complex_unfft macros
;;	distance between fft data elements is 8K
;;	do 22 macros each processing 16 data values

;; Do inverse FFT levels 4,5
;; On input the 64-byte cache lines hold these data values:
;;	0K	8K	16K	24K
;;	2K	...
;;	...
;;	6K
;;	32K	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0K	32K	64K	96K
;;	2K	...
;;	...
;;	30K
;;	128K	...
;;	...
;;
;; Do 8 eight_reals_unfft_2 macros
;; Do 8 nop_two_two_complex_unfft_2 macros
;;	distance between fft data elements is 16K
;;	do 4 macros each processing 32 data values
;;
;; Do 32 four_complex_unfft macros
;;	distance between fft data elements is 16K
;;	do 8 macros each processing 32 data values

;; Do inverse FFT levels 1,2,3
;; On input the 64-byte cache lines hold these data values:
;;	0K	32K	64K	96K
;;	2K	...
;;	...
;;	30K
;;	128K	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0K	96K	192K	288K
;;	...
;;	94K
;;
;; Do 64 six_reals_last_unfft macros
;;	distance between fft data elements is 64K
;;	do 16 macros each processing 24 data values

	start_timer 17
	mov	rdi, sincos2		;; Load sin/cos pointer
	mov	rbx, sincos3		;; Load sin/cos pointer
	mov	eax, 2			;; 2 iterations
	xsix_reals_unfft_preload

c2a:	xtouchlines 12			;; Preload TLB

c4b:	x2cl_eight_reals_unfft_1 rsi, 64, 2*clmblkdst
	add	al, 256/2		;; Test loop counter
	jnc	c4b			;; Iterate if necessary

	bump	rsi, -2*64+4*clmblkdst	;; Next source pointer
	mov	ah, 256/2+6		;; 11 iterations
c5b:	x2cl_two_complex_unfft rsi, 64, 2*clmblkdst
	add	al, 256/2		;; Test loop counter
	jnc	c5b			;; Iterate if necessary
	bump	rsi, -2*64+4*clmblkdst	;; Next source pointer
	bump	rdi, XMM_SCD		;; Next sine/cosine pointer
	add	ah, 256/2		;; Test loop counter
	jnc	c5b			;; Iterate if necessary
	bump	rsi, -2*4*clmblkdst+clmblkdst8;; Next source pointer
	sub	ah, 1			;; Test outer loop counter
	jnz	c5b			;; Iterate if necessary
	bump	rdi, -11*XMM_SCD	;; Restore sine/cosine pointer

	bump	rsi, -6*clmblkdst8	;; Restore source pointer
c2b:	x4cl_eight_reals_unfft_2 rsi, 64, 4*clmblkdst, clmblkdst8
	add	al, 256/2		;; Test loop counter
	jnc	c2b			;; Iterate if necessary
	bump	rsi, -2*64+2*clmblkdst
	add	ah, 256/2		;; Test loop counter
	jnc	c2b			;; Iterate if necessary

	bump	rsi, -2*2*clmblkdst+2*clmblkdst8;; Next source pointer
c3b:	x4cl_four_complex_unfft rsi, 64, 4*clmblkdst, clmblkdst8, rbx
	xloop1sp c3b, 2*clmblkdst
	add	edx, 80000000h		;; Test loop counter
	jnc	c3b			;; Iterate if necessary
	bump	rsi, -2*2*clmblkdst+2*clmblkdst8
	bump	rbx, XMM_SCD		;; Next sine/cosine pointer
	add	ah, 256/2
	jnc	c3b
	bump	rbx, -2*XMM_SCD		;; Restore sine/cosine pointer

	bump	rsi, -3*2*clmblkdst8	;; Restore source pointer
	mov	ah, 2
c1b:	x3cl_six_reals_last_unfft rsi, 64, 2*clmblkdst8
	xloop1sp c1b, 2*clmblkdst
	add	ah, 256/4
	jnc	c1b
	bump	rsi, -4*2*clmblkdst+clmblkdst8;; Next source pointer
	sub	ah, 1
	jnz	c1b

	IF clm GT 1
	bump	rsi, -2*clmblkdst8+2*64	;; Next source pointer
	bump	rcx, -12*blkdst+128	;; Next prefetch pointer
	add	dl, 256/clm		;; clm iterations
	jnc	c4b
	bump	rsi, -clm*2*64+clmblkdst;; Next source pointer
	bump	rcx, -clm*128+12*blkdst	;; Next prefetch pointer
	ELSE
	bump	rsi, -2*clmblkdst8+clmblkdst;; Next source pointer
	ENDIF

	sub	al, 1			;; Test loop counter
	jnz	c2a			;; Iterate if necessary
	end_timer 17

;; Normalize these values

	pass1_normalize 1, yes_fft

;; Copy data from the scratch area back to the FFT data area

	copy_scratch_data_to_fft 48
	jmp	endlp			;; Skip FFT code if flag not set

;; Fire up auxiliary threads to do pass 1 forward FFT

pass1:	pass1_forward_fft_setup b0b

;; Copy data from the FFT data area back to the scratch area

b0b:	copy_fft_data_to_scratch 48

;; Do FFT levels 1,2,3
;;
;; On input the 64-byte cache lines hold these data values:
;;	0K	96K	192K	288K
;;	...
;;	94K
;; On output the 64-byte cache lines hold these data values:
;;	0K	32K	64K	96K
;;	2K	...
;;	...
;;	30K
;;	128K	192K	256K	320K
;;	130K
;;	...
;;	190K
;;
;; Do 64 six_reals_first_fft macros (FFT levels 1,2,3)
;;	distance between fft data elements is 64K
;;	do 16 iterations each processing 24 data values

;; Do FFT levels 4,5
;; Values 0K-63K is real data, 64K-127K is semi-real data, and
;; 128K-383K is complex data.
;;
;; On input the 64-byte cache lines hold these data values:
;;	0K	32K	64K	96K
;;	2K	...
;;	...
;;	30K
;;	128K	192K	256K	320K
;;	130K
;;	...
;;	190K
;; On output the 64-byte cache lines hold these data values:
;;	0K	16K	32K	48K
;;	2K	...
;;	...
;;	14K	...
;;	64K	...
;;	...
;;
;; Do 8 eight_reals_fft_2 macros
;; Do 8 nop_two_two_complex_fft_2 macros
;;	distance between fft data elements is 16K
;;	do 8 macros each processing 16 data values
;;
;; Do 32 four_complex_fft macros
;;	distance between fft data elements is 16K
;;	do 8 macros each processing 32 data values

;; Do FFT levels 6
;; Values 0K-15K is real data, 16K-31K is semi-real data, 32K-383K is
;; complex data.
;;
;; On input the 64-byte cache lines hold these data values:
;;	0K	16K	32K	48K
;;	2K	...
;;	...
;;	14K	...
;;	64K	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0K	8K	16K	24K
;;	2K	...
;;	...
;;	6K	...
;;	32K	...
;;	...
;;
;; Do 4 eight_reals_fft_1 macros
;; Do 4 two_two_complex_fft macros
;;	distance between fft data elements is 8K
;;	do 4 macros each processing 16 data values
;;
;; Do 40 two_two_complex_fft macros
;;	distance between fft data elements is 8K
;;	do 20 macros each processing 16 data values

yes_fft:start_timer 21
	mov	rsi, scratch_area	;; Get address of scratch area
	mov	rcx, DATA_ADDR		;; Load source address for prefetching
pfing	mov	rbp, DATA_PREFETCH	;; Set prefetch distance
pfing	sub	rbp, rcx
pfing	bump	rcx, 24*blkdst		;; Create prefetch pointer
	mov	rdi, sincos3		;; Load sin/cos pointer
	mov	rbx, sincos2		;; Load sin/cos pointer
	mov	eax, 4			;; 4 iterations
	sub	rdx, rdx
	xsix_reals_fft_preload

b1a:	xtouchlines 6			;; Preload TLBs

b1b:	x3cl_six_reals_fft rsi, 64, 2*clmblkdst8
	xloop1sp b1b, 4*clmblkdst
	add	ah, 256/2
	jnc	b1b
	bump	rsi, -2*4*clmblkdst+clmblkdst8;; Next source pointer
	add	eax, 80000000h		;; 2 iterations
	jnc	b1b

	bump	rsi, -2*clmblkdst8	;; Restore source pointer
b2b:	x2cl_eight_reals_fft_2 rsi, 64, clmblkdst8
	add	al, 256/2		;; Test inner loop counter
	jnc	b2b			;; Iterate if necessary
	bump	rsi, -2*64+4*clmblkdst	;; Next source pointer
	add	ah, 256/2
	jnc	b2b

	bump	rsi, -2*4*clmblkdst+2*clmblkdst8;; Next source pointer
b3b:	x4cl_four_complex_fft rsi, 64, clmblkdst8, 2*clmblkdst8, rdi
	xloop1sp b3b, 4*clmblkdst
	add	ah, 256/2
	jnc	b3b

	bump	rsi, -2*4*clmblkdst-2*clmblkdst8;; Restore source pointer
b4b:	x2cl_eight_reals_fft_1 rsi, 64, 4*clmblkdst
	add	al, 256/2		;; Test inner loop counter
	jnc	b4b			;; Iterate if necessary

	bump	rsi, -2*64+clmblkdst8	;; Load source pointer
	bump	rbx, XMM_SCD		;; Next sine/cosine pointer
	mov	ah, 5			;; 5 iterations
b5b:	x2cl_two_complex_fft rsi, 64, 4*clmblkdst, rbx
	add	al, 256/2		;; Test inner loop counter
	jnc	b5b			;; Iterate if necessary
	bump	rsi, -2*64+clmblkdst8	;; Next source pointer
	bump	rbx, 2*XMM_SCD		;; Next sine/cosine pointer
	sub	ah, 1			;; Test outer loop counter
	jnz	b5b			;; Iterate if necessary
	bump	rbx, -11*XMM_SCD	;; Restore sine/cosine pointer

	IF clm GT 1
	bump	rsi, -6*clmblkdst8+2*64	;; Next source pointer
	bump	rcx, -6*blkdst+128	;; Next prefetch pointer
	add	dl, 256/clm		;; clm iterations
	jnc	b1b
	bump	rsi, -clm*2*64+clmblkdst;; Next source pointer
	bump	rcx, -clm*128+6*blkdst	;; Next prefetch pointer
	ELSE
	bump	rsi, -6*clmblkdst8+clmblkdst;; Next source pointer
	ENDIF

	sub	al, 1			;; Test loop counter
	jnz	b1a			;; Iterate if necessary
	bump	rsi, -4*clmblkdst	;; Restore source pointer
pfing	bump	rcx, -48*blkdst		;; Restore prefetch pointer
	end_timer 21

;; Do FFT levels 7,8
;; Values 0K-7K is real data, 8K-15K is semi-real data, 16K-383K is
;; complex data.
;;
;; On input the 64-byte cache lines hold these data values:
;;	0K	8K	16K	24K
;;	2K	...
;;	...
;;	6K	...
;;	32K	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0K	2K	4K	6K
;;	8K	...
;;	...

;; Do 1 eight_reals_fft_2 macros
;; Do 1 nop_two_two_complex_fft_2 macros
;; Do 2 four_complex_fft macros
;;	distance between fft data elements is 2K
;;	do 1 macros each processing 32 data values

	start_timer 24
	mov	rdi, sincos1		;; Load sin/cos pointer
b6b:	g4cl_eight_reals_fft_2 rsi, 64, clmblkdst, 2*clmblkdst, rcx, 64, blkdst, 2*blkdst
	add	al, 256/2/clm		;; Test inner loop counter
	jnc	b6b			;; Iterate if necessary

;; Do 44 four_complex_fft macros
;;	distance between fft data elements is 2K
;;	do 11 macros each processing 32 data values

	bump	rsi, -2*clm*64+4*clmblkdst;; Next source pointer
	bump	rcx, -2*clm*64+4*blkdst	;; Next dest pointer
	bump	rdi, XMM_SCD		;; Next sine/cosine pointer
	mov	eax, 6*256+1		;; 11 iterations of 2*clm
b7b:	g4cl_four_complex_fft rsi, 64, clmblkdst, 2*clmblkdst, rcx, 64, blkdst, 2*blkdst
	L2prefetch128 [rcx-64][rbp]
	L2prefetch128 [rcx-64+blkdst][rbp]
	bump	rbp, -64+2*blkdst
	add	al, 256/2		;; Test inner loop counter
	jnc	b7b			;; Iterate if necessary
	bump	rbp, 128-4*blkdst
	IF clm NE 1
	add	dl, 256/clm
	jnc	b7b
	ENDIF
	bump	rsi, -2*clm*64+4*clmblkdst;; Next source pointer
	bump	rcx, -2*clm*64+4*blkdst	;; Next dest pointer
	bump	rdi, 2*XMM_SCD	;; Next sine/cosine pointer
	sub	al, 1			;; Test outer loop counter
	jnz	b7b
	bump	rsi, -8*clmblkdst+clmblkdst8
	sub	ah, 1			;; Test outer loop counter
	mov	al, 2
	jnz	b7b			;; Iterate if necessary
	end_timer 24

;; Work on next 4 sets of 192 values

endlp:	xpass1_get_next_block pass2, c0b, b0b ;; Get next block and jump
	done:	xfft_footer
	ENDM


;; Perform a 384K-element negacyclic FFT.  This is done in two passes.  Pass 1
;; does 8 levels, pass 2 uses common code to perform the last 11 levels.

hg_pass1sclevels8complex3 MACRO fftname, pass2_levels, clmarg
	LOCAL	b0b, b1a, b1b, b2a, b2b, b3a, b3b, b5a, b5b, b7b
	LOCAL	c0b, c1a, c1b, c3a, c3b, c5a, c5b, c7b
	LOCAL	pass1, pass2, endlp, done, yes_fft

	set_FFT_constants fftname, 0, 1, pass2_levels, clmarg
	xfft_header pass1

;; Do the last 11 FFT levels
;;	do 48 pass2_11_levels_complex

pass2:	start_timer 0
	call	xpass2name
	end_timer 0
	cmp	ffttype, 1		;; We're done if FFTing only
	je	done

;; Do the inverse FFT

	pass1_inverse_fft_setup c0b

;; Do inverse FFT levels 7,8
;; On input the 64-byte cache lines hold these data values:
;;	0K	2K	4K	6K
;;	8K	10K	12K	14K
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0K	4K	8K	12K
;;	2K	...
;;	16K	...
;;	...

;; Do 48 four_complex_unfft macros
;;	distance between fft data elements is 2K
;;	do 24 macros each processing 16 data values

c0b:	start_timer 16
	mov	rsi, DATA_ADDR		;; Get source address
	mov	rcx, scratch_area	;; Get address of scratch area
	mov	rdi, sincos1		;; Load sin/cos pointer
	mov	eax, 6			;; 6 iterations of 4 iters of 2*clm
	sub	rdx, rdx
pfing	mov	rbp, DATA_PREFETCH	;; Set prefetch distance
pfing	sub	rbp, rsi
c7b:	g2cl_four_complex_unfft rsi, 64, blkdst, rcx, 64, clmblkdst
	add	dl, 256/2/clm		;; Test loop counter
	jnc	c7b			;; Iterate if necessary
	bump	rsi, -2*clm*64+2*blkdst	;; Next source pointer
	bump	rcx, -2*clm*64+2*clmblkdst;; Next dest pointer
	bump	rdi, XMM_SCD		;; Next sine/cosine pointer
	add	al, 256/4		;; Test loop counter
	jnc	c7b
	bump	rcx, -4*2*clmblkdst+clmblkdst8
	sub	al, 1			;; Test outer loop counter
	jnz	c7b			;; Iterate if necessary
	lea	rcx, [rsi-24*2*blkdst]	;; Create prefetch pointer
	mov	rsi, scratch_area	;; Restore source pointer
	end_timer 16

;; Do inverse FFT levels 5,6
;; On input the 64-byte cache lines hold these data values:
;;	0K	4K	8K	12K
;;	2K	...
;;	16K	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0K	16K	32K	48K
;;	2K	...
;;	...
;;	14K
;;	64K	...
;;	...

;; Do 48 four_complex_unfft macros
;;	distance between fft data elements is 8K
;;	do 12 macros each processing 32 data values

	start_timer 17
	mov	rdi, sincos2		;; Load sin/cos pointer
	mov	eax, 6*256+2		;; 6 iterations of 2 iters of 2*clm
c5a:	xtouch	[rcx+(clm-1)*128]	;; Preload TLB
c5b:	x4cl_four_complex_unfft rsi, 64, 2*clmblkdst, 4*clmblkdst, rdi
	xloop1	c5b, clmblkdst
	sub	al, 1			;; Test middle loop counter
	jnz	c5a			;; Iterate if necessary
	bump	rsi, -2*clmblkdst+clmblkdst8;; Next source pointer
	bump	rdi, XMM_SCD		;; Next sine/cosine pointer
	sub	eax, 256		;; Test outer loop counter
	mov	al, 2			;; Load middle loop counter
	jnz	c5a			;; Iterate if necessary
	bump	rsi, -6*clmblkdst8	;; Restore source pointer
	end_timer 17

;; Do inverse FFT levels 4
;; On input the 64-byte cache lines hold these data values:
;;	0K	16K	32K	48K
;;	2K	...
;;	...
;;	14K
;;	64K	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0K	32K	64K	96K
;;	2K	...
;;	...
;;	30K
;;	128K	...
;;	...

;; Do 48 two_two_complex_unfft macros
;;	distance between fft data elements is 32K
;;	do 24 macros each processing 16 data values

	start_timer 18
	mov	rdi, sincos3		;; Load sin/cos pointer
	mov	eax, 3*256+2		;; 3 iterations of 2 iters of 4*2*clm
c3a:	xtouch	[rcx+(clm-1)*128]	;; Preload TLB
c3b:	x2cl_two_complex_unfft rsi, 64, clmblkdst8
	xloop4	c3b, clmblkdst		;; 4*2*clm iterations
	sub	al, 1			;; Test loop counter
	jnz	c3a			;; Iterate if necessary
	bump	rsi, -2*4*clmblkdst+2*clmblkdst8;; Next source pointer
	bump	rdi, XMM_SCD		;; Next sine/cosine pointer
	sub	eax, 256		;; Test outer loop counter
	mov	al, 2			;; Reload middle counter
	jnz	c3a			;; Iterate if necessary
	bump	rsi, -3*2*clmblkdst8	;; Restore source pointer
	end_timer 18

;; Do inverse FFT levels 1,2,3
;; On input the 64-byte cache lines hold these data values:
;;	0K	32K	64K	96K
;;	2K	...
;;	...
;;	30K
;;	128K	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0K	96K	192K	288K
;;	...
;;	94K

;; Do 64 three_complex_last_unfft macros
;;	distance between fft data elements is 64K
;;	do 16 macros each processing 24 data values

	start_timer 19
	mov	rdi, plus1_premults	;; Address of premultiplier table
	mov	eax, 2*256+4		;; 8 iterations of 2*2*clm
c1a:	xtouch	[rcx+(clm-1)*128]	;; Preload TLB
c1b:	x3cl_three_complex_last_unfft rsi, 64, 2*clmblkdst8
	xloop2edi c1b, clmblkdst, 192	;; 2*2*clm iterations
	sub	al, 1			;; Test loop counter
	jnz	c1a			;; Iterate if necessary
	bump	rsi, -4*2*clmblkdst+clmblkdst8;; Next source pointer
	sub	ah, 1			;; Test outer loop counter
	mov	al, 4			;; Load middle loop counter
	jnz	c1a
	end_timer 19

;; Normalize these values

	pass1_normalize 1, yes_fft

;; Copy data from the scratch area back to the FFT data area

	copy_scratch_data_to_fft 48
	jmp	endlp			;; Skip FFT code if flag not set

;; Fire up auxiliary threads to do pass 1 forward FFT

pass1:	pass1_forward_fft_setup b0b

;; Copy data from the FFT data area back to the scratch area

b0b:	copy_fft_data_to_scratch 48

;; Do FFT levels 1,2,3
;;
;; On input the 64-byte cache lines hold these data values:
;;	0K	96K	192K	288K
;;	...
;;	94K
;; On output the 64-byte cache lines hold these data values:
;;	0K	32K	64K	96K
;;	2K	...
;;	...
;;	30K
;;	128K	192K	256K	320K
;;	130K
;;	...
;;	190K

;; Do 64 three_complex_first_fft macros (FFT levels 1,2,3)
;;	distance between fft data elements is 64K
;;	do 16 iterations each processing 24 data values

yes_fft:start_timer 21
	mov	rsi, scratch_area	;; Get address of scratch area
	mov	rdi, plus1_premults	;; Address of premultiplier table
	mov	rcx, DATA_ADDR		;; Load source address for prefetching
pfing	mov	rbp, DATA_PREFETCH	;; Set prefetch distance
pfing	sub	rbp, rcx
pfing	bump	rcx, 26*blkdst		;; Create prefetching pointer
	mov	eax, 2			;; 2 iters of 2 iters of 4*2*clm
	sub	rdx, rdx
b1a:	xtouch	[rcx+(clm-1)*128]	;; Preload TLB
b1b:	x3cl_three_complex_fft rsi, 64, 2*clmblkdst8
	xloop4edi b1b, clmblkdst, 192	;; 2 * 2*clm iterations
	add	eax, 80000000h		;; Test loop counter
	jnc	b1a			;; Iterate if necessary
	bump	rsi, -2*4*clmblkdst+clmblkdst8;; Next source pointer
	sub	al, 1			;; Test loop counter
	jnz	b1a			;; Iterate if necessary
	bump	rsi, -2*clmblkdst8	;; Restore source pointer
	end_timer 21

;; Do FFT levels 4
;;
;; On input the 64-byte cache lines hold these data values:
;;	0K	32K	64K	96K
;;	2K	...
;;	...
;;	30K
;;	128K	192K	256K	320K
;;	130K
;;	...
;;	190K
;; On output the 64-byte cache lines hold these data values:
;;	0K	32K	64K	96K
;;	2K	...
;;	...
;;	30K	...
;;	128K	...
;;	...

;; Do 48 two_two_complex_fft macros
;;	distance between fft data elements is 32K
;;	do 24 macros each processing 16 data values

	start_timer 22
	mov	rdi, sincos3		;; Load sin/cos pointer
	mov	al, 2			;; 2 iterations of 4*2*clm
b2a:	xtouch	[rcx+(clm-1)*128]	;; Preload TLB
b2b:	x2cl_two_complex_fft_in_place rsi, 64, clmblkdst8, rdi
	xloop4	b2b, clmblkdst		;; 4 * 2*clm iterations
	sub	al, 1			;; Test loop counter
	jnz	b2a			;; Iterate if necessary
	bump	rsi, -2*4*clmblkdst+2*clmblkdst8;; Next source pointer
	bump	rdi, XMM_SCD		;; Next sin/cos pointer
	mov	al, 2			;; 2 iters of 2 iters of 4*2*clm
b3a:	xtouch	[rcx+(clm-1)*128]	;; Preload TLB
b3b:	x2cl_two_complex_fft rsi, 64, 2*clmblkdst8, rdi
	xloop4	b3b, clmblkdst		;; 4 * 2*clm iterations
	add	eax, 80000000h		;; 2 iterations
	jnc	b3a
	bump	rsi, -2*4*clmblkdst+clmblkdst8
	sub	al, 1			;; Test outer loop counter
	jnz	b3a			;; Iterate if necessary
	bump	rsi, -2*clmblkdst8-2*clmblkdst8;; Restore source pointer
	end_timer 22

;; Do FFT levels 5,6
;;
;; On input the 64-byte cache lines hold these data values:
;;	0K	32K	64K	96K
;;	2K	...
;;	...
;;	30K	...
;;	128K	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0K	8K	16K	24K
;;	2K	...
;;	...
;;	6K	...
;;	32K	...
;;	...

;; Do 48 four_complex_fft macros
;;	distance between fft data elements is 8K
;;	do 12 macros each processing 32 data values

	start_timer 23
	mov	rdi, sincos2		;; Load sin/cos pointer
	mov	eax, 3*256+4		;; 3 iterations of 4 iters of 2*clm
b5a:	xtouch	[rcx+(clm-1)*128]	;; Preload TLB
b5b:	x4cl_four_complex_fft rsi, 64, 4*clmblkdst, clmblkdst8, rdi
	xloop1	b5b, clmblkdst		;; 2 * 2*clm iterations
	sub	al, 1			;; Test middle loop counter
	jnz	b5a			;; Iterate if necessary
	bump	rsi, -4*clmblkdst+2*clmblkdst8;; Next source pointer
	bump	rdi, 2*XMM_SCD		;; Next sine/cosine pointer
	sub	eax, 256		;; Test outer loop counter
	mov	al, 4			;; Load middle loop counter
	jnz	b5a			;; Iterate if necessary
	bump	rsi, -3*2*clmblkdst8	;; Restore source pointer
pfing	bump	rcx, -48*blkdst		;; Restore prefetch pointer
	end_timer 23

;; Do FFT levels 7,8
;;
;; On input the 64-byte cache lines hold these data values:
;;	0K	8K	16K	24K
;;	2K	...
;;	...
;;	6K	...
;;	32K	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0K	2K	4K	6K
;;	8K	...
;;	...

;; Do 48 four_complex_fft macros
;;	distance between fft data elements is 2K
;;	do 12 macros each processing 32 data values

	start_timer 24
	mov	rdi, sincos1		;; Load sin/cos pointer
	mov	eax, 6*256+2		;; 6 iterations of 2 iters of 2*clm
b7b:	g4cl_four_complex_fft rsi, 64, clmblkdst, 2*clmblkdst, rcx, 64, blkdst, 2*blkdst
	L2prefetch128 [rcx-64][rbp]
	L2prefetch128 [rcx-64+blkdst][rbp]
	bump	rbp, -64+2*blkdst
	add	al, 256/2		;; Test inner loop counter
	jnc	b7b			;; Iterate if necessary
	bump	rbp, 128-4*blkdst
	IF clm NE 1
	add	dl, 256/clm
	jnc	b7b
	ENDIF
	bump	rsi, -2*clm*64+4*clmblkdst;; Next source pointer
	bump	rcx, -2*clm*64+4*blkdst	;; Next dest pointer
	bump	rdi, 2*XMM_SCD		;; Next sine/cosine pointer
	sub	al, 1			;; Test outer loop counter
	jnz	b7b
	bump	rsi, -8*clmblkdst+clmblkdst8
	sub	ah, 1			;; Test outer loop counter
	mov	al, 2
	jnz	b7b			;; Iterate if necessary
	end_timer 24

;; Work on next 4 sets of 192 values

endlp:	xpass1_get_next_block pass2, c0b, b0b ;; Get next block and jump
	done:	xfft_footer
	ENDM


;; Perform a 448K-element FFT.  This is done in two passes.  Pass 1
;; does 8 levels, pass 2 uses common code to perform the last 11 levels.

hg_pass1sclevels8pfa7 MACRO fftname, pass2_levels, clmarg
	LOCAL	b0b, b1a, b1b, b2b, b3a, b3b, b4b, b5b, b6b, b7b
	LOCAL	c0b, c1b, c2a, c2b, c3b, c4b, c5b, c6b, c7b
	LOCAL	pass1, pass2, endlp, done, yes_fft

	set_FFT_constants fftname, 0, 0, pass2_levels, clmarg
	xfft_header pass1

;; Do the last 11 FFT levels
;;	do 1 pass2_11_levels_real and 55 pass2_11_levels_complex

pass2:	start_timer 0
	call	xpass2name
	end_timer 0
	cmp	ffttype, 1		;; We're done if FFTing only
	je	done

;; Do the inverse FFT

	pass1_inverse_fft_setup c0b

;; Do inverse FFT levels 7,8
;; On input the 64-byte cache lines hold these data values:
;;	0K	2K	4K	6K
;;	8K	10K	12K	14K
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0K	4K	8K	12K
;;	2K	...
;;	16K	...
;;	...

;; Do 1 eight_reals_unfft_2 macros
;; Do 1 nop_two_two_complex_unfft_2 macros
;;	distance between fft data elements is 2K
;;	do 1 macros each processing 16 data values

c0b:	start_timer 16
	mov	rsi, DATA_ADDR		;; Get source address
	mov	rcx, scratch_area	;; Get address of scratch area
	sub	rax, rax
	sub	rdx, rdx
pfing	mov	rbp, DATA_PREFETCH	;; Set prefetch distance
pfing	sub	rbp, rsi
c6b:	g2cl_eight_reals_unfft_2 rsi, 64, blkdst, rcx, 64, clmblkdst
	add	al, 256/2/clm		;; Test loop counter
	jnc	c6b			;; Iterate if necessary

;; Do 54 four_complex_unfft macros
;;	distance between fft data elements is 2K
;;	do 27 macros each processing 16 data values

	bump	rsi, -2*clm*64+2*blkdst	;; Next source pointer
	bump	rcx, -2*clm*64+2*clmblkdst;; Next dest pointer
	mov	rdi, sincos1		;; Load sin/cos pointer
	mov	al, 256/4+7		;; 27 iterations of 2*clm
c7b:	g2cl_four_complex_unfft rsi, 64, blkdst, rcx, 64, clmblkdst
	add	dl, 256/2/clm		;; Test loop counter
	jnc	c7b			;; Iterate if necessary
	bump	rsi, -2*clm*64+2*blkdst	;; Next source pointer
	bump	rcx, -2*clm*64+2*clmblkdst;; Next dest pointer
	bump	rdi, XMM_SCD		;; Next sine/cosine pointer
	add	al, 256/4		;; Test loop counter
	jnc	c7b			;; Iterate if necessary
	bump	rcx, -8*clmblkdst+clmblkdst8
	sub	al, 1			;; Test outer loop counter
	jnz	c7b			;; Iterate if necessary
	lea	rcx, [rsi-28*2*blkdst]	;; Create prefetch pointer
	mov	rsi, scratch_area	;; Restore source pointer
	end_timer 16

;; Do inverse FFT levels 6
;; On input the 64-byte cache lines hold these data values:
;;	0K	4K	8K	12K
;;	2K	...
;;	16K	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0K	8K	16K	24K
;;	2K	...
;;	...
;;	6K
;;	32K	...
;;	...
;;
;; Do 4 eight_reals_unfft_1 macros
;;	distance between fft data elements is 8K
;;	do 2 macros each processing 16 data values
;;
;; Do 52 two_two_complex_unfft macros
;;	distance between fft data elements is 8K
;;	do 26 macros each processing 16 data values

;; Do inverse FFT levels 4,5
;; On input the 64-byte cache lines hold these data values:
;;	0K	8K	16K	24K
;;	2K	...
;;	...
;;	6K
;;	32K	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0K	16K	32K	48K
;;	2K	...
;;	...
;;	14K
;;	64K	96K	128K	160K
;;	...
;;
;; Do 8 eight_reals_unfft_2 macros
;;	distance between fft data elements is 16K
;;	do 4 macros each processing 16 data values
;;
;; Do 48 four_complex_unfft macros
;;	distance between fft data elements is 16K
;;	do 12 macros each processing 32 data values

;; Do inverse FFT levels 1,2,3
;; On input the 64-byte cache lines hold these data values:
;;	0K	16K	32K	48K
;;	2K	...
;;	...
;;	14K
;;	64K	96K	128K	160K
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0K	16K	224K	240K
;;	2K	...
;;	...
;;	14K
;;	32K	64K	256K	288K
;;	34K
;;	...
;;	62K
;;	96K	160K	320K	384K
;;	98K
;;	...
;;	158K
;;
;; Do 64 seven_reals_last_unfft macros
;;	distance between fft data elements is 64K
;;	do 8 macros each processing 56 data values

	start_timer 17
	mov	rdi, sincos2		;; Load sin/cos pointer
	mov	rbx, sincos3		;; Load sin/cos pointer
	mov	eax, 2			;; 2 iterations
	xseven_reals_unfft_preload

c2a:	xtouchlines 14			;; Preload TLBs

c4b:	x2cl_eight_reals_unfft_1 rsi, 64, 2*clmblkdst
	add	al, 256/2		;; Test loop counter
	jnc	c4b			;; Iterate if necessary

	bump	rsi, -2*64+4*clmblkdst	;; Next source pointer
	mov	ah, 256/2+7		;; 13 iterations
c5b:	x2cl_two_complex_unfft rsi, 64, 2*clmblkdst
	add	al, 256/2		;; Test loop counter
	jnc	c5b			;; Iterate if necessary
	bump	rsi, -2*64+4*clmblkdst	;; Next source pointer
	bump	rdi, XMM_SCD		;; Next sine/cosine pointer
	add	ah, 256/2		;; Test loop counter
	jnc	c5b			;; Iterate if necessary
	bump	rsi, -2*4*clmblkdst+clmblkdst8;; Next source pointer
	sub	ah, 1			;; Test outer loop counter
	jnz	c5b			;; Iterate if necessary
	bump	rdi, -13*XMM_SCD	;; Restore sine/cosine pointer

	bump	rsi, -7*clmblkdst8	;; Restore source pointer
c2b:	x2cl_half_eight_reals_unfft_2 rsi, 64, 4*clmblkdst
	add	al, 256/2		;; Test loop counter
	jnc	c2b			;; Iterate if necessary
	bump	rsi, -2*64+2*clmblkdst
	add	ah, 256/2		;; Test loop counter
	jnc	c2b			;; Iterate if necessary

	bump	rsi, -2*2*clmblkdst+clmblkdst8;; Next source pointer
	mov	ah, 3			;; 3 iterations
c3b:	x4cl_four_complex_unfft rsi, 64, 4*clmblkdst, clmblkdst8, rbx
	xloop1sp c3b, 2*clmblkdst
	add	edx, 80000000h		;; Test loop counter
	jnc	c3b			;; Iterate if necessary
	bump	rsi, -2*2*clmblkdst+2*clmblkdst8
	bump	rbx, XMM_SCD		;; Next sine/cosine pointer
	sub	ah, 1			;; Test outer loop counter
	jnz	c3b			;; Iterate if necessary
	bump	rbx, -3*XMM_SCD		;; Restore sine/cosine pointer

	bump	rsi, -3*2*clmblkdst8-clmblkdst8;; Restore source pointer
c1b:	x7cl_seven_reals_last_unfft rsi, 64, clmblkdst8
	xloop1prefetch2sp c1b, 2*clmblkdst
	add	ah, 256/4
	jnc	c1b

	IF clm GT 1
	bump	rsi, -4*2*clmblkdst+2*64;; Next source pointer
	bump	rcx, -14*blkdst+128	;; Next prefetch pointer
	add	dl, 256/clm		;; clm iterations
	jnc	c4b
	bump	rsi, -clm*2*64+clmblkdst;; Next source pointer
	bump	rcx, -clm*128+14*blkdst	;; Next prefetch pointer
	ELSE
	bump	rsi, -4*2*clmblkdst+clmblkdst;; Next source pointer
	ENDIF

	sub	al, 1			;; Test loop counter
	jnz	c2a			;; Iterate if necessary
	end_timer 17

;; Normalize these values

	pass1_normalize 1, yes_fft

;; Copy data from the scratch area back to the FFT data area

	copy_scratch_data_to_fft 56
	jmp	endlp			;; Skip FFT code if flag not set

;; Fire up auxiliary threads to do pass 1 forward FFT

pass1:	pass1_forward_fft_setup b0b

;; Copy data from the FFT data area back to the scratch area

b0b:	copy_fft_data_to_scratch 56

;; Do FFT levels 1,2,3
;;
;; On input the 64-byte cache lines hold these data values:
;;	0K	16K	224K	240K
;;	2K	...
;;	...
;;	14K
;;	32K	64K	256K	288K
;;	34K
;;	...
;;	62K
;;	96K	160K	320K	384K
;;	98K
;;	...
;;	158K
;; On output the 64-byte cache lines hold these data values:
;;	0K	16K	32K	48K
;;	2K	...
;;	...
;;	14K
;;	64K	96K	128K	160K
;;	66K
;;	...
;;	94K
;;	192K	256K	320K	384K
;;	194K
;;	...
;;	254K
;;
;; Do 64 seven_reals_first_fft macros (FFT levels 1,2,3)
;;	distance between fft data elements is 64K
;;	do 8 iterations each processing 56 data values

;; Do FFT levels 4,5
;; Values 0K-63K is real data, 64K-447K is complex data.
;;
;; On input the 64-byte cache lines hold these data values:
;;	0K	16K	32K	48K
;;	2K	...
;;	...
;;	14K
;;	64K	96K	128K	160K
;;	66K
;;	...
;;	94K
;;	192K	256K	320K	384K
;;	194K
;;	...
;;	254K
;; On output the 64-byte cache lines hold these data values:
;;	0K	16K	32K	48K
;;	2K	...
;;	...
;;	14K	...
;;	64K	...
;;	...
;;
;; Do 8 eight_reals_fft_2 macros
;;	distance between fft data elements is 16K
;;	do 8 macros each processing 8 data values
;;
;; Do 16 four_complex_fft macros
;;	distance between fft data elements is 16K
;;	do 8 macros each processing 16 data values
;;
;; Do 32 four_complex_fft macros
;;	distance between fft data elements is 16K
;;	do 8 macros each processing 32 data values

;; Do FFT levels 6
;; Values 0K-15K is real data, 16K-31K is semi-real data, 32K-447K is
;; complex data.
;;
;; On input the 64-byte cache lines hold these data values:
;;	0K	16K	32K	48K
;;	2K	...
;;	...
;;	14K	...
;;	64K	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0K	8K	16K	24K
;;	2K	...
;;	...
;;	6K	...
;;	32K	...
;;	...

;; Do 4 eight_reals_fft_1 macros
;; Do 4 two_two_complex_fft macros
;;	distance between fft data elements is 8K
;;	do 4 macros each processing 16 data values
;;
;; Do 48 two_two_complex_fft macros
;;	distance between fft data elements is 8K
;;	do 24 macros each processing 16 data values

yes_fft:start_timer 21
	mov	rsi, scratch_area	;; Get address of scratch area
	mov	rcx, DATA_ADDR		;; Load source address for prefetching
pfing	mov	rbp, DATA_PREFETCH	;; Set prefetch distance
pfing	sub	rbp, rcx
pfing	bump	rcx, +28*blkdst		;; Create prefetching pointer
	mov	rdi, sincos3		;; Load sin/cos pointer
	mov	rbx, sincos2		;; Load sin/cos pointer
	mov	eax, 4			;; 4 iterations
	sub	rdx, rdx
	xseven_reals_fft_preload

b1a:	xtouchlines 7			;; Preload TLBs

b1b:	x7cl_seven_reals_fft rsi, 64, clmblkdst8
	xloop1prefetch2sp b1b, 4*clmblkdst
	add	ah, 256/2
	jnc	b1b

	bump	rsi, -2*4*clmblkdst	;; Restore source pointer
b2b:	x1cl_half_eight_reals_fft_2 rsi, 64
	add	al, 256/2		;; Test inner loop counter
	jnc	b2b			;; Iterate if necessary
	bump	rsi, -2*64+4*clmblkdst	;; Next source pointer
	add	ah, 256/2
	jnc	b2b

	bump	rsi, -2*4*clmblkdst+clmblkdst8;; Next source pointer
b3a:	x2cl_four_complex_fft rsi, 64, clmblkdst8
	add	al, 256/2		;; Test inner loop counter
	jnc	b3a			;; Iterate if necessary
	bump	rsi, -2*64+4*clmblkdst	;; Next source pointer
	add	ah, 256/2
	jnc	b3a
	L2prefetch128 [rcx][rbp]
pfing	bump	rcx, blkdst		;; Next prefetch pointer

	bump	rsi, -2*4*clmblkdst+2*clmblkdst8;; Next source pointer
	bump	rdi, XMM_SCD		;; Next sine/cosine pointer
b3b:	x4cl_four_complex_fft rsi, 64, clmblkdst8, 2*clmblkdst8, rdi
	xloop1sp b3b, 4*clmblkdst
	add	ah, 256/2
	jnc	b3b
	bump	rdi, -XMM_SCD		;; Restore sine/cosine pointer

	bump	rsi, -2*4*clmblkdst-3*clmblkdst8;; Restore source pointer
b4b:	x2cl_eight_reals_fft_1 rsi, 64, 4*clmblkdst
	add	al, 256/2		;; Test inner loop counter
	jnc	b4b			;; Iterate if necessary

	bump	rsi, -2*64+clmblkdst8	;; Next source pointer
	bump	rbx, XMM_SCD		;; Next sine/cosine pointer
	mov	ah, 6			;; 6 iterations
b5b:	x2cl_two_complex_fft rsi, 64, 4*clmblkdst, rbx
	add	al, 256/2		;; Test inner loop counter
	jnc	b5b			;; Iterate if necessary
	bump	rsi, -2*64+clmblkdst8	;; Next source pointer
	bump	rbx, 2*XMM_SCD		;; Next sine/cosine pointer
	sub	ah, 1			;; Test outer loop counter
	jnz	b5b			;; Iterate if necessary
	bump	rbx, -13*XMM_SCD	;; Restore sine/cosine pointer

	IF clm GT 1
	bump	rsi, -7*clmblkdst8+2*64	;; Next source pointer
	bump	rcx, -7*blkdst+128	;; Next prefetch pointer
	add	dl, 256/clm		;; clm iterations
	jnc	b1b
	bump	rsi, -clm*2*64+clmblkdst;; Next source pointer
	bump	rcx, -clm*128+7*blkdst	;; Next prefetch pointer
	ELSE
	bump	rsi, -7*clmblkdst8+clmblkdst;; Next source pointer
	ENDIF

	sub	al, 1			;; Test loop counter
	jnz	b1a			;; Iterate if necessary
	bump	rsi, -4*clmblkdst	;; Restore source pointer
pfing	bump	rcx, -56*blkdst		;; Restore prefetch pointer
	end_timer 21

;; Do FFT levels 7,8
;; Values 0K-7K is real data, 8K-15K is semi-real data, 16K-447K is
;; complex data.
;;
;; On input the 64-byte cache lines hold these data values:
;;	0K	8K	16K	24K
;;	2K	...
;;	...
;;	6K	...
;;	32K	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0K	2K	4K	6K
;;	8K	...
;;	...

;; Do 1 eight_reals_fft_2 macros
;; Do 1 nop_two_two_complex_fft_2 macros
;; Do 2 four_complex_fft macros
;;	distance between fft data elements is 2K
;;	do 1 macros each processing 32 data values

	start_timer 24
	mov	rdi, sincos1		;; Load sin/cos pointer
b6b:	g4cl_eight_reals_fft_2 rsi, 64, clmblkdst, 2*clmblkdst, rcx, 64, blkdst, 2*blkdst
	add	al, 256/2/clm		;; Test inner loop counter
	jnc	b6b			;; Iterate if necessary

;; Do 52 four_complex_fft macros
;;	distance between fft data elements is 2K
;;	do 13 macros each processing 32 data values

	bump	rsi, -2*clm*64+4*clmblkdst;; Next source pointer
	bump	rcx, -2*clm*64+4*blkdst	;; Next dest pointer
	bump	rdi, XMM_SCD		;; Next sine/cosine pointer
	mov	eax, 7*256+1		;; 13 iterations of 2*clm
b7b:	g4cl_four_complex_fft rsi, 64, clmblkdst, 2*clmblkdst, rcx, 64, blkdst, 2*blkdst
	L2prefetch128 [rcx-64][rbp]
	L2prefetch128 [rcx-64+blkdst][rbp]
	bump	rbp, -64+2*blkdst
	add	al, 256/2		;; Test inner loop counter
	jnc	b7b			;; Iterate if necessary
	bump	rbp, 128-4*blkdst
	IF clm NE 1
	add	dl, 256/clm
	jnc	b7b
	ENDIF
	bump	rsi, -2*clm*64+4*clmblkdst;; Next source pointer
	bump	rcx, -2*clm*64+4*blkdst	;; Next dest pointer
	bump	rdi, 2*XMM_SCD		;; Next sine/cosine pointer
	sub	al, 1
	jnz	b7b
	bump	rsi, -8*clmblkdst+clmblkdst8
	sub	ah, 1			;; Test outer loop counter
	mov	al, 2
	jnz	b7b			;; Iterate if necessary
	end_timer 24

;; Work on next 4 sets of 224 values

endlp:	xpass1_get_next_block pass2, c0b, b0b ;; Get next block and jump
	done:	xfft_footer
	ENDM


;; Perform a 512K-element FFT.  This is done in two passes.  Pass 1
;; does 8 levels, pass 2 uses common code to perform the last 11 levels.

hg_pass1sclevels8pfa8 MACRO fftname, pass2_levels, clmarg
	LOCAL	b0b, b1a, b1b, b2b, b3b, b4b, b5b, b6b, b7b
	LOCAL	c0b, c1b, c2a, c2b, c3b, c4b, c5b, c6b, c7b
	LOCAL	pass1, pass2, endlp, done, yes_fft

	set_FFT_constants fftname, 0, 0, pass2_levels, clmarg
	xfft_header pass1

;; Do the last 11 FFT levels
;;	do 1 pass2_11_levels_real and 63 pass2_11_levels_complex

pass2:	start_timer 0
	call	xpass2name
	end_timer 0
	cmp	ffttype, 1		;; We're done if FFTing only
	je	done

;; Do the inverse FFT

	pass1_inverse_fft_setup c0b

;; Do inverse FFT levels 7,8
;; On input the 64-byte cache lines hold these data values:
;;	0K	2K	4K	6K
;;	8K	10K	12K	14K
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0K	4K	8K	12K
;;	2K	...
;;	16K	...
;;	...

;; Do 1 eight_reals_unfft_2 macros
;; Do 1 nop_two_two_complex_unfft_2 macros
;;	distance between fft data elements is 2K
;;	do 1 macros each processing 16 data values

c0b:	start_timer 16
	mov	rsi, DATA_ADDR		;; Get source address
	mov	rcx, scratch_area	;; Get address of scratch area
	sub	rax, rax
	sub	rdx, rdx
pfing	mov	rbp, DATA_PREFETCH	;; Set prefetch distance
pfing	sub	rbp, rsi
c6b:	g2cl_eight_reals_unfft_2 rsi, 64, blkdst, rcx, 64, clmblkdst
	add	al, 256/2/clm		;; Test loop counter
	jnc	c6b			;; Iterate if necessary

;; Do 62 four_complex_unfft macros
;;	distance between fft data elements is 2K
;;	do 31 macros each processing 16 data values

	bump	rsi, -2*clm*64+2*blkdst	;; Next source pointer
	bump	rcx, -2*clm*64+2*clmblkdst;; Next dest pointer
	mov	rdi, sincos1		;; Load sin/cos pointer
	mov	al, 256/4+8		;; 31 iterations of 2*clm
c7b:	g2cl_four_complex_unfft rsi, 64, blkdst, rcx, 64, clmblkdst
	add	dl, 256/2/clm		;; Test loop counter
	jnc	c7b			;; Iterate if necessary
	bump	rsi, -2*clm*64+2*blkdst	;; Next source pointer
	bump	rcx, -2*clm*64+2*clmblkdst;; Next dest pointer
	bump	rdi, XMM_SCD		;; Next sine/cosine pointer
	add	al, 256/4		;; Test loop counter
	jnc	c7b			;; Iterate if necessary
	bump	rcx, -8*clmblkdst+clmblkdst8
	sub	al, 1			;; Test outer loop counter
	jnz	c7b			;; Iterate if necessary
	lea	rcx, [rsi-32*2*blkdst]	;; Create prefetch pointer
	mov	rsi, scratch_area	;; Restore source pointer
	end_timer 16

;; Do inverse FFT levels 6
;; On input the 64-byte cache lines hold these data values:
;;	0K	4K	8K	12K
;;	2K	...
;;	16K	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0K	8K	16K	24K
;;	2K	...
;;	...
;;	6K
;;	32K	...
;;	...
;;
;; Do 4 eight_reals_unfft_1 macros
;;	distance between fft data elements is 8K
;;	do 2 macros each processing 16 data values
;;
;; Do 60 two_two_complex_unfft macros
;;	distance between fft data elements is 8K
;;	do 30 macros each processing 16 data values

;; Do inverse FFT levels 4,5
;; On input the 64-byte cache lines hold these data values:
;;	0K	8K	16K	24K
;;	2K	...
;;	...
;;	6K
;;	32K	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0K	32K	64K	96K
;;	2K	...
;;	...
;;	30K
;;	128K	...
;;	...
;;
;; Do 8 eight_reals_unfft_2 macros
;; Do 8 nop_two_two_complex_unfft_2 macros
;;	distance between fft data elements is 16K
;;	do 4 macros each processing 32 data values
;;
;; Do 48 four_complex_unfft macros
;;	distance between fft data elements is 16K
;;	do 12 macros each processing 32 data values

;; Do inverse FFT levels 1,2,3
;; On input the 64-byte cache lines hold these data values:
;;	0K	32K	64K	96K
;;	2K	...
;;	...
;;	30K
;;	128K	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0K	128K	256K	384K
;;	2K	...
;;	...
;;	126K
;;
;; Do 64 eight_reals_last_unfft macros
;;	distance between fft data elements is 64K
;;	do 16 macros each processing 32 data values

	start_timer 17
	mov	rdi, sincos2		;; Load sin/cos pointer
	mov	rbx, sincos3		;; Load sin/cos pointer
	mov	eax, 2			;; 2 iterations

c2a:	xtouchlines 16			;; Preload TLBs

c4b:	x2cl_eight_reals_unfft_1 rsi, 64, 2*clmblkdst
	add	al, 256/2		;; Test loop counter
	jnc	c4b			;; Iterate if necessary

	bump	rsi, -2*64+4*clmblkdst	;; Next source pointer
	mov	ah, 256/2+8		;; 15 iterations
c5b:	x2cl_two_complex_unfft rsi, 64, 2*clmblkdst
	add	al, 256/2		;; Test loop counter
	jnc	c5b			;; Iterate if necessary
	bump	rsi, -2*64+4*clmblkdst	;; Next source pointer
	bump	rdi, XMM_SCD		;; Next sine/cosine pointer
	add	ah, 256/2		;; Test loop counter
	jnc	c5b			;; Iterate if necessary
	bump	rsi, -2*4*clmblkdst+clmblkdst8;; Next source pointer
	sub	ah, 1			;; Test outer loop counter
	jnz	c5b			;; Iterate if necessary
	bump	rdi, -15*XMM_SCD	;; Restore sine/cosine pointer

	bump	rsi, -8*clmblkdst8	;; Restore source pointer
c2b:	x4cl_eight_reals_unfft_2 rsi, 64, 4*clmblkdst, clmblkdst8
	xloop1sp c2b, 2*clmblkdst
	add	ah, 256/2		;; Test loop counter
	jnc	c2b			;; Iterate if necessary

	bump	rsi, -2*2*clmblkdst+2*clmblkdst8;; Next source pointer
	mov	ah, 3			;; 3 iterations
c3b:	x4cl_four_complex_unfft rsi, 64, 4*clmblkdst, clmblkdst8, rbx
	xloop1sp c3b, 2*clmblkdst
	add	edx, 80000000h		;; Test loop counter
	jnc	c3b			;; Iterate if necessary
	bump	rsi, -2*2*clmblkdst+2*clmblkdst8
	bump	rbx, XMM_SCD		;; Next sine/cosine pointer
	sub	ah, 1
	jnz	c3b
	bump	rbx, -3*XMM_SCD		;; Next sine/cosine pointer

	bump	rsi, -4*2*clmblkdst8	;; Restore source pointer
	mov	ah, 2
c1b:	x4cl_eight_reals_last_unfft rsi, 64, 2*clmblkdst8, 4*clmblkdst8
	xloop1sp c1b, 2*clmblkdst
	add	ah, 256/4
	jnc	c1b
	bump	rsi, -4*2*clmblkdst+clmblkdst8;; Next source pointer
	sub	ah, 1
	jnz	c1b

	IF clm GT 1
	bump	rsi, -2*clmblkdst8+2*64	;; Next source pointer
	bump	rcx, -16*blkdst+128	;; Next prefetch pointer
	add	dl, 256/clm		;; clm iterations
	jnc	c4b
	bump	rsi, -clm*2*64+clmblkdst;; Next source pointer
	bump	rcx, -clm*128+16*blkdst	;; Next prefetch pointer
	ELSE
	bump	rsi, -2*clmblkdst8+clmblkdst;; Next source pointer
	ENDIF

	sub	al, 1			;; Test loop counter
	jnz	c2a			;; Iterate if necessary
	end_timer 17

;; Normalize these values

	pass1_normalize 1, yes_fft

;; Copy data from the scratch area back to the FFT data area

	copy_scratch_data_to_fft 64
	jmp	endlp			;; Skip FFT code if flag not set

;; Fire up auxiliary threads to do pass 1 forward FFT

pass1:	pass1_forward_fft_setup b0b

;; Copy data from the FFT data area back to the scratch area

b0b:	copy_fft_data_to_scratch 64

;; Do FFT levels 1,2,3
;;
;; On input the 64-byte cache lines hold these data values:
;;	0K	128K	256K	384K
;;	2K	...
;;	...
;;	126K
;; On output the 64-byte cache lines hold these data values:
;;	0K	64K	128K	192K
;;	2K	...
;;	...
;;	62K	...
;;	256K	...
;;	...
;;
;; Do 64 eight_reals_first_fft macros (FFT levels 1,2,3)
;;	distance between fft data elements is 64K
;;	do 32 iterations each processing 16 data values

;; Do FFT levels 4,5
;; Values 0K-63K is real data, 64K-127K is semi-real data, 128K-511K is
;; complex data.
;;
;; On input the 64-byte cache lines hold these data values:
;;	0K	64K	128K	192K
;;	2K	...
;;	...
;;	62K	...
;;	256K	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0K	16K	32K	48K
;;	2K	...
;;	...
;;	14K	...
;;	64K	...
;;	...
;;
;; Do 8 eight_reals_fft_2 macros
;; Do 8 nop_two_two_complex_fft_2 macros
;; Do 16 four_complex_fft macros
;;	distance between fft data elements is 16K
;;	do 8 macros each processing 32 data values
;;
;; Do 32 four_complex_fft macros
;;	distance between fft data elements is 16K
;;	do 8 macros each processing 32 data values

;; Do FFT levels 6
;; Values 0K-15K is real data, 16K-31K is semi-real data, 32K-511K is
;; complex data.
;;
;; On input the 64-byte cache lines hold these data values:
;;	0K	16K	32K	48K
;;	2K	...
;;	...
;;	14K	...
;;	64K	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0K	8K	16K	24K
;;	2K	...
;;	...
;;	6K	...
;;	32K	...
;;	...
;;
;; Do 4 eight_reals_fft_1 macros
;; Do 4 two_two_complex_fft macros
;;	distance between fft data elements is 8K
;;	do 4 macros each processing 16 data values
;;
;; Do 56 two_two_complex_fft macros
;;	distance between fft data elements is 8K
;;	do 28 macros each processing 16 data values

yes_fft:start_timer 21
	mov	rsi, scratch_area	;; Get address of scratch area
	mov	rcx, DATA_ADDR		;; Load source address for prefetching
pfing	mov	rbp, DATA_PREFETCH	;; Set prefetch distance
pfing	sub	rbp, rcx
pfing	bump	rcx, +32*blkdst		;; Create prefetching pointer
	mov	rdi, sincos3		;; Load sin/cos pointer
	mov	rbx, sincos2		;; Load sin/cos pointer
	mov	eax, 4			;; 4 iterations
	sub	rdx, rdx

b1a:	xtouchlines 8			;; Preload TLBs

b1b:	x2cl_eight_reals_fft rsi, 64, 4*clmblkdst8
	add	al, 256/2		;; Test loop counter
	jnc	b1b			;; Iterate if necessary
	bump	rsi, -2*64+4*clmblkdst	;; Next source pointer
	add	ah, 256/2
	jnc	b1b
	bump	rsi, -2*4*clmblkdst+clmblkdst8;; Next source pointer
	L2prefetch128 [rcx][rbp]
pfing	bump	rcx, blkdst		;; Next prefetch pointer
	add	eax, 80000000h/2	;; 4 iterations
	jnc	b1b

	bump	rsi, -4*clmblkdst8	;; Restore source pointer
b2b:	x4cl_eight_reals_fft_2 rsi, 64, clmblkdst8, 2*clmblkdst8, rdi
	xloop1sp b2b, 4*clmblkdst
	add	ah, 256/2
	jnc	b2b

	bump	rsi, -2*4*clmblkdst+4*clmblkdst8;; Next source pointer
	bump	rdi, XMM_SCD		;; Next sine/cosine pointer
b3b:	x4cl_four_complex_fft rsi, 64, clmblkdst8, 2*clmblkdst8, rdi
	xloop1sp b3b, 4*clmblkdst
	add	ah, 256/2
	jnc	b3b
	bump	rdi, -XMM_SCD		;; Restore sine/cosine pointer

	bump	rsi, -2*4*clmblkdst-4*clmblkdst8;; Restore src pointer
b4b:	x2cl_eight_reals_fft_1 rsi, 64, 4*clmblkdst
	add	al, 256/2		;; Test inner loop counter
	jnc	b4b			;; Iterate if necessary

	bump	rsi, -2*64+clmblkdst8	;; Load source pointer
	bump	rbx, XMM_SCD		;; Next sine/cosine pointer
	mov	ah, 7			;; 7 iterations
b5b:	x2cl_two_complex_fft rsi, 64, 4*clmblkdst, rbx
	add	al, 256/2		;; Test inner loop counter
	jnc	b5b			;; Iterate if necessary
	bump	rsi, -2*64+clmblkdst8	;; Next source pointer
	bump	rbx, 2*XMM_SCD		;; Next sine/cosine pointer
	sub	ah, 1			;; Test outer loop counter
	jnz	b5b			;; Iterate if necessary
	bump	rbx, -15*XMM_SCD	;; Restore sine/cosine pointer

	IF clm GT 1
	bump	rsi, -8*clmblkdst8+2*64	;; Next source pointer
	bump	rcx, -8*blkdst+128	;; Next prefetch pointer
	add	dl, 256/clm		;; clm iterations
	jnc	b1b
	bump	rsi, -clm*2*64+clmblkdst;; Next source pointer
	bump	rcx, -clm*128+8*blkdst	;; Next prefetch pointer
	ELSE
	bump	rsi, -8*clmblkdst8+clmblkdst;; Next source pointer
	ENDIF

	sub	al, 1			;; Test loop counter
	jnz	b1a			;; Iterate if necessary
	bump	rsi, -4*clmblkdst	;; Restore source pointer
pfing	bump	rcx, -64*blkdst		;; Restore prefetch pointer
	end_timer 21

;; Do FFT levels 7,8
;; Values 0K-7K is real data, 8K-15K is semi-real data, 16K-511K is
;; complex data.
;;
;; On input the 64-byte cache lines hold these data values:
;;	0K	8K	16K	24K
;;	2K	...
;;	...
;;	6K	...
;;	32K	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0K	2K	4K	6K
;;	8K	...
;;	...

;; Do 1 eight_reals_fft_2 macros
;; Do 1 nop_two_two_complex_fft_2 macros
;; Do 2 four_complex_fft macros
;;	distance between fft data elements is 2K
;;	do 1 macros each processing 32 data values

	start_timer 24
	mov	rdi, sincos1		;; Load sin/cos pointer
b6b:	g4cl_eight_reals_fft_2 rsi, 64, clmblkdst, 2*clmblkdst, rcx, 64, blkdst, 2*blkdst
	add	al, 256/2/clm		;; Test inner loop counter
	jnc	b6b			;; Iterate if necessary

;; Do 60 four_complex_fft macros
;;	distance between fft data elements is 2K
;;	do 15 macros each processing 32 data values

	bump	rsi, -2*clm*64+4*clmblkdst;; Next source pointer
	bump	rcx, -2*clm*64+4*blkdst	;; Next dest pointer
	bump	rdi, XMM_SCD		;; Next sine/cosine pointer
	mov	eax, 8*256+1		;; 15 iterations of 2*clm
b7b:	g4cl_four_complex_fft rsi, 64, clmblkdst, 2*clmblkdst, rcx, 64, blkdst, 2*blkdst
	L2prefetch128 [rcx-64][rbp]
	L2prefetch128 [rcx-64+blkdst][rbp]
	bump	rbp, -64+2*blkdst
	add	al, 256/2		;; Test inner loop counter
	jnc	b7b			;; Iterate if necessary
	bump	rbp, 128-4*blkdst
	IF clm NE 1
	add	dl, 256/clm
	jnc	b7b
	ENDIF
	bump	rsi, -2*clm*64+4*clmblkdst;; Next source pointer
	bump	rcx, -2*clm*64+4*blkdst	;; Next dest pointer
	bump	rdi, 2*XMM_SCD		;; Next sine/cosine pointer
	sub	al, 1
	jnz	b7b
	bump	rsi, -8*clmblkdst+clmblkdst8
	sub	ah, 1			;; Test outer loop counter
	mov	al, 2
	jnz	b7b			;; Iterate if necessary
	end_timer 24

;; Work on next 4 sets of 256 values

endlp:	xpass1_get_next_block pass2, c0b, b0b ;; Get next block and jump
	done:	xfft_footer
	ENDM


;; Perform a 512K-element negayclic FFT.  This is done in two passes.  Pass 1
;; does 8 levels, pass 2 uses common code to perform the last 11 levels.

hg_pass1sclevels8complex4 MACRO fftname, pass2_levels, clmarg
	LOCAL	b0b, b1a, b1b, b3a, b3b, b5a, b5b, b7b
	LOCAL	c0b, c1a, c1b, c3a, c3b, c5a, c5b, c7b
	LOCAL	pass1, pass2, endlp, done, yes_fft

	set_FFT_constants fftname, 0, 1, pass2_levels, clmarg
	xfft_header pass1

;; Do the last 11 FFT levels
;;	do 64 pass2_11_levels_complex

pass2:	start_timer 0
	call	xpass2name
	end_timer 0
	cmp	ffttype, 1		;; We're done if FFTing only
	je	done

;; Do the inverse FFT

	pass1_inverse_fft_setup c0b

;; Do inverse FFT levels 7,8
;; On input the 64-byte cache lines hold these data values:
;;	0K	2K	4K	6K
;;	8K	10K	12K	14K
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0K	4K	8K	12K
;;	2K	...
;;	16K	...
;;	...

;; Do 64 four_complex_unfft macros
;;	distance between fft data elements is 2K
;;	do 32 macros each processing 16 data values

c0b:	start_timer 16
	mov	rsi, DATA_ADDR		;; Get source address
	mov	rcx, scratch_area	;; Get address of scratch area
	mov	rdi, sincos1		;; Load sin/cos pointer
	mov	eax, 8			;; 32 iterations of 2*clm
	sub	rdx, rdx
pfing	mov	rbp, DATA_PREFETCH	;; Set prefetch distance
pfing	sub	rbp, rsi
c7b:	g2cl_four_complex_unfft rsi, 64, blkdst, rcx, 64, clmblkdst
	add	dl, 256/2/clm		;; Test loop counter
	jnc	c7b			;; Iterate if necessary
	bump	rsi, -2*clm*64+2*blkdst	;; Next source pointer
	bump	rcx, -2*clm*64+2*clmblkdst;; Next dest pointer
	bump	rdi, XMM_SCD		;; Next sine/cosine pointer
	add	al, 256/4		;; Test loop counter
	jnc	c7b			;; Iterate if necessary
	bump	rcx, -8*clmblkdst+clmblkdst8
	sub	al, 1			;; Test outer loop counter
	jnz	c7b			;; Iterate if necessary
	lea	rcx, [rsi-32*2*blkdst]	;; Create prefetch pointer
	mov	rsi, scratch_area	;; Restore source pointer
	end_timer 16

;; Do inverse FFT levels 5,6
;; On input the 64-byte cache lines hold these data values:
;;	0K	4K	8K	12K
;;	2K	...
;;	16K	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0K	16K	32K	48K
;;	2K	...
;;	...
;;	14K
;;	64K	...
;;	...

;; Do 64 four_complex_unfft macros
;;	distance between fft data elements is 8K
;;	do 16 macros each processing 32 data values

	start_timer 17
	mov	rdi, sincos2		;; Load sin/cos pointer
	mov	eax, 8*256+2		;; 8 iterations of 2 iters of 2*clm
c5a:	xtouch	[rcx+(clm-1)*128]	;; Preload TLB
c5b:	x4cl_four_complex_unfft rsi, 64, 2*clmblkdst, 4*clmblkdst, rdi
	xloop1	c5b, clmblkdst
	sub	al, 1			;; Test middle loop counter
	jnz	c5a			;; Iterate if necessary
	bump	rsi, -2*clmblkdst+clmblkdst8;; Next source pointer
	bump	rdi, XMM_SCD		;; Next sine/cosine pointer
	sub	eax, 256		;; Test outer loop counter
	mov	al, 2			;; Load middle loop counter
	jnz	c5a			;; Iterate if necessary
	bump	rsi, -8*clmblkdst8	;; Restore source pointer
	end_timer 17

;; Do inverse FFT levels 4
;; On input the 64-byte cache lines hold these data values:
;;	0K	16K	32K	48K
;;	2K	...
;;	...
;;	14K
;;	64K	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0K	32K	64K	96K
;;	2K	...
;;	...
;;	30K
;;	128K	...
;;	...

;; Do 64 two_two_complex_unfft macros
;;	distance between fft data elements is 32K
;;	do 32 macros each processing 16 data values

	start_timer 18
	mov	rdi, sincos3		;; Load sin/cos pointer
	mov	eax, 4*256+2		;; 4 iterations of 2 iters of 4*2*clm
c3a:	xtouch	[rcx+(clm-1)*128]	;; Preload TLB
c3b:	x2cl_two_complex_unfft rsi, 64, clmblkdst8
	xloop4	c3b, clmblkdst		;; 4* 2*clm iterations
	sub	al, 1			;; Test loop counter
	jnz	c3a			;; Iterate if necessary
	bump	rsi, -8*clmblkdst+2*clmblkdst8;; Next source pointer
	bump	rdi, XMM_SCD		;; Next sine/cosine pointer
	sub	eax, 256		;; Test outer loop counter
	mov	al, 2			;; Reload middle counter
	jnz	c3a			;; Iterate if necessary
	bump	rsi, -4*2*clmblkdst8	;; Restore source pointer
	end_timer 18

;; Do inverse FFT levels 1,2,3
;; On input the 64-byte cache lines hold these data values:
;;	0K	32K	64K	96K
;;	2K	...
;;	...
;;	30K
;;	128K	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0K	128K	256K	384K
;;	2K	...
;;	...
;;	126K

;; Do 64 four_complex_last_unfft macros
;;	distance between fft data elements is 64K
;;	do 16 macros each processing 32 data values

	start_timer 19
	mov	rdi, plus1_premults	;; Address of premultiplier table
	mov	eax, 2*256+4		;; 2 iters of 4 iterations of 2*2*clm
c1a:	xtouch	[rcx+(clm-1)*128]	;; Preload TLB
c1b:	x4cl_four_complex_last_unfft rsi, 64, 2*clmblkdst8, 4*clmblkdst8, 16*128
	xloop2edi c1b, clmblkdst, 128	;; 2 * 2*clm iterations
	sub	al, 1			;; Test loop counter
	jnz	c1a			;; Iterate if necessaryxn
	bump	rsi, -4*2*clmblkdst+clmblkdst8;; Next source pointer
	sub	ah, 1			;; Test outer loop counter
	mov	al, 4			;; Load middle loop counter
	jnz	c1a
	end_timer 19

;; Normalize these values

	pass1_normalize 1, yes_fft

;; Copy data from the scratch area back to the FFT data area

	copy_scratch_data_to_fft 64
	jmp	endlp			;; Skip FFT code if flag not set

;; Fire up auxiliary threads to do pass 1 forward FFT

pass1:	pass1_forward_fft_setup b0b

;; Copy data from the FFT data area back to the scratch area

b0b:	copy_fft_data_to_scratch 64

;; Do FFT levels 1,2,3
;;
;; On input the 64-byte cache lines hold these data values:
;;	0K	128K	256K	384K
;;	2K	...
;;	...
;;	126K
;; On output the 64-byte cache lines hold these data values:
;;	0K	64K	128K	192K
;;	2K	...
;;	...
;;	62K	...
;;	256K	...
;;	...

;; Do 64 four_complex_first_fft macros (FFT levels 1,2,3)
;;	distance between fft data elements is 64K
;;	do 32 iterations each processing 16 data values

yes_fft:start_timer 21
	mov	rsi, scratch_area	;; Get address of scratch area
	mov	rdi, plus1_premults	;; Address of premultiplier table
	mov	rcx, DATA_ADDR		;; Load source address for prefetching
pfing	mov	rbp, DATA_PREFETCH	;; Set prefetch distance
pfing	sub	rbp, rcx
pfing	bump	rcx, 32*blkdst		;; Create prefetching pointer
	mov	eax, 4			;; 4 iters of 2 iters of 4*2*clm
	sub	rdx, rdx
b1a:	xtouch	[rcx+(clm-1)*128]	;; Preload TLB
b1b:	x2cl_four_complex_first_fft_scratch rsi, 64, 4*clmblkdst8
	xloop4edi b1b, clmblkdst, 128	;; 4 * 2*clm iterations
	add	eax, 80000000h		;; 2 iterations
	jnc	b1a
	bump	rsi, -2*4*clmblkdst+clmblkdst8
	sub	al, 1			;; Test loop counter
	jnz	b1a			;; Iterate if necessary
	bump	rsi, -4*clmblkdst8	;; Restore source pointer
	end_timer 21

;; Do FFT levels 4
;;
;; On input the 64-byte cache lines hold these data values:
;;	0K	32K	64K	96K
;;	2K	...
;;	...
;;	30K
;;	128K	192K	256K	320K
;;	130K
;;	...
;;	190K
;; On output the 64-byte cache lines hold these data values:
;;	0K	32K	64K	96K
;;	2K	...
;;	...
;;	30K	...
;;	128K	...
;;	...

;; Do 64 two_two_complex_fft macros
;;	distance between fft data elements is 32K
;;	do 32 macros each processing 16 data values

	start_timer 22
	mov	rdi, sincos3		;; Load sin/cos pointer
	mov	eax, 2*256+2		;; 2 iters of 2 iterations of 4*2*clm
b3a:	xtouch	[rcx+(clm-1)*128]	;; Preload TLB
b3b:	x2cl_two_complex_fft rsi, 64, 2*clmblkdst8, rdi
	xloop4	b3b, clmblkdst		;; 4 * 2*clm iterations
	add	eax, 80000000h		;; 2 iterations
	jnc	b3a
	bump	rsi, -2*4*clmblkdst+clmblkdst8
	sub	al, 1			;; Test outer loop counter
	jnz	b3a			;; Iterate if necessary
	bump	rsi, -2*clmblkdst8+4*clmblkdst8;; Next source pointer
	bump	rdi, 2*XMM_SCD		;; Next sin/cos pointer
	sub	eax, 256		;; Test outer loop counter
	mov	al, 2
	jnz	b3a			;; Iterate if necessary
	bump	rsi, -2*4*clmblkdst8	;; Restore source pointer
	end_timer 22

;; Do FFT levels 5,6
;;
;; On input the 64-byte cache lines hold these data values:
;;	0K	32K	64K	96K
;;	2K	...
;;	...
;;	30K	...
;;	128K	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0K	8K	16K	24K
;;	2K	...
;;	...
;;	6K	...
;;	32K	...
;;	...

;; Do 64 four_complex_fft macros
;;	distance between fft data elements is 8K
;;	do 16 macros each processing 32 data values

	start_timer 23
	mov	rdi, sincos2		;; Load sin/cos pointer
	mov	eax, 4*256+4		;; 4 iterations of 4 iters of 2*2*clm
b5a:	xtouch	[rcx+(clm-1)*128]	;; Preload TLB
b5b:	x4cl_four_complex_fft rsi, 64, 4*clmblkdst, clmblkdst8, rdi
	xloop1	b5b, clmblkdst		;; 2 * 2*clm iterations
	sub	al, 1			;; Test middle loop counter
	jnz	b5a			;; Iterate if necessary
	bump	rsi, -4*clmblkdst+2*clmblkdst8;; Next source pointer
	bump	rdi, 2*XMM_SCD		;; Next sine/cosine pointer
	sub	eax, 256		;; Test outer loop counter
	mov	al, 4			;; Load middle loop counter
	jnz	b5a			;; Iterate if necessary
	bump	rsi, -4*2*clmblkdst8	;; Restore source pointer
pfing	bump	rcx, -64*blkdst		;; Restore prefetch pointer
	end_timer 23

;; Do FFT levels 7,8
;;
;; On input the 64-byte cache lines hold these data values:
;;	0K	8K	16K	24K
;;	2K	...
;;	...
;;	6K	...
;;	32K	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0K	2K	4K	6K
;;	8K	...
;;	...

;; Do 64 four_complex_fft macros
;;	distance between fft data elements is 2K
;;	do 16 macros each processing 32 data values

	start_timer 24
	mov	rdi, sincos1		;; Load sin/cos pointer
	mov	eax, 8*256+2		;; 8 iters of 2 of 2*clm
b7b:	g4cl_four_complex_fft rsi, 64, clmblkdst, 2*clmblkdst, rcx, 64, blkdst, 2*blkdst
	L2prefetch128 [rcx-64][rbp]
	L2prefetch128 [rcx-64+blkdst][rbp]
	bump	rbp, -64+2*blkdst
	add	al, 256/2		;; Test inner loop counter
	jnc	b7b			;; Iterate if necessary
	bump	rbp, 128-4*blkdst
	IF clm NE 1
	add	dl, 256/clm
	jnc	b7b
	ENDIF
	bump	rsi, -2*clm*64+4*clmblkdst;; Next source pointer
	bump	rcx, -2*clm*64+4*blkdst	;; Next dest pointer
	bump	rdi, 2*XMM_SCD		;; Next sine/cosine pointer
	sub	al, 1
	jnz	b7b
	bump	rsi, -8*clmblkdst+clmblkdst8
	sub	ah, 1			;; Test outer loop counter
	mov	al, 2
	jnz	b7b			;; Iterate if necessary
	end_timer 24

;; Work on next 4 sets of 256 values

endlp:	xpass1_get_next_block pass2, c0b, b0b ;; Get next block and jump
	done:	xfft_footer
	ENDM


;; Perform a 640K-element FFT.  This is done in two passes.  Pass 1
;; does 9 levels, pass 2 uses common code to perform the last 11 levels.

hg_pass1sclevels9pfa5 MACRO fftname, pass2_levels, clmarg
	LOCAL	b0b, b1a, b1aa, b1b, b2b, b3b, b4b, b5b, b6b, b7b
	LOCAL	c0b, c1b, c2a, c2b, c3b, c4b, c5a, c5b, c6b, c7b
	LOCAL	pass1, pass2, endlp, done, yes_fft

	set_FFT_constants fftname, 0, 0, pass2_levels, clmarg
	xfft_header pass1

;; Do the last 11 FFT levels
;;	do 1 pass2_11_levels_real and 79 pass2_11_levels_complex

pass2:	start_timer 0
	call	xpass2name
	end_timer 0
	cmp	ffttype, 1		;; We're done if FFTing only
	je	done

;; Do the inverse FFT

	pass1_inverse_fft_setup c0b

;; Do inverse FFT levels 8,9
;; On input the 64-byte cache lines hold these data values:
;;	0K	2K	4K	6K
;;	8K	10K	12K	14K
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0K	4K	8K	12K
;;	2K	...
;;	16K	...
;;	...

;; Do 1 eight_reals_unfft_2 macros
;; Do 1 nop_two_two_complex_unfft_2 macros
;;	distance between fft data elements is 2K
;;	do 1 macros each processing 16 data values

c0b:	start_timer 16
	mov	rsi, DATA_ADDR		;; Get source address
	mov	rcx, scratch_area	;; Get address of scratch area
	sub	rax, rax
	sub	rdx, rdx
pfing	mov	rbp, DATA_PREFETCH	;; Set prefetch distance
pfing	sub	rbp, rsi
c6b:	g2cl_eight_reals_unfft_2 rsi, 64, blkdst, rcx, 64, clmblkdst
	add	al, 256/2/clm		;; Test loop counter
	jnc	c6b			;; Iterate if necessary

;; Do 78 four_complex_unfft macros
;;	distance between fft data elements is 2K
;;	do 39 macros each processing 16 data values

	bump	rsi, -2*clm*64+2*blkdst	;; Next source pointer
	bump	rcx, -2*clm*64+2*clmblkdst;; Next dest pointer
	mov	rdi, sincos1		;; Load sin/cos pointer
	mov	al, 256/4+10		;; 39 iterations of 2*clm
c7b:	g2cl_four_complex_unfft rsi, 64, blkdst, rcx, 64, clmblkdst
	add	dl, 256/2/clm		;; Test loop counter
	jnc	c7b			;; Iterate if necessary
	bump	rsi, -2*clm*64+2*blkdst	;; Next source pointer
	bump	rcx, -2*clm*64+2*clmblkdst;; Next dest pointer
	bump	rdi, XMM_SCD		;; Next sine/cosine pointer
	add	al, 256/4		;; Test loop counter
	jnc	c7b			;; Iterate if necessary
	bump	rcx, -8*clmblkdst+clmblkdst8;; Next dest pointer
	sub	al, 1			;; Test outer loop counter
	jnz	c7b			;; Iterate if necessary
	lea	rcx, [rsi-40*2*blkdst]	;; Create prefetch pointer
	mov	rsi, scratch_area	;; Restore source pointer
	end_timer 16

;; Do inverse FFT levels 6,7
;; On input the 64-byte cache lines hold these data values:
;;	0K	4K	8K	12K
;;	2K	...
;;	16K	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0K	16K	32K	48K
;;	2K	...
;;	...
;;	14K
;;	64K	...
;;	...

;; Do 4 eight_reals_unfft_2 macros
;; Do 4 nop_two_two_complex_unfft_2 macros
;;	distance between fft data elements is 8K
;;	do 2 macros each processing 32 data values

	start_timer 17
	mov	al, 2			;; 2 iterations of 2*clm
c4b:	x4cl_eight_reals_unfft_2 rsi, 64, 2*clmblkdst, 4*clmblkdst
	add	al, 256/2/clm		;; Test loop counter
	jnc	c4b			;; Iterate if necessary
	bump	rsi, -2*clm*64+clmblkdst;; Next source pointer
	sub	al, 1			;; Test outer loop counter
	jnz	c4b			;; Iterate if necessary

;; Do 72 four_complex_unfft macros
;;	distance between fft data elements is 8K
;;	do 18 macros each processing 32 data values

	bump	rsi, -2*clmblkdst+clmblkdst8;; Next source pointer
	mov	rdi, sincos2		;; Load sin/cos pointer
	mov	eax, 9*256+2		;; 9 iterations of 2 iters of 2*clm
c5a:	xtouch	[rcx+(clm-1)*128]	;; Preload TLB
c5b:	x4cl_four_complex_unfft rsi, 64, 2*clmblkdst, 4*clmblkdst, rdi
	xloop1	c5b, clmblkdst
	sub	al, 1			;; Test middle loop counter
	jnz	c5a			;; Iterate if necessary
	bump	rsi, -2*clmblkdst+clmblkdst8;; Next source pointer
	bump	rdi, XMM_SCD		;; Next sine/cosine pointer
	sub	eax, 256		;; Test outer loop counter
	mov	al, 2			;; Load middle loop counter
	jnz	c5a			;; Iterate if necessary
	bump	rsi, -10*clmblkdst8	;; Restore source pointer
	end_timer 17

;; Do inverse FFT levels 4,5
;; On input the 64-byte cache lines hold these data values:
;;	0K	16K	32K	48K
;;	2K	...
;;	...
;;	14K
;;	64K	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0K	32K	64K	96K
;;	2K	...
;;	...
;;	30K
;;	128K	192K	256K	320K
;;	...
;;
;; Do 16 eight_reals_unfft_2 macros
;;	distance between fft data elements is 32K
;;	do 8 macros each processing 16 data values
;;
;; Do 64 four_complex_unfft macros
;;	distance between fft data elements is 32K
;;	do 16 macros each processing 32 data values

;; Do inverse FFT levels 1,2,3
;; On input the 64-byte cache lines hold these data values:
;;	0K	32K	64K	96K
;;	2K	...
;;	...
;;	30K
;;	128K	192K	256K	320K
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0K	32K	320K	352K
;;	2K	...
;;	...
;;	30K
;;	64K	192K	384K	512K
;;	66K
;;	...
;;	190K

;; Do 128 five_reals_last_unfft macros
;;	distance between fft data elements is 128K
;;	do 16 macros each processing 40 data values

	start_timer 18
	mov	rdi, sincos3		;; Load sin/cos pointer
	mov	al, 8			;; 8 iterations

c2a:	xtouchlines 3			;; Preload TLBs

c2b:	x2cl_half_eight_reals_unfft_2 rsi, 64, clmblkdst8
	add	al, 256/2		;; Test loop counter
	jnc	c2b			;; Iterate if necessary

	bump	rsi, -2*64+2*clmblkdst8	;; Next source pointer
c3b:	x4cl_four_complex_unfft rsi, 64, clmblkdst8, 2*clmblkdst8, rdi
	add	al, 256/2		;; Test loop counter
	jnc	c3b			;; Iterate if necessary
	bump	rsi, -2*64+4*clmblkdst8	;; Restore source pointer
	bump	rdi, XMM_SCD		;; Next sine/cosine pointer
	add	ah, 256/2		;; Test outer loop counter
	jnc	c3b			;; Iterate if necessary
	L2prefetch128 [rcx][rbp]
pfing	bump	rcx, blkdst		;; Next prefetch pointer
	bump	rdi, -2*XMM_SCD		;; Restore sine/cosine pointer

	bump	rsi, -2*4*clmblkdst8-2*clmblkdst8;; Restore source pointer
	xfive_reals_unfft_preload	;; Preload consts for five real macros
c1b:	x5cl_five_reals_last_unfft rsi, 64, 2*clmblkdst8
	xloop1sp c1b, clmblkdst8
	add	ah, 256/2
	jnc	c1b

	IF clm GT 1
	bump	rsi, -2*clmblkdst8+2*64	;; Next source pointer
	bump	rcx, -3*blkdst+128	;; Next prefetch pointer
	add	dl, 256/clm		;; clm iterations
	jnc	c2b
	bump	rsi, -clm*2*64+clmblkdst;; Next source pointer
	bump	rcx, -clm*128+3*blkdst	;; Next prefetch pointer
	ELSE
	bump	rsi, -2*clmblkdst8+clmblkdst;; Next source pointer
	ENDIF

	sub	al, 1			;; Test loop counter
	jnz	c2a			;; Iterate if necessary
	end_timer 18

;; Normalize these values

	pass1_normalize 1, yes_fft

;; Copy data from the scratch area back to the FFT data area

	copy_scratch_data_to_fft 80
	jmp	endlp			;; Skip FFT code if flag not set

;; Fire up auxiliary threads to do pass 1 forward FFT

pass1:	pass1_forward_fft_setup b0b

;; Copy data from the FFT data area back to the scratch area

b0b:	copy_fft_data_to_scratch 80

;; Do FFT levels 1,2,3
;;
;; On input the 64-byte cache lines hold these data values:
;;	0K	32K	320K	352K
;;	2K	...
;;	...
;;	30K
;;	64K	192K	384K	512K
;;	66K
;;	...
;;	190K
;; On output the 64-byte cache lines hold these data values:
;;	0K	32K	64K	96K
;;	2K	...
;;	...
;;	30K
;;	128K	256K	384K	512K
;;	130K
;;	...
;;	254K
;;
;; Do 128 five_reals_first_fft macros (FFT levels 1,2,3)
;;	distance between fft data elements is 128K
;;	do 16 iterations each processing 40 data values

;; Do FFT levels 4,5
;; Values 0K-127K is real data, 128K-639K is complex data.
;;
;; On input the 64-byte cache lines hold these data values:
;;	0K	32K	64K	96K
;;	2K	...
;;	...
;;	30K
;;	128K	256K	384K	512K
;;	130K
;;	...
;;	254K
;; On output the 64-byte cache lines hold these data values:
;;	0K	32K	64K	96K
;;	2K	...
;;	...
;;	30K	...
;;	128K	...
;;	...
;;
;; Do 16 eight_reals_fft_2 macros
;;	distance between fft data elements is 32K
;;	do 16 macros each processing 8 data values
;;
;; Do 64 four_complex_fft macros
;;	distance between fft data elements is 32K
;;	do 16 macros each processing 32 data values

;; Do FFT levels 6,7
;; Values 0K-31K is real data, 32K-63K is semi-real data, 64K-639K is
;; complex data.
;;
;; On input the 64-byte cache lines hold these data values:
;;	0K	32K	64K	96K
;;	2K	...
;;	...
;;	30K	...
;;	128K	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0K	8K	16K	24K
;;	2K	...
;;	...
;;	6K	...
;;	32K	...
;;	...
;;
;; Do 8 eight_reals_fft_2 macros
;; Do 8 two_two_complex_fft macros
;;	distance between fft data elements is 8K
;;	do 4 macros each processing 32 data values
;;
;; Do 64 four_complex_fft macros
;;	distance between fft data elements is 8K
;;	do 16 macros each processing 32 data values

yes_fft:start_timer 21
	mov	rsi, scratch_area	;; Get address of scratch area
	mov	rcx, DATA_ADDR		;; Load source address for prefetching
pfing	mov	rbp, DATA_PREFETCH	;; Set prefetch distance
pfing	sub	rbp, rcx
pfing	bump	rcx, 40*blkdst		;; Create prefetching pointer
	mov	rdi, sincos3		;; Load sin/cos pointer
	mov	rbx, sincos2		;; Load sin/cos pointer
	mov	eax, 4			;; 4 iterations
	sub	rdx, rdx

b1a:	xtouchlines 10			;; Preload TLBs

b1aa:	xfive_reals_fft_preload		;; Preload consts for five real macros
b1b:	x5cl_five_reals_fft rsi, 64, 2*clmblkdst8
	xloop1sp b1b, 4*clmblkdst
	add	ah, 256/2		;; Test inner loop counter
	jnc	b1b			;; Iterate if necessary
	bump	rsi, -2*4*clmblkdst+clmblkdst8
	add	eax, 80000000h
	jnc	b1b

	bump	rsi, -2*clmblkdst8	;; Restore source pointer
b2b:	x1cl_half_eight_reals_fft_2 rsi, 64
	add	al, 256/2		;; Test inner loop counter
	jnc	b2b			;; Iterate if necessary
	bump	rsi, -2*64+4*clmblkdst
	add	ah, 256/2		;; Test inner loop counter
	jnc	b2b			;; Iterate if necessary
	bump	rsi, -2*4*clmblkdst+clmblkdst8
	add	eax, 80000000h
	jnc	b2b

;;	bump	rsi, -2*clmblkdst8+2*clmblkdst8;; Next source pointer
b3b:	x4cl_four_complex_fft rsi, 64, 2*clmblkdst8, 4*clmblkdst8, rdi
	add	al, 256/2		;; Test inner loop counter
	jnc	b3b			;; Iterate if necessary
	bump	rsi, -2*64+4*clmblkdst
	add	ah, 256/2		;; Test inner loop counter
	jnc	b3b			;; Iterate if necessary
	L2prefetch128 [rcx][rbp]
pfing	bump	rcx, blkdst		;; Next prefetch pointer
	bump	rsi, -2*4*clmblkdst+clmblkdst8
	add	eax, 80000000h
	jnc	b3b

	bump	rsi, -2*clmblkdst8-2*clmblkdst8;; Restore source pointer
b4b:	x4cl_eight_reals_fft_2 rsi, 64, 4*clmblkdst, clmblkdst8, rbx
	add	al, 256/2		;; Test inner loop counter
	jnc	b4b			;; Iterate if necessary

	bump	rsi, -2*64+2*clmblkdst8	;; Next source pointer
	bump	rbx, XMM_SCD		;; Next sine/cosine pointer
b5b:	x4cl_four_complex_fft rsi, 64, 4*clmblkdst, clmblkdst8, rbx
	xloop1sp b5b, 2*clmblkdst8
	bump	rbx, 2*XMM_SCD		;; Next sine/cosine pointer
	add	ah, 256/4		;; Test middle loop counter
	jnc	b5b			;; Iterate if necessary
	bump	rbx, -9*XMM_SCD		;; Restore sine/cosine pointer

	IF clm GT 1
	bump	rsi, -5*2*clmblkdst8+2*64;; Next source pointer
	bump	rcx, -10*blkdst+128	;; Next prefetch pointer
	add	dl, 256/clm		;; clm iterations
	jnc	b1aa
	bump	rsi, -clm*2*64+clmblkdst;; Next source pointer
	bump	rcx, -clm*128+10*blkdst	;; Next prefetch pointer
	ELSE
	bump	rsi, -5*2*clmblkdst8+clmblkdst;; Next source pointer
	ENDIF

	sub	al, 1			;; Test loop counter
	jnz	b1a			;; Iterate if necessary
	bump	rsi, -4*clmblkdst	;; Restore source pointer
pfing	bump	rcx, -80*blkdst		;; Restore prefetch pointer
	end_timer 21

;; Do FFT levels 8,9
;; Values 0K-7K is real data, 8K-15K is semi-real data, 16K-619K is
;; complex data.
;;
;; On input the 64-byte cache lines hold these data values:
;;	0K	8K	16K	24K
;;	2K	...
;;	...
;;	6K	...
;;	32K	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0K	2K	4K	6K
;;	8K	...
;;	...

;; Do 1 eight_reals_fft_2 macros
;; Do 1 nop_two_two_complex_fft_2 macros
;; Do 2 four_complex_fft macros
;;	distance between fft data elements is 2K
;;	do 1 macros each processing 32 data values

	start_timer 24
	mov	rdi, sincos1		;; Load sin/cos pointer
b6b:	g4cl_eight_reals_fft_2 rsi, 64, clmblkdst, 2*clmblkdst, rcx, 64, blkdst, 2*blkdst
	add	al, 256/2/clm		;; Test inner loop counter
	jnc	b6b			;; Iterate if necessary

;; Do 76 four_complex_fft macros
;;	distance between fft data elements is 2K
;;	do 19 macros each processing 32 data values

	bump	rsi, -2*clm*64+4*clmblkdst;; Next source pointer
	bump	rcx, -2*clm*64+4*blkdst	;; Next dest pointer
	bump	rdi, XMM_SCD		;; Next sine/cosine pointer
	mov	eax, 10*256+1		;; 19 iterations of 2*clm
b7b:	g4cl_four_complex_fft rsi, 64, clmblkdst, 2*clmblkdst, rcx, 64, blkdst, 2*blkdst
	L2prefetch128 [rcx-64][rbp]
	L2prefetch128 [rcx-64+blkdst][rbp]
	bump	rbp, -64+2*blkdst
	add	al, 256/2		;; Test inner loop counter
	jnc	b7b			;; Iterate if necessary
	bump	rbp, 128-4*blkdst	
	IF clm NE 1
	add	dl, 256/clm
	jnc	b7b
	ENDIF
	bump	rsi, -2*clm*64+4*clmblkdst;; Next source pointer
	bump	rcx, -2*clm*64+4*blkdst	;; Next dest pointer
	bump	rdi, 2*XMM_SCD		;; Next sine/cosine pointer
	sub	al, 1			;; Test outer loop counter
	jnz	b7b			;; Iterate if necessary
	bump	rsi, -8*clmblkdst+clmblkdst8
	sub	ah, 1
	mov	al, 2
	jnz	b7b
	end_timer 24

;; Work on next 4 sets of 320 values

endlp:	xpass1_get_next_block pass2, c0b, b0b ;; Get next block and jump
	done:	xfft_footer
	ENDM


;; Perform a 768K-element FFT.  This is done in two passes.  Pass 1
;; does 9 levels, pass 2 uses common code to perform the last 11 levels.

hg_pass1sclevels9pfa6 MACRO fftname, pass2_levels, clmarg
	LOCAL	b0b, b1a, b1b, b2b, b3b, b4b, b5b, b6b, b7b
	LOCAL	c0b, c1b, c2a, c2b, c3b, c4b, c5a, c5b, c6b, c7b
	LOCAL	pass1, pass2, endlp, done, yes_fft

	set_FFT_constants fftname, 0, 0, pass2_levels, clmarg
	xfft_header pass1

;; Do the last 11 FFT levels
;;	do 1 pass2_11_levels_real and 95 pass2_11_levels_complex

pass2:	start_timer 0
	call	xpass2name
	end_timer 0
	cmp	ffttype, 1		;; We're done if FFTing only
	je	done

;; Do the inverse FFT

	pass1_inverse_fft_setup c0b

;; Do inverse FFT levels 8,9
;; On input the 64-byte cache lines hold these data values:
;;	0K	2K	4K	6K
;;	8K	10K	12K	14K
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0K	4K	8K	12K
;;	2K	...
;;	16K	...
;;	...

;; Do 1 eight_reals_unfft_2 macros
;; Do 1 nop_two_two_complex_unfft_2 macros
;;	distance between fft data elements is 2K
;;	do 1 macros each processing 16 data values

c0b:	start_timer 16
	mov	rsi, DATA_ADDR		;; Get source address
	mov	rcx, scratch_area	;; Get address of scratch area
	sub	rax, rax
	sub	rdx, rdx
pfing	mov	rbp, DATA_PREFETCH	;; Set prefetch distance
pfing	sub	rbp, rsi
c6b:	g2cl_eight_reals_unfft_2 rsi, 64, blkdst, rcx, 64, clmblkdst
	add	al, 256/2/clm		;; Test loop counter
	jnc	c6b			;; Iterate if necessary

;; Do 94 four_complex_unfft macros
;;	distance between fft data elements is 2K
;;	do 47 macros each processing 16 data values

	bump	rsi, -2*clm*64+2*blkdst	;; Next source pointer
	bump	rcx, -2*clm*64+2*clmblkdst;; Next dest pointer
	mov	rdi, sincos1		;; Load sin/cos pointer
	mov	al, 256/4+12		;; 47 iterations of 2*clm
c7b:	g2cl_four_complex_unfft rsi, 64, blkdst, rcx, 64, clmblkdst
	add	dl, 256/2/clm		;; Test loop counter
	jnc	c7b			;; Iterate if necessary
	bump	rsi, -2*clm*64+2*blkdst	;; Next source pointer
	bump	rcx, -2*clm*64+2*clmblkdst;; Next dest pointer
	bump	rdi, XMM_SCD		;; Next sine/cosine pointer
	add	al, 256/4
	jnc	c7b
	bump	rcx, -8*clmblkdst+clmblkdst8
	sub	al, 1			;; Test outer loop counter
	jnz	c7b			;; Iterate if necessary
	lea	rcx, [rsi-48*2*blkdst]	;; Create prefetch pointer
	mov	rsi, scratch_area	;; Restore source pointer
	end_timer 16

;; Do inverse FFT levels 6,7
;; On input the 64-byte cache lines hold these data values:
;;	0K	4K	8K	12K
;;	2K	...
;;	16K	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0K	16K	32K	48K
;;	2K	...
;;	...
;;	14K
;;	64K	...
;;	...

;; Do 4 eight_reals_unfft_2 macros
;; Do 4 nop_two_two_complex_unfft_2 macros
;;	distance between fft data elements is 8K
;;	do 2 macros each processing 32 data values

	start_timer 17
	mov	al, 2			;; 2 iterations of 2*clm
c4b:	x4cl_eight_reals_unfft_2 rsi, 64, 2*clmblkdst, 4*clmblkdst
	add	al, 256/2/clm		;; Test loop counter
	jnc	c4b			;; Iterate if necessary
	bump	rsi, -2*clm*64+clmblkdst;; Next source pointer
	sub	al, 1			;; Test outer loop counter
	jnz	c4b			;; Iterate if necessary

;; Do 88 four_complex_unfft macros
;;	distance between fft data elements is 8K
;;	do 22 macros each processing 32 data values

	bump	rsi, -2*clmblkdst+clmblkdst8;; Next source pointer
	mov	rdi, sincos2		;; Load sin/cos pointer
	mov	eax, 11*256+2		;; 11 iterations of 2 iters of 2*clm
c5a:	xtouch	[rcx+(clm-1)*128]	;; Preload TLB
c5b:	x4cl_four_complex_unfft rsi, 64, 2*clmblkdst, 4*clmblkdst, rdi
	xloop1	c5b, clmblkdst
	sub	al, 1			;; Test middle loop counter
	jnz	c5a			;; Iterate if necessary
	bump	rsi, -2*clmblkdst+clmblkdst8;; Next source pointer
	bump	rdi, XMM_SCD		;; Next sine/cosine pointer
	sub	eax, 256		;; Test outer loop counter
	mov	al, 2			;; Load middle loop counter
	jnz	c5a			;; Iterate if necessary
	bump	rsi, -12*clmblkdst8	;; Restore source pointer
	end_timer 17

;; Do inverse FFT levels 4,5
;; On input the 64-byte cache lines hold these data values:
;;	0K	16K	32K	48K
;;	2K	...
;;	...
;;	14K
;;	64K	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0K	64K	128K	192K
;;	2K	...
;;	...
;;	30K
;;	256K	...
;;	...
;;
;; Do 16 eight_reals_unfft_2 macros
;; Do 16 nop_two_two_complex_unfft_2 macros
;;	distance between fft data elements is 32K
;;	do 8 macros each processing 32 data values
;;
;; Do 64 four_complex_unfft macros
;;	distance between fft data elements is 32K
;;	do 16 macros each processing 32 data values

;; Do inverse FFT levels 1,2,3
;; On input the 64-byte cache lines hold these data values:
;;	0K	64K	128K	192K
;;	2K	...
;;	...
;;	30K
;;	256K	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0K	192K	384K	576K
;;	...
;;	190K
;;
;; Do 128 six_reals_last_unfft macros
;;	distance between fft data elements is 128K
;;	do 32 macros each processing 24 data values

	start_timer 18
	mov	rdi, sincos3		;; Load sin/cos pointer
	mov	al, 8			;; 8 iterations
	xsix_reals_unfft_preload

c2a:	xtouchlines 3			;; Preload TLBs

c2b:	x4cl_eight_reals_unfft_2 rsi, 64, clmblkdst8, 2*clmblkdst8
	add	al, 256/2		;; Test loop counter
	jnc	c2b			;; Iterate if necessary

	bump	rsi, -2*64+4*clmblkdst8	;; Next source pointer
c3b:	x4cl_four_complex_unfft rsi, 64, clmblkdst8, 2*clmblkdst8, rdi
	xloop1sp c3b, 4*clmblkdst8
	bump	rdi, XMM_SCD		;; Next sine/cosine pointer
	add	ah, 256/2		;; Test outer loop counter
	jnc	c3b			;; Iterate if necessary
	bump	rdi, -2*XMM_SCD		;; Restore sine/cosine pointer

	bump	rsi, -3*4*clmblkdst8	;; Restore source pointer
c1b:	x3cl_six_reals_last_unfft rsi, 64, 4*clmblkdst8
	add	al, 256/2		;; Test loop counter
	jnc	c1b			;; Iterate if necessary
	bump	rsi, -2*64+clmblkdst8
	add	ah, 256/4
	jnc	c1b
	L2prefetch128 [rcx][rbp]
pfing	bump	rcx, blkdst		;; Next prefetch pointer

	IF clm GT 1
	bump	rsi, -4*clmblkdst8+2*64	;; Next source pointer
	bump	rcx, -3*blkdst+128	;; Next prefetch pointer
	add	dl, 256/clm		;; clm iterations
	jnc	c2b
	bump	rsi, -clm*2*64+clmblkdst;; Next source pointer
	bump	rcx, -clm*128+3*blkdst	;; Next prefetch pointer
	ELSE
	bump	rsi, -4*clmblkdst8+clmblkdst;; Next source pointer
	ENDIF

	sub	al, 1			;; Test loop counter
	jnz	c2a			;; Iterate if necessary
	end_timer 18

;; Normalize these values

	pass1_normalize 1, yes_fft

;; Copy data from the scratch area back to the FFT data area

	copy_scratch_data_to_fft 96
	jmp	endlp			;; Skip FFT code if flag not set

;; Fire up auxiliary threads to do pass 1 forward FFT

pass1:	pass1_forward_fft_setup b0b

;; Copy data from the FFT data area back to the scratch area

b0b:	copy_fft_data_to_scratch 96

;; Do FFT levels 1,2,3
;;
;; On input the 64-byte cache lines hold these data values:
;;	0K	192K	384K	576K
;;	...
;;	190K
;; On output the 64-byte cache lines hold these data values:
;;	0K	64K	128K	192K
;;	2K	...
;;	...
;;	62K
;;	256K	384K	512K	640K
;;	258K
;;	...
;;	382K
;;
;; Do 128 six_reals_first_fft macros (FFT levels 1,2,3)
;;	distance between fft data elements is 128K
;;	do 32 iterations each processing 24 data values

;; Do FFT levels 4,5
;; Values 0K-127K is real data, 128K-255K is semi-real data, and
;; 256K-767K is complex data.
;;
;; On input the 64-byte cache lines hold these data values:
;;	0K	64K	128K	192K
;;	2K	...
;;	...
;;	62K
;;	256K	384K	512K	640K
;;	258K
;;	...
;;	382K
;; On output the 64-byte cache lines hold these data values:
;;	0K	16K	32K	48K
;;	2K	...
;;	...
;;	14K	...
;;	64K	...
;;	...
;;
;; Do 16 eight_reals_fft_2 macros
;; Do 16 nop_two_two_complex_fft_2 macros
;;	distance between fft data elements is 32K
;;	do 16 macros each processing 16 data values
;;
;; Do 64 four_complex_fft macros
;;	distance between fft data elements is 32K
;;	do 16 macros each processing 32 data values

;; Do FFT levels 6,7
;; Values 0K-31K is real data, 32K-63K is semi-real data, 64K-767K is
;; complex data.
;;
;; On input the 64-byte cache lines hold these data values:
;;	0K	32K	64K	96K
;;	2K	...
;;	...
;;	30K	...
;;	128K	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0K	8K	16K	24K
;;	2K	...
;;	...
;;	6K	...
;;	32K	...
;;	...
;;
;; Do 8 eight_reals_fft_2 macros
;; Do 8 two_two_complex_fft macros
;;	distance between fft data elements is 8K
;;	do 4 macros each processing 32 data values
;;
;; Do 80 four_complex_fft macros
;;	distance between fft data elements is 8K
;;	do 20 macros each processing 32 data values

yes_fft:start_timer 21
	mov	rsi, scratch_area	;; Get address of scratch area
	mov	rcx, DATA_ADDR		;; Load source address for prefetching
pfing	mov	rbp, DATA_PREFETCH	;; Set prefetch distance
pfing	sub	rbp, rcx
pfing	bump	rcx, 44*blkdst		;; Create prefetching pointer
	mov	rdi, sincos3		;; Load sin/cos pointer
	mov	rbx, sincos2		;; Load sin/cos pointer
	mov	eax, 4			;; 4 iterations
	sub	rdx, rdx
	xsix_reals_fft_preload

b1a:	xtouchlines 13			;; Preload TLBs

b1b:	x3cl_six_reals_fft rsi, 64, 4*clmblkdst8
	add	al, 256/2		;; Test inner loop counter
	jnc	b1b			;; Iterate if necessary
	bump	rsi, -2*64+4*clmblkdst
	add	ah, 256/2		;; Test inner loop counter
	jnc	b1b			;; Iterate if necessary
	L2prefetch128 [rcx][rbp]
pfing	bump	rcx, blkdst		;; Next prefetch pointer
	bump	rsi, -2*4*clmblkdst+clmblkdst8
	add	eax, 80000000h/4*2
	jnc	b1b

	bump	rsi, -4*clmblkdst8	;; Restore source pointer
b2b:	x2cl_eight_reals_fft_2 rsi, 64, 2*clmblkdst8
	add	al, 256/2		;; Test inner loop counter
	jnc	b2b			;; Iterate if necessary
	bump	rsi, -2*64+4*clmblkdst
	add	ah, 256/2		;; Test inner loop counter
	jnc	b2b			;; Iterate if necessary
	bump	rsi, -2*4*clmblkdst+clmblkdst8
	add	eax, 80000000h
	jnc	b2b

	bump	rsi, -2*clmblkdst8+4*clmblkdst8;; Next source pointer
b3b:	x4cl_four_complex_fft rsi, 64, 2*clmblkdst8, 4*clmblkdst8, rdi
	xloop1sp b3b, 4*clmblkdst
	add	ah, 256/2		;; Test inner loop counter
	jnc	b3b			;; Iterate if necessary
	bump	rsi, -2*4*clmblkdst+clmblkdst8
	add	eax, 80000000h
	jnc	b3b

	bump	rsi, -2*clmblkdst8-4*clmblkdst8;; Restore source pointer
b4b:	x4cl_eight_reals_fft_2 rsi, 64, 4*clmblkdst, clmblkdst8, rbx
	add	al, 256/2		;; Test inner loop counter
	jnc	b4b			;; Iterate if necessary

	bump	rsi, -2*64+2*clmblkdst8	;; Load source pointer
	bump	rbx, XMM_SCD		;; Next sine/cosine pointer
	mov	ah, 5			;; 5 iterations
b5b:	x4cl_four_complex_fft rsi, 64, 4*clmblkdst, clmblkdst8, rbx
	xloop1sp b5b, 2*clmblkdst8
	bump	rbx, 2*XMM_SCD		;; Next sine/cosine pointer
	sub	ah, 1			;; Test outer loop counter
	jnz	b5b			;; Iterate if necessary
	bump	rbx, -11*XMM_SCD	;; Restore sine/cosine pointer

	IF clm GT 1
	bump	rsi, -6*2*clmblkdst8+2*64;; Next source pointer
	bump	rcx, -13*blkdst+128	;; Next prefetch pointer
	add	dl, 256/clm		;; clm iterations
	jnc	b1b
	bump	rsi, -clm*2*64+clmblkdst;; Next source pointer
	bump	rcx, -clm*128+13*blkdst	;; Next prefetch pointer
	ELSE
	bump	rsi, -6*2*clmblkdst8+clmblkdst;; Next source pointer
	ENDIF

	sub	al, 1			;; Test loop counter
	jnz	b1a			;; Iterate if necessary
	bump	rsi, -4*clmblkdst	;; Restore source pointer
pfing	bump	rcx, -96*blkdst		;; Restore prefetch pointer
	end_timer 21

;; Do FFT levels 8,9
;; Values 0K-7K is real data, 8K-15K is semi-real data, 16K-767K is
;; complex data.
;;
;; On input the 64-byte cache lines hold these data values:
;;	0K	8K	16K	24K
;;	2K	...
;;	...
;;	6K	...
;;	32K	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0K	2K	4K	6K
;;	8K	...
;;	...

;; Do 1 eight_reals_fft_2 macros
;; Do 1 nop_two_two_complex_fft_2 macros
;; Do 2 four_complex_fft macros
;;	distance between fft data elements is 2K
;;	do 1 macros each processing 32 data values

	start_timer 24
	mov	rdi, sincos1		;; Load sin/cos pointer
b6b:	g4cl_eight_reals_fft_2 rsi, 64, clmblkdst, 2*clmblkdst, rcx, 64, blkdst, 2*blkdst
	add	al, 256/2/clm		;; Test inner loop counter
	jnc	b6b			;; Iterate if necessary

;; Do 92 four_complex_fft macros
;;	distance between fft data elements is 2K
;;	do 23 macros each processing 32 data values

	bump	rsi, -2*clm*64+4*clmblkdst;; Next source pointer
	bump	rcx, -2*clm*64+4*blkdst	;; Next dest pointer
	bump	rdi, XMM_SCD		;; Next sine/cosine pointer
	mov	eax, 12*256+1		;; 23 iterations of 2*clm
b7b:	g4cl_four_complex_fft rsi, 64, clmblkdst, 2*clmblkdst, rcx, 64, blkdst, 2*blkdst
	L2prefetch128 [rcx-64][rbp]
	L2prefetch128 [rcx-64+blkdst][rbp]
	bump	rbp, -64+2*blkdst
	add	al, 256/2		;; Test inner loop counter
	jnc	b7b			;; Iterate if necessary
	bump	rbp, 128-4*blkdst
	IF clm NE 1
	add	dl, 256/clm
	jnc	b7b
	ENDIF
	bump	rsi, -2*clm*64+4*clmblkdst;; Next source pointer
	bump	rcx, -2*clm*64+4*blkdst	;; Next dest pointer
	bump	rdi, 2*XMM_SCD		;; Next sine/cosine pointer
	sub	al, 1
	jnz	b7b
	bump	rsi, -8*clmblkdst+clmblkdst8
	sub	ah, 1			;; Test outer loop counter
	mov	al, 2
	jnz	b7b			;; Iterate if necessary
	end_timer 24

;; Work on next 4 sets of 384 values

endlp:	xpass1_get_next_block pass2, c0b, b0b ;; Get next block and jump
	done:	xfft_footer
	ENDM


;; Perform a 768K-element negacyclic FFT.  This is done in two passes.  Pass 1
;; does 9 levels, pass 2 uses common code to perform the last 11 levels.

hg_pass1sclevels9complex3 MACRO fftname, pass2_levels, clmarg
	LOCAL	b0b, b1a, b1b, b2a, b2b, b3a, b3b, b5a, b5b, b7b
	LOCAL	c0b, c1a, c1b, c3a, c3b, c5a, c5b, c7b
	LOCAL	pass1, pass2, endlp, done, yes_fft

	set_FFT_constants fftname, 0, 1, pass2_levels, clmarg
	xfft_header pass1

;; Do the last 11 FFT levels
;;	do 96 pass2_11_levels_complex

pass2:	start_timer 0
	call	xpass2name
	end_timer 0
	cmp	ffttype, 1		;; We're done if FFTing only
	je	done

;; Do the inverse FFT

	pass1_inverse_fft_setup c0b

;; Do inverse FFT levels 8,9
;; On input the 64-byte cache lines hold these data values:
;;	0K	2K	4K	6K
;;	8K	10K	12K	14K
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0K	4K	8K	12K
;;	2K	...
;;	16K	...
;;	...

;; Do 96 four_complex_unfft macros
;;	distance between fft data elements is 2K
;;	do 48 macros each processing 16 data values

c0b:	start_timer 16
	mov	rsi, DATA_ADDR		;; Get source address
	mov	rcx, scratch_area	;; Get address of scratch area
	mov	rdi, sincos1		;; Load sin/cos pointer
	mov	eax, 12			;; 12 iterations of 4 iters of 2*clm
	sub	rdx, rdx
pfing	mov	rbp, DATA_PREFETCH	;; Set prefetch distance
pfing	sub	rbp, rsi
c7b:	g2cl_four_complex_unfft rsi, 64, blkdst, rcx, 64, clmblkdst
	add	dl, 256/2/clm		;; Test loop counter
	jnc	c7b			;; Iterate if necessary
	bump	rsi, -2*clm*64+2*blkdst	;; Next source pointer
	bump	rcx, -2*clm*64+2*clmblkdst;; Next dest pointer
	bump	rdi, XMM_SCD		;; Next sine/cosine pointer
	add	al, 256/4
	jnc	c7b
	bump	rcx, -8*clmblkdst+clmblkdst8
	sub	al, 1			;; Test outer loop counter
	jnz	c7b			;; Iterate if necessary
	lea	rcx, [rsi-48*2*blkdst]	;; Create prefetch pointer
	mov	rsi, scratch_area	;; Restore source pointer
	end_timer 16

;; Do inverse FFT levels 6,7
;; On input the 64-byte cache lines hold these data values:
;;	0K	4K	8K	12K
;;	2K	...
;;	16K	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0K	16K	32K	48K
;;	2K	...
;;	...
;;	14K
;;	64K	...
;;	...

;; Do 96 four_complex_unfft macros
;;	distance between fft data elements is 8K
;;	do 24 macros each processing 32 data values

	start_timer 17
	mov	rdi, sincos2		;; Load sin/cos pointer
	mov	al, 12			;; 12 iterations of 2*2*clm
c5a:	xtouch	[rcx+(clm-1)*128]	;; Preload TLB
c5b:	x4cl_four_complex_unfft rsi, 64, 2*clmblkdst, 4*clmblkdst, rdi
	xloop2	c5b, clmblkdst		;; 2 * 2*clm iterations
	bump	rsi, -2*clmblkdst+clmblkdst8;; Next source pointer
	bump	rdi, XMM_SCD		;; Next sine/cosine pointer
	sub	al, 1			;; Test outer loop counter
	jnz	c5a			;; Iterate if necessary
	bump	rsi, -12*clmblkdst8	;; Restore source pointer
	end_timer 17

;; Do inverse FFT levels 4,5
;; On input the 64-byte cache lines hold these data values:
;;	0K	16K	32K	48K
;;	2K	...
;;	...
;;	14K
;;	64K	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0K	64K	128K	192K
;;	2K	...
;;	...
;;	30K
;;	256K	...
;;	...

;; Do 96 four_complex_unfft macros
;;	distance between fft data elements is 32K
;;	do 24 macros each processing 32 data values

	start_timer 18
	mov	rdi, sincos3		;; Load sin/cos pointer
	mov	eax, 3*256+4		;; 3 iterations of 4 iters of 2*2*clm
c3a:	xtouch	[rcx+(clm-1)*128]	;; Preload TLB
c3b:	x4cl_four_complex_unfft rsi, 64, clmblkdst8, 2*clmblkdst8, rdi
	xloop2	c3b, clmblkdst		;; 2* 2*clm iterations
	sub	al, 1			;; Test middle loop counter
	jnz	c3a			;; Iterate if necessary
	bump	rsi, -4*2*clmblkdst+4*clmblkdst8;; Next source pointer
	bump	rdi, XMM_SCD		;; Next sine/cosine pointer
	sub	eax, 256		;; Test outer loop counter
	mov	al, 4			;; Load middle loop counter
	jnz	c3a			;; Iterate if necessary
	bump	rsi, -3*4*clmblkdst8	;; Restore source pointer
	end_timer 18

;; Do inverse FFT levels 1,2,3
;; On input the 64-byte cache lines hold these data values:
;;	0K	64K	128K	192K
;;	2K	...
;;	...
;;	30K
;;	256K	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0K	192K	384K	576K
;;	...
;;	190K

;; Do 128 three_complex_last_unfft macros
;;	distance between fft data elements is 128K
;;	do 32 macros each processing 24 data values

	start_timer 19
	mov	rdi, plus1_premults	;; Address of premultiplier table
	mov	eax, 4*256+8		;; 4 iters of 8 iters of 2*clm
c1a:	xtouch	[rcx+(clm-1)*128]	;; Preload TLB
c1b:	x3cl_three_complex_last_unfft rsi, 64, 4*clmblkdst8
	xloop1	c1b, clmblkdst		;; 2*clm iterations
	bump	rdi, 192		;; Next premultiplier pointer
	sub	al, 1			;; Test loop counter
	jnz	c1a			;; Iterate if necessary
	bump	rsi, -8*clmblkdst+clmblkdst8;; Next source pointer
	sub	ah, 1			;; Test outer loop counter
	mov	al, 8			;; Load middle loop counter
	jnz	c1a
	end_timer 19

;; Normalize these values

	pass1_normalize 1, yes_fft

;; Copy data from the scratch area back to the FFT data area

	copy_scratch_data_to_fft 96
	jmp	endlp			;; Skip FFT code if flag not set

;; Fire up auxiliary threads to do pass 1 forward FFT

pass1:	pass1_forward_fft_setup b0b

;; Copy data from the FFT data area back to the scratch area

b0b:	copy_fft_data_to_scratch 96

;; Do FFT levels 1,2,3
;;
;; On input the 64-byte cache lines hold these data values:
;;	0K	192K	384K	576K
;;	...
;;	190K
;; On output the 64-byte cache lines hold these data values:
;;	0K	64K	128K	192K
;;	2K	...
;;	...
;;	62K
;;	256K	384K	512K	640K
;;	258K
;;	...
;;	382K

;; Do 128 three_complex_first_fft macros (FFT levels 1,2,3)
;;	distance between fft data elements is 128K
;;	do 32 iterations each processing 24 data values

yes_fft:start_timer 21
	mov	rsi, scratch_area	;; Get address of scratch area
	mov	rdi, plus1_premults	;; Address of premultiplier table
	mov	rcx, DATA_ADDR		;; Load source address for prefetching
pfing	mov	rbp, DATA_PREFETCH	;; Set prefetch distance
pfing	sub	rbp, rcx
pfing	bump	rcx, 56*blkdst		;; Create prefetching pointer
	mov	eax, 4*256+4		;; 4 iters of 4 iters of 2*2*clm
	sub	rdx, rdx
b1a:	xtouch	[rcx+(clm-1)*128]	;; Preload TLB
b1b:	x3cl_three_complex_fft rsi, 64, 4*clmblkdst8
	xloop2edi b1b, clmblkdst, 192 ;; 2 * 2*clm iterations
	sub	al, 1			;; Test inner loop counter
	jnz	b1a			;; Iterate if necessary
	bump	rsi, -8*clmblkdst+clmblkdst8;; Next source pointer
	sub	ah, 1			;; Test outer loop counter
	mov	al, 4			;; Load middle loop counter
	jnz	b1a
	bump	rsi, -4*clmblkdst8	;; Restore source pointer
	end_timer 21

;; Do FFT levels 4,5
;;
;; On input the 64-byte cache lines hold these data values:
;;	0K	64K	128K	192K
;;	2K	...
;;	...
;;	62K
;;	256K	384K	512K	640K
;;	258K
;;	...
;;	382K
;; On output the 64-byte cache lines hold these data values:
;;	0K	32K	64K	96K
;;	2K	...
;;	...
;;	30K	...
;;	128K	...
;;	...

;; Do 96 four_complex_fft macros
;;	distance between fft data elements is 32K
;;	do 8 macros each processing 16 data values
;;	do 16 macros each processing 32 data values

	start_timer 22
	mov	rdi, sincos3		;; Load sin/cos pointer
	mov	al, 2			;; 2 iters of 2 of 4*2*clm
b2a:	xtouch	[rcx+(clm-1)*128]	;; Preload TLB
b2b:	x2cl_four_complex_fft rsi, 64, 2*clmblkdst8
	xloop4	b2b, clmblkdst		;; 4 * 2*clm iterations
	add	eax, 80000000h		;; 2 iterations
	jnc	b2a			;; Iterate if necessary
	bump	rsi, -2*4*clmblkdst+clmblkdst8
	sub	al, 1			;; Test inner loop counter
	jnz	b2a
	bump	rsi, -2*clmblkdst8+4*clmblkdst8;; Next source pointer
	bump	rdi, XMM_SCD		;; Next sine/cosine pointer
	mov	al, 2			;; 2 iters of 4 of 2*2*clm
b3a:	xtouch	[rcx+(clm-1)*128]	;; Preload TLB
b3b:	x4cl_four_complex_fft rsi, 64, 2*clmblkdst8, 4*clmblkdst8, rdi
	xloop2	b3b, clmblkdst		;; 2 * 2*clm iterations
	add	eax, 80000000h/2	;; 4 iterations
	jnc	b3a			;; Iterate if necessary
	bump	rsi, -4*2*clmblkdst+clmblkdst8
	sub	al, 1			;; Test inner loop counter
	jnz	b3a
	bump	rsi, -2*clmblkdst8-4*clmblkdst8;; Next source pointer
	end_timer 22

;; Do FFT levels 6,7
;;
;; On input the 64-byte cache lines hold these data values:
;;	0K	32K	64K	96K
;;	2K	...
;;	...
;;	30K	...
;;	128K	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0K	8K	16K	24K
;;	2K	...
;;	...
;;	6K	...
;;	32K	...
;;	...

;; Do 96 four_complex_fft macros
;;	distance between fft data elements is 8K
;;	do 24 macros each processing 32 data values

	start_timer 23
	mov	rdi, sincos2		;; Load sin/cos pointer
	mov	al, 6			;; 6 iters of 2 iters of 2*2*clm
b5a:	xtouch	[rcx+(clm-1)*128]	;; Preload TLB
b5b:	x4cl_four_complex_fft rsi, 64, 4*clmblkdst, clmblkdst8, rdi
	xloop2 b5b, clmblkdst		;; 2 * 2*clm iterations
	add	eax, 80000000h		;; 2 iterations
	jnc	b5a			;; Iterate if necessary
	bump	rsi, -4*clmblkdst+2*clmblkdst8;; Next source pointer
	bump	rdi, 2*XMM_SCD		;; Next sine/cosine pointer
	sub	al, 1			;; Test outer loop counter
	jnz	b5a			;; Iterate if necessary
	bump	rsi, -6*2*clmblkdst8	;; Restore source pointer
pfing	bump	rcx, -96*blkdst		;; Restore prefetch pointer
	end_timer 23

;; Do FFT levels 8,9
;;
;; On input the 64-byte cache lines hold these data values:
;;	0K	8K	16K	24K
;;	2K	...
;;	...
;;	6K	...
;;	32K	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0K	2K	4K	6K
;;	8K	...
;;	...

;; Do 96 four_complex_fft macros
;;	distance between fft data elements is 2K
;;	do 24 macros each processing 32 data values

	start_timer 24
	mov	rdi, sincos1		;; Load sin/cos pointer
	mov	eax, 12*256+2		;; 12 iterations 2 iters of 2*clm
b7b:	g4cl_four_complex_fft rsi, 64, clmblkdst, 2*clmblkdst, rcx, 64, blkdst, 2*blkdst
	L2prefetch128 [rcx-64][rbp]
	L2prefetch128 [rcx-64+blkdst][rbp]
	bump	rbp, -64+2*blkdst
	add	al, 256/2		;; Test inner loop counter
	jnc	b7b			;; Iterate if necessary
	bump	rbp, 128-4*blkdst
	IF clm NE 1
	add	dl, 256/clm
	jnc	b7b
	ENDIF
	bump	rsi, -2*clm*64+4*clmblkdst;; Next source pointer
	bump	rcx, -2*clm*64+4*blkdst	;; Next dest pointer
	bump	rdi, 2*XMM_SCD		;; Next sine/cosine pointer
	sub	al, 1
	jnz	b7b
	bump	rsi, -8*clmblkdst+clmblkdst8
	sub	ah, 1			;; Test outer loop counter
	mov	al, 2
	jnz	b7b			;; Iterate if necessary
	end_timer 24

;; Work on next 4 sets of 384 values

endlp:	xpass1_get_next_block pass2, c0b, b0b ;; Get next block and jump
	done:	xfft_footer
	ENDM


;; Perform a 896K-element FFT.  This is done in two passes.  Pass 1
;; does 9 levels, pass 2 uses common code to perform the last 11 levels.

hg_pass1sclevels9pfa7 MACRO fftname, pass2_levels, clmarg
	LOCAL	b0b, b1a, b1b, b2b, b3a, b3b, b4b, b5b, b6b, b7b
	LOCAL	c0b, c1b, c2a, c2b, c3b, c4b, c5a, c5b, c6b, c7b
	LOCAL	pass1, pass2, endlp, done, yes_fft

	set_FFT_constants fftname, 0, 0, pass2_levels, clmarg
	xfft_header pass1

;; Do the last 11 FFT levels
;;	do 1 pass2_11_levels_real and 111 pass2_11_levels_complex

pass2:	start_timer 0
	call	xpass2name
	end_timer 0
	cmp	ffttype, 1		;; We're done if FFTing only
	je	done

;; Do the inverse FFT

	pass1_inverse_fft_setup c0b

;; Do inverse FFT levels 8,9
;; On input the 64-byte cache lines hold these data values:
;;	0K	2K	4K	6K
;;	8K	10K	12K	14K
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0K	4K	8K	12K
;;	2K	...
;;	16K	...
;;	...

;; Do 1 eight_reals_unfft_2 macros
;; Do 1 nop_two_two_complex_unfft_2 macros
;;	distance between fft data elements is 2K
;;	do 1 macros each processing 16 data values

c0b:	start_timer 16
	mov	rsi, DATA_ADDR		;; Get source address
	mov	rcx, scratch_area	;; Get address of scratch area
	sub	rax, rax
	sub	rdx, rdx
pfing	mov	rbp, DATA_PREFETCH	;; Set prefetch distance
pfing	sub	rbp, rsi
c6b:	g2cl_eight_reals_unfft_2 rsi, 64, blkdst, rcx, 64, clmblkdst
	add	dl, 256/2/clm		;; Test loop counter
	jnc	c6b			;; Iterate if necessary

;; Do 110 four_complex_unfft macros
;;	distance between fft data elements is 2K
;;	do 55 macros each processing 16 data values

	bump	rsi, -2*clm*64+2*blkdst	;; Next source pointer
	bump	rcx, -2*clm*64+2*clmblkdst;; Next dest pointer
	mov	rdi, sincos1		;; Load sin/cos pointer
	mov	al, 256/4+14		;; 55 iterations of 2*clm
c7b:	g2cl_four_complex_unfft rsi, 64, blkdst, rcx, 64, clmblkdst
	add	dl, 256/2/clm		;; Test loop counter
	jnc	c7b			;; Iterate if necessary
	bump	rsi, -2*clm*64+2*blkdst	;; Next source pointer
	bump	rcx, -2*clm*64+2*clmblkdst;; Next dest pointer
	bump	rdi, XMM_SCD		;; Next sine/cosine pointer
	add	al, 256/4		;; Test loop counter
	jnc	c7b
	bump	rcx, -8*clmblkdst+clmblkdst8
	sub	al, 1			;; Test outer loop counter
	jnz	c7b			;; Iterate if necessary
	lea	rcx, [rsi-56*2*blkdst]	;; Create prefetch pointer
	mov	rsi, scratch_area	;; Restore dest pointer
	end_timer 16

;; Do inverse FFT levels 6,7
;; On input the 64-byte cache lines hold these data values:
;;	0K	4K	8K	12K
;;	2K	...
;;	16K	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0K	16K	32K	48K
;;	2K	...
;;	...
;;	14K
;;	64K	...
;;	...

;; Do 4 eight_reals_unfft_2 macros
;; Do 4 nop_two_two_complex_unfft_2 macros
;;	distance between fft data elements is 8K
;;	do 2 macros each processing 32 data values

	start_timer 17
	mov	al, 2			;; 2 iterations of 2*clm
c4b:	x4cl_eight_reals_unfft_2 rsi, 64, 2*clmblkdst, 4*clmblkdst
	add	al, 256/2/clm		;; Test loop counter
	jnc	c4b			;; Iterate if necessary
	bump	rsi, -2*clm*64+clmblkdst;; Next source pointer
	sub	al, 1			;; Test outer loop counter
	jnz	c4b			;; Iterate if necessary

;; Do 104 four_complex_unfft macros
;;	distance between fft data elements is 8K
;;	do 26 macros each processing 32 data values

	bump	rsi, -2*clmblkdst+clmblkdst8;; Next source pointer
	mov	rdi, sincos2		;; Load sin/cos pointer
	mov	eax, 13*256+2		;; 13 iterations of 2 iters of 2*clm
c5a:	xtouch	[rcx+(clm-1)*128]	;; Preload TLB
c5b:	x4cl_four_complex_unfft rsi, 64, 2*clmblkdst, 4*clmblkdst, rdi
	xloop1	c5b, clmblkdst
	sub	al, 1			;; Test middle loop counter
	jnz	c5a			;; Iterate if necessary
	bump	rsi, -2*clmblkdst+clmblkdst8;; Next source pointer
	bump	rdi, XMM_SCD		;; Next sine/cosine pointer
	sub	eax, 256		;; Test outer loop counter
	mov	al, 2			;; Load middle loop counter
	jnz	c5a			;; Iterate if necessary
	bump	rsi, -14*clmblkdst8	;; Restore source pointer
	end_timer 17

;; Do inverse FFT levels 4,5
;; On input the 64-byte cache lines hold these data values:
;;	0K	16K	32K	48K
;;	2K	...
;;	...
;;	14K
;;	64K	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0K	32K	64K	96K
;;	2K	...
;;	...
;;	30K
;;	128K	192K	256K	320K
;;	...
;;
;; Do 16 eight_reals_unfft_2 macros
;;	distance between fft data elements is 32K
;;	do 8 macros each processing 16 data values
;;
;; Do 96 four_complex_unfft macros
;;	distance between fft data elements is 32K
;;	do 24 macros each processing 32 data values

;; Do inverse FFT levels 1,2,3
;; On input the 64-byte cache lines hold these data values:
;;	0K	32K	64K	96K
;;	2K	...
;;	...
;;	30K
;;	128K	192K	256K	320K
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0K	32K	448K	480K
;;	2K	...
;;	...
;;	30K
;;	64K	128K	512K	576K
;;	66K
;;	...
;;	126K
;;	192K	320K	640K	768K
;;	194K
;;	...
;;	318K
;;
;; Do 128 seven_reals_last_unfft macros
;;	distance between fft data elements is 128K
;;	do 32 macros each processing 56 data values

	start_timer 18
	mov	rdi, sincos3		;; Load sin/cos pointer
pfing	bump	rcx, -4*blkdst		;; Adjust prefetch pointer
	mov	al, 8			;; 8 iterations
	xseven_reals_unfft_preload

c2a:	xtouchlines 5			;; Preload TLBs

c2b:	x2cl_half_eight_reals_unfft_2 rsi, 64, clmblkdst8
	add	al, 256/2		;; Test loop counter
	jnc	c2b			;; Iterate if necessary

	bump	rsi, -2*64+2*clmblkdst8	;; Next source pointer
	mov	ah, 3
c3b:	x4cl_four_complex_unfft rsi, 64, clmblkdst8, 2*clmblkdst8, rdi
	xloop1sp c3b, 4*clmblkdst8
	bump	rdi, XMM_SCD		;; Next sine/cosine pointer
	sub	ah, 1			;; Test outer loop counter
	jnz	c3b			;; Iterate if necessary
	bump	rdi, -3*XMM_SCD		;; Restore sine/cosine pointer

	bump	rsi, -3*4*clmblkdst8-2*clmblkdst8;; Restore source pointer
c1b:	x7cl_seven_reals_last_unfft rsi, 64, 2*clmblkdst8
	xloop1sp c1b, clmblkdst8
	add	ah, 256/2
	jnc	c1b

	IF clm GT 1
	bump	rsi, -2*clmblkdst8+2*64	;; Next source pointer
	bump	rcx, -5*blkdst+128	;; Next prefetch pointer
	add	dl, 256/clm		;; clm iterations
	jnc	c2b
	bump	rsi, -clm*2*64+clmblkdst;; Next source pointer
	bump	rcx, -clm*128+5*blkdst	;; Next prefetch pointer
	ELSE
	bump	rsi, -2*clmblkdst8+clmblkdst;; Next source pointer
	ENDIF

	sub	al, 1			;; Test loop counter
	jnz	c2a			;; Iterate if necessary
	end_timer 18

;; Normalize these values

	pass1_normalize 1, yes_fft

;; Copy data from the scratch area back to the FFT data area

	copy_scratch_data_to_fft 112
	jmp	endlp			;; Skip FFT code if flag not set

;; Fire up auxiliary threads to do pass 1 forward FFT

pass1:	pass1_forward_fft_setup b0b

;; Copy data from the FFT data area back to the scratch area

b0b:	copy_fft_data_to_scratch 112

;; Do FFT levels 1,2,3
;;
;; On input the 64-byte cache lines hold these data values:
;;	0K	32K	448K	480K
;;	2K	...
;;	...
;;	30K
;;	64K	128K	512K	576K
;;	66K
;;	...
;;	126K
;;	192K	320K	640K	768K
;;	194K
;;	...
;;	318K
;; On output the 64-byte cache lines hold these data values:
;;	0K	32K	64K	96K
;;	2K	...
;;	...
;;	30K
;;	128K	192K	256K	320K
;;	130K
;;	...
;;	190K
;;	384K	512K	640K	768K
;;	386K
;;	...
;;	510K
;;
;; Do 128 seven_reals_first_fft macros (FFT levels 1,2,3)
;;	distance between fft data elements is 128K
;;	do 16 iterations each processing 56 data values

;; Do FFT levels 4,5
;; Values 0K-127K is real data, 128K-895K is complex data.
;;
;; On input the 64-byte cache lines hold these data values:
;;	0K	32K	64K	96K
;;	2K	...
;;	...
;;	30K
;;	128K	192K	256K	320K
;;	130K
;;	...
;;	190K
;;	384K	512K	640K	768K
;;	386K
;;	...
;;	510K
;; On output the 64-byte cache lines hold these data values:
;;	0K	32K	64K	96K
;;	2K	...
;;	...
;;	30K	...
;;	128K	...
;;	...
;;
;; Do 16 eight_reals_fft_2 macros
;;	distance between fft data elements is 32K
;;	do 16 macros each processing 8 data values
;;
;; Do 64 four_complex_fft macros
;;	distance between fft data elements is 32K
;;	do 16 macros each processing 32 data values

;; Do FFT levels 6,7
;; Values 0K-31K is real data, 32K-63K is semi-real data, 64K-895K is
;; complex data.
;;
;; On input the 64-byte cache lines hold these data values:
;;	0K	32K	64K	96K
;;	2K	...
;;	...
;;	30K	...
;;	128K	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0K	8K	16K	24K
;;	2K	...
;;	...
;;	6K	...
;;	32K	...
;;	...
;;
;; Do 8 eight_reals_fft_2 macros
;; Do 8 two_two_complex_fft macros
;;	distance between fft data elements is 8K
;;	do 4 macros each processing 32 data values
;;
;; Do 96 four_complex_fft macros
;;	distance between fft data elements is 8K
;;	do 24 macros each processing 32 data values

yes_fft:start_timer 21
	mov	rsi, scratch_area	;; Get address of scratch area
	mov	rcx, DATA_ADDR		;; Load source address for prefetching
pfing	mov	rbp, DATA_PREFETCH	;; Set prefetch distance
pfing	sub	rbp, rcx
pfing	bump	rcx, 56*blkdst		;; Create prefetching pointer
	mov	rdi, sincos3		;; Load sin/cos pointer
	mov	rbx, sincos2		;; Load sin/cos pointer
	mov	eax, 4			;; 4 iterations
	sub	rdx, rdx
	xseven_reals_fft_preload

b1a:	xtouchlines 14			;; Preload TLBs

b1b:	x7cl_seven_reals_fft rsi, 64, 2*clmblkdst8
	xloop1sp b1b, 4*clmblkdst
	add	ah, 256/2		;; Test inner loop counter
	jnc	b1b			;; Iterate if necessary
	bump	rsi, -2*4*clmblkdst+clmblkdst8
	add	eax, 80000000h
	jnc	b1b

	bump	rsi, -2*clmblkdst8	;; Restore source pointer
b2b:	x1cl_half_eight_reals_fft_2 rsi, 64
	add	al, 256/2		;; Test inner loop counter
	jnc	b2b			;; Iterate if necessary
	bump	rsi, -2*64+4*clmblkdst
	add	ah, 256/2		;; Test inner loop counter
	jnc	b2b			;; Iterate if necessary
	bump	rsi, -2*4*clmblkdst+clmblkdst8
	add	eax, 80000000h
	jnc	b2b

;;	bump	rsi, -2*clmblkdst8+2*clmblkdst8;; Next source pointer
b3a:	x2cl_four_complex_fft rsi, 64, 2*clmblkdst8
	add	al, 256/2		;; Test inner loop counter
	jnc	b3a			;; Iterate if necessary
	bump	rsi, -2*64+4*clmblkdst
	add	ah, 256/2		;; Test inner loop counter
	jnc	b3a			;; Iterate if necessary
	bump	rsi, -2*4*clmblkdst+clmblkdst8
	add	eax, 80000000h
	jnc	b3a

	bump	rsi, -2*clmblkdst8+4*clmblkdst8;; Next source pointer
	bump	rdi, XMM_SCD		;; Next sine/cosine pointer
b3b:	x4cl_four_complex_fft rsi, 64, 2*clmblkdst8, 4*clmblkdst8, rdi
	xloop1sp b3b, 4*clmblkdst
	add	ah, 256/2		;; Test inner loop counter
	jnc	b3b			;; Iterate if necessary
	bump	rsi, -2*4*clmblkdst+clmblkdst8
	add	eax, 80000000h
	jnc	b3b
	bump	rdi, -XMM_SCD		;; Restore sine/cosine pointer

	bump	rsi, -2*clmblkdst8-6*clmblkdst8;; Restore source pointer
b4b:	x4cl_eight_reals_fft_2 rsi, 64, 4*clmblkdst, clmblkdst8, rbx
	add	al, 256/2		;; Test inner loop counter
	jnc	b4b			;; Iterate if necessary

	bump	rsi, -2*64+2*clmblkdst8	;; Load source pointer
	bump	rbx, XMM_SCD		;; Next sine/cosine pointer
	mov	ah, 6			;; 6 iterations
b5b:	x4cl_four_complex_fft rsi, 64, 4*clmblkdst, clmblkdst8, rbx
	xloop1sp b5b, 2*clmblkdst8
	bump	rbx, 2*XMM_SCD		;; Next sine/cosine pointer
	sub	ah, 1			;; Test outer loop counter
	jnz	b5b			;; Iterate if necessary
	bump	rbx, -13*XMM_SCD	;; Restore sine/cosine pointer

	IF clm GT 1
	bump	rsi, -7*2*clmblkdst8+2*64;; Next source pointer
	bump	rcx, -14*blkdst+128	;; Next prefetch pointer
	add	dl, 256/clm		;; clm iterations
	jnc	b1b
	bump	rsi, -clm*2*64+clmblkdst;; Next source pointer
	bump	rcx, -clm*128+14*blkdst	;; Next prefetch pointer
	ELSE
	bump	rsi, -7*2*clmblkdst8+clmblkdst;; Next source pointer
	ENDIF

	sub	al, 1			;; Test loop counter
	jnz	b1a			;; Iterate if necessary
	bump	rsi, -4*clmblkdst	;; Restore source pointer
pfing	bump	rcx, -112*blkdst	;; Restore prefetch pointer
	end_timer 21

;; Do FFT levels 8,9
;; Values 0K-7K is real data, 8K-15K is semi-real data, 16K-895K is
;; complex data.
;;
;; On input the 64-byte cache lines hold these data values:
;;	0K	8K	16K	24K
;;	2K	...
;;	...
;;	6K	...
;;	32K	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0K	2K	4K	6K
;;	8K	...
;;	...

;; Do 1 eight_reals_fft_2 macros
;; Do 1 nop_two_two_complex_fft_2 macros
;; Do 2 four_complex_fft macros
;;	distance between fft data elements is 2K
;;	do 1 macros each processing 32 data values

	start_timer 24
	mov	rdi, sincos1		;; Load sin/cos pointer
b6b:	g4cl_eight_reals_fft_2 rsi, 64, clmblkdst, 2*clmblkdst, rcx, 64, blkdst, 2*blkdst
	add	al, 256/2/clm		;; Test inner loop counter
	jnc	b6b			;; Iterate if necessary

;; Do 108 four_complex_fft macros
;;	distance between fft data elements is 2K
;;	do 27 macros each processing 32 data values

	bump	rsi, -2*clm*64+4*clmblkdst;; Next source pointer
	bump	rcx, -2*clm*64+4*blkdst	;; Next dest pointer
	bump	rdi, XMM_SCD		;; Next sine/cosine pointer
	mov	eax, 14*256+1		;; 27 iterations of 2*clm
b7b:	g4cl_four_complex_fft rsi, 64, clmblkdst, 2*clmblkdst, rcx, 64, blkdst, 2*blkdst
	L2prefetch128 [rcx-64][rbp]
	L2prefetch128 [rcx-64+blkdst][rbp]
	bump	rbp, -64+2*blkdst
	add	al, 256/2		;; Test inner loop counter
	jnc	b7b			;; Iterate if necessary
	bump	rbp, 128-4*blkdst
	IF clm NE 1
	add	dl, 256/clm
	jnc	b7b
	ENDIF
	bump	rsi, -2*clm*64+4*clmblkdst;; Next source pointer
	bump	rcx, -2*clm*64+4*blkdst	;; Next dest pointer
	bump	rdi, 2*XMM_SCD		;; Next sine/cosine pointer
	sub	al, 1			;; Test outer loop counter
	jnz	b7b
	bump	rsi, -8*clmblkdst+clmblkdst8
	sub	ah, 1			;; Test outer loop counter
	mov	al, 2
	jnz	b7b			;; Iterate if necessary
	end_timer 24

;; Work on next 4 sets of 448 values

endlp:	xpass1_get_next_block pass2, c0b, b0b ;; Get next block and jump
	done:	xfft_footer
	ENDM


;; Perform a 1M-element FFT.  This is done in two passes.  Pass 1
;; does 9 levels, pass 2 uses common code to perform the last 11 levels.

hg_pass1sclevels9pfa8 MACRO fftname, pass2_levels, clmarg
	LOCAL	b0b, b1a, b1b, b2b, b3b, b4b, b5b, b6b, b7b
	LOCAL	c0b, c1b, c2a, c2b, c3b, c4a, c4b, c5a, c5b, c6b, c7b
	LOCAL	pass1, pass2, endlp, done, yes_fft

	set_FFT_constants fftname, 0, 0, pass2_levels, clmarg
	xfft_header pass1

;; Do the last 11 FFT levels
;;	do 1 pass2_11_levels_real and 127 pass2_11_levels_complex

pass2:	start_timer 0
	call	xpass2name
	end_timer 0
	cmp	ffttype, 1		;; We're done if FFTing only
	je	done

;; Do the inverse FFT

	pass1_inverse_fft_setup c0b

;; Do inverse FFT levels 8,9
;; On input the 64-byte cache lines hold these data values:
;;	0K	2K	4K	6K
;;	8K	10K	12K	14K
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0K	4K	8K	12K
;;	2K	...
;;	16K	...
;;	...

;; Do 1 eight_reals_unfft_2 macros
;; Do 1 nop_two_two_complex_unfft_2 macros
;;	distance between fft data elements is 2K
;;	do 1 macros each processing 16 data values

c0b:	start_timer 16
	mov	rsi, DATA_ADDR		;; Get source address
	mov	rcx, scratch_area	;; Get address of scratch area
	sub	rax, rax
	sub	rdx, rdx
pfing	mov	rbp, DATA_PREFETCH	;; Set prefetch distance
pfing	sub	rbp, rsi
c6b:	g2cl_eight_reals_unfft_2 rsi, 64, blkdst, rcx, 64, clmblkdst
	add	al, 256/2/clm		;; Test loop counter
	jnc	c6b			;; Iterate if necessary

;; Do 126 four_complex_unfft macros
;;	distance between fft data elements is 2K
;;	do 63 macros each processing 16 data values

	bump	rsi, -2*clm*64+2*blkdst	;; Next source pointer
	bump	rcx, -2*clm*64+2*clmblkdst;; Next dest pointer
	mov	rdi, sincos1		;; Load sin/cos pointer
	mov	al, 256/4+16		;; 63 iterations of 2*clm
c7b:	g2cl_four_complex_unfft rsi, 64, blkdst, rcx, 64, clmblkdst
	add	dl, 256/2/clm		;; Test loop counter
	jnc	c7b			;; Iterate if necessary
	bump	rsi, -2*clm*64+2*blkdst	;; Next source pointer
	bump	rcx, -2*clm*64+2*clmblkdst;; Next dest pointer
	bump	rdi, XMM_SCD		;; Next sine/cosine pointer
	add	al, 256/4		;; Test loop counter
	jnc	c7b			;; Iterate if necessary
	bump	rcx, -8*clmblkdst+clmblkdst8;; Next dest pointer
	sub	al, 1
	jnz	c7b
	lea	rcx, [rsi-64*2*blkdst]	;; Create prefetch pointer
	mov	rsi, scratch_area	;; Restore source pointer
	end_timer 16

;; Do inverse FFT levels 6,7
;; On input the 64-byte cache lines hold these data values:
;;	0K	4K	8K	12K
;;	2K	...
;;	16K	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0K	16K	32K	48K
;;	2K	...
;;	...
;;	14K
;;	64K	...
;;	...

;; Do 4 eight_reals_unfft_2 macros
;; Do 4 nop_two_two_complex_unfft_2 macros
;;	distance between fft data elements is 8K
;;	do 2 macros each processing 32 data values

	start_timer 17
	mov	al, 2			;; 2 iterations of 2*clm
c4a:	xtouch	[rcx+(clm-1)*128]	;; Preload TLB
c4b:	x4cl_eight_reals_unfft_2 rsi, 64, 2*clmblkdst, 4*clmblkdst
	xloop1	c4b, clmblkdst
	sub	al, 1			;; Test outer loop counter
	jnz	c4a			;; Iterate if necessary

;; Do 120 four_complex_unfft macros
;;	distance between fft data elements is 8K
;;	do 30 macros each processing 32 data values

	bump	rsi, -2*clmblkdst+clmblkdst8;; Next source pointer
	mov	rdi, sincos2		;; Load sin/cos pointer
	mov	eax, 15*256+2		;; 15 iterations of 2 iters of 2*clm
c5a:	xtouch	[rcx+(clm-1)*128]	;; Preload TLB
c5b:	x4cl_four_complex_unfft rsi, 64, 2*clmblkdst, 4*clmblkdst, rdi
	xloop1	c5b, clmblkdst
	sub	al, 1			;; Test middle loop counter
	jnz	c5a			;; Iterate if necessary
	bump	rsi, -2*clmblkdst+clmblkdst8;; Next source pointer
	bump	rdi, XMM_SCD		;; Next sine/cosine pointer
	sub	eax, 256		;; Test outer loop counter
	mov	al, 2			;; Load middle loop counter
	jnz	c5a			;; Iterate if necessary
	bump	rsi, -16*clmblkdst8	;; Restore source pointer
	end_timer 17

;; Do inverse FFT levels 4,5
;; On input the 64-byte cache lines hold these data values:
;;	0K	16K	32K	48K
;;	2K	...
;;	...
;;	14K
;;	64K	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0K	64K	128K	192K
;;	2K	...
;;	...
;;	62K
;;	256K	...
;;	...
;;
;; Do 16 eight_reals_unfft_2 macros
;; Do 16 nop_two_two_complex_unfft_2 macros
;;	distance between fft data elements is 32K
;;	do 8 macros each processing 32 data values
;;
;; Do 96 four_complex_unfft macros
;;	distance between fft data elements is 32K
;;	do 24 macros each processing 32 data values

;; Do inverse FFT levels 1,2,3
;; On input the 64-byte cache lines hold these data values:
;;	0K	64K	128K	192K
;;	2K	...
;;	...
;;	62K
;;	256K	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0K	256K	512K	768K
;;	2K	...
;;	...
;;	254K
;;
;; Do 128 eight_reals_last_unfft macros
;;	distance between fft data elements is 128K
;;	do 32 macros each processing 32 data values

	start_timer 18
	mov	rdi, sincos3		;; Load sin/cos pointer
	mov	eax, 8			;; 8 iterations

c2a:	xtouchlines 4			;; Preload TLBs

c2b:	x4cl_eight_reals_unfft_2 rsi, 64, clmblkdst8, 2*clmblkdst8
	xloop1sp c2b, 4*clmblkdst8

	mov	ah, 3			;; 3 iterations
c3b:	x4cl_four_complex_unfft rsi, 64, clmblkdst8, 2*clmblkdst8, rdi
	xloop1sp c3b, 4*clmblkdst8
	bump	rdi, XMM_SCD		;; Next sine/cosine pointer
	sub	ah, 1			;; Test outer loop counter
	jnz	c3b			;; Iterate if necessary
	bump	rdi, -3*XMM_SCD		;; Next sine/cosine pointer

	bump	rsi, -4*4*clmblkdst8	;; Next source pointer
c1b:	x4cl_eight_reals_last_unfft rsi, 64, 4*clmblkdst8, 8*clmblkdst8
	add	al, 256/2		;; Test loop counter
	jnc	c1b			;; Iterate if necessary
	bump	rsi, -2*64+clmblkdst8	;; Restore source pointer
	add	ah, 256/4
	jnc	c1b

	IF clm GT 1
	bump	rsi, -4*clmblkdst8+2*64	;; Next source pointer
	bump	rcx, -4*blkdst+128	;; Next prefetch pointer
	add	dl, 256/clm		;; clm iterations
	jnc	c2b
	bump	rsi, -clm*2*64+clmblkdst;; Next source pointer
	bump	rcx, -clm*128+4*blkdst	;; Next prefetch pointer
	ELSE
	bump	rsi, -4*clmblkdst8+clmblkdst;; Next source pointer
	ENDIF

	sub	al, 1			;; Test loop counter
	jnz	c2a			;; Iterate if necessary
	end_timer 18

;; Normalize these values

	pass1_normalize 1, yes_fft

;; Copy data from the scratch area back to the FFT data area

	copy_scratch_data_to_fft 128
	jmp	endlp			;; Skip FFT code if flag not set

;; Fire up auxiliary threads to do pass 1 forward FFT

pass1:	pass1_forward_fft_setup b0b

;; Copy data from the FFT data area back to the scratch area

b0b:	copy_fft_data_to_scratch 128

;; Do FFT levels 1,2,3
;;
;; On input the 64-byte cache lines hold these data values:
;;	0K	256K	512K	768K
;;	2K	...
;;	...
;;	254K
;; On output the 64-byte cache lines hold these data values:
;;	0K	128K	256K	384K
;;	2K	...
;;	...
;;	126K	...
;;	512K	...
;;	...
;;
;; Do 128 eight_reals_first_fft macros (FFT levels 1,2,3)
;;	distance between fft data elements is 128K
;;	do 64 iterations each processing 16 data values

;; Do FFT levels 4,5
;; Values 0K-127K is real data, 128K-255K is semi-real data, 256K-1023K is
;; complex data.
;;
;; On input the 64-byte cache lines hold these data values:
;;	0K	128K	256K	384K
;;	2K	...
;;	...
;;	126K	...
;;	512K	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0K	32K	64K	96K
;;	2K	...
;;	...
;;	30K	...
;;	128K	...
;;	...
;;
;; Do 16 eight_reals_fft_2 macros
;; Do 16 nop_two_two_complex_fft_2 macros
;; Do 32 four_complex_fft macros
;;	distance between fft data elements is 32K
;;	do 16 macros each processing 32 data values
;;
;; Do 64 four_complex_fft macros
;;	distance between fft data elements is 32K
;;	do 16 macros each processing 32 data values

;; Do FFT levels 6,7
;; Values 0K-31K is real data, 32K-63K is semi-real data, 64K-1023K is
;; complex data.
;;
;; On input the 64-byte cache lines hold these data values:
;;	0K	32K	64K	96K
;;	2K	...
;;	...
;;	30K	...
;;	128K	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0K	8K	16K	24K
;;	2K	...
;;	...
;;	6K	...
;;	32K	...
;;	...
;;
;; Do 8 eight_reals_fft_2 macros
;; Do 8 two_two_complex_fft macros
;;	distance between fft data elements is 8K
;;	do 4 macros each processing 32 data values
;;
;; Do 112 four_complex_fft macros
;;	distance between fft data elements is 8K
;;	do 28 macros each processing 32 data values

yes_fft:start_timer 21
	mov	rsi, scratch_area	;; Get address of scratch area
	mov	rcx, DATA_ADDR		;; Load source address for prefetching
pfing	mov	rbp, DATA_PREFETCH	;; Set prefetch distance
pfing	sub	rbp, rcx
pfing	bump	rcx, 64*blkdst		;; Create prefetching pointer
	mov	rdi, sincos3		;; Load sin/cos pointer
	mov	rbx, sincos2		;; Load sin/cos pointer
	mov	eax, 4			;; 4 iterations
	sub	rdx, rdx

b1a:	xtouchlines 16			;; Preload TLBs

b1b:	x2cl_eight_reals_fft rsi, 64, 8*clmblkdst8
	add	al, 256/2		;; Test inner loop counter
	jnc	b1b			;; Iterate if necessary
	bump	rsi, -2*64+4*clmblkdst
	add	ah, 256/2		;; Test inner loop counter
	jnc	b1b			;; Iterate if necessary
	L2prefetch128 [rcx][rbp]
pfing	bump	rcx, blkdst		;; Next prefetch pointer
	bump	rsi, -2*4*clmblkdst+clmblkdst8
	add	eax, 80000000h/8*2
	jnc	b1b

	bump	rsi, -8*clmblkdst8	;; Restore source pointer
b2b:	x4cl_eight_reals_fft_2 rsi, 64, 2*clmblkdst8, 4*clmblkdst8, rdi
	add	al, 256/2		;; Test inner loop counter
	jnc	b2b			;; Iterate if necessary
	bump	rsi, -2*64+4*clmblkdst
	add	ah, 256/2		;; Test inner loop counter
	jnc	b2b			;; Iterate if necessary
	bump	rsi, -2*4*clmblkdst+clmblkdst8
	add	eax, 80000000h
	jnc	b2b

	bump	rsi, -2*clmblkdst8+8*clmblkdst8;; Next source pointer
	bump	rdi, XMM_SCD		;; Next sine/cosine pointer
b3b:	x4cl_four_complex_fft rsi, 64, 2*clmblkdst8, 4*clmblkdst8, rdi
	add	al, 256/2		;; Test inner loop counter
	jnc	b3b			;; Iterate if necessary
	bump	rsi, -2*64+4*clmblkdst
	add	ah, 256/2		;; Test inner loop counter
	jnc	b3b			;; Iterate if necessary
	bump	rsi, -2*4*clmblkdst+clmblkdst8
	add	eax, 80000000h
	jnc	b3b
	bump	rdi, -XMM_SCD		;; Restore sine/cosine pointer

	bump	rsi, -2*clmblkdst8-8*clmblkdst8;; Restore source pointer
b4b:	x4cl_eight_reals_fft_2 rsi, 64, 4*clmblkdst, clmblkdst8, rbx
	xloop1sp b4b, 2*clmblkdst8

	bump	rbx, XMM_SCD		;; Next sine/cosine pointer
	mov	ah, 7			;; 7 iterations
b5b:	x4cl_four_complex_fft rsi, 64, 4*clmblkdst, clmblkdst8, rbx
	xloop1sp b5b, 2*clmblkdst8
	bump	rbx, 2*XMM_SCD		;; Next sine/cosine pointer
	sub	ah, 1			;; Test outer loop counter
	jnz	b5b			;; Iterate if necessary
	bump	rbx, -15*XMM_SCD	;; Restore sine/cosine pointer

	IF clm GT 1
	bump	rsi, -8*2*clmblkdst8+2*64;; Next source pointer
	bump	rcx, -16*blkdst+128	;; Next prefetch pointer
	add	dl, 256/clm		;; clm iterations
	jnc	b1b
	bump	rsi, -clm*2*64+clmblkdst;; Next source pointer
	bump	rcx, -clm*128+16*blkdst	;; Next prefetch pointer
	ELSE
	bump	rsi, -8*2*clmblkdst8+clmblkdst;; Next source pointer
	ENDIF

	sub	al, 1			;; Test loop counter
	jnz	b1a			;; Iterate if necessary
	bump	rsi, -4*clmblkdst	;; Restore source pointer
pfing	bump	rcx, -128*blkdst	;; Restore prefetch pointer
	end_timer 21

;; Do FFT levels 8,9
;; Values 0K-7K is real data, 8K-15K is semi-real data, 16K-1023K is
;; complex data.
;;
;; On input the 64-byte cache lines hold these data values:
;;	0K	8K	16K	24K
;;	2K	...
;;	...
;;	6K	...
;;	32K	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0K	2K	4K	6K
;;	8K	...
;;	...

;; Do 1 eight_reals_fft_2 macros
;; Do 1 nop_two_two_complex_fft_2 macros
;; Do 2 four_complex_fft macros
;;	distance between fft data elements is 2K
;;	do 1 macros each processing 32 data values

	start_timer 24
	mov	rdi, sincos1		;; Load sin/cos pointer
b6b:	g4cl_eight_reals_fft_2 rsi, 64, clmblkdst, 2*clmblkdst, rcx, 64, blkdst, 2*blkdst
	add	dl, 256/2/clm		;; Test inner loop counter
	jnc	b6b			;; Iterate if necessary

;; Do 124 four_complex_fft macros
;;	distance between fft data elements is 2K
;;	do 31 macros each processing 32 data values

	bump	rsi, -2*clm*64+4*clmblkdst;; Next source pointer
	bump	rcx, -2*clm*64+4*blkdst	;; Next dest pointer
	bump	rdi, XMM_SCD		;; Next sine/cosine pointer
	mov	eax, 16*256+1		;; 31 iterations of 2*clm
b7b:	g4cl_four_complex_fft rsi, 64, clmblkdst, 2*clmblkdst, rcx, 64, blkdst, 2*blkdst
	L2prefetch128 [rcx-64][rbp]
	L2prefetch128 [rcx-64+blkdst][rbp]
	bump	rbp, -64+2*blkdst
	add	al, 256/2		;; Test inner loop counter
	jnc	b7b			;; Iterate if necessary
	bump	rbp, 128-4*blkdst
	IF clm NE 1
	add	dl, 256/clm
	jnc	b7b
	ENDIF
	bump	rsi, -2*clm*64+4*clmblkdst;; Next source pointer
	bump	rcx, -2*clm*64+4*blkdst	;; Next dest pointer
	bump	rdi, 2*XMM_SCD		;; Next sine/cosine pointer
	sub	al, 1			;; Test outer loop counter
	jnz	b7b			;; Iterate if necessary
	bump	rsi, -8*clmblkdst+clmblkdst8
	sub	ah, 1
	mov	al, 2
	jnz	b7b
	end_timer 24

;; Work on next 4 sets of 512 values

endlp:	xpass1_get_next_block pass2, c0b, b0b ;; Get next block and jump
	done:	xfft_footer
	ENDM


;; Perform a 1M-element negacyclic FFT.  This is done in two passes.  Pass 1
;; does 9 levels, pass 2 uses common code to perform the last 11 levels.

hg_pass1sclevels9complex4 MACRO fftname, pass2_levels, clmarg
	LOCAL	b0b, b1a, b1b, b3a, b3b, b5a, b5b, b7b
	LOCAL	c0b, c1a, c1b, c3a, c3b, c5a, c5b, c7b
	LOCAL	pass1, pass2, endlp, done, yes_fft

	set_FFT_constants fftname, 0, 1, pass2_levels, clmarg
	xfft_header pass1

;; Do the last 11 FFT levels
;;	do 128 pass2_11_levels_complex

pass2:	start_timer 0
	call	xpass2name
	end_timer 0
	cmp	ffttype, 1		;; We're done if FFTing only
	je	done

;; Do the inverse FFT

	pass1_inverse_fft_setup c0b

;; Do inverse FFT levels 8,9
;; On input the 64-byte cache lines hold these data values:
;;	0K	2K	4K	6K
;;	8K	10K	12K	14K
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0K	4K	8K	12K
;;	2K	...
;;	16K	...
;;	...

;; Do 128 four_complex_unfft macros
;;	distance between fft data elements is 2K
;;	do 64 macros each processing 16 data values

c0b:	start_timer 16
	mov	rsi, DATA_ADDR		;; Get source address
	mov	rcx, scratch_area	;; Get address of scratch area
	mov	rdi, sincos1		;; Load sin/cos pointer
	mov	eax, 16			;; 64 iterations
	sub	rdx, rdx
pfing	mov	rbp, DATA_PREFETCH	;; Set prefetch distance
pfing	sub	rbp, rsi
c7b:	g2cl_four_complex_unfft rsi, 64, blkdst, rcx, 64, clmblkdst
	add	dl, 256/2/clm		;; Test loop counter
	jnc	c7b			;; Iterate if necessary
	bump	rsi, -2*clm*64+2*blkdst	;; Next source pointer
	bump	rcx, -2*clm*64+2*clmblkdst;; Next dest pointer
	bump	rdi, XMM_SCD		;; Next sine/cosine pointer
	add	al, 256/4		;; Test loop counter
	jnc	c7b			;; Iterate if necessary
	bump	rcx, -8*clmblkdst+clmblkdst8;; Next dest pointer
	sub	al, 1
	jnz	c7b
	lea	rcx, [rsi-64*2*blkdst]	;; Create prefetch pointer
	mov	rsi, scratch_area	;; Restore source pointer
	end_timer 16

;; Do inverse FFT levels 6,7
;; On input the 64-byte cache lines hold these data values:
;;	0K	4K	8K	12K
;;	2K	...
;;	16K	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0K	16K	32K	48K
;;	2K	...
;;	...
;;	14K
;;	64K	...
;;	...

;; Do 128 four_complex_unfft macros
;;	distance between fft data elements is 8K
;;	do 32 macros each processing 32 data values

	start_timer 17
	mov	rdi, sincos2		;; Load sin/cos pointer
	mov	al, 16			;; 16 iterations of 2 iters of 2*clm
c5a:	xtouch	[rcx+(clm-1)*128]	;; Preload TLB
c5b:	x4cl_four_complex_unfft rsi, 64, 2*clmblkdst, 4*clmblkdst, rdi
	xloop2	c5b, clmblkdst		;; 2 * 2*clm iterations
	bump	rsi, -2*clmblkdst+clmblkdst8;; Next source pointer
	bump	rdi, XMM_SCD		;; Next sine/cosine pointer
	sub	al, 1			;; Test outer loop counter
	jnz	c5a			;; Iterate if necessary
	bump	rsi, -16*clmblkdst8	;; Restore source pointer
	end_timer 17

;; Do inverse FFT levels 4,5
;; On input the 64-byte cache lines hold these data values:
;;	0K	16K	32K	48K
;;	2K	...
;;	...
;;	14K
;;	64K	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0K	64K	128K	192K
;;	2K	...
;;	...
;;	62K
;;	256K	...
;;	...

;; Do 128 four_complex_unfft macros
;;	distance between fft data elements is 32K
;;	do 32 macros each processing 32 data values

	start_timer 18
	mov	rdi, sincos3		;; Load sin/cos pointer
	mov	eax, 4*256+4		;; 4 iterations of 4 iters of 2*2*clm
c3a:	xtouch	[rcx+(clm-1)*128]	;; Preload TLB
c3b:	x4cl_four_complex_unfft rsi, 64, clmblkdst8, 2*clmblkdst8, rdi
	xloop2	c3b, clmblkdst		;; 2* 2*clm iterations
	sub	al, 1			;; Test middle loop counter
	jnz	c3a			;; Iterate if necessary
	bump	rsi, -4*2*clmblkdst+4*clmblkdst8;; Next source pointer
	bump	rdi, XMM_SCD		;; Next sine/cosine pointer
	sub	eax, 256		;; Test outer loop counter
	mov	al, 4			;; Load middle loop counter
	jnz	c3a			;; Iterate if necessary
	bump	rsi, -4*4*clmblkdst8	;; Restore source pointer
	end_timer 18

;; Do inverse FFT levels 1,2,3
;; On input the 64-byte cache lines hold these data values:
;;	0K	64K	128K	192K
;;	2K	...
;;	...
;;	62K
;;	256K	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0K	256K	512K	768K
;;	2K	...
;;	...
;;	254K

;; Do 128 four_complex_last_unfft macros
;;	distance between fft data elements is 128K
;;	do 32 macros each processing 32 data values

	start_timer 19
	mov	rdi, plus1_premults	;; Address of premultiplier table
	mov	eax, 4*256+8		;; 4 iters of 8 iters of 2*clm
c1a:	xtouch	[rcx+(clm-1)*128]	;; Preload TLB
c1b:	x4cl_four_complex_last_unfft rsi, 64, 4*clmblkdst8, 8*clmblkdst8, 32*128
	xloop1	c1b, clmblkdst		;; 2*clm iterations
	bump	rdi, 128		;; Next premultiplier pointer
	sub	al, 1			;; Test loop counter
	jnz	c1a			;; Iterate if necessaryxn
	bump	rsi, -8*clmblkdst+clmblkdst8;; Next source pointer
	sub	ah, 1			;; Test outer loop counter
	mov	al, 8			;; Load middle loop counter
	jnz	c1a
	end_timer 19

;; Normalize these values

	pass1_normalize 1, yes_fft

;; Copy data from the scratch area back to the FFT data area

	copy_scratch_data_to_fft 128
	jmp	endlp			;; Skip FFT code if flag not set

;; Fire up auxiliary threads to do pass 1 forward FFT

pass1:	pass1_forward_fft_setup b0b

;; Copy data from the FFT data area back to the scratch area

b0b:	copy_fft_data_to_scratch 128

;; Do FFT levels 1,2,3
;;
;; On input the 64-byte cache lines hold these data values:
;;	0K	256K	512K	768K
;;	2K	...
;;	...
;;	254K
;; On output the 64-byte cache lines hold these data values:
;;	0K	128K	256K	384K
;;	2K	...
;;	...
;;	126K	...
;;	512K	...
;;	...

;; Do 128 four_complex_first_fft macros (FFT levels 1,2,3)
;;	distance between fft data elements is 128K
;;	do 64 iterations each processing 16 data values

yes_fft:start_timer 21
	mov	rsi, scratch_area	;; Get address of scratch area
	mov	rdi, plus1_premults	;; Address of premultiplier table
	mov	rcx, DATA_ADDR		;; Load source address for prefetching
pfing	mov	rbp, DATA_PREFETCH	;; Set prefetch distance
pfing	sub	rbp, rcx
pfing	bump	rcx, 64*blkdst		;; Create prefetching pointer
	mov	eax, 8*256+4		;; 8 iters of 4 iters of 2*2*clm
	sub	rdx, rdx
b1a:	xtouch	[rcx+(clm-1)*128]	;; Preload TLB
b1b:	x2cl_four_complex_first_fft_scratch rsi, 64, 8*clmblkdst8
	xloop2edi b1b, clmblkdst, 128 ;; 2 * 2*clm iterations
	sub	al, 1			;; Test inner loop counter
	jnz	b1a			;; Iterate if necessary
	bump	rsi, -8*clmblkdst+clmblkdst8;; Next source pointer
	sub	ah, 1			;; Test outer loop counter
	mov	al, 4			;; Load middle loop counter
	jnz	b1a
	bump	rsi, -8*clmblkdst8	;; Restore source pointer
	end_timer 21

;; Do FFT levels 4,5
;;
;; On input the 64-byte cache lines hold these data values:
;;	0K	128K	256K	384K
;;	2K	...
;;	...
;;	126K	...
;;	512K	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0K	32K	64K	96K
;;	2K	...
;;	...
;;	30K	...
;;	128K	...
;;	...

;; Do 128 four_complex_fft macros
;;	distance between fft data elements is 32K
;;	do 32 macros each processing 32 data values

	start_timer 22
	mov	rdi, sincos3		;; Load sin/cos pointer
	mov	eax, 2*256+2		;; 2 iters of 2 iters of 4 of 2*2*clm
b3a:	xtouch	[rcx+(clm-1)*128]	;; Preload TLB
b3b:	x4cl_four_complex_fft rsi, 64, 2*clmblkdst8, 4*clmblkdst8, rdi
	xloop2	b3b, clmblkdst		;; 2 * 2*clm iterations
	add	eax, 80000000h/2	;; 4 iterations
	jnc	b3a			;; Iterate if necessary
	bump	rsi, -8*clmblkdst+clmblkdst8
	sub	al, 1			;; Test inner loop counter
	jnz	b3a
	bump	rsi, -2*clmblkdst8+8*clmblkdst8;; Next source pointer
	bump	rdi, 2*XMM_SCD		;; Next sine/cosine pointer
	sub	ah, 1
	mov	al, 2
	jnz	b3a
	bump	rsi, -2*8*clmblkdst8	;; Restore source pointer
	end_timer 22

;; Do FFT levels 6,7
;;
;; On input the 64-byte cache lines hold these data values:
;;	0K	32K	64K	96K
;;	2K	...
;;	...
;;	30K	...
;;	128K	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0K	8K	16K	24K
;;	2K	...
;;	...
;;	6K	...
;;	32K	...
;;	...

;; Do 128 four_complex_fft macros
;;	distance between fft data elements is 8K
;;	do 32 macros each processing 32 data values

	start_timer 23
	mov	rdi, sincos2		;; Load sin/cos pointer
	mov	al, 8			;; 8 iters of 2 iters of 2*2*clm
b5a:	xtouch	[rcx+(clm-1)*128]	;; Preload TLB
b5b:	x4cl_four_complex_fft rsi, 64, 4*clmblkdst, clmblkdst8, rdi
	xloop2 b5b, clmblkdst		;; 2 * 2*clm iterations
	add	eax, 80000000h		;; 2 iterations
	jnc	b5a			;; Iterate if necessary
	bump	rsi, -4*clmblkdst+2*clmblkdst8;; Next source pointer
	bump	rdi, 2*XMM_SCD		;; Next sine/cosine pointer
	sub	al, 1			;; Test outer loop counter
	jnz	b5a			;; Iterate if necessary
	bump	rsi, -8*2*clmblkdst8	;; Restore source pointer
pfing	bump	rcx, -128*blkdst	;; Restore prefetch pointer
	end_timer 23

;; Do FFT levels 8,9
;;
;; On input the 64-byte cache lines hold these data values:
;;	0K	8K	16K	24K
;;	2K	...
;;	...
;;	6K	...
;;	32K	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0K	2K	4K	6K
;;	8K	...
;;	...

;; Do 128 four_complex_fft macros
;;	distance between fft data elements is 2K
;;	do 32 macros each processing 32 data values

	start_timer 24
	mov	rdi, sincos1		;; Load sin/cos pointer
	mov	eax, 16*256+2		;; 32 iterations of 2*clm
b7b:	g4cl_four_complex_fft rsi, 64, clmblkdst, 2*clmblkdst, rcx, 64, blkdst, 2*blkdst
	L2prefetch128 [rcx-64][rbp]
	L2prefetch128 [rcx-64+blkdst][rbp]
	bump	rbp, -64+2*blkdst
	add	al, 256/2		;; Test inner loop counter
	jnc	b7b			;; Iterate if necessary
	bump	rbp, 128-4*blkdst
	IF clm NE 1
	add	dl, 256/clm
	jnc	b7b
	ENDIF
	bump	rsi, -2*clm*64+4*clmblkdst;; Next source pointer
	bump	rcx, -2*clm*64+4*blkdst	;; Next dest pointer
	bump	rdi, 2*XMM_SCD		;; Next sine/cosine pointer
	sub	al, 1			;; Test outer loop counter
	jnz	b7b			;; Iterate if necessary
	bump	rsi, -8*clmblkdst+clmblkdst8
	sub	ah, 1
	mov	al, 2
	jnz	b7b
	end_timer 24

;; Work on next 4 sets of 512 values

endlp:	xpass1_get_next_block pass2, c0b, b0b ;; Get next block and jump
	done:	xfft_footer
	ENDM


;; Perform a 1280K-element FFT.  This is done in two passes.  Pass 1
;; does 10 levels, pass 2 uses common code to perform the last 11 levels.

hg_pass1sclevels10pfa5 MACRO fftname, pass2_levels, clmarg
	LOCAL	b0b, b1a, b1aa, b1b, b2b, b3b
	LOCAL	b4b, b5b, b6b, b7a, b7b, b8b, b9b
	LOCAL	c0b, c1b, c2a, c2b, c3b, c4b, c5b, c6b, c7a, c7b, c8b, c9b
	LOCAL	pass1, pass2, endlp, done, yes_fft

	set_FFT_constants fftname, 0, 0, pass2_levels, clmarg
	xfft_header pass1

;; Do the last 11 FFT levels
;;	do 1 pass2_11_levels_real and 159 pass2_11_levels_complex

pass2:	start_timer 0
	call	xpass2name
	end_timer 0
	cmp	ffttype, 1		;; We're done if FFTing only
	je	done

;; Do the inverse FFT

	pass1_inverse_fft_setup c0b

;; Do inverse FFT levels 9,10
;; On input the 64-byte cache lines hold these data values:
;;	0K	2K	4K	6K
;;	8K	10K	12K	14K
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0K	4K	8K	12K
;;	2K	...
;;	16K	...
;;	...

;; Do 1 eight_reals_unfft_2 macros
;; Do 1 nop_two_two_complex_unfft_2 macros
;;	distance between fft data elements is 2K
;;	do 1 macros each processing 16 data values

c0b:	start_timer 16
	mov	rsi, DATA_ADDR		;; Get source address
	mov	rcx, scratch_area	;; Get address of scratch area
	sub	rax, rax
	sub	rdx, rdx
pfing	mov	rbp, DATA_PREFETCH	;; Set prefetch distance
pfing	sub	rbp, rsi
c8b:	g2cl_eight_reals_unfft_2 rsi, 64, blkdst, rcx, 64, clmblkdst
	add	al, 256/2/clm		;; Test loop counter
	jnc	c8b			;; Iterate if necessary

;; Do 158 four_complex_unfft macros
;;	distance between fft data elements is 2K
;;	do 79 macros each processing 16 data values

	bump	rsi, -2*clm*64+2*blkdst	;; Next source pointer
	bump	rcx, -2*clm*64+2*clmblkdst;; Next dest pointer
	mov	rdi, sincos1		;; Load sin/cos pointer
	mov	al, 256/4+20		;; 79 iterations of 2*clm
c9b:	g2cl_four_complex_unfft rsi, 64, blkdst, rcx, 64, clmblkdst
	add	dl, 256/2/clm		;; Test loop counter
	jnc	c9b			;; Iterate if necessary
	bump	rsi, -2*clm*64+2*blkdst	;; Next source pointer
	bump	rcx, -2*clm*64+2*clmblkdst;; Next dest pointer
	bump	rdi, XMM_SCD		;; Next sine/cosine pointer
	add	al, 256/4
	jnc	c9b
	bump	rcx, -8*clmblkdst+clmblkdst8
	sub	al, 1			;; Test outer loop counter
	jnz	c9b			;; Iterate if necessary
	lea	rcx, [rsi-80*2*blkdst]	;; Create prefetch pointer
	mov	rsi, scratch_area	;; Restore source pointer
	end_timer 16

;; Do inverse FFT levels 7,8
;; On input the 64-byte cache lines hold these data values:
;;	0K	4K	8K	12K
;;	2K	...
;;	16K	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0K	16K	32K	48K
;;	2K	...
;;	...
;;	14K
;;	64K	...
;;	...

;; Do 4 eight_reals_unfft_2 macros
;; Do 4 nop_two_two_complex_unfft_2 macros
;;	distance between fft data elements is 8K
;;	do 2 macros each processing 32 data values

	start_timer 17
	mov	al, 2			;; 2 iterations of 2*clm
c6b:	x4cl_eight_reals_unfft_2 rsi, 64, 2*clmblkdst, 4*clmblkdst
	add	al, 256/2/clm		;; Test loop counter
	jnc	c6b			;; Iterate if necessary
	bump	rsi, -2*clm*64+clmblkdst;; Next source pointer
	sub	al, 1			;; Test outer loop counter
	jnz	c6b			;; Iterate if necessary

;; Do 152 four_complex_unfft macros
;;	distance between fft data elements is 8K
;;	do 38 macros each processing 32 data values

	bump	rsi, -2*clmblkdst+clmblkdst8;; Next source pointer
	mov	rdi, sincos2		;; Load sin/cos pointer
	mov	eax, 19*256+2		;; 19 iterations of 2 iters of 2*clm
c7a:	xtouch	[rcx+(clm-1)*128]	;; Preload TLB
c7b:	x4cl_four_complex_unfft rsi, 64, 2*clmblkdst, 4*clmblkdst, rdi
	xloop1	c7b, clmblkdst
	sub	al, 1			;; Test middle loop counter
	jnz	c7a			;; Iterate if necessary
	bump	rsi, -2*clmblkdst+clmblkdst8;; Next source pointer
	bump	rdi, XMM_SCD		;; Next sine/cosine pointer
	sub	eax, 256		;; Test outer loop counter
	mov	al, 2			;; Load middle loop counter
	jnz	c7a			;; Iterate if necessary
	bump	rsi, -20*clmblkdst8	;; Restore source pointer
	end_timer 17

;; Do inverse FFT levels 6
;; On input the 64-byte cache lines hold these data values:
;;	0K	16K	32K	48K
;;	2K	...
;;	...
;;	14K
;;	64K	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0K	32K	64K	96K
;;	2K	...
;;	...
;;	30K
;;	128K	...
;;	...
;;
;; Do 16 eight_reals_unfft_1 macros
;;	distance between fft data elements is 32K
;;	do 8 macros each processing 16 data values
;;
;; Do 152 four_complex_unfft macros
;;	distance between fft data elements is 32K
;;	do 76 macros each processing 16 data values

;; Do inverse FFT levels 4,5
;; On input the 64-byte cache lines hold these data values:
;;	0K	32K	64K	96K
;;	2K	...
;;	...
;;	30K
;;	128K	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0K	64K	128K	192K
;;	2K	...
;;	...
;;	62K
;;	256K	384K	512K	768K
;;	...
;;
;; Do 32 eight_reals_unfft_2 macros
;;	distance between fft data elements is 64K
;;	do 16 macros each processing 16 data values
;;
;; Do 128 four_complex_unfft macros
;;	distance between fft data elements is 64K
;;	do 32 macros each processing 32 data values

;; Do inverse FFT levels 1,2,3
;; On input the 64-byte cache lines hold these data values:
;;	0K	64K	128K	192K
;;	2K	...
;;	...
;;	62K
;;	256K	384K	512K	768K
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0K	64K	640K	704K
;;	2K	...
;;	...
;;	62K
;;	128K	384K	768K	1024K
;;	130K
;;	...
;;	382K
;;
;; Do 256 five_reals_last_unfft macros
;;	distance between fft data elements is 256K
;;	do 32 macros each processing 40 data values

	start_timer 18
	mov	rdi, sincos3		;; Load sin/cos pointer
	mov	rbx, sincos4		;; Load sin/cos pointer
	mov	eax, 8			;; 8 iterations 

c2a:	xtouchlines 8			;; Preload TLBs

c4b:	x2cl_eight_reals_unfft_1 rsi, 64, clmblkdst8
	add	al, 256/2		;; Test loop counter
	jnc	c4b			;; Iterate if necessary

	bump	rsi, -2*64+2*clmblkdst8	;; Next source pointer
	mov	ah, 9			;; 9 iterations
c5b:	x2cl_two_complex_unfft rsi, 64, clmblkdst8
	add	al, 256/2		;; Test loop counter
	jnc	c5b			;; Iterate if necessary
	bump	rsi, -2*64+2*clmblkdst8	;; Next source pointer
	bump	rdi, XMM_SCD		;; Next sine/cosine pointer
	sub	ah, 1			;; Test outer loop counter
	jnz	c5b			;; Iterate if necessary
	bump	rdi, -9*XMM_SCD		;; Restore sine/cosine pointer

	bump	rsi, -10*2*clmblkdst8	;; Restore source pointer
c2b:	x2cl_half_eight_reals_unfft_2 rsi, 64, 2*clmblkdst8
	add	al, 256/2		;; Test loop counter
	jnc	c2b			;; Iterate if necessary
	bump	rsi, -2*64+clmblkdst8
	add	ah, 256/2		;; Test loop counter
	jnc	c2b			;; Iterate if necessary

	bump	rsi, -2*clmblkdst8+4*clmblkdst8;; Next source pointer
c3b:	x4cl_four_complex_unfft rsi, 64, 2*clmblkdst8, 4*clmblkdst8, rbx
	xloop1sp c3b, clmblkdst8
	add	edx, 80000000h		;; Test loop counter
	jnc	c3b			;; Iterate if necessary
	bump	rsi, -2*clmblkdst8+8*clmblkdst8
	bump	rbx, XMM_SCD		;; Next sine/cosine pointer
	add	ah, 256/2		;; Test outer loop counter
	jnc	c3b			;; Iterate if necessary
	bump	rbx, -2*XMM_SCD		;; Restore sine/cosine pointer

	bump	rsi, -2*8*clmblkdst8-4*clmblkdst8;; Restore source pointer
	xfive_reals_unfft_preload	;; Preload consts for five real macros
c1b:	x5cl_five_reals_last_unfft rsi, 64, 4*clmblkdst8
	xloop1sp c1b, clmblkdst8
	add	ah, 256/4
	jnc	c1b

	IF clm GT 1
	bump	rsi, -4*clmblkdst8+2*64	;; Next source pointer
	bump	rcx, -8*blkdst+128	;; Next prefetch pointer
	add	dl, 256/clm		;; clm iterations
	jnc	c4b
	bump	rsi, -clm*2*64+clmblkdst;; Next source pointer
	bump	rcx, -clm*128+8*blkdst	;; Next prefetch pointer
	ELSE
	bump	rsi, -4*clmblkdst8+clmblkdst;; Next source pointer
	ENDIF

	sub	al, 1			;; Test loop counter
	jnz	c2a			;; Iterate if necessary
	end_timer 18

;; Normalize these values

	pass1_normalize 1, yes_fft

;; Copy data from the scratch area back to the FFT data area

	copy_scratch_data_to_fft 160
	jmp	endlp			;; Skip FFT code if flag not set

;; Fire up auxiliary threads to do pass 1 forward FFT

pass1:	pass1_forward_fft_setup b0b

;; Copy data from the FFT data area back to the scratch area

b0b:	copy_fft_data_to_scratch 160

;; Do FFT levels 1,2,3
;;
;; On input the 64-byte cache lines hold these data values:
;;	0K	64K	640K	704K
;;	2K	...
;;	...
;;	62K
;;	128K	384K	768K	1024K
;;	130K
;;	...
;;	382K
;; On output the 64-byte cache lines hold these data values:
;;	0K	64K	128K	192K
;;	2K	...
;;	...
;;	62K
;;	256K	512K	768K	1024K
;;	258K
;;	...
;;	510K
;;
;; Do 256 five_reals_first_fft macros (FFT levels 1,2,3)
;;	distance between fft data elements is 256K
;;	do 32 iterations each processing 40 data values

;; Do FFT levels 4,5
;; Values 0K-255K is real data, 256K-1279K is complex data.
;;
;; On input the 64-byte cache lines hold these data values:
;;	0K	64K	128K	192K
;;	2K	...
;;	...
;;	62K
;;	256K	512K	768K	1024K
;;	258K
;;	...
;;	510K
;; On output the 64-byte cache lines hold these data values:
;;	0K	64K	128K	192K
;;	2K	...
;;	...
;;	62K	...
;;	256K	...
;;	...
;;
;; Do 32 eight_reals_fft_2 macros
;;	distance between fft data elements is 64K
;;	do 32 macros each processing 8 data values
;;
;; Do 128 four_complex_fft macros
;;	distance between fft data elements is 64K
;;	do 32 macros each processing 32 data values

;; Do FFT levels 6
;; Values 0K-63K is real data, 64K-127K is semi-real data, 128K-1279K is
;; complex data.
;;
;; On input the 64-byte cache lines hold these data values:
;;	0K	64K	128K	192K
;;	2K	...
;;	...
;;	62K	...
;;	256K	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0K	32K	64K	96K
;;	2K	...
;;	...
;;	30K	...
;;	128K	...
;;	...
;;
;; Do 16 eight_reals_fft_1 macros
;; Do 16 two_two_complex_fft macros
;;	distance between fft data elements is 32K
;;	do 16 macros each processing 16 data values
;;
;; Do 128 two_two_complex_fft macros
;;	distance between fft data elements is 32K
;;	do 64 macros each processing 16 data values

yes_fft:start_timer 22
	mov	rsi, scratch_area	;; Get address of scratch area
	mov	rcx, DATA_ADDR		;; Load source address for prefetching
pfing	mov	rbp, DATA_PREFETCH	;; Set prefetch distance
pfing	sub	rbp, rcx
pfing	bump	rcx, 60*blkdst		;; Create prefetching pointer
	mov	rdi, sincos4		;; Load sin/cos pointer
	mov	rbx, sincos3		;; Load sin/cos pointer
	mov	eax, 2			;; 16 iterations
	sub	rdx, rdx

b1a:	xtouchlines 4			;; Preload TLBs

b1aa:	xfive_reals_fft_preload		;; Preload consts for five real macros
b1b:	x5cl_five_reals_fft rsi, 64, 4*clmblkdst8
	xloop1sp b1b, 2*clmblkdst8
	add	ah, 256/2
	jnc	b1b

	bump	rsi, -2*2*clmblkdst8	;; Restore source pointer
b2b:	x1cl_half_eight_reals_fft_2 rsi, 64
	add	al, 256/2		;; Test inner loop counter
	jnc	b2b			;; Iterate if necessary
	bump	rsi, -2*64+2*clmblkdst8	;; Next source pointer
	add	ah, 256/2
	jnc	b2b

;;	bump	rsi, -2*2*clmblkdst8+4*clmblkdst8;; Next source pointer
b3b:	x4cl_four_complex_fft rsi, 64, 4*clmblkdst8, 8*clmblkdst8, rdi
	xloop1sp b3b, 2*clmblkdst8
	add	ah, 256/2
	jnc	b3b

	bump	rsi, -4*clmblkdst8-4*clmblkdst8;; Restore source pointer
b4b:	x2cl_eight_reals_fft_1 rsi, 64, 2*clmblkdst8
	add	al, 256/2		;; Test inner loop counter
	jnc	b4b			;; Iterate if necessary

	bump	rsi, -2*64+4*clmblkdst8	;; Next source pointer
	bump	rbx, XMM_SCD		;; Next sine/cosine pointer
b5b:	x2cl_two_complex_fft rsi, 64, 2*clmblkdst8, rbx
	add	al, 256/2		;; Test inner loop counter
	jnc	b5b			;; Iterate if necessary
	bump	rsi, -2*64+4*clmblkdst8	;; Next source pointer
	bump	rbx, 2*XMM_SCD		;; Next sine/cosine pointer
	add	ah, 256/4		;; Test outer loop counter
	jnc	b5b			;; Iterate if necessary
	bump	rbx, -9*XMM_SCD		;; Restore sine/cosine pointer

	IF clm GT 1
	bump	rsi, -5*4*clmblkdst8+2*64;; Next source pointer
	bump	rcx, -4*blkdst+128	;; Next prefetch pointer
	add	dl, 256/clm		;; clm iterations
	jnc	b1aa
	bump	rsi, -clm*2*64+clmblkdst;; Next source pointer
	bump	rcx, -clm*128+4*blkdst	;; Next prefetch pointer
	ELSE
	bump	rsi, -5*4*clmblkdst8+clmblkdst;; Next source pointer
	ENDIF

	add	eax, 80000000h/8*2	;; Test loop counter
	jnc	b1a			;; Iterate if necessary
	bump	rsi, -8*clmblkdst+clmblkdst8;; Next source pointer
	sub	al, 1			;; Test loop counter
	jnz	b1a			;; Iterate if necessary
	bump	rsi, -2*clmblkdst8	;; Restore source pointer
	end_timer 22

;; Do FFT levels 7,8
;; Values 0K-31K is real data, 32K-63K is semi-real data, 64K-1279K is
;; complex data.
;;
;; On input the 64-byte cache lines hold these data values:
;;	0K	32K	64K	96K
;;	2K	...
;;	...
;;	30K	...
;;	128K	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0K	8K	16K	24K
;;	2K	...
;;	...
;;	6K	...
;;	32K	...
;;	...

;; Do 8 eight_reals_fft_2 macros
;; Do 8 two_two_complex_fft macros
;;	distance between fft data elements is 8K
;;	do 4 macros each processing 32 data values

	start_timer 25
	mov	rdi, sincos2		;; Load sin/cos pointer
	mov	al, 4			;; 4 iterations of 2*clm
b6b:	x4cl_eight_reals_fft_2 rsi, 64, 4*clmblkdst, clmblkdst8, rdi
	add	al, 256/2/clm		;; Test inner loop counter
	jnc	b6b			;; Iterate if necessary
	bump	rsi, -2*clm*64+clmblkdst;; Next source pointer
	sub	al, 1			;; Test outer loop counter
	jnz	b6b			;; Iterate if necessary

;; Do 144 four_complex_fft macros
;;	distance between fft data elements is 8K
;;	do 36 macros each processing 32 data values

	bump	rsi, -4*clmblkdst+2*clmblkdst8;; Load source pointer
	bump	rdi, XMM_SCD		;; Next sine/cosine pointer
	mov	eax, 9*256+4		;; 9 iterations of 4 iters of 2*clm
b7a:	xtouch	[rcx+(clm-1)*128]	;; Preload TLB
b7b:	x4cl_four_complex_fft rsi, 64, 4*clmblkdst, clmblkdst8, rdi
	xloop1	b7b, clmblkdst
	sub	al, 1			;; Test middle loop counter
	jnz	b7a			;; Iterate if necessary
	bump	rsi, -4*clmblkdst+2*clmblkdst8;; Next source pointer
	bump	rdi, 2*XMM_SCD		;; Next sine/cosine pointer
	sub	eax, 256		;; Test outer loop counter
	mov	al, 4			;; Load middle loop counter
	jnz	b7a			;; Iterate if necessary
	bump	rsi, -10*2*clmblkdst8	;; Restore source pointer
pfing	bump	rcx, -160*blkdst	;; Restore prefetch pointer
	end_timer 25

;; Do FFT levels 9,10
;; Values 0K-7K is real data, 8K-15K is semi-real data, 16K-1279K is
;; complex data.
;;
;; On input the 64-byte cache lines hold these data values:
;;	0K	8K	16K	24K
;;	2K	...
;;	...
;;	6K	...
;;	32K	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0K	2K	4K	6K
;;	8K	...
;;	...

;; Do 1 eight_reals_fft_2 macros
;; Do 1 nop_two_two_complex_fft_2 macros
;; Do 2 four_complex_fft macros
;;	distance between fft data elements is 2K
;;	do 1 macros each processing 32 data values

	start_timer 26
	mov	rdi, sincos1		;; Load sin/cos pointer
b8b:	g4cl_eight_reals_fft_2 rsi, 64, clmblkdst, 2*clmblkdst, rcx, 64, blkdst, 2*blkdst
	add	al, 256/2/clm		;; Test inner loop counter
	jnc	b8b			;; Iterate if necessary

;; Do 156 four_complex_fft macros
;;	distance between fft data elements is 2K
;;	do 39 macros each processing 32 data values

	bump	rsi, -2*clm*64+4*clmblkdst;; Next source pointer
	bump	rcx, -2*clm*64+4*blkdst	;; Next dest pointer
	bump	rdi, XMM_SCD		;; Next sine/cosine pointer
	mov	eax, 20*256+1		;; 39 iterations of 2*clm
b9b:	g4cl_four_complex_fft rsi, 64, clmblkdst, 2*clmblkdst, rcx, 64, blkdst, 2*blkdst
	L2prefetch128 [rcx-64][rbp]
	L2prefetch128 [rcx-64+blkdst][rbp]
	bump	rbp, -64+2*blkdst
	add	al, 256/2		;; Test inner loop counter
	jnc	b9b			;; Iterate if necessary
	bump	rbp, 128-4*blkdst
	IF clm NE 1
	add	dl, 256/clm
	jnc	b9b
	ENDIF
	bump	rsi, -2*clm*64+4*clmblkdst;; Next source pointer
	bump	rcx, -2*clm*64+4*blkdst	;; Next dest pointer
	bump	rdi, 2*XMM_SCD		;; Next sine/cosine pointer
	sub	al, 1			;; Test outer loop counter
	jnz	b9b
	bump	rsi, -8*clmblkdst+clmblkdst8
	sub	ah, 1			;; Test outer loop counter
	mov	al, 2
	jnz	b9b			;; Iterate if necessary
	end_timer 26

;; Work on next 4 sets of 640 values

endlp:	xpass1_get_next_block pass2, c0b, b0b ;; Get next block and jump
	done:	xfft_footer
	ENDM


;; Perform a 1536K-element FFT.  This is done in two passes.  Pass 1
;; does 10 levels, pass 2 uses common code to perform the last 11 levels.

hg_pass1sclevels10pfa6 MACRO fftname, pass2_levels, clmarg
	LOCAL	b0b, b1a, b1b, b2b, b3b, b4b, b5b, b6b, b7a, b7b, b8b, b9b
	LOCAL	c0b, c1b, c2a, c2b, c3b, c4b, c5b, c6b, c7a, c7b, c8b, c9b
	LOCAL	pass1, pass2, endlp, done, yes_fft

	set_FFT_constants fftname, 0, 0, pass2_levels, clmarg
	xfft_header pass1

;; Do the last 11 FFT levels
;;	do 1 pass2_11_levels_real and 191 pass2_11_levels_complex

pass2:	start_timer 0
	call	xpass2name
	end_timer 0
	cmp	ffttype, 1		;; We're done if FFTing only
	je	done

;; Do the inverse FFT

	pass1_inverse_fft_setup c0b

;; Do inverse FFT levels 9,10
;; On input the 64-byte cache lines hold these data values:
;;	0K	2K	4K	6K
;;	8K	10K	12K	14K
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0K	4K	8K	12K
;;	2K	...
;;	16K	...
;;	...

;; Do 1 eight_reals_unfft_2 macros
;; Do 1 nop_two_two_complex_unfft_2 macros
;;	distance between fft data elements is 2K
;;	do 1 macros each processing 16 data values

c0b:	start_timer 16
	mov	rsi, DATA_ADDR		;; Get source address
	mov	rcx, scratch_area	;; Get address of scratch area
	sub	rax, rax
	sub	rdx, rdx
pfing	mov	rbp, DATA_PREFETCH	;; Set prefetch distance
pfing	sub	rbp, rsi
c8b:	g2cl_eight_reals_unfft_2 rsi, 64, blkdst, rcx, 64, clmblkdst
	add	al, 256/2/clm		;; Test loop counter
	jnc	c8b			;; Iterate if necessary

;; Do 190 four_complex_unfft macros
;;	distance between fft data elements is 2K
;;	do 95 macros each processing 16 data values

	bump	rsi, -2*clm*64+2*blkdst	;; Next source pointer
	bump	rcx, -2*clm*64+2*clmblkdst;; Next dest pointer
	mov	rdi, sincos1		;; Load sin/cos pointer
	mov	al, 256/4+24		;; 95 iterations of 2*clm
c9b:	g2cl_four_complex_unfft rsi, 64, blkdst, rcx, 64, clmblkdst
	add	dl, 256/2/clm		;; Test loop counter
	jnc	c9b			;; Iterate if necessary
	bump	rsi, -2*clm*64+2*blkdst	;; Next source pointer
	bump	rcx, -2*clm*64+2*clmblkdst;; Next dest pointer
	bump	rdi, XMM_SCD		;; Next sine/cosine pointer
	add	al, 256/4		;; Test loop counter
	jnc	c9b
	bump	rcx, -8*clmblkdst+clmblkdst8
	sub	al, 1			;; Test outer loop counter
	jnz	c9b			;; Iterate if necessary
	lea	rcx, [rsi-96*2*blkdst]	;; Create prefetch pointer
	mov	rsi, scratch_area	;; Restore source pointer
	end_timer 16

;; Do inverse FFT levels 7,8
;; On input the 64-byte cache lines hold these data values:
;;	0K	4K	8K	12K
;;	2K	...
;;	16K	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0K	16K	32K	48K
;;	2K	...
;;	...
;;	14K
;;	64K	...
;;	...

;; Do 4 eight_reals_unfft_2 macros
;; Do 4 nop_two_two_complex_unfft_2 macros
;;	distance between fft data elements is 8K
;;	do 2 macros each processing 32 data values

	start_timer 17
	mov	al, 2			;; 2 iterations of 2*clm
c6b:	x4cl_eight_reals_unfft_2 rsi, 64, 2*clmblkdst, 4*clmblkdst
	add	al, 256/2/clm		;; Test loop counter
	jnc	c6b			;; Iterate if necessary
	bump	rsi, -2*clm*64+clmblkdst;; Next source pointer
	sub	al, 1			;; Test outer loop counter
	jnz	c6b			;; Iterate if necessary

;; Do 184 four_complex_unfft macros
;;	distance between fft data elements is 8K
;;	do 46 macros each processing 32 data values

	bump	rsi, -2*clmblkdst+clmblkdst8;; Next source pointer
	mov	rdi, sincos2		;; Load sin/cos pointer
	mov	eax, 23*256+2		;; 23 iterations of 2 iters of 2*clm
c7a:	xtouch	[rcx+(clm-1)*128]	;; Preload TLB
c7b:	x4cl_four_complex_unfft rsi, 64, 2*clmblkdst, 4*clmblkdst, rdi
	xloop1	c7b, clmblkdst
	sub	al, 1			;; Test middle loop counter
	jnz	c7a			;; Iterate if necessary
	bump	rsi, -2*clmblkdst+clmblkdst8;; Next source pointer
	bump	rdi, XMM_SCD		;; Next sine/cosine pointer
	sub	eax, 256		;; Test outer loop counter
	mov	al, 2			;; Load middle loop counter
	jnz	c7a			;; Iterate if necessary
	bump	rsi, -24*clmblkdst8	;; Restore source pointer
	end_timer 17

;; Do inverse FFT levels 6
;; On input the 64-byte cache lines hold these data values:
;;	0K	16K	32K	48K
;;	2K	...
;;	...
;;	14K
;;	64K	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0K	32K	64K	96K
;;	2K	...
;;	...
;;	30K
;;	128K	...
;;	...
;;
;; Do 16 eight_reals_unfft_1 macros
;;	distance between fft data elements is 32K
;;	do 8 macros each processing 16 data values
;;
;; Do 176 four_complex_unfft macros
;;	distance between fft data elements is 32K
;;	do 88 macros each processing 16 data values

;; Do inverse FFT levels 4,5
;; On input the 64-byte cache lines hold these data values:
;;	0K	32K	64K	96K
;;	2K	...
;;	...
;;	30K
;;	128K	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0K	128K	256K	384K
;;	2K	...
;;	...
;;	126K
;;	512K	...
;;	...
;;
;; Do 32 eight_reals_unfft_2 macros
;; Do 32 nop_two_two_complex_unfft_2 macros
;;	distance between fft data elements is 64K
;;	do 16 macros each processing 32 data values
;;
;; Do 128 four_complex_unfft macros
;;	distance between fft data elements is 64K
;;	do 32 macros each processing 32 data values

;; Do inverse FFT levels 1,2,3
;; On input the 64-byte cache lines hold these data values:
;;	0K	128K	256K	384K
;;	2K	...
;;	...
;;	126K
;;	512K	768K	1024K	1280K
;;	514K
;;	...
;;	766K
;; On output the 64-byte cache lines hold these data values:
;;	0K	384K	768K	1152K
;;	...
;;	382K
;;
;; Do 256 six_reals_last_unfft macros
;;	distance between fft data elements is 256K
;;	do 64 macros each processing 24 data values

	start_timer 18
	mov	rdi, sincos3		;; Load sin/cos pointer
	mov	rbx, sincos4		;; Load sin/cos pointer
	mov	al, 8			;; 8 iterations 
	xsix_reals_unfft_preload

c2a:	xtouchlines 12			;; Preload TLB

c4b:	x2cl_eight_reals_unfft_1 rsi, 64, clmblkdst8
	add	al, 256/2		;; Test loop counter
	jnc	c4b			;; Iterate if necessary

	bump	rsi, -2*64+2*clmblkdst8	;; Next source pointer
	mov	ah, 11			;; 11 iterations
c5b:	x2cl_two_complex_unfft rsi, 64, clmblkdst8
	add	al, 256/2		;; Test loop counter
	jnc	c5b			;; Iterate if necessary
	bump	rsi, -2*64+2*clmblkdst8	;; Next source pointer
	bump	rdi, XMM_SCD		;; Next sine/cosine pointer
	sub	ah, 1			;; Test outer loop counter
	jnz	c5b			;; Iterate if necessary
	bump	rdi, -11*XMM_SCD	;; Restore sine/cosine pointer

	bump	rsi, -12*2*clmblkdst8	;; Restore source pointer
c2b:	x4cl_eight_reals_unfft_2 rsi, 64, 2*clmblkdst8, 4*clmblkdst8
	add	al, 256/2		;; Test loop counter
	jnc	c2b			;; Iterate if necessary
	bump	rsi, -2*64+clmblkdst8
	add	ah, 256/2		;; Test loop counter
	jnc	c2b			;; Iterate if necessary

	bump	rsi, -2*clmblkdst8+8*clmblkdst8;; Next source pointer
c3b:	x4cl_four_complex_unfft rsi, 64, 2*clmblkdst8, 4*clmblkdst8, rbx
	xloop1sp c3b, clmblkdst8
	add	edx, 80000000h		;; Test loop counter
	jnc	c3b			;; Iterate if necessary
	bump	rsi, -2*clmblkdst8+8*clmblkdst8
	bump	rbx, XMM_SCD		;; Next sine/cosine pointer
	add	ah, 256/2
	jnc	c3b
	bump	rbx, -2*XMM_SCD		;; Restore sine/cosine pointer

	bump	rsi, -3*8*clmblkdst8	;; Restore source pointer
c1b:	x3cl_six_reals_last_unfft rsi, 64, 8*clmblkdst8
	xloop1sp c1b, clmblkdst8
	add	ah, 256/8
	jnc	c1b

	IF clm GT 1
	bump	rsi, -8*clmblkdst8+2*64	;; Next source pointer
	bump	rcx, -12*blkdst+128	;; Next prefetch pointer
	add	dl, 256/clm		;; clm iterations
	jnc	c4b
	bump	rsi, -clm*2*64+clmblkdst;; Next source pointer
	bump	rcx, -clm*128+12*blkdst	;; Next prefetch pointer
	ELSE
	bump	rsi, -8*clmblkdst8+clmblkdst;; Next source pointer
	ENDIF

	sub	al, 1			;; Test loop counter
	jnz	c2a			;; Iterate if necessary
	end_timer 18

;; Normalize these values

	pass1_normalize 1, yes_fft

;; Copy data from the scratch area back to the FFT data area

	copy_scratch_data_to_fft 192
	jmp	endlp			;; Skip FFT code if flag not set

;; Fire up auxiliary threads to do pass 1 forward FFT

pass1:	pass1_forward_fft_setup b0b

;; Copy data from the FFT data area back to the scratch area

b0b:	copy_fft_data_to_scratch 192

;; Do FFT levels 1,2,3
;;
;; On input the 64-byte cache lines hold these data values:
;;	0K	384K	768K	1152K
;;	...
;;	382K
;; On output the 64-byte cache lines hold these data values:
;;	0K	128K	256K	384K
;;	2K	...
;;	...
;;	126K
;;	512K	768K	1024K	1280K
;;	514K
;;	...
;;	766K
;;
;; Do 256 six_reals_first_fft macros (FFT levels 1,2,3)
;;	distance between fft data elements is 256K
;;	do 64 iterations each processing 24 data values

;; Do FFT levels 4,5
;; Values 0K-255K is real data, 256K-511K is semi-real data, 512K-1535K is
;; complex data.
;;
;; On input the 64-byte cache lines hold these data values:
;;	0K	128K	256K	384K
;;	2K	...
;;	...
;;	126K
;;	512K	768K	1024K	1280K
;;	514K
;;	...
;;	766K
;; On output the 64-byte cache lines hold these data values:
;;	0K	64K	128K	192K
;;	2K	...
;;	...
;;	62K	...
;;	256K	...
;;	...
;;
;; Do 32 eight_reals_fft_2 macros
;; Do 32 nop_two_two_complex_fft_2 macros
;;	distance between fft data elements is 64K
;;	do 32 macros each processing 16 data values
;;
;; Do 128 four_complex_fft macros
;;	distance between fft data elements is 64K
;;	do 32 macros each processing 32 data values

;; Do FFT levels 6
;; Values 0K-63K is real data, 64K-127K is semi-real data, 128K-1535K is
;; complex data.
;;
;; On input the 64-byte cache lines hold these data values:
;;	0K	64K	128K	192K
;;	2K	...
;;	...
;;	62K	...
;;	256K	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0K	32K	64K	96K
;;	2K	...
;;	...
;;	30K	...
;;	128K	...
;;	...
;;
;; Do 16 eight_reals_fft_1 macros
;; Do 16 two_two_complex_fft macros
;;	distance between fft data elements is 32K
;;	do 16 macros each processing 16 data values
;;
;; Do 160 two_two_complex_fft macros
;;	distance between fft data elements is 32K
;;	do 80 macros each processing 16 data values

yes_fft:start_timer 22
	mov	rsi, scratch_area	;; Get address of scratch area
	mov	rcx, DATA_ADDR		;; Load source address for prefetching
pfing	mov	rbp, DATA_PREFETCH	;; Set prefetch distance
pfing	sub	rbp, rcx
pfing	bump	rcx, 116*blkdst		;; Create prefetch pointer
	mov	rdi, sincos4		;; Load sin/cos pointer
	mov	rbx, sincos3		;; Load sin/cos pointer
	mov	eax, 2			;; 16 iterations
	sub	rdx, rdx
	xsix_reals_fft_preload

b1a:	xtouchlines 2			;; Preload TLBs

b1b:	x3cl_six_reals_fft rsi, 64, 8*clmblkdst8
	add	al, 256/2		;; Test loop counter
	jnc	b1b			;; Iterate if necessary
	bump	rsi, -2*64+2*clmblkdst8	;; Next source pointer
	add	ah, 256/4
	jnc	b1b

	bump	rsi, -4*2*clmblkdst8	;; Restore source pointer
b2b:	x2cl_eight_reals_fft_2 rsi, 64, 4*clmblkdst8
	add	al, 256/2		;; Test inner loop counter
	jnc	b2b			;; Iterate if necessary
	bump	rsi, -2*64+2*clmblkdst8	;; Next source pointer
	add	ah, 256/2
	jnc	b2b

	bump	rsi, -2*2*clmblkdst8+8*clmblkdst8;; Next source pointer
b3b:	x4cl_four_complex_fft rsi, 64, 4*clmblkdst8, 8*clmblkdst8, rdi
	xloop1sp b3b, 2*clmblkdst8
	add	ah, 256/2
	jnc	b3b

	bump	rsi, -2*2*clmblkdst8-8*clmblkdst8;; Restore source pointer
b4b:	x2cl_eight_reals_fft_1 rsi, 64, 2*clmblkdst8
	add	al, 256/2		;; Test inner loop counter
	jnc	b4b			;; Iterate if necessary

	bump	rsi, -2*64+4*clmblkdst8	;; Load source pointer
	bump	rbx, XMM_SCD		;; Next sine/cosine pointer
	mov	ah, 5			;; 5 iterations
b5b:	x2cl_two_complex_fft rsi, 64, 2*clmblkdst8, rbx
	add	al, 256/2		;; Test inner loop counter
	jnc	b5b			;; Iterate if necessary
	bump	rsi, -2*64+4*clmblkdst8	;; Next source pointer
	bump	rbx, 2*XMM_SCD		;; Next sine/cosine pointer
	sub	ah, 1			;; Test outer loop counter
	jnz	b5b			;; Iterate if necessary
	bump	rbx, -11*XMM_SCD	;; Restore sine/cosine pointer

	IF clm GT 1
	bump	rsi, -6*4*clmblkdst8+2*64;; Next source pointer
	bump	rcx, -2*blkdst+128	;; Next prefetch pointer
	add	dl, 256/clm		;; clm iterations
	jnc	b1b
	bump	rsi, -clm*2*64+clmblkdst;; Next source pointer
	bump	rcx, -clm*128+2*blkdst	;; Next prefetch pointer
	ELSE
	bump	rsi, -6*4*clmblkdst8+clmblkdst;; Next source pointer
	ENDIF

	add	eax, 80000000h/8*2	;; Test loop counter
	jnc	b1a			;; Iterate if necessary
	bump	rsi, -8*clmblkdst+clmblkdst8;; Next source pointer
	sub	al, 1			;; Test loop counter
	jnz	b1a			;; Iterate if necessary
	bump	rsi, -2*clmblkdst8	;; Restore source pointer
	end_timer 22

;; Do FFT levels 7,8
;; Values 0K-31K is real data, 32K-63K is semi-real data, 64K-1535K is
;; complex data.
;;
;; On input the 64-byte cache lines hold these data values:
;;	0K	32K	64K	96K
;;	2K	...
;;	...
;;	30K	...
;;	128K	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0K	8K	16K	24K
;;	2K	...
;;	...
;;	6K	...
;;	32K	...
;;	...

;; Do 8 eight_reals_fft_2 macros
;; Do 8 two_two_complex_fft macros
;;	distance between fft data elements is 8K
;;	do 4 macros each processing 32 data values

	start_timer 25
	mov	rdi, sincos2		;; Load sin/cos pointer
	mov	al, 4			;; 4 iterations of 2*clm
b6b:	x4cl_eight_reals_fft_2 rsi, 64, 4*clmblkdst, clmblkdst8, rdi
	add	al, 256/2/clm		;; Test inner loop counter
	jnc	b6b			;; Iterate if necessary
	bump	rsi, -2*clm*64+clmblkdst;; Next source pointer
	sub	al, 1			;; Test outer loop counter
	jnz	b6b			;; Iterate if necessary

;; Do 176 four_complex_fft macros
;;	distance between fft data elements is 8K
;;	do 44 macros each processing 32 data values

	bump	rsi, -4*clmblkdst+2*clmblkdst8;; Load source pointer
	bump	rdi, XMM_SCD		;; Next sine/cosine pointer
	mov	eax, 11*256+4		;; 11 iterations of 4 iters of 2*clm
b7a:	xtouch	[rcx+(clm-1)*128]	;; Preload TLB
b7b:	x4cl_four_complex_fft rsi, 64, 4*clmblkdst, clmblkdst8, rdi
	xloop1	b7b, clmblkdst
	sub	al, 1			;; Test middle loop counter
	jnz	b7a			;; Iterate if necessary
	bump	rsi, -4*clmblkdst+2*clmblkdst8;; Next source pointer
	bump	rdi, 2*XMM_SCD		;; Next sine/cosine pointer
	sub	eax, 256		;; Test outer loop counter
	mov	al, 4			;; Load middle loop counter
	jnz	b7a			;; Iterate if necessary
	bump	rsi, -12*2*clmblkdst8	;; Restore source pointer
pfing	bump	rcx, -192*blkdst	;; Restore prefetch pointer
	end_timer 25

;; Do FFT levels 9,10
;; Values 0K-7K is real data, 8K-15K is semi-real data, 16K-1535K is
;; complex data.
;;
;; On input the 64-byte cache lines hold these data values:
;;	0K	8K	16K	24K
;;	2K	...
;;	...
;;	6K	...
;;	32K	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0K	2K	4K	6K
;;	8K	...
;;	...

;; Do 1 eight_reals_fft_2 macros
;; Do 1 nop_two_two_complex_fft_2 macros
;; Do 2 four_complex_fft macros
;;	distance between fft data elements is 2K
;;	do 1 macros each processing 32 data values

	start_timer 26
	mov	rdi, sincos1		;; Load sin/cos pointer
b8b:	g4cl_eight_reals_fft_2 rsi, 64, clmblkdst, 2*clmblkdst, rcx, 64, blkdst, 2*blkdst
	add	al, 256/2/clm		;; Test inner loop counter
	jnc	b8b			;; Iterate if necessary

;; Do 188 four_complex_fft macros
;;	distance between fft data elements is 2K
;;	do 47 macros each processing 32 data values

	bump	rsi, -2*clm*64+4*clmblkdst;; Next source pointer
	bump	rcx, -2*clm*64+4*blkdst	;; Next dest pointer
	bump	rdi, XMM_SCD		;; Next sine/cosine pointer
	mov	eax, 24*256+1		;; 47 iterations of 2*clm
b9b:	g4cl_four_complex_fft rsi, 64, clmblkdst, 2*clmblkdst, rcx, 64, blkdst, 2*blkdst
	L2prefetch128 [rcx-64][rbp]
	L2prefetch128 [rcx-64+blkdst][rbp]
	bump	rbp, -64+2*blkdst
	add	al, 256/2		;; Test inner loop counter
	jnc	b9b			;; Iterate if necessary
	bump	rbp, 128-4*blkdst
	IF clm NE 1
	add	dl, 256/clm
	jnc	b9b
	ENDIF
	bump	rsi, -2*clm*64+4*clmblkdst;; Next source pointer
	bump	rcx, -2*clm*64+4*blkdst	;; Next dest pointer
	bump	rdi, 2*XMM_SCD		;; Next sine/cosine pointer
	sub	al, 1			;; Test outer loop counter
	jnz	b9b
	bump	rsi, -8*clmblkdst+clmblkdst8
	sub	ah, 1			;; Test outer loop counter
	mov	al, 2
	jnz	b9b			;; Iterate if necessary
	end_timer 26

;; Work on next 4 sets of 768 values

endlp:	xpass1_get_next_block pass2, c0b, b0b ;; Get next block and jump
	done:	xfft_footer
	ENDM


;; Perform a 1536K-element negacyclic FFT.  This is done in two passes.  Pass
;; 1 does 10 levels, pass 2 uses common code to perform the last 11 levels.

hg_pass1sclevels10complex3 MACRO fftname, pass2_levels, clmarg
	LOCAL	b0b, b1a, b1b, b2a, b2b, b3a, b3b, b5a, b5b, b7a, b7b, b9b
	LOCAL	c0b, c1a, c1b, c3a, c3b, c5a, c5b, c7a, c7b, c9b
	LOCAL	pass1, pass2, endlp, done, yes_fft

	set_FFT_constants fftname, 0, 1, pass2_levels, clmarg
	xfft_header pass1

;; Do the last 11 FFT levels
;;	do 192 pass2_11_levels_complex

pass2:	start_timer 0
	call	xpass2name
	end_timer 0
	cmp	ffttype, 1		;; We're done if FFTing only
	je	done

;; Do the inverse FFT

	pass1_inverse_fft_setup c0b

;; Do inverse FFT levels 9,10
;; On input the 64-byte cache lines hold these data values:
;;	0K	2K	4K	6K
;;	8K	10K	12K	14K
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0K	4K	8K	12K
;;	2K	...
;;	16K	...
;;	...

;; Do 192 four_complex_unfft macros
;;	distance between fft data elements is 2K
;;	do 96 macros each processing 16 data values

c0b:	start_timer 16
	mov	rsi, DATA_ADDR		;; Get source address
	mov	rcx, scratch_area	;; Get address of scratch area
	mov	rdi, sincos1		;; Load sin/cos pointer
	mov	eax, 24			;; 24 iterations of 4 iters of 2*clm
	sub	rdx, rdx
pfing	mov	rbp, DATA_PREFETCH	;; Set prefetch distance
pfing	sub	rbp, rsi
c9b:	g2cl_four_complex_unfft rsi, 64, blkdst, rcx, 64, clmblkdst
	add	dl, 256/2/clm		;; Test loop counter
	jnc	c9b			;; Iterate if necessary
	bump	rsi, -2*clm*64+2*blkdst	;; Next source pointer
	bump	rcx, -2*clm*64+2*clmblkdst;; Next dest pointer
	bump	rdi, XMM_SCD		;; Next sine/cosine pointer
	add	al, 256/4		;; Test loop counter
	jnc	c9b
	bump	rcx, -8*clmblkdst+clmblkdst8
	sub	al, 1			;; Test outer loop counter
	jnz	c9b			;; Iterate if necessary
	lea	rcx, [rsi-96*2*blkdst]	;; Create prefetch pointer
	mov	rsi, scratch_area	;; Restore source pointer
	end_timer 16

;; Do inverse FFT levels 7,8
;; On input the 64-byte cache lines hold these data values:
;;	0K	4K	8K	12K
;;	2K	...
;;	16K	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0K	16K	32K	48K
;;	2K	...
;;	...
;;	14K
;;	64K	...
;;	...

;; Do 192 four_complex_unfft macros
;;	distance between fft data elements is 8K
;;	do 48 macros each processing 32 data values

	start_timer 17
	mov	rdi, sincos2		;; Load sin/cos pointer
	mov	al, 24			;; 24 iterations of 2*2*clm
c7a:	xtouch	[rcx+(clm-1)*128]	;; Preload TLB
c7b:	x4cl_four_complex_unfft rsi, 64, 2*clmblkdst, 4*clmblkdst, rdi
	xloop2	c7b, clmblkdst		;; 2*2*clm iterations
	bump	rsi, -2*clmblkdst+clmblkdst8;; Next source pointer
	bump	rdi, XMM_SCD		;; Next sine/cosine pointer
	sub	al, 1			;; Test outer loop counter
	jnz	c7a			;; Iterate if necessary
	bump	rsi, -24*clmblkdst8	;; Restore source pointer
	end_timer 17

;; Do inverse FFT levels 5,6
;; On input the 64-byte cache lines hold these data values:
;;	0K	16K	32K	48K
;;	2K	...
;;	...
;;	14K
;;	64K	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0K	64K	128K	192K
;;	2K	...
;;	...
;;	62K
;;	256K	...
;;	...

;; Do 192 four_complex_unfft macros
;;	distance between fft data elements is 32K
;;	do 48 macros each processing 32 data values

	start_timer 18
	mov	rdi, sincos3		;; Load sin/cos pointer
	mov	eax, 6*256+4		;; 6 iterations of 4 iters of 2*2*clm
c5a:	xtouch	[rcx+(clm-1)*128]	;; Preload TLB
c5b:	x4cl_four_complex_unfft rsi, 64, clmblkdst8, 2*clmblkdst8, rdi
	xloop2	c5b, clmblkdst		;; 2 * 2*clm iterations
	sub	al, 1			;; Test middle loop counter
	jnz	c5a			;; Iterate if necessary
	bump	rsi, -4*2*clmblkdst+4*clmblkdst8;; Next source pointer
	bump	rdi, XMM_SCD		;; Next sine/cosine pointer
	sub	eax, 256		;; Test outer loop counter
	mov	al, 4			;; Load middle loop counter
	jnz	c5a			;; Iterate if necessary
	bump	rsi, -6*4*clmblkdst8	;; Restore source pointer
	end_timer 18

;; Do inverse FFT levels 4
;; On input the 64-byte cache lines hold these data values:
;;	0K	64K	128K	192K
;;	2K	...
;;	...
;;	62K
;;	256K	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0K	128K	256K	384K
;;	2K	...
;;	...
;;	126K
;;	512K	...
;;	...

;; Do 192 two_two_complex_unfft macros
;;	distance between fft data elements is 64K
;;	do 96 macros each processing 16 data values

	start_timer 19
	mov	rdi, sincos4		;; Load sin/cos pointer
	mov	eax, 3*256+4		;; 3 iters of 4 iters 2 of 4*2*clm
c3a:	xtouch	[rcx+(clm-1)*128]	;; Preload TLB
c3b:	x2cl_two_complex_unfft rsi, 64, 4*clmblkdst8
	xloop4	c3b, clmblkdst		;; 4* 2*clm iterations
	add	eax, 80000000h		;; 2 iterations
	jnc	c3a			;; Iterate if necessary
	bump	rsi, -2*4*clmblkdst+clmblkdst8;; Next source pointer
	sub	al, 1			;; Test middle loop counter
	jnz	c3a
	bump	rsi, -4*clmblkdst8+8*clmblkdst8;; Next source pointer
	bump	rdi, XMM_SCD		;; Next sine/cosine pointer
	sub	eax, 256		;; Test outer loop counter
	mov	al, 4			;; Load middle loop counter
	jnz	c3a			;; Iterate if necessary
	bump	rsi, -3*8*clmblkdst8	;; Restore source pointer
	end_timer 19

;; Do inverse FFT levels 1,2,3
;; On input the 64-byte cache lines hold these data values:
;;	0K	128K	256K	384K
;;	2K	...
;;	...
;;	126K
;;	512K	768K	1024K	1280K
;;	514K
;;	...
;;	766K
;; On output the 64-byte cache lines hold these data values:
;;	0K	384K	768K	1152K
;;	...
;;	382K

;; Do 256 three_complex_last_unfft macros
;;	distance between fft data elements is 256K
;;	do 64 macros each processing 24 data values

	start_timer 20
	mov	rdi, plus1_premults	;; Address of premultiplier table
	mov	eax, 8*256+4		;; 8 iters of 4 iters of 2*2*clm
c1a:	xtouch	[rcx+(clm-1)*128]	;; Preload TLB
c1b:	x3cl_three_complex_last_unfft rsi, 64, 8*clmblkdst8
	xloop2edi c1b, clmblkdst, 192 ;; 2 * 2*clm iterations
	sub	al, 1			;; Test loop counter
	jnz	c1a			;; Iterate if necessaryxn
	bump	rsi, -4*2*clmblkdst+clmblkdst8;; Next source pointer
	sub	ah, 1			;; Test outer loop counter
	mov	al, 4			;; Load middle loop counter
	jnz	c1a
	end_timer 20

;; Normalize these values

	pass1_normalize 1, yes_fft

;; Copy data from the scratch area back to the FFT data area

	copy_scratch_data_to_fft 192
	jmp	endlp			;; Skip FFT code if flag not set

;; Fire up auxiliary threads to do pass 1 forward FFT

pass1:	pass1_forward_fft_setup b0b

;; Copy data from the FFT data area back to the scratch area

b0b:	copy_fft_data_to_scratch 192

;; Do FFT levels 1,2,3
;;
;; On input the 64-byte cache lines hold these data values:
;;	0K	384K	768K	1152K
;;	...
;;	382K
;; On output the 64-byte cache lines hold these data values:
;;	0K	128K	256K	384K
;;	2K	...
;;	...
;;	126K
;;	512K	768K	1024K	1280K
;;	514K
;;	...
;;	766K

;; Do 256 three_complex_first_fft macros (FFT levels 1,2,3)
;;	distance between fft data elements is 256K
;;	do 64 iterations each processing 24 data values

yes_fft:start_timer 22
	mov	rsi, scratch_area	;; Get address of scratch area
	mov	rdi, plus1_premults	;; Address of premultiplier table
	mov	rcx, DATA_ADDR		;; Load source address for prefetching
pfing	mov	rbp, DATA_PREFETCH	;; Set prefetch distance
pfing	sub	rbp, rcx
pfing	bump	rcx, 104*blkdst		;; Create prefetching pointer
	mov	eax, 8			;; 8 iters of 2 iters of 4*2*clm
	sub	rdx, rdx
b1a:	xtouch	[rcx+(clm-1)*128]	;; Preload TLB
b1b:	x3cl_three_complex_fft rsi, 64, 8*clmblkdst8
	xloop4edi b1b, clmblkdst, 192 ;; 4 * 2*clm iterations
	add	eax, 80000000h		;; 2 iterations
	jnc	b1a
	bump	rsi, -2*4*clmblkdst+clmblkdst8
	sub	al, 1			;; Test loop counter
	jnz	b1a			;; Iterate if necessary
	bump	rsi, -8*clmblkdst8	;; Restore source pointer
	end_timer 22

;; Do FFT levels 4
;;
;; On input the 64-byte cache lines hold these data values:
;;	0K	128K	256K	384K
;;	2K	...
;;	...
;;	126K
;;	512K	768K	1024K	1280K
;;	514K
;;	...
;;	766K
;; On output the 64-byte cache lines hold these data values:
;;	0K	128K	256K	384K
;;	2K	...
;;	...
;;	126K	...
;;	512K	...
;;	...

;; Do 192 two_two_complex_fft macros
;;	distance between fft data elements is 64K
;;	do 96 macros each processing 16 data values

	start_timer 23
	mov	rdi, sincos4		;; Load sin/cos pointer
	mov	al, 4			;; 4 iters of 2 of 4*2*clm
b2a:	xtouch	[rcx+(clm-1)*128]	;; Preload TLB
b2b:	x2cl_two_complex_fft_in_place rsi, 64, 4*clmblkdst8, rdi
	xloop4	b2b, clmblkdst		;; 4 * 2*clm iterations
	add	eax, 80000000h		;; 2 iterations
	jnc	b2a
	bump	rsi, -2*4*clmblkdst+clmblkdst8
	sub	al, 1			;; Test outer loop counter
	jnz	b2a			;; Iterate if necessary
	bump	rsi, -4*clmblkdst8+8*clmblkdst8;; Next source pointer
	bump	rdi, XMM_SCD		;; Next sine/cosine pointer
	mov	al, 8			;; 8 iters of 2 of 4*2*clm
b3a:	xtouch	[rcx+(clm-1)*128]	;; Preload TLB
b3b:	x2cl_two_complex_fft rsi, 64, 8*clmblkdst8, rdi
	xloop4	b3b, clmblkdst		;; 4 * 2*clm iterations
	add	eax, 80000000h		;; 2 iterations
	jnc	b3a
	bump	rsi, -2*4*clmblkdst+clmblkdst8
	sub	al, 1			;; Test outer loop counter
	jnz	b3a			;; Iterate if necessary
	bump	rsi, -8*clmblkdst8-8*clmblkdst8;; Restore source pointer
	end_timer 23

;; Do FFT levels 5,6
;;
;; On input the 64-byte cache lines hold these data values:
;;	0K	128K	256K	384K
;;	2K	...
;;	...
;;	126K	...
;;	512K	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0K	32K	64K	96K
;;	2K	...
;;	...
;;	30K	...
;;	128K	...
;;	...

;; Do 192 four_complex_fft macros
;;	distance between fft data elements is 32K
;;	do 48 macros each processing 32 data values

	start_timer 24
	mov	rdi, sincos3		;; Load sin/cos pointer
	mov	eax, 3*256+2		;; 3 iters of 2 iters of 4 of 2*2*clm
b5a:	xtouch	[rcx+(clm-1)*128]	;; Preload TLB
b5b:	x4cl_four_complex_fft rsi, 64, 2*clmblkdst8, 4*clmblkdst8, rdi
	xloop2	b5b, clmblkdst		;; 2 * 2*clm iterations
	add	eax, 80000000h/2	;; 4 iterations
	jnc	b5a
	bump	rsi, -4*2*clmblkdst+clmblkdst8
	sub	al, 1			;; Test middle loop counter
	jnz	b5a			;; Iterate if necessary
	bump	rsi, -2*clmblkdst8+8*clmblkdst8;; Next source pointer
	bump	rdi, 2*XMM_SCD		;; Next sine/cosine pointer
	sub	eax, 256		;; Test outer loop counter
	mov	al, 2			;; Load middle loop counter
	jnz	b5a			;; Iterate if necessary
	bump	rsi, -3*8*clmblkdst8	;; Restore source pointer
	end_timer 24

;; Do FFT levels 7,8
;;
;; On input the 64-byte cache lines hold these data values:
;;	0K	32K	64K	96K
;;	2K	...
;;	...
;;	30K	...
;;	128K	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0K	8K	16K	24K
;;	2K	...
;;	...
;;	6K	...
;;	32K	...
;;	...

;; Do 192 four_complex_fft macros
;;	distance between fft data elements is 8K
;;	do 48 macros each processing 32 data values

	start_timer 25
	mov	rdi, sincos2		;; Load sin/cos pointer
	mov	eax, 12*256+2		;; 12 iterations of 2 iters of 2*2*clm
b7a:	xtouch	[rcx+(clm-1)*128]	;; Preload TLB
b7b:	x4cl_four_complex_fft rsi, 64, 4*clmblkdst, clmblkdst8, rdi
	xloop2	b7b, clmblkdst		;; 2*2*clm iterations
	sub	al, 1			;; Test middle loop counter
	jnz	b7a			;; Iterate if necessary
	bump	rsi, -2*2*clmblkdst+2*clmblkdst8;; Next source pointer
	bump	rdi, 2*XMM_SCD		;; Next sine/cosine pointer
	sub	eax, 256		;; Test outer loop counter
	mov	al, 2			;; Load middle loop counter
	jnz	b7a			;; Iterate if necessary
	bump	rsi, -12*2*clmblkdst8	;; Restore source pointer
pfing	bump	rcx, -192*blkdst	;; Restore prefetch pointer
	end_timer 25

;; Do FFT levels 9,10
;;
;; On input the 64-byte cache lines hold these data values:
;;	0K	8K	16K	24K
;;	2K	...
;;	...
;;	6K	...
;;	32K	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0K	2K	4K	6K
;;	8K	...
;;	...

;; Do 192 four_complex_fft macros
;;	distance between fft data elements is 2K
;;	do 48 macros each processing 32 data values

	start_timer 26
	mov	rdi, sincos1		;; Load sin/cos pointer
	mov	eax, 24*256+2		;; 24 iterations of 2 iters of 2*clm
b9b:	g4cl_four_complex_fft rsi, 64, clmblkdst, 2*clmblkdst, rcx, 64, blkdst, 2*blkdst
	L2prefetch128 [rcx-64][rbp]
	L2prefetch128 [rcx-64+blkdst][rbp]
	bump	rbp, -64+2*blkdst
	add	al, 256/2		;; Test inner loop counter
	jnc	b9b			;; Iterate if necessary
	bump	rbp, 128-4*blkdst
	IF clm NE 1
	add	dl, 256/clm
	jnc	b9b
	ENDIF
	bump	rsi, -2*clm*64+4*clmblkdst;; Next source pointer
	bump	rcx, -2*clm*64+4*blkdst	;; Next dest pointer
	bump	rdi, 2*XMM_SCD		;; Next sine/cosine pointer
	sub	al, 1			;; Test outer loop counter
	jnz	b9b
	bump	rsi, -8*clmblkdst+clmblkdst8
	sub	ah, 1			;; Test outer loop counter
	mov	al, 2
	jnz	b9b			;; Iterate if necessary
	end_timer 26

;; Work on next 4 sets of 768 values

endlp:	xpass1_get_next_block pass2, c0b, b0b ;; Get next block and jump
	done:	xfft_footer
	ENDM


;; Perform a 1792K-element FFT.  This is done in two passes.  Pass 1
;; does 10 levels, pass 2 uses common code to perform the last 11 levels.

hg_pass1sclevels10pfa7 MACRO fftname, pass2_levels, clmarg
	LOCAL	b0b, b1a, b1b, b2b, b3a, b3b, b4b
	LOCAL	b5b, b6a, b6b, b7a, b7b, b8b, b9b
	LOCAL	c0b, c1b, c2a, c2b, c3b, c4b, c5b
	LOCAL	c6a, c6b, c7a, c7b, c8b, c9b
	LOCAL	pass1, pass2, endlp, done, yes_fft

	set_FFT_constants fftname, 0, 0, pass2_levels, clmarg
	xfft_header pass1

;; Do the last 11 FFT levels
;;	do 1 pass2_11_levels_real and 223 pass2_11_levels_complex

pass2:	start_timer 0
	call	xpass2name
	end_timer 0
	cmp	ffttype, 1		;; We're done if FFTing only
	je	done

;; Do the inverse FFT

	pass1_inverse_fft_setup c0b

;; Do inverse FFT levels 9,10
;; On input the 64-byte cache lines hold these data values:
;;	0K	2K	4K	6K
;;	8K	10K	12K	14K
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0K	4K	8K	12K
;;	2K	...
;;	16K	...
;;	...

;; Do 1 eight_reals_unfft_2 macros
;; Do 1 nop_two_two_complex_unfft_2 macros
;;	distance between fft data elements is 2K
;;	do 1 macros each processing 16 data values

c0b:	start_timer 16
	mov	rsi, DATA_ADDR		;; Get source address
	mov	rcx, scratch_area	;; Get address of scratch area
	sub	rax, rax
	sub	rdx, rdx
pfing	mov	rbp, DATA_PREFETCH	;; Set prefetch distance
pfing	sub	rbp, rsi
c8b:	g2cl_eight_reals_unfft_2 rsi, 64, blkdst, rcx, 64, clmblkdst
	add	al, 256/2/clm		;; Test loop counter
	jnc	c8b			;; Iterate if necessary

;; Do 222 four_complex_unfft macros
;;	distance between fft data elements is 2K
;;	do 111 macros each processing 16 data values

	bump	rsi, -2*clm*64+2*blkdst	;; Next source pointer
	bump	rcx, -2*clm*64+2*clmblkdst;; Next dest pointer
	mov	rdi, sincos1		;; Load sin/cos pointer
	mov	al, 256/4+28		;; 111 iterations of 2*clm
c9b:	g2cl_four_complex_unfft rsi, 64, blkdst, rcx, 64, clmblkdst
	add	dl, 256/2/clm		;; Test loop counter
	jnc	c9b			;; Iterate if necessary
	bump	rsi, -2*clm*64+2*blkdst	;; Next source pointer
	bump	rcx, -2*clm*64+2*clmblkdst;; Next dest pointer
	bump	rdi, XMM_SCD		;; Next sine/cosine pointer
	add	al, 256/4		;; Test loop counter
	jnc	c9b
	bump	rcx, -8*clmblkdst+clmblkdst8
	sub	al, 1			;; Test outer loop counter
	jnz	c9b			;; Iterate if necessary
	lea	rcx, [rsi-112*2*blkdst]	;; Create prefetch pointer
	mov	rsi, scratch_area	;; Restore source pointer
	end_timer 16

;; Do inverse FFT levels 7,8
;; On input the 64-byte cache lines hold these data values:
;;	0K	4K	8K	12K
;;	2K	...
;;	16K	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0K	16K	32K	48K
;;	2K	...
;;	...
;;	14K
;;	64K	...
;;	...

;; Do 4 eight_reals_unfft_2 macros
;; Do 4 nop_two_two_complex_unfft_2 macros
;;	distance between fft data elements is 8K
;;	do 2 macros each processing 32 data values

	start_timer 17
 	mov	al, 2			;; 2 iterations of 2*clm
c6a:	xtouch	[rcx+(clm-1)*128]	;; Preload TLB
c6b:	x4cl_eight_reals_unfft_2 rsi, 64, 2*clmblkdst, 4*clmblkdst
	xloop1	c6b, clmblkdst
	sub	al, 1			;; Test outer loop counter
	jnz	c6a			;; Iterate if necessary

;; Do 216 four_complex_unfft macros
;;	distance between fft data elements is 8K
;;	do 54 macros each processing 32 data values

	bump	rsi, -2*clmblkdst+clmblkdst8;; Next source pointer
	mov	rdi, sincos2		;; Load sin/cos pointer
	mov	eax, 27*256+2		;; 27 iterations of 2 iters of 2*clm
c7a:	xtouch	[rcx+(clm-1)*128]	;; Preload TLB
c7b:	x4cl_four_complex_unfft rsi, 64, 2*clmblkdst, 4*clmblkdst, rdi
	xloop1	c7b, clmblkdst
	sub	al, 1			;; Test middle loop counter
	jnz	c7a			;; Iterate if necessary
	bump	rsi, -2*clmblkdst+clmblkdst8;; Next source pointer
	bump	rdi, XMM_SCD		;; Next sine/cosine pointer
	sub	eax, 256		;; Test outer loop counter
	mov	al, 2			;; Load middle loop counter
	jnz	c7a			;; Iterate if necessary
	bump	rsi, -28*clmblkdst8	;; Restore source pointer
	end_timer 17

;; Do inverse FFT levels 6
;; On input the 64-byte cache lines hold these data values:
;;	0K	16K	32K	48K
;;	2K	...
;;	...
;;	14K
;;	64K	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0K	32K	64K	96K
;;	2K	...
;;	...
;;	30K
;;	128K	...
;;	...

;; Do 16 eight_reals_unfft_1 macros
;;	distance between fft data elements is 32K
;;	do 8 macros each processing 16 data values
;;
;; Do 208 four_complex_unfft macros
;;	distance between fft data elements is 32K
;;	do 104 macros each processing 16 data values

;; Do inverse FFT levels 4,5
;; On input the 64-byte cache lines hold these data values:
;;	0K	32K	64K	96K
;;	2K	...
;;	...
;;	30K
;;	128K	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0K	64K	128K	192K
;;	2K	...
;;	...
;;	62K
;;	256K	384K	512K	640K
;;	...
;;
;; Do 32 eight_reals_unfft_2 macros
;;	distance between fft data elements is 64K
;;	do 16 macros each processing 16 data values
;;
;; Do 192 four_complex_unfft macros
;;	distance between fft data elements is 64K
;;	do 48 macros each processing 32 data values
;;
;; Do inverse FFT levels 1,2,3
;; On input the 64-byte cache lines hold these data values:
;;	0K	64K	128K	192K
;;	2K	...
;;	...
;;	62K
;;	256K	384K	512K	640K
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0K	64K	896K	960K
;;	2K	...
;;	...
;;	62K
;;	128K	256K	1024K	1152K
;;	130K
;;	...
;;	254K
;;	384K	640K	1280K	1536K
;;	386K
;;	...
;;	638K
;;
;; Do 256 seven_reals_last_unfft macros
;;	distance between fft data elements is 256K
;;	do 32 macros each processing 56 data values

	start_timer 18
	mov	rdi, sincos3		;; Load sin/cos pointer
	mov	rbx, sincos4		;; Load sin/cos pointer
	mov	al, 8			;; 8 iterations
	xseven_reals_unfft_preload

c2a:	xtouchlines 14			;; Preload TLBs

c4b:	x2cl_eight_reals_unfft_1 rsi, 64, clmblkdst8
	add	al, 256/2		;; Test loop counter
	jnc	c4b			;; Iterate if necessary

	bump	rsi, -2*64+2*clmblkdst8	;; Next source pointer
	mov	ah, 13			;; 13 iterations
c5b:	x2cl_two_complex_unfft rsi, 64, clmblkdst8
	add	al, 256/2		;; Test loop counter
	jnc	c5b			;; Iterate if necessary
	bump	rsi, -2*64+2*clmblkdst8	;; Next source pointer
	bump	rdi, XMM_SCD		;; Next sine/cosine pointer
	sub	ah, 1			;; Test outer loop counter
	jnz	c5b			;; Iterate if necessary
	bump	rdi, -13*XMM_SCD	;; Restore sine/cosine pointer

	bump	rsi, -14*2*clmblkdst8	;; Restore source pointer
c2b:	x2cl_half_eight_reals_unfft_2 rsi, 64, 2*clmblkdst8
	add	al, 256/2		;; Test loop counter
	jnc	c2b			;; Iterate if necessary
	bump	rsi, -2*64+clmblkdst8
	add	ah, 256/2		;; Test loop counter
	jnc	c2b			;; Iterate if necessary

	bump	rsi, -2*clmblkdst8+4*clmblkdst8;; Next source pointer
	mov	ah, 3			;; 3 iterations
c3b:	x4cl_four_complex_unfft rsi, 64, 2*clmblkdst8, 4*clmblkdst8, rbx
	xloop1sp c3b, clmblkdst8
	add	edx, 80000000h		;; Test loop counter
	jnc	c3b			;; Iterate if necessary
	bump	rsi, -2*clmblkdst8+8*clmblkdst8
	bump	rbx, XMM_SCD		;; Next sine/cosine pointer
	sub	ah, 1			;; Test outer loop counter
	jnz	c3b			;; Iterate if necessary
	bump	rbx, -3*XMM_SCD		;; Restore sine/cosine pointer

	bump	rsi, -3*8*clmblkdst8-4*clmblkdst8;; Restore source pointer
c1b:	x7cl_seven_reals_last_unfft rsi, 64, 4*clmblkdst8
	xloop1prefetch2sp c1b, clmblkdst8
	add	ah, 256/4
	jnc	c1b

	IF clm GT 1
	bump	rsi, -4*clmblkdst8+2*64	;; Next source pointer
	bump	rcx, -14*blkdst+128	;; Next prefetch pointer
	add	dl, 256/clm		;; clm iterations
	jnc	c4b
	bump	rsi, -clm*2*64+clmblkdst;; Next source pointer
	bump	rcx, -clm*128+14*blkdst	;; Next prefetch pointer
	ELSE
	bump	rsi, -4*clmblkdst8+clmblkdst;; Next source pointer
	ENDIF

	sub	al, 1			;; Test loop counter
	jnz	c2a			;; Iterate if necessary
	end_timer 18

;; Normalize these values

	pass1_normalize 1, yes_fft

;; Copy data from the scratch area back to the FFT data area

	copy_scratch_data_to_fft 224
	jmp	endlp			;; Skip FFT code if flag not set

;; Fire up auxiliary threads to do pass 1 forward FFT

pass1:	pass1_forward_fft_setup b0b

;; Copy data from the FFT data area back to the scratch area

b0b:	copy_fft_data_to_scratch 224

;; Do FFT levels 1,2,3
;;
;; On input the 64-byte cache lines hold these data values:
;;	0K	64K	896K	960K
;;	2K	...
;;	...
;;	62K
;;	128K	256K	1024K	1152K
;;	130K
;;	...
;;	254K
;;	384K	640K	1280K	1536K
;;	386K
;;	...
;;	638K
;; On output the 64-byte cache lines hold these data values:
;;	0K	64K	128K	192K
;;	2K	...
;;	...
;;	62K
;;	256K	384K	512K	640K
;;	258K
;;	...
;;	382K
;;	768K	1024K	1280K	1536K
;;	770K
;;	...
;;	1022K
;;
;; Do 256 seven_reals_first_fft macros (FFT levels 1,2,3)
;;	distance between fft data elements is 256K
;;	do 32 iterations each processing 56 data values

;; Do FFT levels 4,5
;; Values 0K-255K is real data, 256K-511K is semi-real data, 512K-1791K is
;; complex data.
;;
;; On input the 64-byte cache lines hold these data values:
;;	0K	64K	128K	192K
;;	2K	...
;;	...
;;	62K
;;	256K	384K	512K	640K
;;	258K
;;	...
;;	382K
;;	768K	1024K	1280K	1536K
;;	770K
;;	...
;;	1022K
;; On output the 64-byte cache lines hold these data values:
;;	0K	64K	128K	192K
;;	2K	...
;;	...
;;	62K	...
;;	256K	...
;;	...
;;
;; Do 32 eight_reals_fft_2 macros
;;	distance between fft data elements is 64K
;;	do 32 macros each processing 8 data values
;;
;; Do 192 four_complex_fft macros
;;	distance between fft data elements is 64K
;;	do 48 macros each processing 32 data values

;; Do FFT levels 6
;; Values 0K-63K is real data, 64K-127K is semi-real data, 128K-1791K is
;; complex data.
;;
;; On input the 64-byte cache lines hold these data values:
;;	0K	64K	128K	192K
;;	2K	...
;;	...
;;	62K	...
;;	256K	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0K	32K	64K	96K
;;	2K	...
;;	...
;;	30K	...
;;	128K	...
;;	...
;;
;; Do 16 eight_reals_fft_1 macros
;; Do 16 two_two_complex_fft macros
;;	distance between fft data elements is 32K
;;	do 16 macros each processing 16 data values
;;
;; Do 192 two_two_complex_fft macros
;;	distance between fft data elements is 32K
;;	do 96 macros each processing 16 data values

yes_fft:start_timer 22
	mov	rsi, scratch_area	;; Get address of scratch area
	mov	rcx, DATA_ADDR		;; Load source address for prefetching
pfing	mov	rbp, DATA_PREFETCH	;; Set prefetch distance
pfing	sub	rbp, rcx
pfing	bump	rcx, 72*blkdst		;; Create prefetching pointer
	mov	rdi, sincos4		;; Load sin/cos pointer
	mov	rbx, sincos3		;; Load sin/cos pointer
	mov	eax, 2			;; 16 iterations
	sub	rdx, rdx
	xseven_reals_fft_preload

b1a:	xtouchlines 6			;; Preload TLBs

b1b:	x7cl_seven_reals_fft rsi, 64, 4*clmblkdst8
	xloop1prefetch2sp b1b, 2*clmblkdst8
	add	ah, 256/2
	jnc	b1b

	bump	rsi, -2*2*clmblkdst8	;; Restore source pointer
b2b:	x1cl_half_eight_reals_fft_2 rsi, 64
	add	al, 256/2		;; Test inner loop counter
	jnc	b2b			;; Iterate if necessary
	bump	rsi, -2*64+2*clmblkdst8	;; Next source pointer
	add	ah, 256/2
	jnc	b2b

;;	bump	rsi, -2*2*clmblkdst8+4*clmblkdst8;; Next source pointer
b3a:	x2cl_four_complex_fft rsi, 64, 4*clmblkdst8
	add	al, 256/2		;; Test inner loop counter
	jnc	b3a			;; Iterate if necessary
	bump	rsi, -2*64+2*clmblkdst8	;; Next source pointer
	add	ah, 256/2
	jnc	b3a

	bump	rsi, -2*2*clmblkdst8+8*clmblkdst8;; Next source pointer
	bump	rdi, XMM_SCD		;; Next sine/cosine pointer
b3b:	x4cl_four_complex_fft rsi, 64, 4*clmblkdst8, 8*clmblkdst8, rdi
	xloop1sp b3b, 2*clmblkdst8
	add	ah, 256/2
	jnc	b3b
	bump	rdi, -XMM_SCD		;; Restore sine/cosine pointer

	bump	rsi, -2*2*clmblkdst8-12*clmblkdst8;; Restore src pointer
b4b:	x2cl_eight_reals_fft_1 rsi, 64, 2*clmblkdst8
	add	al, 256/2		;; Test inner loop counter
	jnc	b4b			;; Iterate if necessary

	bump	rsi, -2*64+4*clmblkdst8	;; Next source pointer
	bump	rbx, XMM_SCD		;; Next sine/cosine pointer
	mov	ah, 6			;; 6 iterations
b5b:	x2cl_two_complex_fft rsi, 64, 2*clmblkdst8, rbx
	add	al, 256/2		;; Test inner loop counter
	jnc	b5b			;; Iterate if necessary
	bump	rsi, -2*64+4*clmblkdst8	;; Next source pointer
	bump	rbx, 2*XMM_SCD		;; Next sine/cosine pointer
	sub	ah, 1			;; Test outer loop counter
	jnz	b5b			;; Iterate if necessary
	bump	rbx, -13*XMM_SCD	;; Restore sine/cosine pointer

	IF clm GT 1
	bump	rsi, -7*4*clmblkdst8+2*64;; Next source pointer
	bump	rcx, -6*blkdst+128	;; Next prefetch pointer
	add	dl, 256/clm		;; clm iterations
	jnc	b1b
	bump	rsi, -clm*2*64+clmblkdst;; Next source pointer
	bump	rcx, -clm*128+6*blkdst	;; Next prefetch pointer
	ELSE
	bump	rsi, -7*4*clmblkdst8+clmblkdst;; Next source pointer
	ENDIF

	add	eax, 80000000h/8*2	;; Test loop counter
	jnc	b1a			;; Iterate if necessary
	bump	rsi, -8*clmblkdst+clmblkdst8;; Next source pointer
	sub	al, 1			;; Test loop counter
	jnz	b1a			;; Iterate if necessary
	bump	rsi, -2*clmblkdst8	;; Restore source pointer
	end_timer 22

;; Do FFT levels 7,8
;; Values 0K-31K is real data, 32K-63K is semi-real data, 64K-1791K is
;; complex data.
;;
;; On input the 64-byte cache lines hold these data values:
;;	0K	32K	64K	96K
;;	2K	...
;;	...
;;	30K	...
;;	128K	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0K	8K	16K	24K
;;	2K	...
;;	...
;;	6K	...
;;	32K	...
;;	...

;; Do 8 eight_reals_fft_2 macros
;; Do 8 two_two_complex_fft macros
;;	distance between fft data elements is 8K
;;	do 4 macros each processing 32 data values

	start_timer 25
	mov	rdi, sincos2		;; Load sin/cos pointer
	mov	al, 4			;; 4 iterations of 2*clm
b6a:	xtouch	[rcx+(clm-1)*128]	;; Preload TLB
b6b:	x4cl_eight_reals_fft_2 rsi, 64, 4*clmblkdst, clmblkdst8, rdi
	xloop1	b6b, clmblkdst
	sub	al, 1			;; Test outer loop counter
	jnz	b6a			;; Iterate if necessary

;; Do 208 four_complex_fft macros
;;	distance between fft data elements is 8K
;;	do 52 macros each processing 32 data values

	bump	rsi, -4*clmblkdst+2*clmblkdst8;; Load source pointer
	bump	rdi, XMM_SCD		;; Next sine/cosine pointer
	mov	eax, 13*256+4		;; 13 iterations of 4 iters of 2*clm
b7a:	xtouch	[rcx+(clm-1)*128]	;; Preload TLB
b7b:	x4cl_four_complex_fft rsi, 64, 4*clmblkdst, clmblkdst8, rdi
	xloop1	b7b, clmblkdst
	sub	al, 1			;; Test middle loop counter
	jnz	b7a			;; Iterate if necessary
	bump	rsi, -4*clmblkdst+2*clmblkdst8;; Next source pointer
	bump	rdi, 2*XMM_SCD		;; Next sine/cosine pointer
	sub	eax, 256		;; Test outer loop counter
	mov	al, 4			;; Load middle loop counter
	jnz	b7a			;; Iterate if necessary
	bump	rsi, -14*2*clmblkdst8	;; Restore source pointer
pfing	bump	rcx, -224*blkdst	;; Restore prefetch pointer
	end_timer 25

;; Do FFT levels 9,10
;; Values 0K-7K is real data, 8K-15K is semi-real data, 16K-1791K is
;; complex data.
;;
;; On input the 64-byte cache lines hold these data values:
;;	0K	8K	16K	24K
;;	2K	...
;;	...
;;	6K	...
;;	32K	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0K	2K	4K	6K
;;	8K	...
;;	...

;; Do 1 eight_reals_fft_2 macros
;; Do 1 nop_two_two_complex_fft_2 macros
;; Do 2 four_complex_fft macros
;;	distance between fft data elements is 2K
;;	do 1 macros each processing 32 data values

	start_timer 26
	mov	rdi, sincos1		;; Load sin/cos pointer
b8b:	g4cl_eight_reals_fft_2 rsi, 64, clmblkdst, 2*clmblkdst, rcx, 64, blkdst, 2*blkdst
	add	al, 256/2/clm		;; Test inner loop counter
	jnc	b8b			;; Iterate if necessary

;; Do 220 four_complex_fft macros
;;	distance between fft data elements is 2K
;;	do 55 macros each processing 32 data values

	bump	rsi, -2*clm*64+4*clmblkdst;; Next source pointer
	bump	rcx, -2*clm*64+4*blkdst	;; Next dest pointer
	bump	rdi, XMM_SCD		;; Next sine/cosine pointer
	mov	eax, 28*256+1		;; 55 iterations of 2*clm
b9b:	g4cl_four_complex_fft rsi, 64, clmblkdst, 2*clmblkdst, rcx, 64, blkdst, 2*blkdst
	L2prefetch128 [rcx-64][rbp]
	L2prefetch128 [rcx-64+blkdst][rbp]
	bump	rbp, -64+2*blkdst
	add	al, 256/2		;; Test inner loop counter
	jnc	b9b			;; Iterate if necessary
	bump	rbp, 128-4*blkdst
	IF clm NE 1
	add	dl, 256/clm
	jnc	b9b
	ENDIF
	bump	rsi, -2*clm*64+4*clmblkdst;; Next source pointer
	bump	rcx, -2*clm*64+4*blkdst	;; Next dest pointer
	bump	rdi, 2*XMM_SCD		;; Next sine/cosine pointer
	sub	al, 1
	jnz	b9b
	bump	rsi, -8*clmblkdst+clmblkdst8
	sub	ah, 1			;; Test outer loop counter
	mov	al, 2
	jnz	b9b			;; Iterate if necessary
	end_timer 26

;; Work on next 4 sets of 896 values

endlp:	xpass1_get_next_block pass2, c0b, b0b ;; Get next block and jump
	done:	xfft_footer
	ENDM


;; Perform a 2M-element FFT.  This is done in two passes.  Pass 1
;; does 10 levels, pass 2 uses common code to perform the last 11 levels.

hg_pass1sclevels10pfa8 MACRO fftname, pass2_levels, clmarg
	LOCAL	b0b, b1a, b1b, b2b, b3b, b4b, b5b, b6a, b6b, b7a, b7b, b8b, b9b
	LOCAL	c0b, c1b, c2a, c2b, c3b, c4b, c5b, c6a, c6b, c7a, c7b, c8b, c9b
	LOCAL	pass1, pass2, endlp, done, yes_fft

	set_FFT_constants fftname, 0, 0, pass2_levels, clmarg
	xfft_header pass1

;; Do the last 11 FFT levels
;;	do 1 pass2_11_levels_real and 255 pass2_11_levels_complex

pass2:	start_timer 0
	call	xpass2name
	end_timer 0
	cmp	ffttype, 1		;; We're done if FFTing only
	je	done

;; Do the inverse FFT

	pass1_inverse_fft_setup c0b

;; Do inverse FFT levels 9,10
;; On input the 64-byte cache lines hold these data values:
;;	0K	2K	4K	6K
;;	8K	10K	12K	14K
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0K	4K	8K	12K
;;	2K	...
;;	16K	...
;;	...

;; Do 1 eight_reals_unfft_2 macros
;; Do 1 nop_two_two_complex_unfft_2 macros
;;	distance between fft data elements is 2K
;;	do 1 macros each processing 16 data values

c0b:	start_timer 16
	mov	rsi, DATA_ADDR		;; Get source address
	mov	rcx, scratch_area	;; Get address of scratch area
	sub	rax, rax
	sub	rdx, rdx
pfing	mov	rbp, DATA_PREFETCH	;; Set prefetch distance
pfing	sub	rbp, rsi
c8b:	g2cl_eight_reals_unfft_2 rsi, 64, blkdst, rcx, 64, clmblkdst
	add	al, 256/2/clm		;; Test loop counter
	jnc	c8b			;; Iterate if necessary

;; Do 254 four_complex_unfft macros
;;	distance between fft data elements is 2K
;;	do 127 macros each processing 16 data values

	bump	rsi, -2*clm*64+2*blkdst	;; Next source pointer
	bump	rcx, -2*clm*64+2*clmblkdst;; Next dest pointer
	mov	rdi, sincos1		;; Load sin/cos pointer
	mov	al, 256/4+32		;; 127 iterations of 2*clm
c9b:	g2cl_four_complex_unfft rsi, 64, blkdst, rcx, 64, clmblkdst
	add	dl, 256/2/clm		;; Test loop counter
	jnc	c9b			;; Iterate if necessary
	bump	rsi, -2*clm*64+2*blkdst	;; Next source pointer
	bump	rcx, -2*clm*64+2*clmblkdst;; Next dest pointer
	bump	rdi, XMM_SCD		;; Next sine/cosine pointer
	add	al, 256/4
	jnc	c9b
	bump	rcx, -8*clmblkdst+clmblkdst8
	sub	al, 1			;; Test outer loop counter
	jnz	c9b			;; Iterate if necessary
	lea	rcx, [rsi-128*2*blkdst]	;; Create prefetch pointer
	mov	rsi, scratch_area	;; Restore source pointer
	end_timer 16

;; Do inverse FFT levels 7,8
;; On input the 64-byte cache lines hold these data values:
;;	0K	4K	8K	12K
;;	2K	...
;;	16K	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0K	16K	32K	48K
;;	2K	...
;;	...
;;	14K
;;	64K	...
;;	...

;; Do 4 eight_reals_unfft_2 macros
;; Do 4 nop_two_two_complex_unfft_2 macros
;;	distance between fft data elements is 8K
;;	do 2 macros each processing 32 data values

	start_timer 17
	mov	al, 2			;; 2 iterations of 2*clm
c6a:	xtouch	[rcx+(clm-1)*128]	;; Preload TLB
c6b:	x4cl_eight_reals_unfft_2 rsi, 64, 2*clmblkdst, 4*clmblkdst
	xloop1	c6b, clmblkdst
	sub	al, 1			;; Test outer loop counter
	jnz	c6a			;; Iterate if necessary

;; Do 248 four_complex_unfft macros
;;	distance between fft data elements is 8K
;;	do 62 macros each processing 32 data values

	bump	rsi, -2*clmblkdst+clmblkdst8;; Next source pointer
	mov	rdi, sincos2		;; Load sin/cos pointer
	mov	eax, 31*256+2		;; 31 iterations of 2 iters of 2*clm
c7a:	xtouch	[rcx+(clm-1)*128]	;; Preload TLB
c7b:	x4cl_four_complex_unfft rsi, 64, 2*clmblkdst, 4*clmblkdst, rdi
	xloop1	c7b, clmblkdst
	sub	al, 1			;; Test middle loop counter
	jnz	c7a			;; Iterate if necessary
	bump	rsi, -2*clmblkdst+clmblkdst8;; Next source pointer
	bump	rdi, XMM_SCD		;; Next sine/cosine pointer
	sub	eax, 256		;; Test outer loop counter
	mov	al, 2			;; Load middle loop counter
	jnz	c7a			;; Iterate if necessary
	bump	rsi, -32*clmblkdst8	;; Restore source pointer
	end_timer 17

;; Do inverse FFT levels 6
;; On input the 64-byte cache lines hold these data values:
;;	0K	16K	32K	48K
;;	2K	...
;;	...
;;	14K
;;	64K	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0K	32K	64K	96K
;;	2K	...
;;	...
;;	30K
;;	128K	...
;;	...
;;
;; Do 16 eight_reals_unfft_1 macros
;;	distance between fft data elements is 32K
;;	do 8 macros each processing 16 data values
;;
;; Do 240 four_complex_unfft macros
;;	distance between fft data elements is 32K
;;	do 120 macros each processing 16 data values

;; Do inverse FFT levels 4,5
;; On input the 64-byte cache lines hold these data values:
;;	0K	32K	64K	96K
;;	2K	...
;;	...
;;	30K
;;	128K	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0K	128K	256K	384K
;;	2K	...
;;	...
;;	126K
;;	512K	...
;;	...
;;
;; Do 32 eight_reals_unfft_2 macros
;; Do 32 nop_two_two_complex_unfft_2 macros
;;	distance between fft data elements is 64K
;;	do 16 macros each processing 32 data values
;;
;; Do 192 four_complex_unfft macros
;;	distance between fft data elements is 64K
;;	do 48 macros each processing 32 data values

;; Do inverse FFT levels 1,2,3
;; On input the 64-byte cache lines hold these data values:
;;	0K	128K	256K	384K
;;	2K	...
;;	...
;;	126K
;;	512K	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0K	512K	1024K	1536K
;;	2K	...
;;	...
;;	510K
;;
;; Do 256 eight_reals_last_unfft macros
;;	distance between fft data elements is 256K
;;	do 64 macros each processing 32 data values

	start_timer 18
	mov	rdi, sincos3		;; Load sin/cos pointer
	mov	rbx, sincos4		;; Load sin/cos pointer
	mov	al, 8			;; 8 iterations

c2a:	xtouchlines 16			;; Preload TLBs

c4b:	x2cl_eight_reals_unfft_1 rsi, 64, clmblkdst8
	add	al, 256/2		;; Test loop counter
	jnc	c4b			;; Iterate if necessary

	bump	rsi, -2*64+2*clmblkdst8	;; Next source pointer
	mov	ah, 15			;; 15 iterations
c5b:	x2cl_two_complex_unfft rsi, 64, clmblkdst8
	add	al, 256/2		;; Test loop counter
	jnc	c5b			;; Iterate if necessary
	bump	rsi, -2*64+2*clmblkdst8	;; Next source pointer
	bump	rdi, XMM_SCD		;; Next sine/cosine pointer
	sub	ah, 1			;; Test outer loop counter
	jnz	c5b			;; Iterate if necessary
	bump	rdi, -15*XMM_SCD	;; Restore sine/cosine pointer

	bump	rsi, -16*2*clmblkdst8	;; Restore source pointer
c2b:	x4cl_eight_reals_unfft_2 rsi, 64, 2*clmblkdst8, 4*clmblkdst8
	xloop1sp c2b, clmblkdst8
	add	ah, 256/2		;; Test loop counter
	jnc	c2b			;; Iterate if necessary

	bump	rsi, -2*clmblkdst8+8*clmblkdst8;; Next source pointer
	mov	ah, 3			;; 3 iterations
c3b:	x4cl_four_complex_unfft rsi, 64, 2*clmblkdst8, 4*clmblkdst8, rbx
	xloop1sp c3b, clmblkdst8
	add	edx, 80000000h		;; Test loop counter
	jnc	c3b			;; Iterate if necessary
	bump	rsi, -2*clmblkdst8+8*clmblkdst8
	bump	rbx, XMM_SCD		;; Next sine/cosine pointer
	sub	ah, 1
	jnz	c3b
	bump	rbx, -3*XMM_SCD		;; Next sine/cosine pointer

	bump	rsi, -4*8*clmblkdst8	;; Restore source pointer
c1b:	x4cl_eight_reals_last_unfft rsi, 64, 8*clmblkdst8, 16*clmblkdst8
	xloop1sp c1b, clmblkdst8
	add	ah, 256/8
	jnc	c1b

	IF clm GT 1
	bump	rsi, -8*clmblkdst8+2*64	;; Next source pointer
	bump	rcx, -16*blkdst+128	;; Next prefetch pointer
	add	dl, 256/clm		;; clm iterations
	jnc	c4b
	bump	rsi, -clm*2*64+clmblkdst;; Next source pointer
	bump	rcx, -clm*128+16*blkdst	;; Next prefetch pointer
	ELSE
	bump	rsi, -8*clmblkdst8+clmblkdst;; Next source pointer
	ENDIF

	sub	al, 1			;; Test loop counter
	jnz	c2a			;; Iterate if necessary
	end_timer 18

;; Normalize these values

	pass1_normalize 1, yes_fft

;; Copy data from the scratch area back to the FFT data area

	copy_scratch_data_to_fft 256
	jmp	endlp			;; Skip FFT code if flag not set

;; Fire up auxiliary threads to do pass 1 forward FFT

pass1:	pass1_forward_fft_setup b0b

;; Copy data from the FFT data area back to the scratch area

b0b:	copy_fft_data_to_scratch 256

;; Do FFT levels 1,2,3
;;
;; On input the 64-byte cache lines hold these data values:
;;	0K	512K	1024K	1536K
;;	2K	...
;;	...
;;	510K
;; On output the 64-byte cache lines hold these data values:
;;	0K	256K	512K	768K
;;	2K	...
;;	...
;;	254K	...
;;	1024K	...
;;	...
;;
;; Do 256 eight_reals_first_fft macros (FFT levels 1,2,3)
;;	distance between fft data elements is 256K
;;	do 64 iterations each processing 16 data values

;; Do FFT levels 4,5
;; Values 0K-255K is real data, 256K-511K is semi-real data, 512K-2047K is
;; complex data.
;;
;; On input the 64-byte cache lines hold these data values:
;;	0K	256K	512K	768K
;;	2K	...
;;	...
;;	254K	...
;;	1024K	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0K	64K	128K	192K
;;	2K	...
;;	...
;;	62K	...
;;	256K	...
;;	...
;;
;; Do 32 eight_reals_fft_2 macros
;; Do 32 nop_two_two_complex_fft_2 macros
;; Do 64 four_complex_fft macros
;;	distance between fft data elements is 64K
;;	do 32 macros each processing 32 data values
;;
;; Do 128 four_complex_fft macros
;;	distance between fft data elements is 64K
;;	do 32 macros each processing 32 data values

;; Do FFT levels 6
;; Values 0K-63K is real data, 64K-127K is semi-real data, 128K-2047K is
;; complex data.
;;
;; On input the 64-byte cache lines hold these data values:
;;	0K	64K	128K	192K
;;	2K	...
;;	...
;;	62K	...
;;	256K	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0K	32K	64K	96K
;;	2K	...
;;	...
;;	30K	...
;;	128K	...
;;	...
;;
;; Do 16 eight_reals_fft_1 macros
;; Do 16 two_two_complex_fft macros
;;	distance between fft data elements is 32K
;;	do 16 macros each processing 16 data values
;;
;; Do 224 two_two_complex_fft macros
;;	distance between fft data elements is 32K
;;	do 112 macros each processing 16 data values

yes_fft:start_timer 22
	mov	rsi, scratch_area	;; Get address of scratch area
	mov	rcx, DATA_ADDR		;; Load source address for prefetching
pfing	mov	rbp, DATA_PREFETCH	;; Set prefetch distance
pfing	sub	rbp, rcx
pfing	bump	rcx, 128*blkdst		;; Create prefetching pointer
	mov	rdi, sincos4		;; Load sin/cos pointer
	mov	rbx, sincos3		;; Load sin/cos pointer
	mov	eax, 2			;; 16 iterations
	sub	rdx, rdx

b1a:	xtouchlines 4			;; Preload TLBs

b1b:	x2cl_eight_reals_fft rsi, 64, 16*clmblkdst8
	add	al, 256/2		;; Test loop counter
	jnc	b1b			;; Iterate if necessary
	bump	rsi, -2*64+2*clmblkdst8	;; Next source pointer
	add	ah, 256/8
	jnc	b1b

	bump	rsi, -8*2*clmblkdst8	;; Restore source pointer
b2b:	x4cl_eight_reals_fft_2 rsi, 64, 4*clmblkdst8, 8*clmblkdst8, rdi
	xloop1sp b2b, 2*clmblkdst8
	add	ah, 256/2
	jnc	b2b

	bump	rsi, -2*2*clmblkdst8+16*clmblkdst8;; Next source pointer
	bump	rdi, XMM_SCD		;; Next sine/cosine pointer
b3b:	x4cl_four_complex_fft rsi, 64, 4*clmblkdst8, 8*clmblkdst8, rdi
	xloop1sp b3b, 2*clmblkdst8
	add	ah, 256/2
	jnc	b3b
	bump	rdi, -XMM_SCD		;; Restore sine/cosine pointer

	bump	rsi, -2*2*clmblkdst8-16*clmblkdst8;; Restore src pointer
b4b:	x2cl_eight_reals_fft_1 rsi, 64, 2*clmblkdst8
	add	al, 256/2		;; Test inner loop counter
	jnc	b4b			;; Iterate if necessary

	bump	rsi, -2*64+4*clmblkdst8	;; Load source pointer
	bump	rbx, XMM_SCD		;; Next sine/cosine pointer
	mov	ah, 7			;; 7 iterations
b5b:	x2cl_two_complex_fft rsi, 64, 2*clmblkdst8, rbx
	add	al, 256/2		;; Test inner loop counter
	jnc	b5b			;; Iterate if necessary
	bump	rsi, -2*64+4*clmblkdst8	;; Next source pointer
	bump	rbx, 2*XMM_SCD		;; Next sine/cosine pointer
	sub	ah, 1			;; Test outer loop counter
	jnz	b5b			;; Iterate if necessary
	bump	rbx, -15*XMM_SCD	;; Restore sine/cosine pointer

	IF clm GT 1
	bump	rsi, -8*4*clmblkdst8+2*64;; Next source pointer
	bump	rcx, -4*blkdst+128	;; Next prefetch pointer
	add	dl, 256/clm		;; clm iterations
	jnc	b1b
	bump	rsi, -clm*2*64+clmblkdst;; Next source pointer
	bump	rcx, -clm*128+4*blkdst	;; Next prefetch pointer
	ELSE
	bump	rsi, -8*4*clmblkdst8+clmblkdst;; Next source pointer
	ENDIF

	add	eax, 80000000h/8*2	;; Test loop counter
	jnc	b1a			;; Iterate if necessary
	bump	rsi, -8*clmblkdst+clmblkdst8;; Next source pointer
	sub	al, 1			;; Test loop counter
	jnz	b1a			;; Iterate if necessary
	bump	rsi, -2*clmblkdst8	;; Restore source pointer
	end_timer 22

;; Do FFT levels 7,8
;; Values 0K-31K is real data, 32K-63K is semi-real data, 64K-2047K is
;; complex data.
;;
;; On input the 64-byte cache lines hold these data values:
;;	0K	32K	64K	96K
;;	2K	...
;;	...
;;	30K	...
;;	128K	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0K	8K	16K	24K
;;	2K	...
;;	...
;;	6K	...
;;	32K	...
;;	...

;; Do 8 eight_reals_fft_2 macros
;; Do 8 two_two_complex_fft macros
;;	distance between fft data elements is 8K
;;	do 4 macros each processing 32 data values

	start_timer 25
	mov	rdi, sincos2		;; Load sin/cos pointer
	mov	al, 4			;; 4 iterations of 2*clm
b6a:	xtouch	[rcx+(clm-1)*128]	;; Preload TLB
b6b:	x4cl_eight_reals_fft_2 rsi, 64, 4*clmblkdst, clmblkdst8, rdi
	xloop1	b6b, clmblkdst
	sub	al, 1			;; Test outer loop counter
	jnz	b6a			;; Iterate if necessary

;; Do 240 four_complex_fft macros
;;	distance between fft data elements is 8K
;;	do 60 macros each processing 32 data values

	bump	rsi, -4*clmblkdst+2*clmblkdst8;; Load source pointer
	bump	rdi, XMM_SCD		;; Next sine/cosine pointer
	mov	eax, 15*256+4		;; 15 iterations of 4 iters of 2*clm
b7a:	xtouch	[rcx+(clm-1)*128]	;; Preload TLB
b7b:	x4cl_four_complex_fft rsi, 64, 4*clmblkdst, clmblkdst8, rdi
	xloop1	b7b, clmblkdst
	sub	al, 1			;; Test middle loop counter
	jnz	b7a			;; Iterate if necessary
	bump	rsi, -4*clmblkdst+2*clmblkdst8;; Next source pointer
	bump	rdi, 2*XMM_SCD		;; Next sine/cosine pointer
	sub	eax, 256		;; Test outer loop counter
	mov	al, 4			;; Load middle loop counter
	jnz	b7a			;; Iterate if necessary
	bump	rsi, -16*2*clmblkdst8	;; Restore source pointer
pfing	bump	rcx, -256*blkdst	;; Restore prefetch pointer
	end_timer 25

;; Do FFT levels 9,10
;; Values 0K-7K is real data, 8K-15K is semi-real data, 16K-2047K is
;; complex data.
;;
;; On input the 64-byte cache lines hold these data values:
;;	0K	8K	16K	24K
;;	2K	...
;;	...
;;	6K	...
;;	32K	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0K	2K	4K	6K
;;	8K	...
;;	...

;; Do 1 eight_reals_fft_2 macros
;; Do 1 nop_two_two_complex_fft_2 macros
;; Do 2 four_complex_fft macros
;;	distance between fft data elements is 2K
;;	do 1 macros each processing 32 data values

	start_timer 26
	mov	rdi, sincos1		;; Load sin/cos pointer
b8b:	g4cl_eight_reals_fft_2 rsi, 64, clmblkdst, 2*clmblkdst, rcx, 64, blkdst, 2*blkdst
	add	al, 256/2/clm		;; Test inner loop counter
	jnc	b8b			;; Iterate if necessary

;; Do 252 four_complex_fft macros
;;	distance between fft data elements is 2K
;;	do 63 macros each processing 32 data values

	bump	rsi, -2*clm*64+4*clmblkdst;; Next source pointer
	bump	rcx, -2*clm*64+4*blkdst	;; Next dest pointer
	bump	rdi, XMM_SCD		;; Next sine/cosine pointer
	mov	eax, 32*256+1		;; 63 iterations of 2*clm
b9b:	g4cl_four_complex_fft rsi, 64, clmblkdst, 2*clmblkdst, rcx, 64, blkdst, 2*blkdst
	L2prefetch128 [rcx-64][rbp]
	L2prefetch128 [rcx-64+blkdst][rbp]
	bump	rbp, -64+2*blkdst
	add	al, 256/2		;; Test inner loop counter
	jnc	b9b			;; Iterate if necessary
	bump	rbp, 128-4*blkdst
	IF clm NE 1
	add	dl, 256/clm
	jnc	b9b
	ENDIF
	bump	rsi, -2*clm*64+4*clmblkdst;; Next source pointer
	bump	rcx, -2*clm*64+4*blkdst	;; Next dest pointer
	bump	rdi, 2*XMM_SCD		;; Next sine/cosine pointer
	sub	al, 1
	jnz	b9b
	bump	rsi, -8*clmblkdst+clmblkdst8
	sub	ah, 1			;; Test outer loop counter
	mov	al, 2
	jnz	b9b			;; Iterate if necessary
	end_timer 26

;; Work on next 4 sets of 1024 values

endlp:	xpass1_get_next_block pass2, c0b, b0b ;; Get next block and jump
	done:	xfft_footer
	ENDM


;; Perform a 2M-element negacyclic FFT.  This is done in two passes.  Pass 1
;; does 10 levels, pass 2 uses common code to perform the last 11 levels.

hg_pass1sclevels10complex4 MACRO fftname, pass2_levels, clmarg
	LOCAL	b0b, b1a, b1b, b3a, b3b, b5a, b5b, b7a, b7b, b9b
	LOCAL	c0b, c1a, c1b, c3a, c3b, c5a, c5b, c7a, c7b, c9b
	LOCAL	pass1, pass2, endlp, done, yes_fft

	set_FFT_constants fftname, 0, 1, pass2_levels, clmarg
	xfft_header pass1

;; Do the last 11 FFT levels
;;	do 256 pass2_11_levels_complex

pass2:	start_timer 0
	call	xpass2name
	end_timer 0
	cmp	ffttype, 1		;; We're done if FFTing only
	je	done

;; Do the inverse FFT

	pass1_inverse_fft_setup c0b

;; Do inverse FFT levels 9,10
;; On input the 64-byte cache lines hold these data values:
;;	0K	2K	4K	6K
;;	8K	10K	12K	14K
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0K	4K	8K	12K
;;	2K	...
;;	16K	...
;;	...

;; Do 256 four_complex_unfft macros
;;	distance between fft data elements is 2K
;;	do 128 macros each processing 16 data values

c0b:	start_timer 16
	mov	rsi, DATA_ADDR		;; Get source address
	mov	rcx, scratch_area	;; Get address of scratch area
	mov	rdi, sincos1		;; Load sin/cos pointer
	mov	eax, 32			;; 128 iterations
	sub	rdx, rdx
pfing	mov	rbp, DATA_PREFETCH	;; Set prefetch distance
pfing	sub	rbp, rsi
c9b:	g2cl_four_complex_unfft rsi, 64, blkdst, rcx, 64, clmblkdst
	add	dl, 256/2/clm		;; Test loop counter
	jnc	c9b			;; Iterate if necessary
	bump	rsi, -2*clm*64+2*blkdst	;; Next source pointer
	bump	rcx, -2*clm*64+2*clmblkdst;; Next dest pointer
	bump	rdi, XMM_SCD		;; Next sine/cosine pointer
	add	al, 256/4
	jnc	c9b
	bump	rcx, -8*clmblkdst+clmblkdst8
	sub	al, 1			;; Test outer loop counter
	jnz	c9b			;; Iterate if necessary
	lea	rcx, [rsi-128*2*blkdst]	;; Create prefetch pointer
	mov	rsi, scratch_area	;; Restore source pointer
	end_timer 16

;; Do inverse FFT levels 7,8
;; On input the 64-byte cache lines hold these data values:
;;	0K	4K	8K	12K
;;	2K	...
;;	16K	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0K	16K	32K	48K
;;	2K	...
;;	...
;;	14K
;;	64K	...
;;	...

;; Do 256 four_complex_unfft macros
;;	distance between fft data elements is 8K
;;	do 64 macros each processing 32 data values

	start_timer 17
	mov	rdi, sincos2		;; Load sin/cos pointer
	mov	eax, 32			;; 32 iterations of 2*2*clm
c7a:	xtouch	[rcx+(clm-1)*128]	;; Preload TLB
c7b:	x4cl_four_complex_unfft rsi, 64, 2*clmblkdst, 4*clmblkdst, rdi
	xloop2	c7b, clmblkdst		;; 2 * 2*clm iterations
	bump	rsi, -2*clmblkdst+clmblkdst8;; Next source pointer
	bump	rdi, XMM_SCD		;; Next sine/cosine pointer
	sub	eax, 1			;; Test outer loop counter
	jnz	c7a			;; Iterate if necessary
	bump	rsi, -32*clmblkdst8	;; Restore source pointer
	end_timer 17

;; Do inverse FFT levels 5,6
;; On input the 64-byte cache lines hold these data values:
;;	0K	16K	32K	48K
;;	2K	...
;;	...
;;	14K
;;	64K	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0K	64K	128K	192K
;;	2K	...
;;	...
;;	62K
;;	256K	...
;;	...

;; Do 256 four_complex_unfft macros
;;	distance between fft data elements is 32K
;;	do 64 macros each processing 32 data values

	start_timer 18
	mov	rdi, sincos3		;; Load sin/cos pointer
	mov	eax, 8*256+4		;; 8 iterations of 4 iters of 2*2*clm
c5a:	xtouch	[rcx+(clm-1)*128]	;; Preload TLB
c5b:	x4cl_four_complex_unfft rsi, 64, clmblkdst8, 2*clmblkdst8, rdi
	xloop2	c5b, clmblkdst		;; 2 * 2*clm iterations
	sub	al, 1			;; Test middle loop counter
	jnz	c5a			;; Iterate if necessary
	bump	rsi, -4*2*clmblkdst+4*clmblkdst8;; Next source pointer
	bump	rdi, XMM_SCD		;; Next sine/cosine pointer
	sub	eax, 256		;; Test outer loop counter
	mov	al, 4			;; Load middle loop counter
	jnz	c5a			;; Iterate if necessary
	bump	rsi, -8*4*clmblkdst8	;; Restore source pointer
	end_timer 18

;; Do inverse FFT levels 4
;; On input the 64-byte cache lines hold these data values:
;;	0K	64K	128K	192K
;;	2K	...
;;	...
;;	62K
;;	256K	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0K	128K	256K	384K
;;	2K	...
;;	...
;;	126K
;;	512K	...
;;	...

;; Do 256 two_two_complex_unfft macros
;;	distance between fft data elements is 64K
;;	do 128 macros each processing 16 data values

	start_timer 19
	mov	rdi, sincos4		;; Load sin/cos pointer
	mov	eax, 4*256+4		;; 4 iters of 4 iters 2 of 4*2*clm
c3a:	xtouch	[rcx+(clm-1)*128]	;; Preload TLB
c3b:	x2cl_two_complex_unfft rsi, 64, 4*clmblkdst8
	xloop4	c3b, clmblkdst		;; 4* 2*clm iterations
	add	eax, 80000000h		;; 2 iterations
	jnc	c3a			;; Iterate if necessary
	bump	rsi, -2*4*clmblkdst+clmblkdst8;; Next source pointer
	sub	al, 1			;; Test middle loop counter
	jnz	c3a
	bump	rsi, -4*clmblkdst8+8*clmblkdst8;; Next source pointer
	bump	rdi, XMM_SCD		;; Next sine/cosine pointer
	sub	eax, 256		;; Test outer loop counter
	mov	al, 4			;; Load middle loop counter
	jnz	c3a			;; Iterate if necessary
	bump	rsi, -4*8*clmblkdst8	;; Restore source pointer
	end_timer 19

;; Do inverse FFT levels 1,2,3
;; On input the 64-byte cache lines hold these data values:
;;	0K	128K	256K	384K
;;	2K	...
;;	...
;;	126K
;;	512K	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0K	512K	1024K	1536K
;;	2K	...
;;	...
;;	510K

;; Do 256 four_complex_unfft macros
;;	distance between fft data elements is 256K
;;	do 64 macros each processing 32 data values

	start_timer 20
	mov	rdi, plus1_premults	;; Address of premultiplier table
	mov	eax, 8*256+4		;; 8 iters of 4 iters of 2*2*clm
c1a:	xtouch	[rcx+(clm-1)*128]	;; Preload TLB
c1b:	x4cl_four_complex_last_unfft rsi, 64, 8*clmblkdst8, 16*clmblkdst8, 64*128
	xloop2edi c1b, clmblkdst, 128	;; 2 * 2*clm iterations
	sub	al, 1			;; Test loop counter
	jnz	c1a			;; Iterate if necessaryxn
	bump	rsi, -4*2*clmblkdst+clmblkdst8;; Next source pointer
	sub	ah, 1			;; Test outer loop counter
	mov	al, 4			;; Load middle loop counter
	jnz	c1a
	end_timer 20

;; Normalize these values

	pass1_normalize 1, yes_fft

;; Copy data from the scratch area back to the FFT data area

	copy_scratch_data_to_fft 256
	jmp	endlp			;; Skip FFT code if flag not set

;; Fire up auxiliary threads to do pass 1 forward FFT

pass1:	pass1_forward_fft_setup b0b

;; Copy data from the FFT data area back to the scratch area

b0b:	copy_fft_data_to_scratch 256

;; Do FFT levels 1,2,3
;;
;; On input the 64-byte cache lines hold these data values:
;;	0K	512K	1024K	1536K
;;	2K	...
;;	...
;;	510K
;; On output the 64-byte cache lines hold these data values:
;;	0K	256K	512K	768K
;;	2K	...
;;	...
;;	254K	...
;;	1024K	...
;;	...

;; Do 256 four_complex_first_fft macros (FFT levels 1,2,3)
;;	distance between fft data elements is 256K
;;	do 128 iterations each processing 16 data values

yes_fft:start_timer 22
	mov	rsi, scratch_area	;; Get address of scratch area
	mov	rdi, plus1_premults	;; Address of premultiplier table
	mov	rcx, DATA_ADDR		;; Load source address for prefetching
pfing	mov	rbp, DATA_PREFETCH	;; Set prefetch distance
pfing	sub	rbp, rcx
pfing	bump	rcx, 128*blkdst		;; Create prefetching pointer
	mov	eax, 16			;; 16 iters of 2 iters of 4*2*clm
	sub	rdx, rdx
b1a:	xtouch	[rcx+(clm-1)*128]	;; Preload TLB
b1b:	x2cl_four_complex_first_fft_scratch rsi, 64, 16*clmblkdst8
	xloop4edi b1b, clmblkdst, 128	;; 4 * 2*clm iterations
	add	eax, 80000000h		;; 2 iterations
	jnc	b1a
	bump	rsi, -2*4*clmblkdst+clmblkdst8
	sub	al, 1			;; Test loop counter
	jnz	b1a			;; Iterate if necessary
	bump	rsi, -16*clmblkdst8	;; Restore source pointer
	end_timer 22

;; Do FFT level 4
;;
;; On input the 64-byte cache lines hold these data values:
;;	0K	256K	512K	768K
;;	2K	...
;;	...
;;	254K	...
;;	1024K	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0K	128K	256K	384K
;;	2K	...
;;	...
;;	126K	...
;;	512K	...
;;	...

;; Do 256 two_two_complex_fft macros
;;	distance between fft data elements is 64K
;;	do 128 macros each processing 16 data values

	start_timer 23
	mov	rdi, sincos4		;; Load sin/cos pointer
	mov	eax, 2*256+8		;; 2 iters of 8 iters of 2 of 4*2*clm
b3a:	xtouch	[rcx+(clm-1)*128]	;; Preload TLB
b3b:	x2cl_two_complex_fft rsi, 64, 8*clmblkdst8, rdi
	xloop4	b3b, clmblkdst		;; 4 * 2*clm iterations
	add	eax, 80000000h		;; 2 iterations
	jnc	b3a
	bump	rsi, -2*4*clmblkdst+clmblkdst8
	sub	al, 1			;; Test outer loop counter
	jnz	b3a			;; Iterate if necessary
	bump	rsi, -8*clmblkdst8+16*clmblkdst8;; Next source pointer
	bump	rdi, 2*XMM_SCD		;; Next sine/cosine pointer
	sub	ah, 1
	mov	al, 8
	jnz	b3a
	bump	rsi, -2*16*clmblkdst8	;; Restore source pointer
	end_timer 23

;; Do FFT levels 5,6
;;
;; On input the 64-byte cache lines hold these data values:
;;	0K	128K	256K	384K
;;	2K	...
;;	...
;;	126K	...
;;	512K	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0K	32K	64K	96K
;;	2K	...
;;	...
;;	30K	...
;;	128K	...
;;	...

;; Do 256 four_complex_fft macros
;;	distance between fft data elements is 32K
;;	do 64 macros each processing 32 data values

	start_timer 24
	mov	rdi, sincos3		;; Load sin/cos pointer
	mov	eax, 4*256+2		;; 4 iters of 2 iters of 4 of 2*2*clm
b5a:	xtouch	[rcx+(clm-1)*128]	;; Preload TLB
b5b:	x4cl_four_complex_fft rsi, 64, 2*clmblkdst8, 4*clmblkdst8, rdi
	xloop2	b5b, clmblkdst		;; 2 * 2*clm iterations
	add	eax, 80000000h/2	;; 4 iterations
	jnc	b5a
	bump	rsi, -4*2*clmblkdst+clmblkdst8
	sub	al, 1			;; Test middle loop counter
	jnz	b5a			;; Iterate if necessary
	bump	rsi, -2*clmblkdst8+8*clmblkdst8;; Next source pointer
	bump	rdi, 2*XMM_SCD		;; Next sine/cosine pointer
	sub	eax, 256		;; Test outer loop counter
	mov	al, 2			;; Load middle loop counter
	jnz	b5a			;; Iterate if necessary
	bump	rsi, -4*8*clmblkdst8	;; Restore source pointer
	end_timer 24

;; Do FFT levels 7,8
;;
;; On input the 64-byte cache lines hold these data values:
;;	0K	32K	64K	96K
;;	2K	...
;;	...
;;	30K	...
;;	128K	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0K	8K	16K	24K
;;	2K	...
;;	...
;;	6K	...
;;	32K	...
;;	...

;; Do 256 four_complex_fft macros
;;	distance between fft data elements is 8K
;;	do 64 macros each processing 32 data values

	start_timer 25
	mov	rdi, sincos2		;; Load sin/cos pointer
	mov	eax, 16*256+2		;; 16 iterations of 2 iters of 2*2*clm
b7a:	xtouch	[rcx+(clm-1)*128]	;; Preload TLB
b7b:	x4cl_four_complex_fft rsi, 64, 4*clmblkdst, clmblkdst8, rdi
	xloop2	b7b, clmblkdst		;; 2 * 2*clm iterations
	sub	al, 1			;; Test middle loop counter
	jnz	b7a			;; Iterate if necessary
	bump	rsi, -2*2*clmblkdst+2*clmblkdst8;; Next source pointer
	bump	rdi, 2*XMM_SCD		;; Next sine/cosine pointer
	sub	eax, 256		;; Test outer loop counter
	mov	al, 2			;; Load middle loop counter
	jnz	b7a			;; Iterate if necessary
	bump	rsi, -16*2*clmblkdst8	;; Restore source pointer
pfing	bump	rcx, -256*blkdst	;; Restore prefetch pointer
	end_timer 25

;; Do FFT levels 9,10
;;
;; On input the 64-byte cache lines hold these data values:
;;	0K	8K	16K	24K
;;	2K	...
;;	...
;;	6K	...
;;	32K	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0K	2K	4K	6K
;;	8K	...
;;	...

;; Do 256 four_complex_fft macros
;;	distance between fft data elements is 2K
;;	do 64 macros each processing 32 data values

	start_timer 26
	mov	rdi, sincos1		;; Load sin/cos pointer
	mov	eax, 32*256+2		;; 32 iters of 2 of 2*clm
b9b:	g4cl_four_complex_fft rsi, 64, clmblkdst, 2*clmblkdst, rcx, 64, blkdst, 2*blkdst
	L2prefetch128 [rcx-64][rbp]
	L2prefetch128 [rcx-64+blkdst][rbp]
	bump	rbp, -64+2*blkdst
	add	al, 256/2		;; Test inner loop counter
	jnc	b9b			;; Iterate if necessary
	bump	rbp, +128-4*blkdst
	IF clm NE 1
	add	dl, 256/clm
	jnc	b9b
	ENDIF
	bump	rsi, -2*clm*64+4*clmblkdst;; Next source pointer
	bump	rcx, -2*clm*64+4*blkdst	;; Next dest pointer
	bump	rdi, 2*XMM_SCD		;; Next sine/cosine pointer
	sub	al, 1
	jnz	b9b
	bump	rsi, -8*clmblkdst+clmblkdst8
	sub	ah, 1			;; Test outer loop counter
	mov	al, 2
	jnz	b9b			;; Iterate if necessary
	end_timer 26

;; Work on next 4 sets of 1024 values

endlp:	xpass1_get_next_block pass2, c0b, b0b ;; Get next block and jump
	done:	xfft_footer
	ENDM


;; Perform a 2560K-element FFT.  This is done in two passes.  Pass 1
;; does 11 levels, pass 2 uses common code to perform the last 11 levels.

hg_pass1sclevels11pfa5 MACRO fftname, pass2_levels, clmarg
	LOCAL	b0b, b1a, b1aa, b1b, b2b, b3b, b4b, b5b, b6b, b7a, b7b, b8b, b9b
	LOCAL	c0b, c1b, c2a, c2b, c3b, c4b
	LOCAL	c5a, c5b, c6b, c7a, c7b, c8b, c9b
	LOCAL	pass1, pass2, endlp, done, yes_fft

	set_FFT_constants fftname, 0, 0, pass2_levels, clmarg
	xfft_header pass1

;; Do the last 11 FFT levels
;;	do 1 pass2_11_levels_real and 319 pass2_11_levels_complex

pass2:	start_timer 0
	call	xpass2name
	end_timer 0
	cmp	ffttype, 1		;; We're done if FFTing only
	je	done

;; Do the inverse FFT

	pass1_inverse_fft_setup c0b

;; Do inverse FFT levels 10,11
;; On input the 64-byte cache lines hold these data values:
;;	0K	2K	4K	6K
;;	8K	10K	12K	14K
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0K	4K	8K	12K
;;	2K	...
;;	16K	...
;;	...

;; Do 1 eight_reals_unfft_2 macros
;; Do 1 nop_two_two_complex_unfft_2 macros
;;	distance between fft data elements is 2K
;;	do 1 macros each processing 16 data values

c0b:	start_timer 16
	mov	rsi, DATA_ADDR		;; Get source address
	mov	rcx, scratch_area	;; Get address of scratch area
	sub	rax, rax
	sub	rdx, rdx
pfing	mov	rbp, DATA_PREFETCH	;; Set prefetch distance
pfing	sub	rbp, rsi
c8b:	g2cl_eight_reals_unfft_2 rsi, 64, blkdst, rcx, 64, clmblkdst
	add	al, 256/2/clm		;; Test loop counter
	jnc	c8b			;; Iterate if necessary

;; Do 318 four_complex_unfft macros
;;	distance between fft data elements is 2K
;;	do 159 macros each processing 16 data values

	bump	rsi, -2*clm*64+2*blkdst	;; Next source pointer
	bump	rcx, -2*clm*64+2*clmblkdst;; Next dest pointer
	mov	rdi, sincos1		;; Load sin/cos pointer
	mov	al, 256/4+40		;; 159 iterations of 2*clm
c9b:	g2cl_four_complex_unfft rsi, 64, blkdst, rcx, 64, clmblkdst
	add	dl, 256/2/clm		;; Test loop counter
	jnc	c9b			;; Iterate if necessary
	bump	rsi, -2*clm*64+2*blkdst	;; Next source pointer
	bump	rcx, -2*clm*64+2*clmblkdst;; Next dest pointer
	bump	rdi, XMM_SCD		;; Next sine/cosine pointer
	add	al, 256/4
	jnc	c9b
	bump	rcx, -8*clmblkdst+clmblkdst8
	sub	al, 1			;; Test outer loop counter
	jnz	c9b			;; Iterate if necessary
	lea	rcx, [rsi-160*2*blkdst]	;; Create prefetch pointer
	mov	rsi, scratch_area	;; Restore source pointer
	end_timer 16

;; Do inverse FFT levels 8,9
;; On input the 64-byte cache lines hold these data values:
;;	0K	4K	8K	12K
;;	2K	...
;;	16K	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0K	16K	32K	48K
;;	2K	...
;;	...
;;	14K
;;	64K	...
;;	...

;; Do 4 eight_reals_unfft_2 macros
;; Do 4 nop_two_two_complex_unfft_2 macros
;;	distance between fft data elements is 8K
;;	do 2 macros each processing 32 data values

	start_timer 17
	mov	al, 2			;; 2 iterations of 2*clm
c6b:	x4cl_eight_reals_unfft_2 rsi, 64, 2*clmblkdst, 4*clmblkdst
	add	al, 256/2/clm		;; Test loop counter
	jnc	c6b			;; Iterate if necessary
	bump	rsi, -2*clm*64+clmblkdst;; Next source pointer
	sub	al, 1			;; Test outer loop counter
	jnz	c6b			;; Iterate if necessary

;; Do 312 four_complex_unfft macros
;;	distance between fft data elements is 8K
;;	do 78 macros each processing 32 data values

	bump	rsi, -2*clmblkdst+clmblkdst8;; Next source pointer
	mov	rdi, sincos2		;; Load sin/cos pointer
	mov	al, 39			;; 39 iterations of 2*2*clm
c7a:	xtouch	[rcx+(clm-1)*128]	;; Preload TLB
c7b:	x4cl_four_complex_unfft rsi, 64, 2*clmblkdst, 4*clmblkdst, rdi
	xloop2	c7b, clmblkdst
	bump	rsi, -2*clmblkdst+clmblkdst8;; Next source pointer
	bump	rdi, XMM_SCD		;; Next sine/cosine pointer
	sub	al, 1			;; Test outer loop counter
	jnz	c7a			;; Iterate if necessary
	bump	rsi, -40*clmblkdst8	;; Restore source pointer
	end_timer 17

;; Do inverse FFT levels 6,7
;; On input the 64-byte cache lines hold these data values:
;;	0K	16K	32K	48K
;;	2K	...
;;	...
;;	14K
;;	64K	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0K	64K	128K	192K
;;	2K	...
;;	...
;;	62K
;;	256K	...
;;	...

;; Do 16 eight_reals_unfft_2 macros
;; Do 16 nop_two_two_complex_unfft_2 macros
;;	distance between fft data elements is 32K
;;	do 8 macros each processing 32 data values

	start_timer 18
	mov	al, 8			;; 8 iterations of 2*clm
c4b:	x4cl_eight_reals_unfft_2 rsi, 64, clmblkdst8, 2*clmblkdst8
	add	al, 256/2/clm		;; Test loop counter
	jnc	c4b			;; Iterate if necessary
	bump	rsi, -2*clm*64+clmblkdst;; Next source pointer
	sub	al, 1			;; Test outer loop counter
	jnz	c4b			;; Iterate if necessary

;; Do 288 four_complex_unfft macros
;;	distance between fft data elements is 8K
;;	do 72 macros each processing 32 data values

	bump	rsi, -8*clmblkdst+4*clmblkdst8;; Next source pointer
	mov	rdi, sincos3		;; Load sin/cos pointer
	mov	eax, 9*256+4		;; 9 iterations of 4 iters of 2*2*clm
c5a:	xtouch	[rcx+(clm-1)*128]	;; Preload TLB
c5b:	x4cl_four_complex_unfft rsi, 64, clmblkdst8, 2*clmblkdst8, rdi
	xloop2	c5b, clmblkdst
	sub	al, 1			;; Test middle loop counter
	jnz	c5a			;; Iterate if necessary
	bump	rsi, -4*2*clmblkdst+4*clmblkdst8;; Next source pointer
	bump	rdi, XMM_SCD		;; Next sine/cosine pointer
	sub	eax, 256		;; Test outer loop counter
	mov	al, 4			;; Load middle loop counter
	jnz	c5a			;; Iterate if necessary
	bump	rsi, -10*4*clmblkdst8	;; Restore source pointer
	end_timer 18

;; Do inverse FFT levels 4,5
;; On input the 64-byte cache lines hold these data values:
;;	0K	64K	128K	192K
;;	2K	...
;;	...
;;	62K
;;	256K	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0K	128K	256K	384K
;;	2K	...
;;	...
;;	126K
;;	512K	768K	1024K	1280K
;;	...
;;
;; Do 64 eight_reals_unfft_2 macros
;;	distance between fft data elements is 128K
;;	do 32 macros each processing 16 data values
;;
;; Do 256 four_complex_unfft macros
;;	distance between fft data elements is 128K
;;	do 64 macros each processing 32 data values

;; Do inverse FFT levels 1,2,3
;; On input the 64-byte cache lines hold these data values:
;;	0K	128K	256K	384K
;;	2K	...
;;	...
;;	126K
;;	512K	768K	1024K	1280K
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0K	128K	1280K	1408K
;;	2K	...
;;	...
;;	126K
;;	256K	768K	1536K	2048K
;;	258K
;;	...
;;	766K
;;
;; Do 512 five_reals_last_unfft macros
;;	distance between fft data elements is 512K
;;	do 64 macros each processing 40 data values

	start_timer 19
	mov	rdi, sincos4		;; Load sin/cos pointer
	mov	al, 4			;; 32 iterations

c2a:	xtouchlines 3			;; Preload TLBs

c2b:	x2cl_half_eight_reals_unfft_2 rsi, 64, 4*clmblkdst8
	add	al, 256/2		;; Test loop counter
	jnc	c2b			;; Iterate if necessary

	bump	rsi, -2*64+8*clmblkdst8	;; Next source pointer
c3b:	x4cl_four_complex_unfft rsi, 64, 4*clmblkdst8, 8*clmblkdst8, rdi
	add	al, 256/2		;; Test inner loop counter
	jnc	c3b			;; Iterate if necessary
	bump	rsi, -2*64+16*clmblkdst8;; Next source pointer
	bump	rdi, XMM_SCD		;; Next sine/cosine pointer
	add	ah, 256/2		;; Test middle loop counter
	jnc	c3b			;; Iterate if necessary
	L2prefetch128 [rcx][rbp]
pfing	bump	rcx, blkdst		;; Next prefetch pointer
	bump	rdi, -2*XMM_SCD		;; Restore sine/cosine pointer

	bump	rsi, -2*16*clmblkdst8-8*clmblkdst8;; Restore src pointer
	xfive_reals_unfft_preload	;; Preload consts for five real macros
c1b:	x5cl_five_reals_last_unfft rsi, 64, 8*clmblkdst8
	xloop1sp c1b, 4*clmblkdst8
	add	ah, 256/2
	jnc	c1b

	IF clm GT 1
	bump	rsi, -2*4*clmblkdst8+2*64;; Next source pointer
	bump	rcx, -3*blkdst+128	;; Next prefetch pointer
	add	dl, 256/clm		;; clm iterations
	jnc	c2b
	bump	rsi, -clm*2*64+clmblkdst;; Next source pointer
	bump	rcx, -clm*128+3*blkdst	;; Next prefetch pointer
	ELSE
	bump	rsi, -2*4*clmblkdst8+clmblkdst;; Next source pointer
	ENDIF

	add	eax, 80000000h/8*2	;; Test loop counter
	jnc	c2a			;; Iterate if necessary
	bump	rsi, -8*clmblkdst+clmblkdst8;; Next source pointer
	sub	al, 1			;; Test loop counter
	jnz	c2a			;; Iterate if necessary
	end_timer 19

;; Normalize these values

	pass1_normalize 1, yes_fft

;; Copy data from the scratch area back to the FFT data area

	copy_scratch_data_to_fft 320
	jmp	endlp			;; Skip FFT code if flag not set

;; Fire up auxiliary threads to do pass 1 forward FFT

pass1:	pass1_forward_fft_setup b0b

;; Copy data from the FFT data area back to the scratch area

b0b:	copy_fft_data_to_scratch 320

;; Do FFT levels 1,2,3
;;
;; On input the 64-byte cache lines hold these data values:
;;	0K	128K	1280K	1408K
;;	2K	...
;;	...
;;	126K
;;	256K	768K	1536K	2048K
;;	258K
;;	...
;;	766K
;; On output the 64-byte cache lines hold these data values:
;;	0K	128K	256K	384K
;;	2K	...
;;	...
;;	126K
;;	512K	1024K	1536K	2048K
;;	514K
;;	...
;;	1022K
;;
;; Do 512 five_reals_first_fft macros (FFT levels 1,2,3)
;;	distance between fft data elements is 512K
;;	do 64 iterations each processing 40 data values

;; Do FFT levels 4,5
;; Values 0K-511K is real data, 512K-2559K is complex data.
;;
;; On input the 64-byte cache lines hold these data values:
;;	0K	128K	256K	384K
;;	2K	...
;;	...
;;	126K
;;	512K	1024K	1536K	2048K
;;	514K
;;	...
;;	1022K
;; On output the 64-byte cache lines hold these data values:
;;	0K	128K	256K	384K
;;	2K	...
;;	...
;;	126K	...
;;	512K	...
;;	...
;;
;; Do 64 eight_reals_fft_2 macros
;;	distance between fft data elements is 128K
;;	do 64 macros each processing 8 data values
;;
;; Do 256 four_complex_fft macros
;;	distance between fft data elements is 128K
;;	do 64 macros each processing 32 data values

;; Do FFT levels 6,7
;; Values 0K-127K is real data, 128K-255K is semi-real data, 256K-2559K is
;; complex data.
;;
;; On input the 64-byte cache lines hold these data values:
;;	0K	128K	256K	384K
;;	2K	...
;;	...
;;	126K	...
;;	512K	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0K	32K	64K	96K
;;	2K	...
;;	...
;;	30K	...
;;	128K	...
;;	...
;;
;; Do 32 eight_reals_fft_2 macros
;; Do 32 two_two_complex_fft macros
;;	distance between fft data elements is 32K
;;	do 16 macros each processing 32 data values
;;
;; Do 256 four_complex_fft macros
;;	distance between fft data elements is 8K
;;	do 64 macros each processing 32 data values

yes_fft:start_timer 22
	mov	rsi, scratch_area	;; Get address of scratch area
	mov	rcx, DATA_ADDR		;; Load source address for prefetching
pfing	mov	rbp, DATA_PREFETCH	;; Set prefetch distance
pfing	sub	rbp, rcx
pfing	bump	rcx, 170*blkdst		;; Create prefetching pointer
	mov	rdi, sincos4		;; Load sin/cos pointer
	mov	rbx, sincos3		;; Load sin/cos pointer
	mov	eax, 2			;; 16 iterations
	sub	rdx, rdx

b1a:	xtouchlines 7			;; Preload TLBs

b1aa:	xfive_reals_fft_preload		;; Preload consts for five real macros
b1b:	x5cl_five_reals_fft rsi, 64, 8*clmblkdst8
	xloop1sp b1b, 2*clmblkdst8
	add	ah, 256/4		;; Test inner loop counter
	jnc	b1b			;; Iterate if necessary

	bump	rsi, -4*2*clmblkdst8	;; Restore source pointer
b2b:	x1cl_half_eight_reals_fft_2 rsi, 64
	add	al, 256/2		;; Test inner loop counter
	jnc	b2b			;; Iterate if necessary
	bump	rsi, -2*64+2*clmblkdst8
	add	ah, 256/4		;; Test inner loop counter
	jnc	b2b			;; Iterate if necessary
	L2prefetch128 [rcx][rbp]
pfing	bump	rcx, blkdst		;; Next prefetch pointer

;;	bump	rsi, -4*2*clmblkdst8+8*clmblkdst8
b3b:	x4cl_four_complex_fft rsi, 64, 8*clmblkdst8, 16*clmblkdst8, rdi
	add	al, 256/2		;; Test inner loop counter
	jnc	b3b			;; Iterate if necessary
	bump	rsi, -2*64+2*clmblkdst8
	add	ah, 256/4		;; Test inner loop counter
	jnc	b3b			;; Iterate if necessary
	L2prefetch128 [rcx][rbp]
pfing	bump	rcx, blkdst		;; Next prefetch pointer

	bump	rsi, -4*2*clmblkdst8-8*clmblkdst8;;Restore src pointer
b4b:	x4cl_eight_reals_fft_2 rsi, 64, 2*clmblkdst8, 4*clmblkdst8, rbx
	add	al, 256/2		;; Test inner loop counter
	jnc	b4b			;; Iterate if necessary
	L2prefetch128 [rcx][rbp]
pfing	bump	rcx, blkdst		;; Next prefetch pointer

	bump	rsi, -2*64+8*clmblkdst8	;; Next source pointer
	bump	rbx, XMM_SCD		;; Next sine/cosine pointer
b5b:	x4cl_four_complex_fft rsi, 64, 2*clmblkdst8, 4*clmblkdst8, rbx
	add	al, 256/2		;; Test inner loop counter
	jnc	b5b			;; Iterate if necessary
	bump	rsi, -2*64+8*clmblkdst8
	bump	rbx, 2*XMM_SCD		;; Next sine/cosine pointer
	add	ah, 256/4		;; Test outer loop counter
	jnc	b5b			;; Iterate if necessary
	bump	rbx, -9*XMM_SCD		;; Restore sine/cosine pointer

	IF clm GT 1
	bump	rsi, -5*8*clmblkdst8+2*64;; Next source pointer
	bump	rcx, -7*blkdst+128	;; Next prefetch pointer
	add	dl, 256/clm		;; clm iterations
	jnc	b1aa
	bump	rsi, -clm*2*64+clmblkdst;; Next source pointer
	bump	rcx, -clm*128+7*blkdst	;; Next prefetch pointer
	ELSE
	bump	rsi, -5*8*clmblkdst8+clmblkdst;; Next source pointer
	ENDIF

	add	eax, 80000000h/8*2	;; Test loop counter
	jnc	b1a			;; Iterate if necessary
	bump	rsi, -8*clmblkdst+clmblkdst8;; Next source pointer
	sub	al, 1			;; Test loop counter
	jnz	b1a			;; Iterate if necessary
	bump	rsi, -2*clmblkdst8	;; Restore source pointer
	end_timer 22

;; Do FFT levels 8,9
;; Values 0K-31K is real data, 32K-63K is semi-real data, 64K-2559K is
;; complex data.
;;
;; On input the 64-byte cache lines hold these data values:
;;	0K	32K	64K	96K
;;	2K	...
;;	...
;;	30K	...
;;	128K	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0K	8K	16K	24K
;;	2K	...
;;	...
;;	6K	...
;;	32K	...
;;	...

;; Do 8 eight_reals_fft_2 macros
;; Do 8 two_two_complex_fft macros
;;	distance between fft data elements is 8K
;;	do 4 macros each processing 32 data values

	start_timer 25
	mov	rdi, sincos2		;; Load sin/cos pointer
	mov	al, 4			;; 4 iterations of 2*clm
b6b:	x4cl_eight_reals_fft_2 rsi, 64, 4*clmblkdst, clmblkdst8, rdi
	add	al, 256/2/clm		;; Test inner loop counter
	jnc	b6b			;; Iterate if necessary
	bump	rsi, -2*clm*64+clmblkdst;; Next source pointer
	sub	al, 1			;; Test outer loop counter
	jnz	b6b			;; Iterate if necessary

;; Do 304 four_complex_fft macros
;;	distance between fft data elements is 8K
;;	do 76 macros each processing 32 data values

	bump	rsi, -4*clmblkdst+2*clmblkdst8;; Load source pointer
	bump	rdi, XMM_SCD		;; Next sine/cosine pointer
	mov	eax, 19*256+2		;; 19 iterations of 2 iters of 2*2*clm
b7a:	xtouch	[rcx+(clm-1)*128]	;; Preload TLB
b7b:	x4cl_four_complex_fft rsi, 64, 4*clmblkdst, clmblkdst8, rdi
	xloop2	b7b, clmblkdst
	sub	al, 1			;; Test middle loop counter
	jnz	b7a			;; Iterate if necessary
	bump	rsi, -2*2*clmblkdst+2*clmblkdst8;; Next source pointer
	bump	rdi, 2*XMM_SCD		;; Next sine/cosine pointer
	sub	eax, 256		;; Test outer loop counter
	mov	al, 2			;; Load middle loop counter
	jnz	b7a			;; Iterate if necessary
	bump	rsi, -20*2*clmblkdst8	;; Restore source pointer
pfing	bump	rcx, -320*blkdst	;; Restore prefetch pointer
	end_timer 25

;; Do FFT levels 10,11
;; Values 0K-7K is real data, 8K-15K is semi-real data, 16K-2559K is
;; complex data.
;;
;; On input the 64-byte cache lines hold these data values:
;;	0K	8K	16K	24K
;;	2K	...
;;	...
;;	6K	...
;;	32K	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0K	2K	4K	6K
;;	8K	...
;;	...

;; Do 1 eight_reals_fft_2 macros
;; Do 1 nop_two_two_complex_fft_2 macros
;; Do 2 four_complex_fft macros
;;	distance between fft data elements is 2K
;;	do 1 macros each processing 32 data values

	start_timer 26
	mov	rdi, sincos1		;; Load sin/cos pointer
b8b:	g4cl_eight_reals_fft_2 rsi, 64, clmblkdst, 2*clmblkdst, rcx, 64, blkdst, 2*blkdst
	add	al, 256/2/clm		;; Test inner loop counter
	jnc	b8b			;; Iterate if necessary

;; Do 316 four_complex_fft macros
;;	distance between fft data elements is 2K
;;	do 79 macros each processing 32 data values

	bump	rsi, -2*clm*64+4*clmblkdst;; Next source pointer
	bump	rcx, -2*clm*64+4*blkdst	;; Next dest pointer
	bump	rdi, XMM_SCD		;; Next sine/cosine pointer
	mov	eax, 40*256+1		;; 79 iterations of 2*clm
b9b:	g4cl_four_complex_fft rsi, 64, clmblkdst, 2*clmblkdst, rcx, 64, blkdst, 2*blkdst
	L2prefetch128 [rcx-64][rbp]
	L2prefetch128 [rcx-64+blkdst][rbp]
	bump	rbp, -64+2*blkdst
	add	al, 256/2		;; Test inner loop counter
	jnc	b9b			;; Iterate if necessary
	bump	rbp, 128-4*blkdst
	IF clm NE 1
	add	dl, 256/clm
	jnc	b9b
	ENDIF
	bump	rsi, -2*clm*64+4*clmblkdst;; Next source pointer
	bump	rcx, -2*clm*64+4*blkdst	;; Next source pointer
	bump	rdi, 2*XMM_SCD		;; Next sine/cosine pointer
	sub	al, 1
	jnz	b9b
	bump	rsi, -8*clmblkdst+clmblkdst8
	sub	ah, 1			;; Test outer loop counter
	mov	al, 2
	jnz	b9b			;; Iterate if necessary
	end_timer 26

;; Work on next 4 sets of 1280 values

endlp:	xpass1_get_next_block pass2, c0b, b0b ;; Get next block and jump
	done:	xfft_footer
	ENDM


;; Perform a 3M-element FFT.  This is done in two passes.  Pass 1
;; does 11 levels, pass 2 uses common code to perform the last 11 levels.

hg_pass1sclevels11pfa6 MACRO fftname, pass2_levels, clmarg
	LOCAL	b0b, b1a, b1b, b2a, b2b, b3a, b3b, b4b
	LOCAL	b5a, b5b, b6b, b7a, b7b, b8b, b9b
	LOCAL	c0b, c1b, c2a, c2b, c3b, c4b
	LOCAL	c5a, c5b, c6b, c7a, c7b, c8b, c9b
	LOCAL	pass1, pass2, endlp, done, yes_fft

	set_FFT_constants fftname, 0, 0, pass2_levels, clmarg
	xfft_header pass1

;; Do the last 11 FFT levels
;;	do 1 pass2_11_levels_real and 383 pass2_11_levels_complex

pass2:	start_timer 0
	call	xpass2name
	end_timer 0
	cmp	ffttype, 1		;; We're done if FFTing only
	je	done

;; Do the inverse FFT

	pass1_inverse_fft_setup c0b

;; Do inverse FFT levels 10,11
;; On input the 64-byte cache lines hold these data values:
;;	0K	2K	4K	6K
;;	8K	10K	12K	14K
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0K	4K	8K	12K
;;	2K	...
;;	16K	...
;;	...

;; Do 1 eight_reals_unfft_2 macros
;; Do 1 nop_two_two_complex_unfft_2 macros
;;	distance between fft data elements is 2K
;;	do 1 macros each processing 16 data values

c0b:	start_timer 16
	mov	rsi, DATA_ADDR		;; Get source address
	mov	rcx, scratch_area	;; Get address of scratch area
	sub	rax, rax
	sub	rdx, rdx
pfing	mov	rbp, DATA_PREFETCH	;; Set prefetch distance
pfing	sub	rbp, rsi
c8b:	g2cl_eight_reals_unfft_2 rsi, 64, blkdst, rcx, 64, clmblkdst
	add	al, 256/2/clm		;; Test loop counter
	jnc	c8b			;; Iterate if necessary

;; Do 382 four_complex_unfft macros
;;	distance between fft data elements is 2K
;;	do 191 macros each processing 16 data values

	bump	rsi, -2*clm*64+2*blkdst	;; Next source pointer
	bump	rcx, -2*clm*64+2*clmblkdst;; Next dest pointer
	mov	rdi, sincos1		;; Load sin/cos pointer
	mov	al, 256/4+48		;; 191 iterations of 2*clm
c9b:	g2cl_four_complex_unfft rsi, 64, blkdst, rcx, 64, clmblkdst
	add	dl, 256/2/clm		;; Test loop counter
	jnc	c9b			;; Iterate if necessary
	bump	rsi, -2*clm*64+2*blkdst	;; Next source pointer
	bump	rcx, -2*clm*64+2*clmblkdst;; Next dest pointer
	bump	rdi, XMM_SCD		;; Next sine/cosine pointer
	add	al, 256/4
	jnc	c9b
	bump	rcx, -8*clmblkdst+clmblkdst8
	sub	al, 1			;; Test outer loop counter
	jnz	c9b			;; Iterate if necessary
	lea	rcx, [rsi-192*2*blkdst]	;; Create prefetch pointer
	mov	rsi, scratch_area	;; Restore source pointer
	end_timer 16

;; Do inverse FFT levels 8,9
;; On input the 64-byte cache lines hold these data values:
;;	0K	4K	8K	12K
;;	2K	...
;;	16K	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0K	16K	32K	48K
;;	2K	...
;;	...
;;	14K
;;	64K	...
;;	...

;; Do 4 eight_reals_unfft_2 macros
;; Do 4 nop_two_two_complex_unfft_2 macros
;;	distance between fft data elements is 8K
;;	do 2 macros each processing 32 data values

	start_timer 17
	mov	al, 2			;; 2 iterations 
c6b:	x4cl_eight_reals_unfft_2 rsi, 64, 2*clmblkdst, 4*clmblkdst
	add	al, 256/2/clm		;; Test loop counter
	jnc	c6b			;; Iterate if necessary
	bump	rsi, -2*clm*64+clmblkdst;; Next source pointer
	sub	al, 1			;; Test outer loop counter
	jnz	c6b			;; Iterate if necessary

;; Do 376 four_complex_unfft macros
;;	distance between fft data elements is 8K
;;	do 94 macros each processing 32 data values

	bump	rsi, -2*clmblkdst+clmblkdst8;; Next source pointer
	mov	rdi, sincos2		;; Load sin/cos pointer
	mov	al, 47			;; 47 iterations of 2*2*clm
c7a:	xtouch	[rcx+(clm-1)*128]	;; Preload TLB
c7b:	x4cl_four_complex_unfft rsi, 64, 2*clmblkdst, 4*clmblkdst, rdi
	xloop2	c7b, clmblkdst
	bump	rsi, -2*clmblkdst+clmblkdst8;; Next source pointer
	bump	rdi, XMM_SCD		;; Next sine/cosine pointer
	sub	al, 1			;; Test outer loop counter
	jnz	c7a			;; Iterate if necessary
	bump	rsi, -48*clmblkdst8	;; Restore source pointer
	end_timer 17

;; Do inverse FFT levels 6,7
;; On input the 64-byte cache lines hold these data values:
;;	0K	16K	32K	48K
;;	2K	...
;;	...
;;	14K
;;	64K	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0K	64K	128K	192K
;;	2K	...
;;	...
;;	62K
;;	256K	...
;;	...

;; Do 16 eight_reals_unfft_2 macros
;; Do 16 nop_two_two_complex_unfft_2 macros
;;	distance between fft data elements is 32K
;;	do 8 macros each processing 32 data values

	start_timer 18
	mov	al, 8			;; 8 iterations of 2*clm
c4b:	x4cl_eight_reals_unfft_2 rsi, 64, clmblkdst8, 2*clmblkdst8
	add	al, 256/2/clm		;; Test loop counter
	jnc	c4b			;; Iterate if necessary
	bump	rsi, -2*clm*64+clmblkdst;; Next source pointer
	sub	al, 1			;; Test outer loop counter
	jnz	c4b			;; Iterate if necessary

;; Do 352 four_complex_unfft macros
;;	distance between fft data elements is 32K
;;	do 88 macros each processing 32 data values

	bump	rsi, -8*clmblkdst+4*clmblkdst8;; Next source pointer
	mov	rdi, sincos3		;; Load sin/cos pointer
	mov	eax, 11*256+4		;; 11 iterations of 4 iters of 2*2*clm
c5a:	xtouch	[rcx+(clm-1)*128]	;; Preload TLB
c5b:	x4cl_four_complex_unfft rsi, 64, clmblkdst8, 2*clmblkdst8, rdi
	xloop2	c5b, clmblkdst
	sub	al, 1			;; Test middle loop counter
	jnz	c5a			;; Iterate if necessary
	bump	rsi, -4*2*clmblkdst+4*clmblkdst8;; Next source pointer
	bump	rdi, XMM_SCD		;; Next sine/cosine pointer
	sub	eax, 256		;; Test outer loop counter
	mov	al, 4			;; Load middle loop counter
	jnz	c5a			;; Iterate if necessary
	bump	rsi, -12*4*clmblkdst8	;; Restore source pointer
	end_timer 18

;; Do inverse FFT levels 4,5
;; On input the 64-byte cache lines hold these data values:
;;	0K	64K	128K	192K
;;	2K	...
;;	...
;;	62K
;;	256K	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0K	256K	512K	768K
;;	2K	...
;;	...
;;	254K
;;	1024K	...
;;	...
;;
;; Do 64 eight_reals_unfft_2 macros
;; Do 64 nop_two_two_complex_unfft_2 macros
;;	distance between fft data elements is 128K
;;	do 32 macros each processing 32 data values
;;
;; Do 256 four_complex_unfft macros
;;	distance between fft data elements is 128K
;;	do 64 macros each processing 32 data values

;; Do inverse FFT levels 1,2,3
;; On input the 64-byte cache lines hold these data values:
;;	0K	256K	512K	768K
;;	2K	...
;;	...
;;	254K
;;	1024K	1536K	2048K	2560K
;;	1026K
;;	...
;;	1534K
;; On output the 64-byte cache lines hold these data values:
;;	0K	768K	1536K	2304K
;;	...
;;	766K
;;
;; Do 512 six_reals_last_unfft macros
;;	distance between fft data elements is 512K
;;	do 128 macros each processing 24 data values

	start_timer 19
	mov	rdi, sincos4		;; Load sin/cos pointer
	mov	al, 4			;; 32 iterations
	xsix_reals_unfft_preload

c2a:	xtouchlines 3			;; Preload TLBs

c2b:	x4cl_eight_reals_unfft_2 rsi, 64, 4*clmblkdst8, 8*clmblkdst8
	add	al, 256/2		;; Test loop counter
	jnc	c2b			;; Iterate if necessary

	bump	rsi, -2*64+16*clmblkdst8;; Next source pointer
c3b:	x4cl_four_complex_unfft rsi, 64, 4*clmblkdst8, 8*clmblkdst8, rdi
	add	al, 256/2		;; Test inner loop counter
	jnc	c3b			;; Iterate if necessary
	bump	rsi, -2*64+16*clmblkdst8;; Next source pointer
	bump	rdi, XMM_SCD		;; Next sine/cosine pointer
	add	ah, 256/2		;; Test middle loop counter
	jnc	c3b			;; Iterate if necessary
	L2prefetch128 [rcx][rbp]
pfing	bump	rcx, blkdst		;; Next prefetch pointer
	bump	rdi, -2*XMM_SCD		;; Restore sine/cosine pointer

	bump	rsi, -3*16*clmblkdst8	;; Restore source pointer
c1b:	x3cl_six_reals_last_unfft rsi, 64, 16*clmblkdst8
	add	al, 256/2		;; Test inner loop counter
	jnc	c1b			;; Iterate if necessary
	bump	rsi, -2*64+4*clmblkdst8	;; Next source pointer
	add	ah, 256/2		;; Test middle loop counter
	jnc	c1b			;; Iterate if necessary
	L2prefetch128 [rcx][rbp]
pfing	bump	rcx, blkdst		;; Next prefetch pointer
	add	edx, 80000000h		;; Test middle loop counter
	jnc	c1b			;; Iterate if necessary

	IF clm GT 1
	bump	rsi, -4*4*clmblkdst8+2*64;; Next source pointer
	bump	rcx, -3*blkdst+128	;; Next prefetch pointer
	add	dl, 256/clm		;; clm iterations
	jnc	c2b
	bump	rsi, -clm*2*64+clmblkdst;; Next source pointer
	bump	rcx, -clm*128+3*blkdst	;; Next prefetch pointer
	ELSE
	bump	rsi, -4*4*clmblkdst8+clmblkdst;; Next source pointer
	ENDIF

	add	eax, 80000000h/8*2	;; Test loop counter
	jnc	c2a			;; Iterate if necessary
	bump	rsi, -8*clmblkdst+clmblkdst8;; Next source pointer
	sub	al, 1			;; Test loop counter
	jnz	c2a			;; Iterate if necessary
	end_timer 19

;; Normalize these values

	pass1_normalize 1, yes_fft

;; Copy data from the scratch area back to the FFT data area

	copy_scratch_data_to_fft 384
	jmp	endlp			;; Skip FFT code if flag not set

;; Fire up auxiliary threads to do pass 1 forward FFT

pass1:	pass1_forward_fft_setup b0b

;; Copy data from the FFT data area back to the scratch area

b0b:	copy_fft_data_to_scratch 384

;; Do FFT levels 1,2,3
;;
;; On input the 64-byte cache lines hold these data values:
;;	0K	768K	1536K	2304K
;;	...
;;	766K
;; On output the 64-byte cache lines hold these data values:
;;	0K	256K	512K	768K
;;	2K	...
;;	...
;;	254K
;;	1024K	1536K	2048K	2560K
;;	1026K
;;	...
;;	1534K

;; Do 512 six_reals_first_fft macros (FFT levels 1,2,3)
;;	distance between fft data elements is 512K
;;	do 128 iterations each processing 24 data values

yes_fft:start_timer 22
	mov	rsi, scratch_area	;; Get address of scratch area
	mov	rcx, DATA_ADDR		;; Load source address for prefetching
pfing	mov	rbp, DATA_PREFETCH	;; Set prefetch distance
pfing	sub	rbp, rcx
pfing	bump	rcx, 186*blkdst		;; Create prefetching pointer
	mov	eax, 16			;; 16 iterations of 4 iters of 2*2*clm
	sub	rdx, rdx
	xsix_reals_fft_preload
b1a:	xtouch	[rcx+(clm-1)*128]	;; Preload TLB
b1b:	x3cl_six_reals_fft rsi, 64, 16*clmblkdst8
	xloop2	b1b, clmblkdst
	add	eax, 80000000h/4*2	;; 4 iterations
	jnc	b1a
	bump	rsi, -8*clmblkdst+clmblkdst8
	sub	al, 1			;; Test loop counter
	jnz	b1a			;; Iterate if necessary
	bump	rsi, -16*clmblkdst8	;; Restore source pointer
	end_timer 22

;; Do FFT levels 4,5
;; Values 0K-511K is real data, 512K-1023K is semi-real data, 1024K-3071K is
;; complex data.
;;
;; On input the 64-byte cache lines hold these data values:
;;	0K	256K	512K	768K
;;	2K	...
;;	...
;;	254K
;;	1024K	1536K	2048K	2560K
;;	1026K
;;	...
;;	1534K
;; On output the 64-byte cache lines hold these data values:
;;	0K	128K	256K	384K
;;	2K	...
;;	...
;;	126K	...
;;	512K	...
;;	...

;; Do 32 eight_reals_fft_2 macros
;; Do 32 nop_two_two_complex_fft_2 macros
;; Do 64 four_complex_fft macros
;;	distance between fft data elements is 128K
;;	do 64 macros each processing 16 data values

	start_timer 23
	mov	al, 8			;; 8 iterations of 2 iters of 4*2*clm
b2a:	xtouch	[rcx+(clm-1)*128]	;; Preload TLB
b2b:	x2cl_eight_reals_fft_2 rsi, 64, 8*clmblkdst8
	xloop4	b2b, clmblkdst
	add	eax, 80000000h
	jnc	b2a
	bump	rsi, -8*clmblkdst+clmblkdst8
	sub	al, 1			;; Test outer loop counter
	jnz	b2a			;; Iterate if necessary

;; Do 256 four_complex_fft macros
;;	distance between fft data elements is 128K
;;	do 64 macros each processing 32 data values

	bump	rsi, -8*clmblkdst8+16*clmblkdst8;; Next source pointer
	mov	rdi, sincos4		;; Load sin/cos pointer
	mov	al, 8			;; 8 iters of 4 iters of 2*2*clm
b3a:	xtouch	[rcx+(clm-1)*128]	;; Preload TLB
b3b:	x4cl_four_complex_fft rsi, 64, 8*clmblkdst8, 16*clmblkdst8, rdi
	xloop2	b3b, clmblkdst
	add	eax, 80000000h/4*2	;; 4 iterations
	jnc	b3a
	bump	rsi, -8*clmblkdst+clmblkdst8
	sub	al, 1			;; Test outer loop counter
	jnz	b3a			;; Iterate if necessary
	bump	rsi, -8*clmblkdst8-16*clmblkdst8;; Restore src pointer
	end_timer 23

;; Do FFT levels 6,7
;; Values 0K-127K is real data, 128K-255K is semi-real data, 256K-4095K is
;; complex data.
;;
;; On input the 64-byte cache lines hold these data values:
;;	0K	128K	256K	384K
;;	2K	...
;;	...
;;	126K	...
;;	512K	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0K	32K	64K	96K
;;	2K	...
;;	...
;;	30K	...
;;	128K	...
;;	...

;; Do 32 eight_reals_fft_2 macros
;; Do 32 two_two_complex_fft macros
;;	distance between fft data elements is 32K
;;	do 16 macros each processing 32 data values

	start_timer 24
	mov	rdi, sincos3		;; Load sin/cos pointer
	mov	al, 2			;; 2 iters of 8 iters of 2*clm
b4b:	x4cl_eight_reals_fft_2 rsi, 64, 2*clmblkdst8, 4*clmblkdst8, rdi
	add	al, 256/2/clm		;; Test inner loop counter
	jnc	b4b			;; Iterate if necessary
	bump	rsi, -2*clm*64+clmblkdst;; Next source pointer
	add	ah, 256/8
	jnc	b4b
	bump	rsi, -8*clmblkdst+clmblkdst8
	sub	al, 1			;; Test outer loop counter
	jnz	b4b			;; Iterate if necessary

;; Do 320 four_complex_fft macros
;;	distance between fft data elements is 8K
;;	do 80 macros each processing 32 data values

	bump	rsi, -2*clmblkdst8+8*clmblkdst8;; Load source pointer
	bump	rdi, XMM_SCD		;; Next sine/cosine pointer
	mov	eax, 5*256+2		;; 5 iterations of 8 iters of 2*2*clm
b5a:	xtouch	[rcx+(clm-1)*128]	;; Preload TLB
b5b:	x4cl_four_complex_fft rsi, 64, 2*clmblkdst8, 4*clmblkdst8, rdi
	xloop2	b5b, clmblkdst
	add	eax, 80000000h/4*2	;; 4 iterations
	jnc	b5a
	bump	rsi, -8*clmblkdst+clmblkdst8
	sub	al, 1			;; Test middle loop counter
	jnz	b5a			;; Iterate if necessary
	bump	rsi, -2*clmblkdst8+8*clmblkdst8;; Next source pointer
	bump	rdi, 2*XMM_SCD		;; Next sine/cosine pointer
	sub	eax, 256		;; Test outer loop counter
	mov	al, 2			;; Load middle loop counter
	jnz	b5a			;; Iterate if necessary
	bump	rsi, -6*8*clmblkdst8	;; Restore source pointer
	end_timer 24

;; Do FFT levels 8,9
;; Values 0K-31K is real data, 32K-63K is semi-real data, 64K-3071K is
;; complex data.
;;
;; On input the 64-byte cache lines hold these data values:
;;	0K	32K	64K	96K
;;	2K	...
;;	...
;;	30K	...
;;	128K	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0K	8K	16K	24K
;;	2K	...
;;	...
;;	6K	...
;;	32K	...
;;	...

;; Do 8 eight_reals_fft_2 macros
;; Do 8 two_two_complex_fft macros
;;	distance between fft data elements is 8K
;;	do 4 macros each processing 32 data values

	start_timer 25
	mov	rdi, sincos2		;; Load sin/cos pointer
	mov	al, 4			;; 4 iterations of 2*clm
b6b:	x4cl_eight_reals_fft_2 rsi, 64, 4*clmblkdst, clmblkdst8, rdi
	add	al, 256/2/clm		;; Test inner loop counter
	jnc	b6b			;; Iterate if necessary
	bump	rsi, -2*clm*64+clmblkdst;; Next source pointer
	sub	al, 1			;; Test outer loop counter
	jnz	b6b			;; Iterate if necessary

;; Do 368 two_two_complex_fft macros
;;	distance between fft data elements is 8K
;;	do 92 macros each processing 32 data values

	bump	rsi, -4*clmblkdst+2*clmblkdst8;; Load source pointer
	bump	rdi, XMM_SCD		;; Next sine/cosine pointer
	mov	eax, 23*256+2		;; 23 iterations of 2 iters of 2*2*clm
b7a:	xtouch	[rcx+(clm-1)*128]	;; Preload TLB
b7b:	x4cl_four_complex_fft rsi, 64, 4*clmblkdst, clmblkdst8, rdi
	xloop2	b7b, clmblkdst
	sub	al, 1			;; Test middle loop counter
	jnz	b7a			;; Iterate if necessary
	bump	rsi, -2*2*clmblkdst+2*clmblkdst8;; Next source pointer
	bump	rdi, 2*XMM_SCD		;; Next sine/cosine pointer
	sub	eax, 256		;; Test outer loop counter
	mov	al, 2			;; Load middle loop counter
	jnz	b7a			;; Iterate if necessary
	bump	rsi, -24*2*clmblkdst8	;; Restore source pointer
pfing	bump	rcx, -384*blkdst	;; Restore prefetch pointer
	end_timer 25

;; Do FFT levels 10,11
;; Values 0K-7K is real data, 8K-15K is semi-real data, 16K-3071K is
;; complex data.
;;
;; On input the 64-byte cache lines hold these data values:
;;	0K	8K	16K	24K
;;	2K	...
;;	...
;;	6K	...
;;	32K	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0K	2K	4K	6K
;;	8K	...
;;	...

;; Do 1 eight_reals_fft_2 macros
;; Do 1 nop_two_two_complex_fft_2 macros
;; Do 2 four_complex_fft macros
;;	distance between fft data elements is 2K
;;	do 1 macros each processing 32 data values

	start_timer 26
	mov	rdi, sincos1		;; Load sin/cos pointer
b8b:	g4cl_eight_reals_fft_2 rsi, 64, clmblkdst, 2*clmblkdst, rcx, 64, blkdst, 2*blkdst
	add	al, 256/2/clm		;; Test inner loop counter
	jnc	b8b			;; Iterate if necessary

;; Do 380 four_complex_fft macros
;;	distance between fft data elements is 2K
;;	do 95 macros each processing 32 data values

	bump	rsi, -2*clm*64+4*clmblkdst;; Next source pointer
	bump	rcx, -2*clm*64+4*blkdst	;; Next dest pointer
	bump	rdi, XMM_SCD		;; Next sine/cosine pointer
	mov	eax, 48*256+1		;; 95 iterations of 2*clm
b9b:	g4cl_four_complex_fft rsi, 64, clmblkdst, 2*clmblkdst, rcx, 64, blkdst, 2*blkdst
	L2prefetch128 [rcx-64][rbp]
	L2prefetch128 [rcx-64+blkdst][rbp]
	bump	rbp, -64+2*blkdst
	add	al, 256/2		;; Test inner loop counter
	jnc	b9b			;; Iterate if necessary
	bump	rbp, 128-4*blkdst
	IF clm NE 1
	add	dl, 256/clm
	jnc	b9b
	ENDIF
	bump	rsi, -2*clm*64+4*clmblkdst;; Next source pointer
	bump	rcx, -2*clm*64+4*blkdst	;; Next dest pointer
	bump	rdi, 2*XMM_SCD		;; Next sine/cosine pointer
	sub	al, 1
	jnz	b9b
	bump	rsi, -8*clmblkdst+clmblkdst8
	sub	ah, 1			;; Test outer loop counter
	mov	al, 2
	jnz	b9b			;; Iterate if necessary
	end_timer 26

;; Work on next 4 sets of 1536 values

endlp:	xpass1_get_next_block pass2, c0b, b0b ;; Get next block and jump
	done:	xfft_footer
	ENDM


;; Perform a 3M-element negacyclic FFT.  This is done in two passes.  Pass 1
;; does 11 levels, pass 2 uses common code to perform the last 11 levels.

hg_pass1sclevels11complex3 MACRO fftname, pass2_levels, clmarg
	LOCAL	b0b, b1a, b1b, b2a, b2b, b3a, b3b, b5a, b5b, b7a, b7b, b9b
	LOCAL	c0b, c1a, c1b, c3a, c3b, c5a, c5b, c7a, c7b, c9b
	LOCAL	pass1, pass2, endlp, done, yes_fft

	set_FFT_constants fftname, 0, 1, pass2_levels, clmarg
	xfft_header pass1

;; Do the last 11 FFT levels
;;	do 384 pass2_11_levels_complex

pass2:	start_timer 0
	call	xpass2name
	end_timer 0
	cmp	ffttype, 1		;; We're done if FFTing only
	je	done

;; Do the inverse FFT

	pass1_inverse_fft_setup c0b

;; Do inverse FFT levels 10,11
;; On input the 64-byte cache lines hold these data values:
;;	0K	2K	4K	6K
;;	8K	10K	12K	14K
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0K	4K	8K	12K
;;	2K	...
;;	16K	...
;;	...

;; Do 384 four_complex_unfft macros
;;	distance between fft data elements is 2K
;;	do 192 macros each processing 16 data values

c0b:	start_timer 16
	mov	rsi, DATA_ADDR		;; Get source address
	mov	rcx, scratch_area	;; Get address of scratch area
	mov	rdi, sincos1		;; Load sin/cos pointer
	mov	eax, 48			;; 48 iterations of 4 of 2*clm
	sub	rdx, rdx
pfing	mov	rbp, DATA_PREFETCH	;; Set prefetch distance
pfing	sub	rbp, rsi
c9b:	g2cl_four_complex_unfft rsi, 64, blkdst, rcx, 64, clmblkdst
	add	dl, 256/2/clm		;; Test loop counter
	jnc	c9b			;; Iterate if necessary
	bump	rsi, -2*clm*64+2*blkdst	;; Next source pointer
	bump	rcx, -2*clm*64+2*clmblkdst;; Next dest pointer
	bump	rdi, XMM_SCD		;; Next sine/cosine pointer
	add	al, 256/4
	jnc	c9b
	bump	rcx, -8*clmblkdst+clmblkdst8
	sub	al, 1			;; Test outer loop counter
	jnz	c9b			;; Iterate if necessary
	lea	rcx, [rsi-192*2*blkdst]	;; Create prefetch pointer
	mov	rsi, scratch_area	;; Restore source pointer
	end_timer 16

;; Do inverse FFT levels 8,9
;; On input the 64-byte cache lines hold these data values:
;;	0K	4K	8K	12K
;;	2K	...
;;	16K	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0K	16K	32K	48K
;;	2K	...
;;	...
;;	14K
;;	64K	...
;;	...

;; Do 384 four_complex_unfft macros
;;	distance between fft data elements is 8K
;;	do 96 macros each processing 32 data values

	start_timer 17
	mov	rdi, sincos2		;; Load sin/cos pointer
	mov	al, 48			;; 48 iterations of 2*2*clm
c7a:	xtouch	[rcx+(clm-1)*128]	;; Preload TLB
c7b:	x4cl_four_complex_unfft rsi, 64, 2*clmblkdst, 4*clmblkdst, rdi
	xloop2	c7b, clmblkdst
	bump	rsi, -2*clmblkdst+clmblkdst8;; Next source pointer
	bump	rdi, XMM_SCD		;; Next sine/cosine pointer
	sub	al, 1			;; Test outer loop counter
	jnz	c7a			;; Iterate if necessary
	bump	rsi, -48*clmblkdst8	;; Restore source pointer
	end_timer 17

;; Do inverse FFT levels 6,7
;; On input the 64-byte cache lines hold these data values:
;;	0K	16K	32K	48K
;;	2K	...
;;	...
;;	14K
;;	64K	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0K	64K	128K	192K
;;	2K	...
;;	...
;;	62K
;;	256K	...
;;	...

;; Do 384 four_complex_unfft macros
;;	distance between fft data elements is 8K
;;	do 96 macros each processing 32 data values

	start_timer 18
	mov	rdi, sincos3		;; Load sin/cos pointer
	mov	eax, 12*256+4		;; 12 iterations of 4 iters of 2*2*clm
c5a:	xtouch	[rcx+(clm-1)*128]	;; Preload TLB
c5b:	x4cl_four_complex_unfft rsi, 64, clmblkdst8, 2*clmblkdst8, rdi
	xloop2	c5b, clmblkdst		;; 2*2*clm iterations
	sub	al, 1			;; Test middle loop counter
	jnz	c5a			;; Iterate if necessary
	bump	rsi, -4*2*clmblkdst+4*clmblkdst8;; Next source pointer
	bump	rdi, XMM_SCD		;; Next sine/cosine pointer
	sub	eax, 256		;; Test outer loop counter
	mov	al, 4			;; Load middle loop counter
	jnz	c5a			;; Iterate if necessary
	bump	rsi, -12*4*clmblkdst8	;; Restore source pointer
	end_timer 18

;; Do inverse FFT levels 4,5
;; On input the 64-byte cache lines hold these data values:
;;	0K	64K	128K	192K
;;	2K	...
;;	...
;;	62K
;;	256K	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0K	256K	512K	768K
;;	2K	...
;;	...
;;	254K
;;	1024K	...
;;	...

;; Do 384 four_complex_unfft macros
;;	distance between fft data elements is 128K
;;	do 96 macros each processing 32 data values

	start_timer 19
	mov	rdi, sincos4		;; Load sin/cos pointer
	mov	eax, 3*256+4		;; 3 iters of 4 iters of 4 of 2*2*clm
c3a:	xtouch	[rcx+(clm-1)*128]	;; Preload TLB
c3b:	x4cl_four_complex_unfft rsi, 64, 4*clmblkdst8, 8*clmblkdst8, rdi
	xloop2	c3b, clmblkdst		;; 2* 2*clm iterations
	add	eax, 80000000h/2	;; 4 iterations
	jnc	c3b			;; Iterate if necessary
	bump	rsi, -4*2*clmblkdst+clmblkdst8;; Next source pointer
	sub	al, 1			;; Test middle loop counter
	jnz	c3a			;; Iterate if necessary
	bump	rsi, -4*clmblkdst8+16*clmblkdst8;; Next source pointer
	bump	rdi, XMM_SCD		;; Next sine/cosine pointer
	sub	eax, 256		;; Test outer loop counter
	mov	al, 4			;; Load middle loop counter
	jnz	c3a			;; Iterate if necessary
	bump	rsi, -3*16*clmblkdst8	;; Restore source pointer
	end_timer 19

;; Do inverse FFT levels 1,2,3
;; On input the 64-byte cache lines hold these data values:
;;	0K	256K	512K	768K
;;	2K	...
;;	...
;;	254K
;;	1024K	1536K	2048K	2560K
;;	1026K
;;	...
;;	1534K
;; On output the 64-byte cache lines hold these data values:
;;	0K	768K	1536K	2304K
;;	...
;;	766K
;;
;; Do 512 three_complex_last_unfft macros
;;	distance between fft data elements is 512K
;;	do 128 macros each processing 24 data values

	start_timer 20
	mov	rdi, plus1_premults	;; Address of premultiplier table
	mov	eax, 16*256+4		;; 16 iters of 4 iters of 2*2*clm
c1a:	xtouch	[rcx+(clm-1)*128]	;; Preload TLB
c1b:	x3cl_three_complex_last_unfft rsi, 64, 16*clmblkdst8
	xloop2edi c1b, clmblkdst, 192 ;; 2*2*clm iterations
	sub	al, 1			;; Test loop counter
	jnz	c1a			;; Iterate if necessary
	bump	rsi, -8*clmblkdst+clmblkdst8;; Next source pointer
	sub	ah, 1			;; Test outer loop counter
	mov	al, 4			;; Load middle loop counter
	jnz	c1a
	end_timer 20

;; Normalize these values

	pass1_normalize 1, yes_fft

;; Copy data from the scratch area back to the FFT data area

	copy_scratch_data_to_fft 384
	jmp	endlp			;; Skip FFT code if flag not set

;; Fire up auxiliary threads to do pass 1 forward FFT

pass1:	pass1_forward_fft_setup b0b

;; Copy data from the FFT data area back to the scratch area

b0b:	copy_fft_data_to_scratch 384

;; Do FFT levels 1,2,3
;;
;; On input the 64-byte cache lines hold these data values:
;;	0K	768K	1536K	2304K
;;	...
;;	766K
;; On output the 64-byte cache lines hold these data values:
;;	0K	256K	512K	768K
;;	2K	...
;;	...
;;	254K
;;	1024K	1536K	2048K	2560K
;;	1026K
;;	...
;;	1534K

;; Do 512 three_complex_first_fft macros (FFT levels 1,2,3)
;;	distance between fft data elements is 512K
;;	do 128 iterations each processing 24 data values

yes_fft:start_timer 22
	mov	rsi, scratch_area	;; Get address of scratch area
	mov	rdi, plus1_premults	;; Address of premultiplier table
	mov	rcx, DATA_ADDR		;; Load source address for prefetching
pfing	mov	rbp, DATA_PREFETCH	;; Set prefetch distance
pfing	sub	rbp, rcx
pfing	bump	rcx, 208*blkdst		;; Create prefetching pointer
	mov	eax, 16			;; 16 iterations of 2 iters of 4*2*clm
	sub	rdx, rdx
b1a:	xtouch	[rcx+(clm-1)*128]	;; Preload TLB
b1b:	x3cl_three_complex_fft rsi, 64, 16*clmblkdst8
	xloop4edi b1b, clmblkdst, 192 ;; 4*2*clm iterations
	add	eax, 80000000h		;; 2 iterations
	jnc	b1a
	bump	rsi, -2*4*clmblkdst+clmblkdst8
	sub	al, 1			;; Test loop counter
	jnz	b1a			;; Iterate if necessary
	bump	rsi, -16*clmblkdst8	;; Restore source pointer
	end_timer 22

;; Do FFT levels 4,5
;;
;; On input the 64-byte cache lines hold these data values:
;;	0K	256K	512K	768K
;;	2K	...
;;	...
;;	254K
;;	1024K	1536K	2048K	2560K
;;	1026K
;;	...
;;	1534K
;; On output the 64-byte cache lines hold these data values:
;;	0K	128K	256K	384K
;;	2K	...
;;	...
;;	126K	...
;;	512K	...
;;	...

;; Do 384 four_complex_fft macros
;;	distance between fft data elements is 128K
;;	do 32 macros each processing 16 data values
;;	do 64 macros each processing 32 data values

	start_timer 23
	mov	rdi, sincos4		;; Load sin/cos pointer
	mov	al, 8			;; 8 iterations of 2 iters of 4*2*clm
b2a:	xtouch	[rcx+(clm-1)*128]	;; Preload TLB
b2b:	x2cl_four_complex_fft rsi, 64, 8*clmblkdst8
	xloop4	b2b, clmblkdst		;; 4*2*clm iterations
	add	eax, 80000000h		;; 2 iterations
	jnc	b2a
	bump	rsi, -8*clmblkdst+clmblkdst8
	sub	al, 1			;; Test outer loop counter
	jnz	b2a			;; Iterate if necessary
	bump	rsi, -8*clmblkdst8+16*clmblkdst8;; Next source pointer
	bump	rdi, XMM_SCD		;; Next sine/cosine pointer
	mov	al, 8			;; 8 iters of 4 iters of 2*2*clm
b3a:	xtouch	[rcx+(clm-1)*128]	;; Preload TLB
b3b:	x4cl_four_complex_fft rsi, 64, 8*clmblkdst8, 16*clmblkdst8, rdi
	xloop2	b3b, clmblkdst		;; 2*2*clm iterations
	add	eax, 80000000h/4*2	;; 4 iterations
	jnc	b3a
	bump	rsi, -8*clmblkdst+clmblkdst8
	sub	al, 1			;; Test outer loop counter
	jnz	b3a			;; Iterate if necessary
	bump	rsi, -8*clmblkdst8-16*clmblkdst8;; Restore src pointer
	end_timer 23

;; Do FFT levels 6,7
;;
;; On input the 64-byte cache lines hold these data values:
;;	0K	128K	256K	384K
;;	2K	...
;;	...
;;	126K	...
;;	512K	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0K	32K	64K	96K
;;	2K	...
;;	...
;;	30K	...
;;	128K	...
;;	...

;; Do 384 four_complex_fft macros
;;	distance between fft data elements is 8K
;;	do 96 macros each processing 32 data values

	start_timer 24
	mov	rdi, sincos3		;; Load sin/cos pointer
	mov	eax, 6*256+2		;; 6 iters of 2 iters of 4 of 2*2*clm
b5a:	xtouch	[rcx+(clm-1)*128]	;; Preload TLB
b5b:	x4cl_four_complex_fft rsi, 64, 2*clmblkdst8, 4*clmblkdst8, rdi
	xloop2	b5b, clmblkdst
	add	eax, 80000000h/4*2	;; 4 iterations
	jnc	b5a
	bump	rsi, -8*clmblkdst+clmblkdst8
	sub	al, 1			;; Test middle loop counter
	jnz	b5a			;; Iterate if necessary
	bump	rsi, -2*clmblkdst8+8*clmblkdst8;; Next source pointer
	bump	rdi, 2*XMM_SCD		;; Next sine/cosine pointer
	sub	eax, 256		;; Test outer loop counter
	mov	al, 2			;; Load middle loop counter
	jnz	b5a			;; Iterate if necessary
	bump	rsi, -6*8*clmblkdst8	;; Restore source pointer
	end_timer 24

;; Do FFT levels 8,9
;;
;; On input the 64-byte cache lines hold these data values:
;;	0K	32K	64K	96K
;;	2K	...
;;	...
;;	30K	...
;;	128K	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0K	8K	16K	24K
;;	2K	...
;;	...
;;	6K	...
;;	32K	...
;;	...

;; Do 384 two_two_complex_fft macros
;;	distance between fft data elements is 8K
;;	do 96 macros each processing 32 data values

	start_timer 25
	mov	rdi, sincos2		;; Load sin/cos pointer
	mov	eax, 24*256+2		;; 24 iterations of 2 iters of 2*2*clm
b7a:	xtouch	[rcx+(clm-1)*128]	;; Preload TLB
b7b:	x4cl_four_complex_fft rsi, 64, 4*clmblkdst, clmblkdst8, rdi
	xloop2	b7b, clmblkdst		;; 2*2*clm iterations
	sub	al, 1			;; Test middle loop counter
	jnz	b7a			;; Iterate if necessary
	bump	rsi, -2*2*clmblkdst+2*clmblkdst8;; Next source pointer
	bump	rdi, 2*XMM_SCD		;; Next sine/cosine pointer
	sub	eax, 256		;; Test outer loop counter
	mov	al, 2			;; Load middle loop counter
	jnz	b7a			;; Iterate if necessary
	bump	rsi, -24*2*clmblkdst8	;; Restore source pointer
pfing	bump	rcx, -384*blkdst	;; Restore prefetch pointer
	end_timer 25

;; Do FFT levels 10,11
;;
;; On input the 64-byte cache lines hold these data values:
;;	0K	8K	16K	24K
;;	2K	...
;;	...
;;	6K	...
;;	32K	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0K	2K	4K	6K
;;	8K	...
;;	...

;; Do 384 four_complex_fft macros
;;	distance between fft data elements is 2K
;;	do 96 macros each processing 32 data values

	start_timer 26
	mov	rdi, sincos1		;; Load sin/cos pointer
	mov	eax, 48*256+2		;; 48 iterations of 2 iters of 2*clm
b9b:	g4cl_four_complex_fft rsi, 64, clmblkdst, 2*clmblkdst, rcx, 64, blkdst, 2*blkdst
	L2prefetch128 [rcx-64][rbp]
	L2prefetch128 [rcx-64+blkdst][rbp]
	bump	rbp, -64+2*blkdst
	add	al, 256/2		;; Test inner loop counter
	jnc	b9b			;; Iterate if necessary
	bump	rbp, 128-4*blkdst
	IF clm NE 1
	add	dl, 256/clm
	jnc	b9b
	ENDIF
	bump	rsi, -2*clm*64+4*clmblkdst;; Next source pointer
	bump	rcx, -2*clm*64+4*blkdst	;; Next dest pointer
	bump	rdi, 2*XMM_SCD		;; Next sine/cosine pointer
	sub	al, 1
	jnz	b9b
	bump	rsi, -8*clmblkdst+clmblkdst8
	sub	ah, 1			;; Test outer loop counter
	mov	al, 2
	jnz	b9b			;; Iterate if necessary
	end_timer 26

;; Work on next 4 sets of 1536 values

endlp:	xpass1_get_next_block pass2, c0b, b0b ;; Get next block and jump
	done:	xfft_footer
	ENDM


;; Perform a 3584K-element FFT.  This is done in two passes.  Pass 1
;; does 11 levels, pass 2 uses common code to perform the last 11 levels.

hg_pass1sclevels11pfa7 MACRO fftname, pass2_levels, clmarg
	LOCAL	b0b, b1a, b1b, b2b, b3a0, b3a, b3, b3b, b4a, b4b
	LOCAL	b5a, b5b, b6b, b7a, b7b, b8b, b9b
	LOCAL	c0b, c1b, c2a, c2b, c3b, c4a, c4b, c5a, c5b
	LOCAL	c6a, c6b, c7a, c7b, c8b, c9b
	LOCAL	pass1, pass2, endlp, done, yes_fft

	set_FFT_constants fftname, 0, 0, pass2_levels, clmarg
	xfft_header pass1

;; Do the last 11 FFT levels
;;	do 1 pass2_11_levels_real and 447 pass2_11_levels_complex

pass2:	start_timer 0
	call	xpass2name
	end_timer 0
	cmp	ffttype, 1		;; We're done if FFTing only
	je	done

;; Do the inverse FFT

	pass1_inverse_fft_setup c0b

;; Do inverse FFT levels 10,11
;; On input the 64-byte cache lines hold these data values:
;;	0K	2K	4K	6K
;;	8K	10K	12K	14K
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0K	4K	8K	12K
;;	2K	...
;;	16K	...
;;	...

;; Do 1 eight_reals_unfft_2 macros
;; Do 1 nop_two_two_complex_unfft_2 macros
;;	distance between fft data elements is 2K
;;	do 1 macros each processing 16 data values

c0b:	start_timer 16
	mov	rsi, DATA_ADDR		;; Get source address
	mov	rcx, scratch_area	;; Get address of scratch area
	sub	rax, rax
	sub	rdx, rdx
pfing	mov	rbp, DATA_PREFETCH	;; Set prefetch distance
pfing	sub	rbp, rsi
c8b:	g2cl_eight_reals_unfft_2 rsi, 64, blkdst, rcx, 64, clmblkdst
	add	al, 256/2/clm		;; Test loop counter
	jnc	c8b			;; Iterate if necessary

;; Do 446 four_complex_unfft macros
;;	distance between fft data elements is 2K
;;	do 223 macros each processing 16 data values

	bump	rsi, -2*clm*64+2*blkdst	;; Next source pointer
	bump	rcx, -2*clm*64+2*clmblkdst;; Next dest pointer
	mov	rdi, sincos1		;; Load sin/cos pointer
	mov	al, 256/4+56		;; 223 iterations of 2*clm
c9b:	g2cl_four_complex_unfft rsi, 64, blkdst, rcx, 64, clmblkdst
	add	dl, 256/2/clm		;; Test loop counter
	jnc	c9b			;; Iterate if necessary
	bump	rsi, -2*clm*64+2*blkdst	;; Next source pointer
	bump	rcx, -2*clm*64+2*clmblkdst;; Next dest pointer
	bump	rdi, XMM_SCD		;; Next sine/cosine pointer
	add	al, 256/4
	jnc	c9b
	bump	rcx, -8*clmblkdst+clmblkdst8
	sub	al, 1			;; Test outer loop counter
	jnz	c9b			;; Iterate if necessary
	lea	rcx, [rsi-224*2*blkdst]	;; Create prefetch pointer
	mov	rsi, scratch_area	;; Restore source pointer
	end_timer 16

;; Do inverse FFT levels 8,9
;; On input the 64-byte cache lines hold these data values:
;;	0K	4K	8K	12K
;;	2K	...
;;	16K	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0K	16K	32K	48K
;;	2K	...
;;	...
;;	14K
;;	64K	...
;;	...

;; Do 4 eight_reals_unfft_2 macros
;; Do 4 nop_two_two_complex_unfft_2 macros
;;	distance between fft data elements is 8K
;;	do 2 macros each processing 32 data values

	start_timer 17
	mov	al, 2			;; 2 iterations of 2*clm
c6a:	xtouch	[rcx+(clm-1)*128]	;; Preload TLB
c6b:	x4cl_eight_reals_unfft_2 rsi, 64, 2*clmblkdst, 4*clmblkdst
	xloop1	c6b, clmblkdst
	sub	al, 1			;; Test outer loop counter
	jnz	c6a			;; Iterate if necessary

;; Do 440 four_complex_unfft macros
;;	distance between fft data elements is 8K
;;	do 110 macros each processing 32 data values

	bump	rsi, -2*clmblkdst+clmblkdst8;; Next source pointer
	mov	rdi, sincos2		;; Load sin/cos pointer
	mov	al, 55			;; 55 iterations of 2*2*clm
c7a:	xtouch	[rcx+(clm-1)*128]	;; Preload TLB
c7b:	x4cl_four_complex_unfft rsi, 64, 2*clmblkdst, 4*clmblkdst, rdi
	xloop2	c7b, clmblkdst
	bump	rsi, -2*clmblkdst+clmblkdst8;; Next source pointer
	bump	rdi, XMM_SCD		;; Next sine/cosine pointer
	sub	al, 1			;; Test outer loop counter
	jnz	c7a			;; Iterate if necessary
	bump	rsi, -56*clmblkdst8	;; Restore source pointer
	end_timer 17

;; Do inverse FFT levels 6,7
;; On input the 64-byte cache lines hold these data values:
;;	0K	16K	32K	48K
;;	2K	...
;;	...
;;	14K
;;	64K	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0K	64K	128K	192K
;;	2K	...
;;	...
;;	62K
;;	256K	...
;;	...

;; Do 16 eight_reals_unfft_2 macros
;; Do 16 nop_two_two_complex_unfft_2 macros
;;	distance between fft data elements is 32K
;;	do 8 macros each processing 32 data values

	start_timer 18
	mov	al, 8			;; 8 iterations of 2*clm
c4a:	xtouch	[rcx+(clm-1)*128]	;; Preload TLB
c4b:	x4cl_eight_reals_unfft_2 rsi, 64, clmblkdst8, 2*clmblkdst8
	xloop1	c4b, clmblkdst
	sub	al, 1			;; Test outer loop counter
	jnz	c4a			;; Iterate if necessary

;; Do 416 four_complex_unfft macros
;;	distance between fft data elements is 32K
;;	do 104 macros each processing 32 data values

	bump	rsi, -8*clmblkdst+4*clmblkdst8;; Next source pointer
	mov	rdi, sincos3		;; Load sin/cos pointer
	mov	eax, 13*256+4		;; 13 iterations of 4 iters of 2*2*clm
c5a:	xtouch	[rcx+(clm-1)*128]	;; Preload TLB
c5b:	x4cl_four_complex_unfft rsi, 64, clmblkdst8, 2*clmblkdst8, rdi
	xloop2	c5b, clmblkdst
	sub	al, 1			;; Test middle loop counter
	jnz	c5a			;; Iterate if necessary
	bump	rsi, -4*2*clmblkdst+4*clmblkdst8;; Next source pointer
	bump	rdi, XMM_SCD		;; Next sine/cosine pointer
	sub	eax, 256		;; Test outer loop counter
	mov	al, 4			;; Load middle loop counter
	jnz	c5a			;; Iterate if necessary
	bump	rsi, -14*4*clmblkdst8	;; Restore source pointer
	end_timer 18

;; Do inverse FFT levels 4,5
;; On input the 64-byte cache lines hold these data values:
;;	0K	64K	128K	192K
;;	2K	...
;;	...
;;	62K
;;	256K	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0K	128K	256K	384K
;;	2K	...
;;	...
;;	126K
;;	512K	768K	1024K	1280K
;;	...
;;
;; Do 64 eight_reals_unfft_2 macros
;;	distance between fft data elements is 128K
;;	do 32 macros each processing 16 data values
;;
;; Do 384 four_complex_unfft macros
;;	distance between fft data elements is 128K
;;	do 96 macros each processing 32 data values

;; Do inverse FFT levels 1,2,3
;; On input the 64-byte cache lines hold these data values:
;;	0K	128K	256K	384K
;;	2K	...
;;	...
;;	126K
;;	512K	768K	1024K	1280K
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0K	128K	1792K	1920K
;;	2K	...
;;	...
;;	126K
;;	256K	512K	2048K	2304K
;;	258K
;;	...
;;	510K
;;	768K	1280K	2560K	3072K
;;	770K
;;	...
;;	1278K
;;
;; Do 512 seven_reals_last_unfft macros
;;	distance between fft data elements is 512K
;;	do 64 macros each processing 56 data values

	start_timer 19
	mov	rdi, sincos4		;; Load sin/cos pointer
	mov	al, 4			;; 32 iterations
	xseven_reals_unfft_preload

c2a:	xtouchlines 5			;; Preload TLBs

c2b:	x2cl_half_eight_reals_unfft_2 rsi, 64, 4*clmblkdst8
	add	al, 256/2		;; Test loop counter
	jnc	c2b			;; Iterate if necessary

	bump	rsi, -2*64+8*clmblkdst8	;; Next source pointer
	mov	ah, 3			;; 3 iterations
c3b:	x4cl_four_complex_unfft rsi, 64, 4*clmblkdst8, 8*clmblkdst8, rdi
	add	al, 256/2		;; Test loop counter
	jnc	c3b			;; Iterate if necessary
	bump	rsi, -2*64+16*clmblkdst8;; Next source pointer
	bump	rdi, XMM_SCD		;; Next sine/cosine pointer
	sub	ah, 1			;; Test middle loop counter
	jnz	c3b			;; Iterate if necessary
	L2prefetch128 [rcx][rbp]
pfing	bump	rcx, blkdst		;; Next prefetch pointer
	bump	rdi, -3*XMM_SCD		;; Restore sine/cosine pointer

	bump	rsi, -3*16*clmblkdst8-8*clmblkdst8;; Restore src pointer
c1b:	x7cl_seven_reals_last_unfft rsi, 64, 8*clmblkdst8
	xloop1prefetch2sp c1b, 4*clmblkdst8
	add	ah, 256/2
	jnc	c1b

	IF clm GT 1
	bump	rsi, -2*4*clmblkdst8+2*64;; Next source pointer
	bump	rcx, -5*blkdst+128	;; Next prefetch pointer
	add	dl, 256/clm		;; clm iterations
	jnc	c2b
	bump	rsi, -clm*2*64+clmblkdst;; Next source pointer
	bump	rcx, -clm*128+5*blkdst	;; Next prefetch pointer
	ELSE
	bump	rsi, -2*4*clmblkdst8+clmblkdst;; Next source pointer
	ENDIF

	add	eax, 80000000h/8*2	;; Test loop counter
	jnc	c2a			;; Iterate if necessary
	bump	rsi, -8*clmblkdst+clmblkdst8;; Next source pointer
	sub	al, 1			;; Test loop counter
	jnz	c2a			;; Iterate if necessary
	end_timer 19

;; Normalize these values

	pass1_normalize 1, yes_fft

;; Copy data from the scratch area back to the FFT data area

	copy_scratch_data_to_fft 448
	jmp	endlp			;; Skip FFT code if flag not set

;; Fire up auxiliary threads to do pass 1 forward FFT

pass1:	pass1_forward_fft_setup b0b

;; Copy data from the FFT data area back to the scratch area

b0b:	copy_fft_data_to_scratch 448

;; Do FFT levels 1,2,3
;;
;; On input the 64-byte cache lines hold these data values:
;;	0K	128K	1792K	1920K
;;	2K	...
;;	...
;;	126K
;;	256K	512K	2048K	2304K
;;	258K
;;	...
;;	510K
;;	768K	1280K	2560K	3072K
;;	770K
;;	...
;;	1278K
;; On output the 64-byte cache lines hold these data values:
;;	0K	128K	256K	384K
;;	2K	...
;;	...
;;	126K
;;	512K	768K	1024K	1280K
;;	514K
;;	...
;;	766K
;;	1536K	2048K	2560K	3072K
;;	1538K
;;	...
;;	2046K
;;
;; Do 512 seven_reals_first_fft macros (FFT levels 1,2,3)
;;	distance between fft data elements is 512K
;;	do 64 iterations each processing 56 data values

yes_fft:start_timer 22
	mov	rsi, scratch_area	;; Get address of scratch area
	mov	rcx, DATA_ADDR		;; Load source address for prefetching
pfing	mov	rbp, DATA_PREFETCH	;; Set prefetch distance
pfing	sub	rbp, rcx
pfing	bump	rcx, 162*blkdst		;; Create prefetching pointer
	mov	eax, 8			;; 8 iterations 8 iters of 2*clm
	sub	rdx, rdx
	xseven_reals_fft_preload
b1a:	xtouch	[rcx+(clm-1)*128]	;; Preload TLB
	xtouch	[rcx+(clm-1)*128+blkdst]
b1b:	x7cl_seven_reals_fft rsi, 64, 8*clmblkdst8
	xloop1prefetch2 b1b, clmblkdst
	add	ah, 256/8
	jnc	b1a
	bump	rsi, -8*clmblkdst+clmblkdst8
	sub	al, 1			;; Test loop counter
	jnz	b1a			;; Iterate if necessary
	bump	rsi, -8*clmblkdst8	;; Restore source pointer
	end_timer 22

;; Do FFT levels 4,5
;; Values 0K-511K is real data, 512K-1023K is semi-real data, 1024K-3583K is
;; complex data.
;;
;; On input the 64-byte cache lines hold these data values:
;;	0K	128K	256K	384K
;;	2K	...
;;	...
;;	126K
;;	512K	768K	1024K	1280K
;;	514K
;;	...
;;	766K
;;	1536K	2048K	2560K	3072K
;;	1538K
;;	...
;;	2046K
;; On output the 64-byte cache lines hold these data values:
;;	0K	128K	256K	384K
;;	2K	...
;;	...
;;	126K	...
;;	512K	...
;;	...

;; Do 64 eight_reals_fft_2 macros
;;	distance between fft data elements is 128K
;;	do 64 macros each processing 8 data values

	start_timer 23
	mov	al, 8			;; 8 iterations of 8 iters of 2*clm
b2b:	x1cl_half_eight_reals_fft_2 rsi, 64
	add	dl, 256/2/clm		;; Test inner loop counter
	jnc	b2b			;; Iterate if necessary
	bump	rsi, -2*clm*64+clmblkdst;; Next source pointer
	add	ah, 256/8
	jnc	b2b
	bump	rsi, -8*clmblkdst+clmblkdst8
	sub	al, 1			;; Test outer loop counter
	jnz	b2b			;; Iterate if necessary

;; Do 128 four_complex_fft macros
;;	distance between fft data elements is 128K
;;	do 64 macros each processing 16 data values

	mov	rdi, sincos4		;; Load sin/cos pointer
	mov	al, 8			;; 8 iterations of 2 iters of 4*2*clm
b3a0:	xtouch	[rcx+(clm-1)*128]	;; Preload TLB
b3a:	x2cl_four_complex_fft rsi, 64, 8*clmblkdst8
	xloop4	b3a, clmblkdst
	add	eax, 80000000h
	jnc	b3a0
	bump	rsi, -8*clmblkdst+clmblkdst8
	sub	al, 1			;; Test outer loop counter
	jnz	b3a0			;; Iterate if necessary

;; Do 256 four_complex_fft macros
;;	distance between fft data elements is 128K
;;	do 64 macros each processing 32 data values

	bump	rsi, -8*clmblkdst8+16*clmblkdst8;; Next source pointer
	bump	rdi, XMM_SCD		;; Next sine/cosine pointer
	mov	al, 8			;; 8 iterations of 4 of 2*2*clm
b3:	xtouch	[rcx+(clm-1)*128]	;; Preload TLB
b3b:	x4cl_four_complex_fft rsi, 64, 8*clmblkdst8, 16*clmblkdst8, rdi
	xloop2	b3b, clmblkdst
	add	eax, 80000000h/4*2	;; 4 iterations
	jnc	b3
	bump	rsi, -8*clmblkdst+clmblkdst8
	sub	al, 1			;; Test outer loop counter
	jnz	b3			;; Iterate if necessary
	bump	rsi, -8*clmblkdst8-24*clmblkdst8;; Restore src pointer
	end_timer 23

;; Do FFT levels 6,7
;; Values 0K-127K is real data, 128K-255K is semi-real data, 256K-3583K is
;; complex data.
;;
;; On input the 64-byte cache lines hold these data values:
;;	0K	128K	256K	384K
;;	2K	...
;;	...
;;	126K	...
;;	512K	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0K	32K	64K	96K
;;	2K	...
;;	...
;;	30K	...
;;	128K	...
;;	...

;; Do 32 eight_reals_fft_2 macros
;; Do 32 two_two_complex_fft macros
;;	distance between fft data elements is 32K
;;	do 16 macros each processing 32 data values

	start_timer 24
	mov	rdi, sincos3		;; Load sin/cos pointer
	mov	al, 2			;; 2 iterations of 4 iters of 2*2*clm
b4a:	xtouch	[rcx+(clm-1)*128]	;; Preload TLB
b4b:	x4cl_eight_reals_fft_2 rsi, 64, 2*clmblkdst8, 4*clmblkdst8, rdi
	xloop2	b4b, clmblkdst
	add	eax, 80000000h/4*2	;; 4 iterations
	jnc	b4a
	bump	rsi, -8*clmblkdst+clmblkdst8
	sub	al, 1			;; Test outer loop counter
	jnz	b4a			;; Iterate if necessary

;; Do 384 four_complex_fft macros
;;	distance between fft data elements is 8K
;;	do 96 macros each processing 32 data values

	bump	rsi, -2*clmblkdst8+8*clmblkdst8;; Load source pointer
	bump	rdi, XMM_SCD		;; Next sine/cosine pointer
	mov	eax, 6*256+2		;; 6 iters of 2 iters of 4 of 2*2*clm
b5a:	xtouch	[rcx+(clm-1)*128]	;; Preload TLB
b5b:	x4cl_four_complex_fft rsi, 64, 2*clmblkdst8, 4*clmblkdst8, rdi
	xloop2	b5b, clmblkdst
	add	eax, 80000000h/4*2	;; 4 iterations
	jnc	b5a
	bump	rsi, -8*clmblkdst+clmblkdst8
	sub	al, 1			;; Test middle loop counter
	jnz	b5a			;; Iterate if necessary
	bump	rsi, -2*clmblkdst8+8*clmblkdst8;; Next source pointer
	bump	rdi, 2*XMM_SCD		;; Next sine/cosine pointer
	sub	eax, 256		;; Test outer loop counter
	mov	al, 2			;; Load middle loop counter
	jnz	b5a			;; Iterate if necessary
	bump	rsi, -7*8*clmblkdst8	;; Restore source pointer
	end_timer 24

;; Do FFT levels 8,9
;; Values 0K-31K is real data, 32K-63K is semi-real data, 64K-3583K is
;; complex data.
;;
;; On input the 64-byte cache lines hold these data values:
;;	0K	32K	64K	96K
;;	2K	...
;;	...
;;	30K	...
;;	128K	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0K	8K	16K	24K
;;	2K	...
;;	...
;;	6K	...
;;	32K	...
;;	...

;; Do 8 eight_reals_fft_2 macros
;; Do 8 two_two_complex_fft macros
;;	distance between fft data elements is 8K
;;	do 4 macros each processing 32 data values

	start_timer 25
	mov	rdi, sincos2		;; Load sin/cos pointer
	mov	al, 4			;; 4 iterations of 2*clm
b6b:	x4cl_eight_reals_fft_2 rsi, 64, 4*clmblkdst, clmblkdst8, rdi
	add	al, 256/2/clm		;; Test inner loop counter
	jnc	b6b			;; Iterate if necessary
	bump	rsi, -2*clm*64+clmblkdst;; Next source pointer
	sub	al, 1			;; Test outer loop counter
	jnz	b6b			;; Iterate if necessary

;; Do 432 four_complex_fft macros
;;	distance between fft data elements is 8K
;;	do 108 macros each processing 32 data values

	bump	rsi, -4*clmblkdst+2*clmblkdst8;; Load source pointer
	bump	rdi, XMM_SCD		;; Next sine/cosine pointer
	mov	eax, 27*256+2		;; 27 iterations of 2 iters of 2*2*clm
b7a:	xtouch	[rcx+(clm-1)*128]	;; Preload TLB
b7b:	x4cl_four_complex_fft rsi, 64, 4*clmblkdst, clmblkdst8, rdi
	xloop2	b7b, clmblkdst
	sub	al, 1			;; Test middle loop counter
	jnz	b7a			;; Iterate if necessary
	bump	rsi, -2*2*clmblkdst+2*clmblkdst8;; Next source pointer
	bump	rdi, 2*XMM_SCD		;; Next sine/cosine pointer
	sub	eax, 256		;; Test outer loop counter
	mov	al, 2			;; Load middle loop counter
	jnz	b7a			;; Iterate if necessary
	bump	rsi, -28*2*clmblkdst8	;; Restore source pointer
pfing	bump	rcx, -448*blkdst	;; Restore prefetching pointer
	end_timer 25

;; Do FFT levels 10,11
;; Values 0K-7K is real data, 8K-15K is semi-real data, 16K-3583K is
;; complex data.
;;
;; On input the 64-byte cache lines hold these data values:
;;	0K	8K	16K	24K
;;	2K	...
;;	...
;;	6K	...
;;	32K	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0K	2K	4K	6K
;;	8K	...
;;	...

;; Do 1 eight_reals_fft_2 macros
;; Do 1 nop_two_two_complex_fft_2 macros
;; Do 2 four_complex_fft macros
;;	distance between fft data elements is 2K
;;	do 1 macros each processing 32 data values

	start_timer 26
	mov	rdi, sincos1		;; Load sin/cos pointer
b8b:	g4cl_eight_reals_fft_2 rsi, 64, clmblkdst, 2*clmblkdst, rcx, 64, blkdst, 2*blkdst
	add	al, 256/2/clm		;; Test inner loop counter
	jnc	b8b			;; Iterate if necessary

;; Do 444 four_complex_fft macros
;;	distance between fft data elements is 2K
;;	do 111 macros each processing 32 data values

	bump	rsi, -2*clm*64+4*clmblkdst;; Next source pointer
	bump	rcx, -2*clm*64+4*blkdst	;; Next dest pointer
	bump	rdi, XMM_SCD		;; Next sine/cosine pointer
	mov	eax, 56*256+1		;; 111 iterations of 2*clm
b9b:	g4cl_four_complex_fft rsi, 64, clmblkdst, 2*clmblkdst, rcx, 64, blkdst, 2*blkdst
	L2prefetch128 [rcx-64][rbp]
	L2prefetch128 [rcx-64+blkdst][rbp]
	bump	rbp, -64+2*blkdst
	add	al, 256/2		;; Test inner loop counter
	jnc	b9b			;; Iterate if necessary
	bump	rbp, 128-4*blkdst
	IF clm NE 1
	add	dl, 256/clm
	jnc	b9b
	ENDIF
	bump	rsi, -2*clm*64+4*clmblkdst;; Next source pointer
	bump	rcx, -2*clm*64+4*blkdst	;; Next dest pointer
	bump	rdi, 2*XMM_SCD		;; Next sine/cosine pointer
	sub	al, 1
	jnz	b9b
	bump	rsi, -8*clmblkdst+clmblkdst8
	sub	ah, 1			;; Test outer loop counter
	mov	al, 2
	jnz	b9b			;; Iterate if necessary
	end_timer 26

;; Work on next 4 sets of 1792 values

endlp:	xpass1_get_next_block pass2, c0b, b0b ;; Get next block and jump
	done:	xfft_footer
	ENDM


;; Perform a 4M-element FFT.  This is done in two passes.  Pass 1
;; does 11 levels, pass 2 uses common code to perform the last 11 levels.

hg_pass1sclevels11pfa8 MACRO fftname, pass2_levels, clmarg
	LOCAL	b0b, b1a, b1b, b2a, b2b, b3a, b3b, b4a, b4b
	LOCAL	b5a, b5b, b6a, b6b, b7a, b7b, b8b, b9b
	LOCAL	c0b, c1b, c2a, c2b, c3b, c4a, c4b, c5a, c5b
	LOCAL	c6b, c7a, c7b, c8b, c9b
	LOCAL	pass1, pass2, endlp, done, yes_fft

	set_FFT_constants fftname, 0, 0, pass2_levels, clmarg
	xfft_header pass1

;; Do the last 11 FFT levels
;;	do 1 pass2_11_levels_real and 511 pass2_11_levels_complex

pass2:	start_timer 0
	call	xpass2name
	end_timer 0
	cmp	ffttype, 1		;; We're done if FFTing only
	je	done

;; Do the inverse FFT

	pass1_inverse_fft_setup c0b

;; Do inverse FFT levels 10,11
;; On input the 64-byte cache lines hold these data values:
;;	0K	2K	4K	6K
;;	8K	10K	12K	14K
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0K	4K	8K	12K
;;	2K	...
;;	16K	...
;;	...

;; Do 1 eight_reals_unfft_2 macros
;; Do 1 nop_two_two_complex_unfft_2 macros
;;	distance between fft data elements is 2K
;;	do 1 macros each processing 16 data values

c0b:	start_timer 16
	mov	rsi, DATA_ADDR		;; Get source address
	mov	rcx, scratch_area	;; Get address of scratch area
	sub	rax, rax
	sub	rdx, rdx
pfing	mov	rbp, DATA_PREFETCH	;; Set prefetch distance
pfing	sub	rbp, rsi
c8b:	g2cl_eight_reals_unfft_2 rsi, 64, blkdst, rcx, 64, clmblkdst
	add	al, 256/2/clm		;; Test loop counter
	jnc	c8b			;; Iterate if necessary

;; Do 510 four_complex_unfft macros
;;	distance between fft data elements is 2K
;;	do 255 macros each processing 16 data values

	bump	rsi, -2*clm*64+2*blkdst	;; Next source pointer
	bump	rcx, -2*clm*64+2*clmblkdst;; Next dest pointer
	mov	rdi, sincos1		;; Load sin/cos pointer
	mov	eax, 64*256+256/4	;; 255 iterations of 2*clm
c9b:	g2cl_four_complex_unfft rsi, 64, blkdst, rcx, 64, clmblkdst
	add	dl, 256/2/clm		;; Test loop counter
	jnc	c9b			;; Iterate if necessary
	bump	rsi, -2*clm*64+2*blkdst	;; Next source pointer
	bump	rcx, -2*clm*64+2*clmblkdst;; Next dest pointer
	bump	rdi, XMM_SCD		;; Next sine/cosine pointer
	add	al, 256/4
	jnc	c9b
	bump	rcx, -8*clmblkdst+clmblkdst8
	sub	ah, 1			;; Test outer loop counter
	jnz	c9b			;; Iterate if necessary
	lea	rcx, [rsi-256*2*blkdst]	;; Create prefetch pointer
	mov	rsi, scratch_area	;; Restore source pointer
	end_timer 16

;; Do inverse FFT levels 8,9
;; On input the 64-byte cache lines hold these data values:
;;	0K	4K	8K	12K
;;	2K	...
;;	16K	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0K	16K	32K	48K
;;	2K	...
;;	...
;;	14K
;;	64K	...
;;	...

;; Do 4 eight_reals_unfft_2 macros
;; Do 4 nop_two_two_complex_unfft_2 macros
;;	distance between fft data elements is 8K
;;	do 2 macros each processing 32 data values

	start_timer 17
	xtouch	[rcx+(clm-1)*128]	;; Preload TLB
c6b:	x4cl_eight_reals_unfft_2 rsi, 64, 2*clmblkdst, 4*clmblkdst
	xloop2	c6b, clmblkdst		;; 2*2*clm iterations

;; Do 504 four_complex_unfft macros
;;	distance between fft data elements is 8K
;;	do 126 macros each processing 32 data values

	bump	rsi, -2*clmblkdst+clmblkdst8;; Next source pointer
	mov	rdi, sincos2		;; Load sin/cos pointer
	mov	al, 63			;; 63 iterations of 2*2*clm
c7a:	xtouch	[rcx+(clm-1)*128]	;; Preload TLB
c7b:	x4cl_four_complex_unfft rsi, 64, 2*clmblkdst, 4*clmblkdst, rdi
	xloop2	c7b, clmblkdst
	bump	rsi, -2*clmblkdst+clmblkdst8;; Next source pointer
	bump	rdi, XMM_SCD		;; Next sine/cosine pointer
	sub	al, 1			;; Test outer loop counter
	jnz	c7a			;; Iterate if necessary
	bump	rsi, -64*clmblkdst8	;; Restore source pointer
	end_timer 17

;; Do inverse FFT levels 6,7
;; On input the 64-byte cache lines hold these data values:
;;	0K	16K	32K	48K
;;	2K	...
;;	...
;;	14K
;;	64K	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0K	64K	128K	192K
;;	2K	...
;;	...
;;	62K
;;	256K	...
;;	...

;; Do 16 eight_reals_unfft_2 macros
;; Do 16 nop_two_two_complex_unfft_2 macros
;;	distance between fft data elements is 32K
;;	do 8 macros each processing 32 data values

	start_timer 18
	mov	al, 4			;; 4 iterations of 2*2*clm
c4a:	xtouch	[rcx+(clm-1)*128]	;; Preload TLB
c4b:	x4cl_eight_reals_unfft_2 rsi, 64, clmblkdst8, 2*clmblkdst8
	xloop2	c4b, clmblkdst
	sub	al, 1			;; Test outer loop counter
	jnz	c4a			;; Iterate if necessary

;; Do 480 four_complex_unfft macros
;;	distance between fft data elements is 32K
;;	do 120 macros each processing 32 data values

	bump	rsi, -8*clmblkdst+4*clmblkdst8;; Next source pointer
	mov	rdi, sincos3		;; Load sin/cos pointer
	mov	eax, 15*256+4		;; 15 iterations of 4 iters of 2*2*clm
c5a:	xtouch	[rcx+(clm-1)*128]	;; Preload TLB
c5b:	x4cl_four_complex_unfft rsi, 64, clmblkdst8, 2*clmblkdst8, rdi
	xloop2	c5b, clmblkdst
	sub	al, 1			;; Test middle loop counter
	jnz	c5a			;; Iterate if necessary
	bump	rsi, -4*2*clmblkdst+4*clmblkdst8;; Next source pointer
	bump	rdi, XMM_SCD		;; Next sine/cosine pointer
	sub	eax, 256		;; Test outer loop counter
	mov	al, 4			;; Load middle loop counter
	jnz	c5a			;; Iterate if necessary
	bump	rsi, -16*4*clmblkdst8	;; Restore source pointer
	end_timer 18

;; Do inverse FFT levels 4,5
;; On input the 64-byte cache lines hold these data values:
;;	0K	64K	128K	192K
;;	2K	...
;;	...
;;	62K
;;	256K	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0K	256K	512K	768K
;;	2K	...
;;	...
;;	254K
;;	1024K	...
;;	...
;;
;; Do 64 eight_reals_unfft_2 macros
;; Do 64 nop_two_two_complex_unfft_2 macros
;;	distance between fft data elements is 128K
;;	do 32 macros each processing 32 data values
;;
;; Do 384 four_complex_unfft macros
;;	distance between fft data elements is 128K
;;	do 96 macros each processing 32 data values

;; Do inverse FFT levels 1,2,3
;; On input the 64-byte cache lines hold these data values:
;;	0K	256K	512K	768K
;;	2K	...
;;	...
;;	254K
;;	1024K	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0K	1024K	2048K	3072K
;;	2K	...
;;	...
;;	1022K
;;
;; Do 512 eight_reals_last_unfft macros
;;	distance between fft data elements is 512K
;;	do 128 macros each processing 32 data values

	start_timer 19
	mov	rdi, sincos4		;; Load sin/cos pointer
	mov	al, 4			;; 32 iterations

c2a:	xtouchlines 4			;; Preload TLBs

c2b:	x4cl_eight_reals_unfft_2 rsi, 64, 4*clmblkdst8, 8*clmblkdst8
	add	al, 256/2
	jnc	c2b
	L2prefetch128 [rcx][rbp]
pfing	bump	rcx, blkdst		;; Next prefetch pointer

	bump	rsi, -2*64+16*clmblkdst8;; Next source pointer
	mov	ah, 3			;; 3 iterations
c3b:	x4cl_four_complex_unfft rsi, 64, 4*clmblkdst8, 8*clmblkdst8, rdi
	add	al, 256/2		;; Test loop counter
	jnc	c3b			;; Iterate if necessary
	bump	rsi, -2*64+16*clmblkdst8;; Next source pointer
	bump	rdi, XMM_SCD		;; Next sine/cosine pointer
	sub	ah, 1			;; Test middle loop counter
	jnz	c3b			;; Iterate if necessary
	L2prefetch128 [rcx][rbp]
pfing	bump	rcx, blkdst		;; Next prefetch pointer
	bump	rdi, -3*XMM_SCD		;; Restore sine/cosine pointer

	bump	rsi, -4*16*clmblkdst8	;; Restore source pointer
c1b:	x4cl_eight_reals_last_unfft rsi, 64, 16*clmblkdst8, 32*clmblkdst8
	add	al, 256/2
	jnc	c1b
	bump	rsi, -2*64+4*clmblkdst8
	add	ah, 256/2
	jnc	c1b
	L2prefetch128 [rcx][rbp]
pfing	bump	rcx, blkdst		;; Next prefetch pointer
	add	edx, 80000000h
	jnc	c1b

	IF clm GT 1
	bump	rsi, -4*4*clmblkdst8+2*64;; Next source pointer
	bump	rcx, -4*blkdst+128	;; Next prefetch pointer
	add	dl, 256/clm		;; clm iterations
	jnc	c2b
	bump	rsi, -clm*2*64+clmblkdst;; Next source pointer
	bump	rcx, -clm*128+4*blkdst	;; Next prefetch pointer
	ELSE
	bump	rsi, -4*4*clmblkdst8+clmblkdst;; Next source pointer
	ENDIF

	add	eax, 80000000h/8*2	;; Test loop counter
	jnc	c2a			;; Iterate if necessary
	bump	rsi, -8*clmblkdst+clmblkdst8;; Next source pointer
	sub	al, 1			;; Test loop counter
	jnz	c2a			;; Iterate if necessary
	end_timer 19

;; Normalize these values

	pass1_normalize 1, yes_fft

;; Copy data from the scratch area back to the FFT data area

	copy_scratch_data_to_fft 512
	jmp	endlp			;; Skip FFT code if flag not set

;; Fire up auxiliary threads to do pass 1 forward FFT

pass1:	pass1_forward_fft_setup b0b

;; Copy data from the FFT data area back to the scratch area

b0b:	copy_fft_data_to_scratch 512

;; Do FFT levels 1,2,3
;;
;; On input the 64-byte cache lines hold these data values:
;;	0K	1024K	2048K	3072K
;;	2K	...
;;	...
;;	1022K
;; On output the 64-byte cache lines hold these data values:
;;	0K	512K	1024K	1536K
;;	2K	...
;;	...
;;	510K	...
;;	2048K	...
;;	...
;;
;; Do 512 eight_reals_first_fft macros (FFT levels 1,2,3)
;;	distance between fft data elements is 512K
;;	do 256 iterations each processing 16 data values

yes_fft:start_timer 22
	mov	rsi, scratch_area	;; Get address of scratch area
	mov	rcx, DATA_ADDR		;; Load source address for prefetching
pfing	mov	rbp, DATA_PREFETCH	;; Set prefetch distance
pfing	sub	rbp, rcx
pfing	bump	rcx, 256*blkdst		;; Create prefetching pointer
	mov	eax, 32*256		;; 32 iterations of 2 iters of 4*2*clm
	sub	rdx, rdx
b1a:	xtouch	[rcx+(clm-1)*128]	;; Preload TLB
b1b:	x2cl_eight_reals_fft rsi, 64, 32*clmblkdst8
	xloop4	b1b, clmblkdst
	add	eax, 80000000h
	jnc	b1a
	bump	rsi, -8*clmblkdst+clmblkdst8
	sub	eax, 256		;; Test loop counter
	jnz	b1a			;; Iterate if necessary
	bump	rsi, -32*clmblkdst8	;; Restore source pointer
	end_timer 22

;; Do FFT levels 4,5
;; Values 0K-511K is real data, 512K-1023K is semi-real data, 1024K-4095K is
;; complex data.
;;
;; On input the 64-byte cache lines hold these data values:
;;	0K	512K	1024K	1536K
;;	2K	...
;;	...
;;	510K	...
;;	2048K	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0K	128K	256K	384K
;;	2K	...
;;	...
;;	126K	...
;;	512K	...
;;	...

;; Do 64 eight_reals_fft_2 macros
;; Do 64 nop_two_two_complex_fft_2 macros
;; Do 128 four_complex_fft macros
;;	distance between fft data elements is 128K
;;	do 64 macros each processing 32 data values

	start_timer 23
	mov	rdi, sincos4		;; Load sin/cos pointer
	mov	al, 8			;; 8 iterations of 4 iters of 2*2*clm
b2a:	xtouch	[rcx+(clm-1)*128]	;; Preload TLB
b2b:	x4cl_eight_reals_fft_2 rsi, 64, 8*clmblkdst8, 16*clmblkdst8, rdi
	xloop2	b2b, clmblkdst
	add	eax, 80000000h/4*2	;; 4 iterations
	jnc	b2a
	bump	rsi, -8*clmblkdst+clmblkdst8
	sub	al, 1			;; Test outer loop counter
	jnz	b2a			;; Iterate if necessary

;; Do 256 four_complex_fft macros
;;	distance between fft data elements is 128K
;;	do 64 macros each processing 32 data values

	bump	rsi, -8*clmblkdst8+32*clmblkdst8;; Next source pointer
	bump	rdi, XMM_SCD		;; Next sine/cosine pointer
	mov	al, 8			;; 8 iterations of 4 iters of 2*2*clm
b3a:	xtouch	[rcx+(clm-1)*128]	;; Preload TLB
b3b:	x4cl_four_complex_fft rsi, 64, 8*clmblkdst8, 16*clmblkdst8, rdi
	xloop2	b3b, clmblkdst
	add	eax, 80000000h/4*2	;; 4 iterations
	jnc	b3a
	bump	rsi, -8*clmblkdst+clmblkdst8
	sub	al, 1			;; Test outer loop counter
	jnz	b3a			;; Iterate if necessary
	bump	rsi, -8*clmblkdst8-32*clmblkdst8;; Restore source pointer
	end_timer 23

;; Do FFT levels 6,7
;; Values 0K-127K is real data, 128K-255K is semi-real data, 256K-4095K is
;; complex data.
;;
;; On input the 64-byte cache lines hold these data values:
;;	0K	128K	256K	384K
;;	2K	...
;;	...
;;	126K	...
;;	512K	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0K	32K	64K	96K
;;	2K	...
;;	...
;;	30K	...
;;	128K	...
;;	...

;; Do 32 eight_reals_fft_2 macros
;; Do 32 two_two_complex_fft macros
;;	distance between fft data elements is 32K
;;	do 16 macros each processing 32 data values

	start_timer 24
	mov	rdi, sincos3		;; Load sin/cos pointer
	mov	al, 2			;; 2 iterations of 4 iters of 2*2*clm
b4a:	xtouch	[rcx+(clm-1)*128]	;; Preload TLB
b4b:	x4cl_eight_reals_fft_2 rsi, 64, 2*clmblkdst8, 4*clmblkdst8, rdi
	xloop2	b4b, clmblkdst
	add	eax, 80000000h/4*2	;; 4 iterations
	jnc	b4a
	bump	rsi, -8*clmblkdst+clmblkdst8
	sub	al, 1			;; Test outer loop counter
	jnz	b4a			;; Iterate if necessary

;; Do 448 four_complex_fft macros
;;	distance between fft data elements is 8K
;;	do 112 macros each processing 32 data values

	bump	rsi, -2*clmblkdst8+8*clmblkdst8;; Load source pointer
	bump	rdi, XMM_SCD		;; Next sine/cosine pointer
	mov	eax, 7*256+2		;; 7 iters of 2 iters of 4 of 2*2*clm
b5a:	xtouch	[rcx+(clm-1)*128]	;; Preload TLB
b5b:	x4cl_four_complex_fft rsi, 64, 2*clmblkdst8, 4*clmblkdst8, rdi
	xloop2	b5b, clmblkdst
	add	eax, 80000000h/4*2	;; 4 iterations
	jnc	b5a
	bump	rsi, -8*clmblkdst+clmblkdst8
	sub	al, 1			;; Test middle loop counter
	jnz	b5a			;; Iterate if necessary
	bump	rsi, -2*clmblkdst8+8*clmblkdst8;; Next source pointer
	bump	rdi, 2*XMM_SCD		;; Next sine/cosine pointer
	sub	eax, 256		;; Test outer loop counter
	mov	al, 2			;; Load middle loop counter
	jnz	b5a			;; Iterate if necessary
	bump	rsi, -8*8*clmblkdst8	;; Restore source pointer
	end_timer 24

;; Do FFT levels 8,9
;; Values 0K-31K is real data, 32K-63K is semi-real data, 64K-4095K is
;; complex data.
;;
;; On input the 64-byte cache lines hold these data values:
;;	0K	32K	64K	96K
;;	2K	...
;;	...
;;	30K	...
;;	128K	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0K	8K	16K	24K
;;	2K	...
;;	...
;;	6K	...
;;	32K	...
;;	...

;; Do 8 eight_reals_fft_2 macros
;; Do 8 two_two_complex_fft macros
;;	distance between fft data elements is 8K
;;	do 4 macros each processing 32 data values

	start_timer 25
	mov	rdi, sincos2		;; Load sin/cos pointer
	mov	al, 2			;; 2 iterations of 2*2*clm
b6a:	xtouch	[rcx+(clm-1)*128]	;; Preload TLB
b6b:	x4cl_eight_reals_fft_2 rsi, 64, 4*clmblkdst, clmblkdst8, rdi
	xloop2	b6b, clmblkdst
	sub	al, 1			;; Test outer loop counter
	jnz	b6a			;; Iterate if necessary

;; Do 496 four_complex_fft macros
;;	distance between fft data elements is 8K
;;	do 124 macros each processing 32 data values

	bump	rsi, -2*2*clmblkdst+2*clmblkdst8;; Load source pointer
	bump	rdi, XMM_SCD		;; Next sine/cosine pointer
	mov	eax, 31*256+2		;; 31 iterations of 2 iters of 2*2*clm
b7a:	xtouch	[rcx+(clm-1)*128]	;; Preload TLB
b7b:	x4cl_four_complex_fft rsi, 64, 4*clmblkdst, clmblkdst8, rdi
	xloop2	b7b, clmblkdst
	sub	al, 1			;; Test middle loop counter
	jnz	b7a			;; Iterate if necessary
	bump	rsi, -2*2*clmblkdst+2*clmblkdst8;; Next source pointer
	bump	rdi, 2*XMM_SCD		;; Next sine/cosine pointer
	sub	eax, 256		;; Test outer loop counter
	mov	al, 2			;; Load middle loop counter
	jnz	b7a			;; Iterate if necessary
	bump	rsi, -32*2*clmblkdst8	;; Restore source pointer
pfing	bump	rcx, -512*blkdst	;; Restore prefetch pointer
	end_timer 25

;; Do FFT levels 10,11
;; Values 0K-7K is real data, 8K-15K is semi-real data, 16K-4095K is
;; complex data.
;;
;; On input the 64-byte cache lines hold these data values:
;;	0K	8K	16K	24K
;;	2K	...
;;	...
;;	6K	...
;;	32K	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0K	2K	4K	6K
;;	8K	...
;;	...

;; Do 1 eight_reals_fft_2 macros
;; Do 1 nop_two_two_complex_fft_2 macros
;; Do 2 four_complex_fft macros
;;	distance between fft data elements is 2K
;;	do 1 macros each processing 32 data values

	start_timer 26
	mov	rdi, sincos1		;; Load sin/cos pointer
b8b:	g4cl_eight_reals_fft_2 rsi, 64, clmblkdst, 2*clmblkdst, rcx, 64, blkdst, 2*blkdst
	add	al, 256/2/clm		;; Test inner loop counter
	jnc	b8b			;; Iterate if necessary

;; Do 508 four_complex_fft macros
;;	distance between fft data elements is 2K
;;	do 127 macros each processing 32 data values

	bump	rsi, -2*clm*64+4*clmblkdst;; Next source pointer
	bump	rcx, -2*clm*64+4*blkdst	;; Next dest pointer
	bump	rdi, XMM_SCD		;; Next sine/cosine pointer
	mov	eax, 64*256+1		;; 127 iterations of 2*clm
b9b:	g4cl_four_complex_fft rsi, 64, clmblkdst, 2*clmblkdst, rcx, 64, blkdst, 2*blkdst
	L2prefetch128 [rcx-64][rbp]
	L2prefetch128 [rcx-64+blkdst][rbp]
	bump	rbp, -64+2*blkdst
	add	al, 256/2		;; Test inner loop counter
	jnc	b9b			;; Iterate if necessary
	bump	rbp, 128-4*blkdst
	IF clm NE 1
	add	dl, 256/clm
	jnc	b9b
	ENDIF
	bump	rsi, -2*clm*64+4*clmblkdst;; Next source pointer
	bump	rcx, -2*clm*64+4*blkdst	;; Next dest pointer
	bump	rdi, 2*XMM_SCD		;; Next sine/cosine pointer
	sub	al, 1
	jnz	b9b
	bump	rsi, -8*clmblkdst+clmblkdst8
	sub	ah, 1			;; Test outer loop counter
	mov	al, 2
	jnz	b9b			;; Iterate if necessary
	end_timer 26

;; Work on next 4 sets of 2048 values

endlp:	xpass1_get_next_block pass2, c0b, b0b ;; Get next block and jump
	done:	xfft_footer
	ENDM


;; Perform a 4M-element negacyclic FFT.  This is done in two passes.  Pass 1
;; does 11 levels, pass 2 uses common code to perform the last 11 levels.

hg_pass1sclevels11complex4 MACRO fftname, pass2_levels, clmarg
	LOCAL	b0b, b1a, b1b, b3a, b3b, b5a, b5b, b7a, b7b, b9b
	LOCAL	c0b, c1a, c1b, c3a, c3b, c5a, c5b, c7a, c7b, c9b
	LOCAL	pass1, pass2, endlp, done, yes_fft

	set_FFT_constants fftname, 0, 1, pass2_levels, clmarg
	xfft_header pass1

;; Do the last 11 FFT levels
;;	do 512 pass2_11_levels_complex

pass2:	start_timer 0
	call	xpass2name
	end_timer 0
	cmp	ffttype, 1		;; We're done if FFTing only
	je	done

;; Do the inverse FFT

	pass1_inverse_fft_setup c0b

;; Do inverse FFT levels 10,11
;; On input the 64-byte cache lines hold these data values:
;;	0K	2K	4K	6K
;;	8K	10K	12K	14K
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0K	4K	8K	12K
;;	2K	...
;;	16K	...
;;	...

;; Do 512 four_complex_unfft macros
;;	distance between fft data elements is 2K
;;	do 256 macros each processing 16 data values

c0b:	start_timer 16
	mov	rsi, DATA_ADDR		;; Get source address
	mov	rcx, scratch_area	;; Get address of scratch area
	mov	rdi, sincos1		;; Load sin/cos pointer
	mov	eax, 64*256		;; 256 iterations of 2*clm
	sub	rdx, rdx
pfing	mov	rbp, DATA_PREFETCH	;; Set prefetch distance
pfing	sub	rbp, rsi
c9b:	g2cl_four_complex_unfft rsi, 64, blkdst, rcx, 64, clmblkdst
	add	dl, 256/2/clm		;; Test loop counter
	jnc	c9b			;; Iterate if necessary
	bump	rsi, -2*clm*64+2*blkdst	;; Next source pointer
	bump	rcx, -2*clm*64+2*clmblkdst;; Next dest pointer
	bump	rdi, XMM_SCD		;; Next sine/cosine pointer
	add	al, 256/4
	jnc	c9b
	bump	rcx, -8*clmblkdst+clmblkdst8
	sub	ah, 1			;; Test outer loop counter
	jnz	c9b			;; Iterate if necessary
	lea	rcx, [rsi-256*2*blkdst]	;; Create prefetch pointer
	mov	rsi, scratch_area	;; Restore source pointer
	end_timer 16

;; Do inverse FFT levels 8,9
;; On input the 64-byte cache lines hold these data values:
;;	0K	4K	8K	12K
;;	2K	...
;;	16K	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0K	16K	32K	48K
;;	2K	...
;;	...
;;	14K
;;	64K	...
;;	...

;; Do 512 four_complex_unfft macros
;;	distance between fft data elements is 8K
;;	do 128 macros each processing 32 data values

	start_timer 17
	mov	rdi, sincos2		;; Load sin/cos pointer
	mov	eax, 64*256		;; 64 iterations of 2*2*clm
c7a:	xtouch	[rcx+(clm-1)*128]	;; Preload TLB
c7b:	x4cl_four_complex_unfft rsi, 64, 2*clmblkdst, 4*clmblkdst, rdi
	xloop2	c7b, clmblkdst		;; 2 * 2*clm iterations
	bump	rsi, -2*clmblkdst+clmblkdst8;; Next source pointer
	bump	rdi, XMM_SCD		;; Next sine/cosine pointer
	sub	eax, 256		;; Test outer loop counter
	jnz	c7a			;; Iterate if necessary
	bump	rsi, -64*clmblkdst8	;; Restore source pointer
	end_timer 17

;; Do inverse FFT levels 6,7
;; On input the 64-byte cache lines hold these data values:
;;	0K	16K	32K	48K
;;	2K	...
;;	...
;;	14K
;;	64K	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0K	64K	128K	192K
;;	2K	...
;;	...
;;	62K
;;	256K	...
;;	...

;; Do 512 four_complex_unfft macros
;;	distance between fft data elements is 8K
;;	do 128 macros each processing 32 data values

	start_timer 18
	mov	rdi, sincos3		;; Load sin/cos pointer
	mov	eax, 16*256+4		;; 16 iterations of 4 iters of 2*2*clm
c5a:	xtouch	[rcx+(clm-1)*128]	;; Preload TLB
c5b:	x4cl_four_complex_unfft rsi, 64, clmblkdst8, 2*clmblkdst8, rdi
	xloop2	c5b, clmblkdst		;; 2 * 2*clm iterations
	sub	al, 1			;; Test middle loop counter
	jnz	c5a			;; Iterate if necessary
	bump	rsi, -4*2*clmblkdst+4*clmblkdst8;; Next source pointer
	bump	rdi, XMM_SCD		;; Next sine/cosine pointer
	sub	eax, 256		;; Test outer loop counter
	mov	al, 4			;; Load middle loop counter
	jnz	c5a			;; Iterate if necessary
	bump	rsi, -16*4*clmblkdst8	;; Restore source pointer
	end_timer 18

;; Do inverse FFT levels 4,5
;; On input the 64-byte cache lines hold these data values:
;;	0K	64K	128K	192K
;;	2K	...
;;	...
;;	62K
;;	256K	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0K	256K	512K	768K
;;	2K	...
;;	...
;;	254K
;;	1024K	...
;;	...

;; Do 512 four_complex_unfft macros
;;	distance between fft data elements is 128K
;;	do 128 macros each processing 32 data values

	start_timer 19
	mov	rdi, sincos4		;; Load sin/cos pointer
	mov	eax, 4*256+4		;; 4 iters of 4 iters 4 of 2*2*clm
c3a:	xtouch	[rcx+(clm-1)*128]	;; Preload TLB
c3b:	x4cl_four_complex_unfft rsi, 64, 4*clmblkdst8, 8*clmblkdst8, rdi
	xloop2	c3b, clmblkdst		;; 2* 2*clm iterations
	add	eax, 80000000h/2	;; 4 iterations
	jnc	c3a			;; Iterate if necessary
	bump	rsi, -4*2*clmblkdst+clmblkdst8;; Next source pointer
	sub	al, 1			;; Test middle loop counter
	jnz	c3a
	bump	rsi, -4*clmblkdst8+16*clmblkdst8;; Next source pointer
	bump	rdi, XMM_SCD		;; Next sine/cosine pointer
	sub	eax, 256		;; Test outer loop counter
	mov	al, 4			;; Load middle loop counter
	jnz	c3a			;; Iterate if necessary
	bump	rsi, -4*16*clmblkdst8	;; Restore source pointer
	end_timer 19

;; Do inverse FFT levels 1,2,3
;; On input the 64-byte cache lines hold these data values:
;;	0K	256K	512K	768K
;;	2K	...
;;	...
;;	254K
;;	1024K	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0K	1024K	2048K	3072K
;;	2K	...
;;	...
;;	1022K

;; Do 512 four_complex_last_unfft macros
;;	distance between fft data elements is 512K
;;	do 128 macros each processing 32 data values

	start_timer 20
	mov	rdi, plus1_premults	;; Address of premultiplier table
	mov	eax, 16*256+4		;; 16 iters of 4 iters of 2*2*clm
c1a:	xtouch	[rcx+(clm-1)*128]	;; Preload TLB
c1b:	x4cl_four_complex_last_unfft rsi, 64, 16*clmblkdst8, 32*clmblkdst8, 128*128
	xloop2edi c1b, clmblkdst, 128 ;; 2 * 2*clm iterations
	sub	al, 1			;; Test loop counter
	jnz	c1a			;; Iterate if necessaryxn
	bump	rsi, -4*2*clmblkdst+clmblkdst8;; Next source pointer
	sub	ah, 1			;; Test outer loop counter
	mov	al, 4			;; Load middle loop counter
	jnz	c1a
	end_timer 20

;; Normalize these values

	pass1_normalize 1, yes_fft

;; Copy data from the scratch area back to the FFT data area

	copy_scratch_data_to_fft 512
	jmp	endlp			;; Skip FFT code if flag not set

;; Fire up auxiliary threads to do pass 1 forward FFT

pass1:	pass1_forward_fft_setup b0b

;; Copy data from the FFT data area back to the scratch area

b0b:	copy_fft_data_to_scratch 512

;; Do FFT levels 1,2,3
;;
;; On input the 64-byte cache lines hold these data values:
;;	0K	1024K	2048K	3072K
;;	2K	...
;;	...
;;	1022K
;; On output the 64-byte cache lines hold these data values:
;;	0K	512K	1024K	1536K
;;	2K	...
;;	...
;;	510K	...
;;	2048K	...
;;	...

;; Do 512 four_complex_first_fft macros (FFT levels 1,2,3)
;;	distance between fft data elements is 512K
;;	do 256 iterations each processing 16 data values

yes_fft:start_timer 22
	mov	rsi, scratch_area	;; Get address of scratch area
	mov	rdi, plus1_premults	;; Address of premultiplier table
	mov	rcx, DATA_ADDR		;; Load source address for prefetching
pfing	mov	rbp, DATA_PREFETCH	;; Set prefetch distance
pfing	sub	rbp, rcx
pfing	bump	rcx, 256*blkdst		;; Create prefetching pointer
	mov	eax, 32*256		;; 32 iters of 2 iters of 4*2*clm
	sub	rdx, rdx
b1a:	xtouch	[rcx+(clm-1)*128]	;; Preload TLB
b1b:	x2cl_four_complex_first_fft_scratch rsi, 64, 32*clmblkdst8
	xloop4edi b1b, clmblkdst, 128 ;; 4 * 2*clm iterations
	add	eax, 80000000h		;; 2 iterations
	jnc	b1a
	bump	rsi, -2*4*clmblkdst+clmblkdst8
	sub	eax, 256		;; Test loop counter
	jnz	b1a			;; Iterate if necessary
	bump	rsi, -32*clmblkdst8	;; Restore source pointer
	end_timer 22

;; Do FFT levels 4,5
;;
;; On input the 64-byte cache lines hold these data values:
;;	0K	512K	1024K	1536K
;;	2K	...
;;	...
;;	510K	...
;;	2048K	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0K	128K	256K	384K
;;	2K	...
;;	...
;;	126K	...
;;	512K	...
;;	...

;; Do 512 four_complex_fft macros
;;	distance between fft data elements is 128K
;;	do 128 macros each processing 32 data values

	start_timer 23
	mov	rdi, sincos4		;; Load sin/cos pointer
	mov	eax, 2*256+8		;; 2 iters of 8 iters of 4 of 2*2*clm
b3a:	xtouch	[rcx+(clm-1)*128]	;; Preload TLB
b3b:	x4cl_four_complex_fft rsi, 64, 8*clmblkdst8, 16*clmblkdst8, rdi
	xloop2	b3b, clmblkdst		;; 2 * 2*clm iterations
	add	eax, 80000000h/2	;; 4 iterations
	jnc	b3a
	bump	rsi, -4*2*clmblkdst+clmblkdst8
	sub	al, 1			;; Test outer loop counter
	jnz	b3a			;; Iterate if necessary
	bump	rsi, -8*clmblkdst8+32*clmblkdst8;; Next source pointer
	bump	rdi, 2*XMM_SCD		;; Next sine/cosine pointer
	sub	ah, 1
	mov	al, 8
	jnz	b3a
	bump	rsi, -2*32*clmblkdst8	;; Restore source pointer
	end_timer 23

;; Do FFT levels 6,7
;;
;; On input the 64-byte cache lines hold these data values:
;;	0K	128K	256K	384K
;;	2K	...
;;	...
;;	126K	...
;;	512K	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0K	32K	64K	96K
;;	2K	...
;;	...
;;	30K	...
;;	128K	...
;;	...

;; Do 4512 four_complex_fft macros
;;	distance between fft data elements is 8K
;;	do 128 macros each processing 32 data values

	start_timer 24
	mov	rdi, sincos3		;; Load sin/cos pointer
	mov	eax, 8*256+2		;; 8 iters of 2 iters of 4 of 2*2*clm
b5a:	xtouch	[rcx+(clm-1)*128]	;; Preload TLB
b5b:	x4cl_four_complex_fft rsi, 64, 2*clmblkdst8, 4*clmblkdst8, rdi
	xloop2	b5b, clmblkdst		;; 2 * 2*clm iterations
	add	eax, 80000000h/2	;; 4 iterations
	jnc	b5a
	bump	rsi, -4*2*clmblkdst+clmblkdst8
	sub	al, 1			;; Test middle loop counter
	jnz	b5a			;; Iterate if necessary
	bump	rsi, -2*clmblkdst8+8*clmblkdst8;; Next source pointer
	bump	rdi, 2*XMM_SCD		;; Next sine/cosine pointer
	sub	eax, 256		;; Test outer loop counter
	mov	al, 2			;; Load middle loop counter
	jnz	b5a			;; Iterate if necessary
	bump	rsi, -8*8*clmblkdst8	;; Restore source pointer
	end_timer 24

;; Do FFT levels 8,9
;;
;; On input the 64-byte cache lines hold these data values:
;;	0K	32K	64K	96K
;;	2K	...
;;	...
;;	30K	...
;;	128K	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0K	8K	16K	24K
;;	2K	...
;;	...
;;	6K	...
;;	32K	...
;;	...

;; Do 512 four_complex_fft macros
;;	distance between fft data elements is 8K
;;	do 128 macros each processing 32 data values

	start_timer 25
	mov	rdi, sincos2		;; Load sin/cos pointer
	mov	eax, 32*256+2		;; 32 iterations of 2 iters of 2*2*clm
b7a:	xtouch	[rcx+(clm-1)*128]	;; Preload TLB
b7b:	x4cl_four_complex_fft rsi, 64, 4*clmblkdst, clmblkdst8, rdi
	xloop2	b7b, clmblkdst		;; 2 * 2*clm iterations
	sub	al, 1			;; Test middle loop counter
	jnz	b7a			;; Iterate if necessary
	bump	rsi, -2*2*clmblkdst+2*clmblkdst8;; Next source pointer
	bump	rdi, 2*XMM_SCD		;; Next sine/cosine pointer
	sub	eax, 256		;; Test outer loop counter
	mov	al, 2			;; Load middle loop counter
	jnz	b7a			;; Iterate if necessary
	bump	rsi, -32*2*clmblkdst8	;; Restore source pointer
pfing	bump	rcx, -512*blkdst	;; Restore prefetch pointer
	end_timer 25

;; Do FFT levels 10,11
;;
;; On input the 64-byte cache lines hold these data values:
;;	0K	8K	16K	24K
;;	2K	...
;;	...
;;	6K	...
;;	32K	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0K	2K	4K	6K
;;	8K	...
;;	...

;; Do 512 four_complex_fft macros
;;	distance between fft data elements is 2K
;;	do 128 macros each processing 32 data values

	start_timer 26
	mov	rdi, sincos1		;; Load sin/cos pointer
	mov	eax, 64*256+2		;; 64 iters of 2 of 2*clm
b9b:	g4cl_four_complex_fft rsi, 64, clmblkdst, 2*clmblkdst, rcx, 64, blkdst, 2*blkdst
	L2prefetch128 [rcx-64][rbp]
	L2prefetch128 [rcx-64+blkdst][rbp]
	bump	rbp, -64+2*blkdst
	add	al, 256/2		;; Test inner loop counter
	jnc	b9b			;; Iterate if necessary
	bump	rbp, 128-4*blkdst
	IF clm NE 1
	add	dl, 256/clm
	jnc	b9b
	ENDIF
	bump	rsi, -2*clm*64+4*clmblkdst;; Next source pointer
	bump	rcx, -2*clm*64+4*blkdst	;; Next dest pointer
	bump	rdi, 2*XMM_SCD		;; Next sine/cosine pointer
	sub	al, 1
	jnz	b9b
	bump	rsi, -8*clmblkdst+clmblkdst8
	sub	ah, 1			;; Test outer loop counter
	mov	al, 2
	jnz	b9b			;; Iterate if necessary
	end_timer 26

;; Work on next 4 sets of 2048 values

endlp:	xpass1_get_next_block pass2, c0b, b0b ;; Get next block and jump
	done:	xfft_footer
	ENDM


;; Perform a 5120K-element FFT.  This is done in two passes.  Pass 1
;; does 12 levels, pass 2 uses common code to perform the last 11 levels.

hg_pass1sclevels12pfa5 MACRO fftname, pass2_levels, clmarg
	LOCAL	b0b, b1a, b1aa, b1b, b2b, b3b
	LOCAL	b4b, b4c, b5b, b5c, b5d, b6b, b7a, b7b, b8b, b9b
	LOCAL	c0b, c1b, c2a, c2b, c3b
	LOCAL	c4b, c4c, c5b, c5c, c5d, c6b, c7a, c7b, c8b, c9b
	LOCAL	pass1, pass2, endlp, done, yes_fft

	set_FFT_constants fftname, 0, 0, pass2_levels, clmarg
	xfft_header pass1

;; Do the last 11 FFT levels
;;	do 1 pass2_11_levels_real and 159 pass2_11_levels_complex

pass2:	start_timer 0
	call	xpass2name
	end_timer 0
	cmp	ffttype, 1		;; We're done if FFTing only
	je	done

;; Do the inverse FFT

	pass1_inverse_fft_setup c0b

;; Do inverse FFT levels 11,12
;; On input the 64-byte cache lines hold these data values:
;;	0K	2K	4K	6K
;;	8K	10K	12K	14K
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0K	4K	8K	12K
;;	2K	...
;;	16K	...
;;	...

;; Do 1 eight_reals_unfft_2 macros
;; Do 1 nop_two_two_complex_unfft_2 macros
;;	distance between fft data elements is 2K
;;	do 1 macros each processing 16 data values

c0b:	start_timer 16
	mov	rsi, DATA_ADDR		;; Get source address
	mov	rcx, scratch_area	;; Get address of scratch area
	sub	rax, rax
	sub	rdx, rdx
pfing	mov	rbp, DATA_PREFETCH	;; Set prefetch distance
pfing	sub	rbp, rsi
c8b:	g2cl_eight_reals_unfft_2 rsi, 64, blkdst, rcx, 64, clmblkdst
	add	al, 256/2/clm		;; Test loop counter
	jnc	c8b			;; Iterate if necessary

;; Do 638 four_complex_unfft macros
;;	distance between fft data elements is 2K
;;	do 319 macros each processing 16 data values

	bump	rsi, -2*clm*64+2*blkdst	;; Next source pointer
	bump	rcx, -2*clm*64+2*clmblkdst;; Next dest pointer
	mov	rdi, sincos1		;; Load sin/cos pointer
	mov	eax, 80*256+256/4	;; 319 iterations of 2*clm
c9b:	g2cl_four_complex_unfft rsi, 64, blkdst, rcx, 64, clmblkdst
	add	dl, 256/2/clm		;; Test loop counter
	jnc	c9b			;; Iterate if necessary
	bump	rsi, -2*clm*64+2*blkdst	;; Next source pointer
	bump	rcx, -2*clm*64+2*clmblkdst;; Next dest pointer
	bump	rdi, XMM_SCD		;; Next sine/cosine pointer
	add	al, 256/4
	jnc	c9b
	bump	rcx, -4*2*clmblkdst+clmblkdst8
	sub	ah, 1			;; Test outer loop counter
	jnz	c9b			;; Iterate if necessary
	lea	rcx, [rsi-320*2*blkdst]	;; Create prefetch pointer
	mov	rsi, scratch_area	;; Restore source pointer
	end_timer 16

;; Do inverse FFT levels 9,10
;; On input the 64-byte cache lines hold these data values:
;;	0K	4K	8K	12K
;;	2K	...
;;	16K	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0K	16K	32K	48K
;;	2K	...
;;	...
;;	14K
;;	64K	...
;;	...

;; Do 4 eight_reals_unfft_2 macros
;; Do 4 nop_two_two_complex_unfft_2 macros
;;	distance between fft data elements is 8K
;;	do 2 macros each processing 32 data values

	start_timer 17
	mov	al, 2			;; 2 iterations of 2*clm
c6b:	x4cl_eight_reals_unfft_2 rsi, 64, 2*clmblkdst, 4*clmblkdst
	add	al, 256/2/clm		;; Test loop counter
	jnc	c6b			;; Iterate if necessary
	bump	rsi, -2*clm*64+clmblkdst;; Next source pointer
	sub	al, 1			;; Test outer loop counter
	jnz	c6b			;; Iterate if necessary

;; Do 632 four_complex_unfft macros
;;	distance between fft data elements is 8K
;;	do 158 macros each processing 32 data values

	bump	rsi, -2*clmblkdst+clmblkdst8;; Next source pointer
	mov	rdi, sincos2		;; Load sin/cos pointer
	mov	eax, 79*65536		;; 79 iterations of 2*2*clm
c7a:	xtouch	[rcx+(clm-1)*128]	;; Preload TLB
c7b:	x4cl_four_complex_unfft rsi, 64, 2*clmblkdst, 4*clmblkdst, rdi
	xloop2	c7b, clmblkdst
	bump	rsi, -2*clmblkdst+clmblkdst8;; Next source pointer
	bump	rdi, XMM_SCD		;; Next sine/cosine pointer
	sub	eax, 65536		;; Test outer loop counter
	jnz	c7a			;; Iterate if necessary
	bump	rsi, -80*clmblkdst8	;; Restore source pointer
	end_timer 17

;; Do inverse FFT levels 7,8
;; On input the 64-byte cache lines hold these data values:
;;	0K	16K	32K	48K
;;	2K	...
;;	...
;;	14K
;;	64K	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0K	64K	128K	192K
;;	2K	...
;;	...
;;	62K
;;	256K	...
;;	...

;; Do 16 eight_reals_unfft_2 macros
;; Do 16 nop_two_two_complex_unfft_2 macros
;;	distance between fft data elements is 32K
;;	do 8 macros each processing 32 data values

	start_timer 18
	mov	al, 8			;; 8 iterations of 2*clm
c4c:	x4cl_eight_reals_unfft_2 rsi, 64, clmblkdst8, 2*clmblkdst8
	add	al, 256/2/clm		;; Test loop counter
	jnc	c4c			;; Iterate if necessary
	bump	rsi, -2*clm*64+clmblkdst;; Next source pointer
	sub	al, 1			;; Test outer loop counter
	jnz	c4c			;; Iterate if necessary

;; Do 608 four_complex_unfft macros
;;	distance between fft data elements is 8K
;;	do 152 macros each processing 32 data values

	bump	rsi, -8*clmblkdst+4*clmblkdst8;; Next source pointer
	mov	rdi, sincos3		;; Load sin/cos pointer
	mov	eax, 19*256+4		;; 19 iterations of 4 iters of 2*2*clm
c5c:	xtouch	[rcx+(clm-1)*128]	;; Preload TLB
c5d:	x4cl_four_complex_unfft rsi, 64, clmblkdst8, 2*clmblkdst8, rdi
	xloop2	c5d, clmblkdst
	sub	al, 1			;; Test middle loop counter
	jnz	c5c			;; Iterate if necessary
	bump	rsi, -4*2*clmblkdst+4*clmblkdst8;; Next source pointer
	bump	rdi, XMM_SCD		;; Next sine/cosine pointer
	sub	eax, 256		;; Test outer loop counter
	mov	al, 4			;; Load middle loop counter
	jnz	c5c			;; Iterate if necessary
	bump	rsi, -20*4*clmblkdst8	;; Restore source pointer
	end_timer 18

;; Do inverse FFT levels 6
;; On input the 64-byte cache lines hold these data values:
;;	0K	64K	128K	192K
;;	2K	...
;;	...
;;	62K
;;	256K	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0K	128K	256K	384K
;;	2K	...
;;	...
;;	126K
;;	512K	...
;;	...
;;
;; Do 64 eight_reals_unfft_1 macros
;;	distance between fft data elements is 128K
;;	do 32 macros each processing 16 data values
;;
;; Do 576 four_complex_unfft macros
;;	distance between fft data elements is 128K
;;	do 288 macros each processing 16 data values

;; Do inverse FFT levels 4,5
;; On input the 64-byte cache lines hold these data values:
;;	0K	128K	256K	384K
;;	2K	...
;;	...
;;	126K
;;	512K	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0K	256K	512K	768K
;;	2K	...
;;	...
;;	254K
;;	1024K	1536K	2048K	2304K
;;	...
;;
;; Do 128 eight_reals_unfft_2 macros
;;	distance between fft data elements is 256K
;;	do 64 macros each processing 16 data values
;;
;; Do 512 four_complex_unfft macros
;;	distance between fft data elements is 256K
;;	do 128 macros each processing 32 data values

;; Do inverse FFT levels 1,2,3
;; On input the 64-byte cache lines hold these data values:
;;	0K	256K	512K	768K
;;	2K	...
;;	...
;;	254K
;;	1024K	1536K	2048K	2560K
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0K	256K	2560K	2816K
;;	2K	...
;;	...
;;	254K
;;	512K	1536K	3072K	4096K
;;	514K
;;	...
;;	1534K
;;
;; Do 1024 five_reals_last_unfft macros
;;	distance between fft data elements is 1024K
;;	do 128 macros each processing 40 data values

	start_timer 19
	mov	rdi, sincos4		;; Load sin/cos pointer
	mov	rbx, sincos5		;; Load sin/cos pointer
	mov	eax, 4			;; 4 iterations 

c2a:	xtouchlines 6			;; Preload TLBs

c4b:	x2cl_eight_reals_unfft_1 rsi, 64, 4*clmblkdst8
	add	al, 256/2		;; Test loop counter
	jnc	c4b			;; Iterate if necessary

	bump	rsi, -2*64+8*clmblkdst8	;; Next source pointer
	mov	ah, 9			;; 9 iterations
c5b:	x2cl_two_complex_unfft rsi, 64, 4*clmblkdst8
	add	al, 256/2		;; Test loop counter
	jnc	c5b			;; Iterate if necessary
	bump	rsi, -2*64+8*clmblkdst8	;; Next source pointer
	bump	rdi, XMM_SCD		;; Next sine/cosine pointer
	sub	ah, 1			;; Test outer loop counter
	jnz	c5b			;; Iterate if necessary
	bump	rdi, -9*XMM_SCD		;; Restore sine/cosine pointer

	bump	rsi, -10*8*clmblkdst8	;; Restore source pointer
c2b:	x2cl_half_eight_reals_unfft_2 rsi, 64, 8*clmblkdst8
	add	al, 256/2		;; Test loop counter
	jnc	c2b			;; Iterate if necessary
	bump	rsi, -2*64+4*clmblkdst8
	add	ah, 256/2		;; Test loop counter
	jnc	c2b			;; Iterate if necessary

	bump	rsi, -2*4*clmblkdst8+16*clmblkdst8;; Next source pointer
c3b:	x4cl_four_complex_unfft rsi, 64, 8*clmblkdst8, 16*clmblkdst8, rbx
	xloop2sp c3b, 4*clmblkdst8
	bump	rsi, -2*4*clmblkdst8+32*clmblkdst8
	bump	rbx, XMM_SCD		;; Next sine/cosine pointer
	add	ah, 256/2		;; Test outer loop counter
	jnc	c3b			;; Iterate if necessary
	bump	rbx, -2*XMM_SCD		;; Restore sine/cosine pointer

	bump	rsi, -2*32*clmblkdst8-16*clmblkdst8;; Restore src pointer
	xfive_reals_unfft_preload	;; Preload consts for five real macros
c1b:	x5cl_five_reals_last_unfft rsi, 64, 16*clmblkdst8
	xloop1sp c1b, 4*clmblkdst8
	add	ah, 256/4
	jnc	c1b

	IF clm GT 1
	bump	rsi, -4*4*clmblkdst8+2*64;; Next source pointer
	bump	rcx, -6*blkdst+128	;; Next prefetch pointer
	add	dl, 256/clm		;; clm iterations
	jnc	c4b
	bump	rsi, -clm*2*64+clmblkdst;; Next source pointer
	bump	rcx, -clm*128+6*blkdst	;; Next prefetch pointer
	ELSE
	bump	rsi, -4*4*clmblkdst8+clmblkdst;; Next source pointer
	ENDIF

	add	eax, 80000000h/4	;; Test loop counter
	jnc	c2a			;; Iterate if necessary
	bump	rsi, -8*clmblkdst+clmblkdst8;; Next source pointer
	sub	al, 1			;; Test loop counter
	jnz	c2a			;; Iterate if necessary
	end_timer 19

;; Normalize these values

	pass1_normalize 1, yes_fft

;; Copy data from the scratch area back to the FFT data area

	copy_scratch_data_to_fft 640
	jmp	endlp			;; Skip FFT code if flag not set

;; Fire up auxiliary threads to do pass 1 forward FFT

pass1:	pass1_forward_fft_setup b0b

;; Copy data from the FFT data area back to the scratch area

b0b:	copy_fft_data_to_scratch 640

;; Do FFT levels 1,2,3
;;
;; On input the 64-byte cache lines hold these data values:
;;	0K	256K	2560K	2816K
;;	2K	...
;;	...
;;	254K
;;	512K	1536K	3072K	4096K
;;	514K
;;	...
;;	1534K
;; On output the 64-byte cache lines hold these data values:
;;	0K	256K	512K	768K
;;	2K	...
;;	...
;;	254K
;;	1024K	2048K	3072K	4096K
;;	1026K
;;	...
;;	2046K
;;
;; Do 1024 five_reals_first_fft macros (FFT levels 1,2,3)
;;	distance between fft data elements is 1024K
;;	do 128 iterations each processing 40 data values

;; Do FFT levels 4,5
;; Values 0K-1023K is real data, 1024K-5119K is complex data.
;;
;; On input the 64-byte cache lines hold these data values:
;;	0K	256K	512K	768K
;;	2K	...
;;	...
;;	254K
;;	1024K	2048K	3072K	4096K
;;	1026K
;;	...
;;	2046K
;; On output the 64-byte cache lines hold these data values:
;;	0K	256K	512K	768K
;;	2K	...
;;	...
;;	254K	...
;;	1024K	...
;;	...
;;
;; Do 128 eight_reals_fft_2 macros
;;	distance between fft data elements is 256K
;;	do 128 macros each processing 8 data values
;;
;; Do 512 four_complex_fft macros
;;	distance between fft data elements is 256K
;;	do 128 macros each processing 32 data values

;; Do FFT levels 6
;; Values 0K-255K is real data, 256K-511K is semi-real data, 512K-5119K is
;; complex data.
;;
;; On input the 64-byte cache lines hold these data values:
;;	0K	256K	512K	768K
;;	2K	...
;;	...
;;	254K	...
;;	1024K	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0K	128K	256K	384K
;;	2K	...
;;	...
;;	126K	...
;;	512K	...
;;	...
;;
;; Do 64 eight_reals_fft_1 macros
;; Do 64 two_two_complex_fft macros
;;	distance between fft data elements is 128K
;;	do 64 macros each processing 16 data values
;;
;; Do 512 two_two_complex_fft macros
;;	distance between fft data elements is 128K
;;	do 256 macros each processing 16 data values

yes_fft:start_timer 22
	mov	rsi, scratch_area	;; Get address of scratch area
	mov	rcx, DATA_ADDR		;; Load source address for prefetching
pfing	mov	rbp, DATA_PREFETCH	;; Set prefetch distance
pfing	sub	rbp, rcx
pfing	bump	rcx, 337*blkdst		;; Create prefetching pointer
	mov	rdi, sincos5		;; Load sin/cos pointer
	mov	rbx, sincos4		;; Load sin/cos pointer
	mov	eax, 8			;; 8 iterations
	sub	rdx, rdx

b1a:	xtouchlines 3			;; Preload TLBs

b1aa:	xfive_reals_fft_preload		;; Preload consts for five real macros
b1b:	x5cl_five_reals_fft rsi, 64, 16*clmblkdst8
	xloop1sp b1b, 8*clmblkdst8
	add	ah, 256/2
	jnc	b1b

	bump	rsi, -2*8*clmblkdst8	;; Restore source pointer
b2b:	x1cl_half_eight_reals_fft_2 rsi, 64
	add	al, 256/2		;; Test inner loop counter
	jnc	b2b			;; Iterate if necessary
	bump	rsi, -2*64+8*clmblkdst8	;; Next source pointer
	add	ah, 256/2
	jnc	b2b

	bump	rsi, -2*8*clmblkdst8+16*clmblkdst8 ;; Next source pointer
b3b:	x4cl_four_complex_fft rsi, 64, 16*clmblkdst8, 32*clmblkdst8, rdi
	xloop2sp b3b, 8*clmblkdst8

	bump	rsi, -2*8*clmblkdst8-16*clmblkdst8;; Restore src pointer
b4b:	x2cl_eight_reals_fft_1 rsi, 64, 8*clmblkdst8
	add	al, 256/2		;; Test inner loop counter
	jnc	b4b			;; Iterate if necessary

	bump	rsi, -2*64+16*clmblkdst8;; Next source pointer
	bump	rbx, XMM_SCD		;; Next sine/cosine pointer
b5b:	x2cl_two_complex_fft rsi, 64, 8*clmblkdst8, rbx
	add	al, 256/2		;; Test inner loop counter
	jnc	b5b			;; Iterate if necessary
	bump	rsi, -2*64+16*clmblkdst8;; Next source pointer
	bump	rbx, +2*XMM_SCD		;; Next sine/cosine pointer
	add	ah, 256/4		;; Test outer loop counter
	jnc	b5b			;; Iterate if necessary
	bump	rbx, -9*XMM_SCD		;; Restore sine/cosine pointer

	IF clm GT 1
	bump	rsi, -5*16*clmblkdst8+2*64;; Next source pointer
	bump	rcx, -3*blkdst+128	;; Next prefetch pointer
	add	dl, 256/clm		;; clm iterations
	jnc	b1aa
	bump	rsi, -clm*2*64+clmblkdst;; Next source pointer
	bump	rcx, -clm*128+3*blkdst	;; Next prefetch pointer
	ELSE
	bump	rsi, -5*16*clmblkdst8+clmblkdst;; Next source pointer
	ENDIF

	add	eax, 80000000h/4	;; Test loop counter
	jnc	b1a			;; Iterate if necessary
	bump	rsi, -8*clmblkdst+clmblkdst8;; Next source pointer
	sub	al, 1			;; Test loop counter
	jnz	b1a			;; Iterate if necessary
	bump	rsi, -8*clmblkdst8	;; Restore source pointer
	end_timer 22

;; Do FFT levels 7,8
;; Values 0K-127K is real data, 128K-255K is semi-real data, 256K-5119K is
;; complex data.
;;
;; On input the 64-byte cache lines hold these data values:
;;	0K	128K	256K	384K
;;	2K	...
;;	...
;;	126K	...
;;	512K	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0K	32K	64K	96K
;;	2K	...
;;	...
;;	30K	...
;;	128K	...
;;	...

;; Do 32 eight_reals_fft_2 macros
;; Do 32 two_two_complex_fft macros
;;	distance between fft data elements is 32K
;;	do 16 macros each processing 32 data values

	start_timer 24
	mov	rdi, sincos3		;; Load sin/cos pointer
	mov	al, 2			;; 2 iterations of 8 iters of 2*clm
b4c:	x4cl_eight_reals_fft_2 rsi, 64, 2*clmblkdst8, 4*clmblkdst8, rdi
	add	al, 256/2/clm		;; Test inner loop counter
	jnc	b4c			;; Iterate if necessary
	bump	rsi, -2*clm*64+clmblkdst;; Next source pointer
	add	ah, 256/8		;; Test loop counter
	jnc	b4c			;; Iterate if necessary
	bump	rsi, -8*clmblkdst+clmblkdst8;; Next source pointer
	sub	al, 1			;; Test outer loop counter
	jnz	b4c			;; Iterate if necessary

;; Do 576 four_complex_fft macros
;;	distance between fft data elements is 32K
;;	do 144 macros each processing 32 data values

	bump	rsi, -2*clmblkdst8+8*clmblkdst8;; Load source pointer
	bump	rdi, XMM_SCD		;; Next sine/cosine pointer
	mov	eax, 9*256+2		;; 9 iters of 2*4 iters of 2*2*clm
b5c:	xtouch	[rcx+(clm-1)*128]	;; Preload TLB
b5d:	x4cl_four_complex_fft rsi, 64, 2*clmblkdst8, 4*clmblkdst8, rdi
	xloop2	b5d, clmblkdst
	add	eax, 80000000h/2	;; 4 iterations
	jnc	b5c
	bump	rsi, -8*clmblkdst+clmblkdst8;; Next source pointer
	sub	al, 1			;; Test middle loop counter
	jnz	b5c			;; Iterate if necessary
	bump	rsi, -2*clmblkdst8+8*clmblkdst8;; Next source pointer
	bump	rdi, 2*XMM_SCD		;; Next sine/cosine pointer
	sub	eax, 256		;; Test outer loop counter
	mov	al, 2			;; Load middle loop counter
	jnz	b5c			;; Iterate if necessary
	bump	rsi, -10*8*clmblkdst8	;; Restore source pointer
	end_timer 24

;; Do FFT levels 9,10
;; Values 0K-31K is real data, 32K-63K is semi-real data, 64K-5119K is
;; complex data.
;;
;; On input the 64-byte cache lines hold these data values:
;;	0K	32K	64K	96K
;;	2K	...
;;	...
;;	30K	...
;;	128K	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0K	8K	16K	24K
;;	2K	...
;;	...
;;	6K	...
;;	32K	...
;;	...

;; Do 8 eight_reals_fft_2 macros
;; Do 8 two_two_complex_fft macros
;;	distance between fft data elements is 8K
;;	do 4 macros each processing 32 data values

	start_timer 25
	mov	rdi, sincos2		;; Load sin/cos pointer
	mov	al, 4			;; 4 iterations of 2*clm
b6b:	x4cl_eight_reals_fft_2 rsi, 64, 4*clmblkdst, clmblkdst8, rdi
	add	al, 256/2/clm		;; Test inner loop counter
	jnc	b6b			;; Iterate if necessary
	bump	rsi, -2*clm*64+clmblkdst;; Next source pointer
	sub	al, 1			;; Test outer loop counter
	jnz	b6b			;; Iterate if necessary

;; Do 624 four_complex_fft macros
;;	distance between fft data elements is 8K
;;	do 156 macros each processing 32 data values

	bump	rsi, -4*clmblkdst+2*clmblkdst8;; Load source pointer
	bump	rdi, XMM_SCD		;; Next sine/cosine pointer
	mov	eax, 39*256		;; 39 iterations of 2 iters of 2*2*clm
b7a:	xtouch	[rcx+(clm-1)*128]	;; Preload TLB
b7b:	x4cl_four_complex_fft rsi, 64, 4*clmblkdst, clmblkdst8, rdi
	xloop4	b7b, clmblkdst
	bump	rsi, -4*clmblkdst+2*clmblkdst8;; Next source pointer
	bump	rdi, 2*XMM_SCD		;; Next sine/cosine pointer
	sub	eax, 256		;; Test outer loop counter
	jnz	b7a			;; Iterate if necessary
	bump	rsi, -40*2*clmblkdst8	;; Restore source pointer
pfing	bump	rcx, -640*blkdst	;; Restore prefetch pointer
	end_timer 25

;; Do FFT levels 11,12
;; Values 0K-7K is real data, 8K-15K is semi-real data, 16K-5119K is
;; complex data.
;;
;; On input the 64-byte cache lines hold these data values:
;;	0K	8K	16K	24K
;;	2K	...
;;	...
;;	6K	...
;;	32K	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0K	2K	4K	6K
;;	8K	...
;;	...

;; Do 1 eight_reals_fft_2 macros
;; Do 1 nop_two_two_complex_fft_2 macros
;; Do 2 four_complex_fft macros
;;	distance between fft data elements is 2K
;;	do 1 macros each processing 32 data values

	start_timer 26
	mov	rdi, sincos1		;; Load sin/cos pointer
b8b:	g4cl_eight_reals_fft_2 rsi, 64, clmblkdst, 2*clmblkdst, rcx, 64, blkdst, 2*blkdst
	add	al, 256/2/clm		;; Test inner loop counter
	jnc	b8b			;; Iterate if necessary

;; Do 636 four_complex_fft macros
;;	distance between fft data elements is 2K
;;	do 159 macros each processing 32 data values

	bump	rsi, -2*clm*64+4*clmblkdst;; Next source pointer
	bump	rcx, -2*clm*64+4*blkdst	;; Next dest pointer
	bump	rdi, XMM_SCD		;; Next sine/cosine pointer
	mov	eax, 80*256+1		;; 159 iterations of 2*clm
b9b:	g4cl_four_complex_fft rsi, 64, clmblkdst, 2*clmblkdst, rcx, 64, blkdst, 2*blkdst
	L2prefetch128 [rcx-64][rbp]
	L2prefetch128 [rcx-64+blkdst][rbp]
	bump	rbp, -64+2*blkdst
	add	al, 256/2		;; Test inner loop counter
	jnc	b9b			;; Iterate if necessary
	bump	rbp, 128-4*blkdst
	IF clm NE 1
	add	dl, 256/clm
	jnc	b9b
	ENDIF
	bump	rsi, -2*clm*64+4*clmblkdst;; Next source pointer
	bump	rcx, -2*clm*64+4*blkdst	;; Next dest pointer
	bump	rdi, 2*XMM_SCD		;; Next sine/cosine pointer
	sub	al, 1			;; Test outer loop counter
	jnz	b9b
	bump	rsi, -8*clmblkdst+clmblkdst8
	sub	ah, 1			;; Test outer loop counter
	mov	al, 2
	jnz	b9b			;; Iterate if necessary
	end_timer 26

;; Work on next 4 sets of 2560 values

endlp:	xpass1_get_next_block pass2, c0b, b0b ;; Get next block and jump
	done:	xfft_footer
	ENDM


;; Perform a 6144K-element FFT.  This is done in two passes.  Pass 1
;; does 12 levels, pass 2 uses common code to perform the last 11 levels.

hg_pass1sclevels12pfa6 MACRO fftname, pass2_levels, clmarg
	LOCAL	b0b, b1a, b1b, b2b, b3b, b4b, b4c, b5b, b5c, b5d
	LOCAL	b6b, b7a, b7b, b8b, b9b
	LOCAL	c0b, c1b, c2a, c2b, c3b, c4b, c4c, c5b, c5c, c5d
	LOCAL	c6b, c7a, c7b, c8b, c9b
	LOCAL	pass1, pass2, endlp, done, yes_fft

	set_FFT_constants fftname, 0, 0, pass2_levels, clmarg
	xfft_header pass1

;; Do the last 11 FFT levels
;;	do 1 pass2_11_levels_real and 191 pass2_11_levels_complex

pass2:	start_timer 0
	call	xpass2name
	end_timer 0
	cmp	ffttype, 1		;; We're done if FFTing only
	je	done

;; Do the inverse FFT

	pass1_inverse_fft_setup c0b

;; Do inverse FFT levels 11,12
;; On input the 64-byte cache lines hold these data values:
;;	0K	2K	4K	6K
;;	8K	10K	12K	14K
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0K	4K	8K	12K
;;	2K	...
;;	16K	...
;;	...

;; Do 1 eight_reals_unfft_2 macros
;; Do 1 nop_two_two_complex_unfft_2 macros
;;	distance between fft data elements is 2K
;;	do 1 macros each processing 16 data values

c0b:	start_timer 16
	mov	rsi, DATA_ADDR		;; Get source address
	mov	rcx, scratch_area	;; Get address of scratch area
	sub	rax, rax
	sub	rdx, rdx
pfing	mov	rbp, DATA_PREFETCH	;; Set prefetch distance
pfing	sub	rbp, rsi
c8b:	g2cl_eight_reals_unfft_2 rsi, 64, blkdst, rcx, 64, clmblkdst
	add	al, 256/2/clm		;; Test loop counter
	jnc	c8b			;; Iterate if necessary

;; Do 766 four_complex_unfft macros
;;	distance between fft data elements is 2K
;;	do 383 macros each processing 16 data values

	bump	rsi, -2*clm*64+2*blkdst	;; Next source pointer
	bump	rcx, -2*clm*64+2*clmblkdst;; Next dest pointer
	mov	rdi, sincos1		;; Load sin/cos pointer
	mov	eax, 96*256+256/4	;; 383 iterations of 2*clm
c9b:	g2cl_four_complex_unfft rsi, 64, blkdst, rcx, 64, clmblkdst
	add	dl, 256/2/clm		;; Test loop counter
	jnc	c9b			;; Iterate if necessary
	bump	rsi, -2*clm*64+2*blkdst	;; Next source pointer
	bump	rcx, -2*clm*64+2*clmblkdst;; Next dest pointer
	bump	rdi, XMM_SCD		;; Next sine/cosine pointer
	add	al, 256/4		;; Test loop counter
	jnc	c9b
	bump	rcx, -8*clmblkdst+clmblkdst8
	sub	eax, 256		;; Test outer loop counter
	jnz	c9b			;; Iterate if necessary
	lea	rcx, [rsi-384*2*blkdst]	;; Create prefetch pointer
	mov	rsi, scratch_area	;; Restore source pointer
	end_timer 16

;; Do inverse FFT levels 9,10
;; On input the 64-byte cache lines hold these data values:
;;	0K	4K	8K	12K
;;	2K	...
;;	16K	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0K	16K	32K	48K
;;	2K	...
;;	...
;;	14K
;;	64K	...
;;	...

;; Do 4 eight_reals_unfft_2 macros
;; Do 4 nop_two_two_complex_unfft_2 macros
;;	distance between fft data elements is 8K
;;	do 2 macros each processing 32 data values

	start_timer 17
	mov	al, 2			;; 2 iterations of 2*clm
c6b:	x4cl_eight_reals_unfft_2 rsi, 64, 2*clmblkdst, 4*clmblkdst
	add	al, 256/2/clm		;; Test loop counter
	jnc	c6b			;; Iterate if necessary
	bump	rsi, -2*clm*64+clmblkdst;; Next source pointer
	sub	al, 1			;; Test outer loop counter
	jnz	c6b			;; Iterate if necessary

;; Do 760 four_complex_unfft macros
;;	distance between fft data elements is 8K
;;	do 190 macros each processing 32 data values

	bump	rsi, -2*clmblkdst+clmblkdst8;; Next source pointer
	mov	rdi, sincos2		;; Load sin/cos pointer
	mov	eax, 95*65536		;; 95 iterations of 2*2*clm
c7a:	xtouch	[rcx+(clm-1)*128]	;; Preload TLB
c7b:	x4cl_four_complex_unfft rsi, 64, 2*clmblkdst, 4*clmblkdst, rdi
	xloop2	c7b, clmblkdst
	bump	rsi, -2*clmblkdst+clmblkdst8;; Next source pointer
	bump	rdi, XMM_SCD		;; Next sine/cosine pointer
	sub	eax, 65536		;; Test outer loop counter
	jnz	c7a			;; Iterate if necessary
	bump	rsi, -96*clmblkdst8	;; Restore source pointer
	end_timer 17

;; Do inverse FFT levels 7,8
;; On input the 64-byte cache lines hold these data values:
;;	0K	16K	32K	48K
;;	2K	...
;;	...
;;	14K
;;	64K	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0K	64K	128K	192K
;;	2K	...
;;	...
;;	62K
;;	256K	...
;;	...

;; Do 16 eight_reals_unfft_2 macros
;; Do 16 nop_two_two_complex_unfft_2 macros
;;	distance between fft data elements is 32K
;;	do 8 macros each processing 32 data values

	start_timer 18
	mov	al, 8			;; 8 iterations of 2*clm
c4c:	x4cl_eight_reals_unfft_2 rsi, 64, clmblkdst8, 2*clmblkdst8
	add	al, 256/2/clm		;; Test loop counter
	jnc	c4c			;; Iterate if necessary
	bump	rsi, -2*clm*64+clmblkdst;; Next source pointer
	sub	al, 1			;; Test outer loop counter
	jnz	c4c			;; Iterate if necessary

;; Do 736 four_complex_unfft macros
;;	distance between fft data elements is 32K
;;	do 184 macros each processing 32 data values

	bump	rsi, -8*clmblkdst+4*clmblkdst8;; Next source pointer
	mov	rdi, sincos3		;; Load sin/cos pointer
	mov	eax, 23*256+4		;; 23 iterations of 4 iters of 2*2*clm
c5c:	xtouch	[rcx+(clm-1)*128]	;; Preload TLB
c5d:	x4cl_four_complex_unfft rsi, 64, clmblkdst8, 2*clmblkdst8, rdi
	xloop2	c5d, clmblkdst
	sub	al, 1			;; Test middle loop counter
	jnz	c5c			;; Iterate if necessary
	bump	rsi, -4*2*clmblkdst+4*clmblkdst8;; Next source pointer
	bump	rdi, XMM_SCD		;; Next sine/cosine pointer
	sub	eax, 256		;; Test outer loop counter
	mov	al, 4			;; Load middle loop counter
	jnz	c5c			;; Iterate if necessary
	bump	rsi, -24*4*clmblkdst8	;; Restore source pointer
	end_timer 18

;; Do inverse FFT levels 6
;; On input the 64-byte cache lines hold these data values:
;;	0K	64K	128K	192K
;;	2K	...
;;	...
;;	62K
;;	256K	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0K	128K	256K	384K
;;	2K	...
;;	...
;;	126K
;;	512K	...
;;	...
;;
;; Do 64 eight_reals_unfft_1 macros
;;	distance between fft data elements is 128K
;;	do 32 macros each processing 16 data values
;;
;; Do 704 four_complex_unfft macros
;;	distance between fft data elements is 128K
;;	do 352 macros each processing 16 data values

;; Do inverse FFT levels 4,5
;; On input the 64-byte cache lines hold these data values:
;;	0K	128K	256K	384K
;;	2K	...
;;	...
;;	126K
;;	512K	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0K	512K	1024K	1536K
;;	2K	...
;;	...
;;	510K
;;	2048K	3072K	4096K	5120K
;;	2050K
;;	...
;;	3070K
;;
;; Do 128 eight_reals_unfft_2 macros
;; Do 128 nop_two_two_complex_unfft_2 macros
;;	distance between fft data elements is 256K
;;	do 64 macros each processing 32 data values
;;
;; Do 512 four_complex_unfft macros
;;	distance between fft data elements is 256K
;;	do 128 macros each processing 32 data values

;; Do inverse FFT levels 1,2,3
;; On input the 64-byte cache lines hold these data values:
;;	0K	512K	1024K	1536K
;;	2K	...
;;	...
;;	510K
;;	2048K	3072K	4096K	5120K
;;	2050K
;;	...
;;	3070K
;; On output the 64-byte cache lines hold these data values:
;;	0K	1536K	3072K	4608K
;;	...
;;	1534K
;;
;; Do 1024 six_reals_last_unfft macros
;;	distance between fft data elements is 1024K
;;	do 256 macros each processing 24 data values

	start_timer 19
	mov	rdi, sincos4		;; Load sin/cos pointer
	mov	rbx, sincos5		;; Load sin/cos pointer
	mov	eax, 4			;; 4 iterations 
	xsix_reals_unfft_preload

c2a:	xtouchlines 7			;; Preload TLBs

c4b:	x2cl_eight_reals_unfft_1 rsi, 64, 4*clmblkdst8
	xloop1sp c4b, 8*clmblkdst8

	mov	ah, 11			;; 11 iterations
c5b:	x2cl_two_complex_unfft rsi, 64, 4*clmblkdst8
	add	al, 256/2		;; Test loop counter
	jnc	c5b			;; Iterate if necessary
	bump	rsi, -2*64+8*clmblkdst8	;; Next source pointer
	bump	rdi, XMM_SCD		;; Next sine/cosine pointer
	sub	ah, 1			;; Test outer loop counter
	jnz	c5b			;; Iterate if necessary
	bump	rdi, -11*XMM_SCD	;; Restore sine/cosine pointer

	bump	rsi, -12*8*clmblkdst8	;; Restore source pointer
c2b:	x4cl_eight_reals_unfft_2 rsi, 64, 8*clmblkdst8, 16*clmblkdst8
	add	al, 256/2		;; Test loop counter
	jnc	c2b			;; Iterate if necessary
	bump	rsi, -2*64+4*clmblkdst8
	add	ah, 256/2		;; Test loop counter
	jnc	c2b			;; Iterate if necessary

	bump	rsi, -8*clmblkdst8+32*clmblkdst8;; Next source pointer
c3b:	x4cl_four_complex_unfft rsi, 64, 8*clmblkdst8, 16*clmblkdst8, rbx
	xloop2sp c3b, 4*clmblkdst8
	bump	rsi, -2*4*clmblkdst8+32*clmblkdst8
	bump	rbx, XMM_SCD		;; Next sine/cosine pointer
	add	ah, 256/2
	jnc	c3b
	bump	rbx, -2*XMM_SCD		;; Restore sine/cosine pointer

	bump	rsi, -3*32*clmblkdst8	;; Restore source pointer
c1b:	x3cl_six_reals_last_unfft rsi, 64, 32*clmblkdst8
	xloop2sp c1b, 4*clmblkdst8
	add	ah, 256/4
	jnc	c1b

	IF clm GT 1
	bump	rsi, -8*4*clmblkdst8+2*64;; Next source pointer
	bump	rcx, -7*blkdst+128	;; Next prefetch pointer
	add	dl, 256/clm		;; clm iterations
	jnc	c4b
	bump	rsi, -clm*2*64+clmblkdst;; Next source pointer
	bump	rcx, -clm*128+7*blkdst	;; Next prefetch pointer
	ELSE
	bump	rsi, -8*4*clmblkdst8+clmblkdst;; Next source pointer
	ENDIF

	add	eax, 80000000h/4	;; Test loop counter
	jnc	c2a			;; Iterate if necessary
	bump	rsi, -8*clmblkdst+clmblkdst8;; Next source pointer
	sub	al, 1			;; Test loop counter
	jnz	c2a			;; Iterate if necessary
	end_timer 19

;; Normalize these values

	pass1_normalize 1, yes_fft

;; Copy data from the scratch area back to the FFT data area

	copy_scratch_data_to_fft 768
	jmp	endlp			;; Skip FFT code if flag not set

;; Fire up auxiliary threads to do pass 1 forward FFT

pass1:	pass1_forward_fft_setup b0b

;; Copy data from the FFT data area back to the scratch area

b0b:	copy_fft_data_to_scratch 768

;; Do FFT levels 1,2,3
;;
;; On input the 64-byte cache lines hold these data values:
;;	0K	1536K	3072K	4608K
;;	...
;;	1534K
;; On output the 64-byte cache lines hold these data values:
;;	0K	512K	1024K	1536K
;;	2K	...
;;	...
;;	510K
;;	2048K	3072K	4096K	5120K
;;	2050K
;;	...
;;	3070K
;;
;; Do 1024 six_reals_first_fft macros (FFT levels 1,2,3)
;;	distance between fft data elements is 1024K
;;	do 256 iterations each processing 24 data values

;; Do FFT levels 4,5
;; Values 0K-1023K is real data, 1024K-2047K is semi-real data, 2048K-6143K is
;; complex data.
;;
;; On input the 64-byte cache lines hold these data values:
;;	0K	512K	1024K	1536K
;;	2K	...
;;	...
;;	510K
;;	2048K	3072K	4096K	5120K
;;	2050K
;;	...
;;	3070K
;; On output the 64-byte cache lines hold these data values:
;;	0K	256K	512K	768K
;;	2K	...
;;	...
;;	254K	...
;;	1024K	...
;;	...
;;
;; Do 128 eight_reals_fft_2 macros
;; Do 128 nop_two_two_complex_fft_2 macros
;;	distance between fft data elements is 256K
;;	do 128 macros each processing 16 data values
;;
;; Do 512 four_complex_fft macros
;;	distance between fft data elements is 256K
;;	do 128 macros each processing 32 data values

;; Do FFT levels 6
;; Values 0K-255K is real data, 256K-511K is semi-real data, 512K-6143K is
;; complex data.
;;
;; On input the 64-byte cache lines hold these data values:
;;	0K	256K	512K	768K
;;	2K	...
;;	...
;;	254K	...
;;	1024K	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0K	128K	256K	384K
;;	2K	...
;;	...
;;	126K	...
;;	512K	...
;;	...
;;
;; Do 64 eight_reals_fft_1 macros
;; Do 64 two_two_complex_fft macros
;;	distance between fft data elements is 128K
;;	do 64 macros each processing 16 data values
;;
;; Do 640 two_two_complex_fft macros
;;	distance between fft data elements is 128K
;;	do 320 macros each processing 16 data values

yes_fft:start_timer 22
	mov	rsi, scratch_area	;; Get address of scratch area
	mov	rcx, DATA_ADDR		;; Load source address for prefetching
pfing	mov	rbp, DATA_PREFETCH	;; Set prefetch distance
pfing	sub	rbp, rcx
pfing	bump	rcx, 394*blkdst		;; Create prefetch pointer
	mov	rdi, sincos5		;; Load sin/cos pointer
	mov	rbx, sincos4		;; Load sin/cos pointer
	mov	eax, 8			;; 8 iterations
	sub	rdx, rdx
	xsix_reals_fft_preload

b1a:	xtouchlines 3			;; Preload TLBs

b1b:	x3cl_six_reals_fft rsi, 64, 32*clmblkdst8
	xloop4sp b1b, 8*clmblkdst8

	bump	rsi, -4*8*clmblkdst8	;; Restore source pointer
b2b:	x2cl_eight_reals_fft_2 rsi, 64, 16*clmblkdst8
	add	al, 256/2		;; Test inner loop counter
	jnc	b2b			;; Iterate if necessary
	bump	rsi, -2*64+8*clmblkdst8	;; Next source pointer
	add	ah, 256/2
	jnc	b2b

	bump	rsi, -2*8*clmblkdst8+32*clmblkdst8;; Next source pointer
b3b:	x4cl_four_complex_fft rsi, 64, 16*clmblkdst8, 32*clmblkdst8, rdi
	xloop1sp b3b, 8*clmblkdst8
	add	ah, 256/2
	jnc	b3b

	bump	rsi, -2*8*clmblkdst8-32*clmblkdst8;; Restore src pointer
b4b:	x2cl_eight_reals_fft_1 rsi, 64, 8*clmblkdst8
	add	al, 256/2		;; Test inner loop counter
	jnc	b4b			;; Iterate if necessary

	bump	rsi, -2*64+16*clmblkdst8;; Load source pointer
	bump	rbx, XMM_SCD		;; Next sine/cosine pointer
	mov	ah, 5			;; 5 iterations
b5b:	x2cl_two_complex_fft rsi, 64, 8*clmblkdst8, rbx
	add	al, 256/2		;; Test inner loop counter
	jnc	b5b			;; Iterate if necessary
	bump	rsi, -2*64+16*clmblkdst8;; Next source pointer
	bump	rbx, 2*XMM_SCD		;; Next sine/cosine pointer
	sub	ah, 1			;; Test outer loop counter
	jnz	b5b			;; Iterate if necessary
	bump	rbx, -11*XMM_SCD	;; Restore sine/cosine pointer

	IF clm GT 1
	bump	rsi, -6*16*clmblkdst8+2*64;; Next source pointer
	bump	rcx, -3*blkdst+128	;; Next prefetch pointer
	add	dl, 256/clm		;; clm iterations
	jnc	b1b
	bump	rsi, -clm*2*64+clmblkdst;; Next source pointer
	bump	rcx, -clm*128+3*blkdst	;; Next prefetch pointer
	ELSE
	bump	rsi, -6*16*clmblkdst8+clmblkdst;; Next source pointer
	ENDIF

	add	eax, 80000000h/4	;; Test loop counter
	jnc	b1a			;; Iterate if necessary
	bump	rsi, -8*clmblkdst+clmblkdst8;; Next source pointer
	sub	al, 1			;; Test loop counter
	jnz	b1a			;; Iterate if necessary
	bump	rsi, -8*clmblkdst8	;; Restore source pointer
	end_timer 22

;; Do FFT levels 7,8
;; Values 0K-127K is real data, 128K-255K is semi-real data, 256K-6143K is
;; complex data.
;;
;; On input the 64-byte cache lines hold these data values:
;;	0K	128K	256K	384K
;;	2K	...
;;	...
;;	126K	...
;;	512K	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0K	32K	64K	96K
;;	2K	...
;;	...
;;	30K	...
;;	128K	...
;;	...

;; Do 32 eight_reals_fft_2 macros
;; Do 32 two_two_complex_fft macros
;;	distance between fft data elements is 32K
;;	do 16 macros each processing 32 data values

	start_timer 24
	mov	rdi, sincos3		;; Load sin/cos pointer
	mov	al, 2			;; 2 iterations of 8 iters of 2*clm
b4c:	x4cl_eight_reals_fft_2 rsi, 64, 2*clmblkdst8, 4*clmblkdst8, rdi
	add	al, 256/2/clm		;; Test inner loop counter
	jnc	b4c			;; Iterate if necessary
	bump	rsi, -2*clm*64+clmblkdst;; Next source pointer
	add	ah, 256/8		;; Test loop counter
	jnc	b4c			;; Iterate if necessary
	bump	rsi, -8*clmblkdst+clmblkdst8;; Next source pointer
	sub	al, 1			;; Test outer loop counter
	jnz	b4c			;; Iterate if necessary

;; Do 704 four_complex_fft macros
;;	distance between fft data elements is 32K
;;	do 176 macros each processing 32 data values

	bump	rsi, -2*clmblkdst8+8*clmblkdst8;; Load source pointer
	bump	rdi, XMM_SCD		;; Next sine/cosine pointer
	mov	eax, 11*256+2		;; 11 iters of 2*4 iters of 2*2*clm
b5c:	xtouch	[rcx+(clm-1)*128]	;; Preload TLB
b5d:	x4cl_four_complex_fft rsi, 64, 2*clmblkdst8, 4*clmblkdst8, rdi
	xloop2	b5d, clmblkdst
	add	eax, 80000000h/2	;; 4 iterations
	jnc	b5c
	bump	rsi, -8*clmblkdst+clmblkdst8;; Next source pointer
	sub	al, 1			;; Test middle loop counter
	jnz	b5c			;; Iterate if necessary
	bump	rsi, -2*clmblkdst8+8*clmblkdst8;; Next source pointer
	bump	rdi, 2*XMM_SCD		;; Next sine/cosine pointer
	sub	eax, 256		;; Test outer loop counter
	mov	al, 2			;; Load middle loop counter
	jnz	b5c			;; Iterate if necessary
	bump	rsi, -12*8*clmblkdst8	;; Restore source pointer
	end_timer 24

;; Do FFT levels 9,10
;; Values 0K-31K is real data, 32K-63K is semi-real data, 64K-6143K is
;; complex data.
;;
;; On input the 64-byte cache lines hold these data values:
;;	0K	32K	64K	96K
;;	2K	...
;;	...
;;	30K	...
;;	128K	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0K	8K	16K	24K
;;	2K	...
;;	...
;;	6K	...
;;	32K	...
;;	...

;; Do 8 eight_reals_fft_2 macros
;; Do 8 two_two_complex_fft macros
;;	distance between fft data elements is 8K
;;	do 4 macros each processing 32 data values

	start_timer 25
	mov	rdi, sincos2		;; Load sin/cos pointer
	mov	al, 4			;; 4 iterations of 2*clm
b6b:	x4cl_eight_reals_fft_2 rsi, 64, 4*clmblkdst, clmblkdst8, rdi
	add	al, 256/2/clm		;; Test inner loop counter
	jnc	b6b			;; Iterate if necessary
	bump	rsi, -2*clm*64+clmblkdst;; Next source pointer
	sub	al, 1			;; Test outer loop counter
	jnz	b6b			;; Iterate if necessary

;; Do 752 four_complex_fft macros
;;	distance between fft data elements is 8K
;;	do 188 macros each processing 32 data values

	bump	rsi, -4*clmblkdst+2*clmblkdst8;; Load source pointer
	bump	rdi, XMM_SCD		;; Next sine/cosine pointer
	mov	eax, 47*256+2		;; 47 iterations of 4 iters of 2*clm
b7a:	xtouch	[rcx+(clm-1)*128]	;; Preload TLB
b7b:	x4cl_four_complex_fft rsi, 64, 4*clmblkdst, clmblkdst8, rdi
	xloop2	b7b, clmblkdst
	sub	al, 1			;; Test middle loop counter
	jnz	b7a			;; Iterate if necessary
	bump	rsi, -4*clmblkdst+2*clmblkdst8;; Next source pointer
	bump	rdi, 2*XMM_SCD		;; Next sine/cosine pointer
	sub	eax, 256		;; Test outer loop counter
	mov	al, 2			;; Load middle loop counter
	jnz	b7a			;; Iterate if necessary
	bump	rsi, -48*2*clmblkdst8	;; Restore source pointer
pfing	bump	rcx, -768*blkdst	;; Restore prefetch pointer
	end_timer 25

;; Do FFT levels 11,12
;; Values 0K-7K is real data, 8K-15K is semi-real data, 16K-6143K is
;; complex data.
;;
;; On input the 64-byte cache lines hold these data values:
;;	0K	8K	16K	24K
;;	2K	...
;;	...
;;	6K	...
;;	32K	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0K	2K	4K	6K
;;	8K	...
;;	...

;; Do 1 eight_reals_fft_2 macros
;; Do 1 nop_two_two_complex_fft_2 macros
;; Do 2 four_complex_fft macros
;;	distance between fft data elements is 2K
;;	do 1 macros each processing 32 data values

	start_timer 26
	mov	rdi, sincos1		;; Load sin/cos pointer
b8b:	g4cl_eight_reals_fft_2 rsi, 64, clmblkdst, 2*clmblkdst, rcx, 64, blkdst, 2*blkdst
	add	al, 256/2/clm		;; Test inner loop counter
	jnc	b8b			;; Iterate if necessary

;; Do 764 four_complex_fft macros
;;	distance between fft data elements is 2K
;;	do 191 macros each processing 32 data values

	bump	rsi, -2*clm*64+4*clmblkdst;; Next source pointer
	bump	rcx, -2*clm*64+4*blkdst	;; Next dest pointer
	bump	rdi, XMM_SCD		;; Next sine/cosine pointer
	mov	eax, 96*256+1		;; 95 iterations of 2*clm
b9b:	g4cl_four_complex_fft rsi, 64, clmblkdst, 2*clmblkdst, rcx, 64, blkdst, 2*blkdst
	L2prefetch128 [rcx-64][rbp]
	L2prefetch128 [rcx-64+blkdst][rbp]
	bump	rbp, -64+2*blkdst
	add	al, 256/2		;; Test inner loop counter
	jnc	b9b			;; Iterate if necessary
	bump	rbp, 128-4*blkdst
	IF clm NE 1
	add	dl, 256/clm
	jnc	b9b
	ENDIF
	bump	rsi, -2*clm*64+4*clmblkdst;; Next source pointer
	bump	rcx, -2*clm*64+4*blkdst	;; Next dest pointer
	bump	rdi, 2*XMM_SCD		;; Next sine/cosine pointer
	sub	al, 1			;; Test outer loop counter
	jnz	b9b
	bump	rsi, -8*clmblkdst+clmblkdst8
	sub	ah, 1			;; Test outer loop counter
	mov	al, 2
	jnz	b9b			;; Iterate if necessary
	end_timer 26

;; Work on next 4 sets of 768 values

endlp:	xpass1_get_next_block pass2, c0b, b0b ;; Get next block and jump
	done:	xfft_footer
	ENDM


;; Perform a 6144K-element negacyclic FFT.  This is done in two passes.  Pass
;; 1 does 12 levels, pass 2 uses common code to perform the last 11 levels.

hg_pass1sclevels12complex3 MACRO fftname, pass2_levels, clmarg
	LOCAL	b0b, b1a, b1b, b2a, b2b, b3a, b3b
	LOCAL	b4a, b4b, b5a, b5b, b7a, b7b, b9b
	LOCAL	c0b, c1a, c1b, c3a, c3b, c4a, c4b, c5a, c5b, c7a, c7b, c9b
	LOCAL	pass1, pass2, endlp, done, yes_fft

	set_FFT_constants fftname, 0, 1, pass2_levels, clmarg
	xfft_header pass1

;; Do the last 11 FFT levels
;;	do 192 pass2_11_levels_complex

pass2:	start_timer 0
	call	xpass2name
	end_timer 0
	cmp	ffttype, 1		;; We're done if FFTing only
	je	done

;; Do the inverse FFT

	pass1_inverse_fft_setup c0b

;; Do inverse FFT levels 11,12
;; On input the 64-byte cache lines hold these data values:
;;	0K	2K	4K	6K
;;	8K	10K	12K	14K
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0K	4K	8K	12K
;;	2K	...
;;	16K	...
;;	...

;; Do 768 four_complex_unfft macros
;;	distance between fft data elements is 2K
;;	do 384 macros each processing 16 data values

c0b:	start_timer 15
	mov	rsi, DATA_ADDR		;; Get source address
	mov	rcx, scratch_area	;; Get address of scratch area
	mov	rdi, sincos1		;; Load sin/cos pointer
	mov	eax, 96*256		;; 96 iterations of 4 iters of 2*clm
	sub	rdx, rdx
pfing	mov	rbp, DATA_PREFETCH	;; Set prefetch distance
pfing	sub	rbp, rsi
c9b:	g2cl_four_complex_unfft rsi, 64, blkdst, rcx, 64, clmblkdst
	add	dl, 256/2/clm		;; Test loop counter
	jnc	c9b			;; Iterate if necessary
	bump	rsi, -2*clm*64+2*blkdst	;; Next source pointer
	bump	rcx, -2*clm*64+2*clmblkdst;; Next dest pointer
	bump	rdi, XMM_SCD		;; Next sine/cosine pointer
	add	al, 256/4		;; Test loop counter
	jnc	c9b
	bump	rcx, -8*clmblkdst+clmblkdst8
	sub	ah, 1			;; Test outer loop counter
	jnz	c9b			;; Iterate if necessary
	lea	rcx, [rsi-384*2*blkdst]	;; Create prefetch pointer
	mov	rsi, scratch_area	;; Restore source pointer
	end_timer 15

;; Do inverse FFT levels 9,10
;; On input the 64-byte cache lines hold these data values:
;;	0K	4K	8K	12K
;;	2K	...
;;	16K	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0K	16K	32K	48K
;;	2K	...
;;	...
;;	14K
;;	64K	...
;;	...

;; Do 768 four_complex_unfft macros
;;	distance between fft data elements is 8K
;;	do 192 macros each processing 32 data values

	start_timer 16
	mov	rdi, sincos2		;; Load sin/cos pointer
	mov	eax, 96*256		;; 96 iterations of 2*2*clm
c7a:	xtouch	[rcx+(clm-1)*128]	;; Preload TLB
c7b:	x4cl_four_complex_unfft rsi, 64, 2*clmblkdst, 4*clmblkdst, rdi
	xloop2	c7b, clmblkdst		;; 2*2*clm iterations
	bump	rsi, -2*clmblkdst+clmblkdst8;; Next source pointer
	bump	rdi, XMM_SCD		;; Next sine/cosine pointer
	sub	ah, 1			;; Test outer loop counter
	jnz	c7a			;; Iterate if necessary
	bump	rsi, -96*clmblkdst8	;; Restore source pointer
	end_timer 16

;; Do inverse FFT levels 7,8
;; On input the 64-byte cache lines hold these data values:
;;	0K	16K	32K	48K
;;	2K	...
;;	...
;;	14K
;;	64K	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0K	64K	128K	192K
;;	2K	...
;;	...
;;	62K
;;	256K	...
;;	...

;; Do 768 four_complex_unfft macros
;;	distance between fft data elements is 32K
;;	do 192 macros each processing 32 data values

	start_timer 17
	mov	rdi, sincos3		;; Load sin/cos pointer
	mov	eax, 24*256+4		;; 24 iterations of 4 iters of 2*2*clm
c5a:	xtouch	[rcx+(clm-1)*128]	;; Preload TLB
c5b:	x4cl_four_complex_unfft rsi, 64, clmblkdst8, 2*clmblkdst8, rdi
	xloop2	c5b, clmblkdst		;; 2*2*clm iterations
	sub	al, 1			;; Test middle loop counter
	jnz	c5a			;; Iterate if necessary
	bump	rsi, -4*2*clmblkdst+4*clmblkdst8;; Next source pointer
	bump	rdi, XMM_SCD		;; Next sine/cosine pointer
	sub	eax, 256		;; Test outer loop counter
	mov	al, 4			;; Load middle loop counter
	jnz	c5a			;; Iterate if necessary
	bump	rsi, -24*4*clmblkdst8	;; Restore source pointer
	end_timer 17

;; Do inverse FFT levels 5,6
;; On input the 64-byte cache lines hold these data values:
;;	0K	64K	128K	192K
;;	2K	...
;;	...
;;	62K
;;	256K	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0K	256K	512K	768K
;;	2K	...
;;	...
;;	254K
;;	1024K	...
;;	...

;; Do 768 four_complex_unfft macros
;;	distance between fft data elements is 128K
;;	do 192 macros each processing 32 data values

	start_timer 18
	mov	rdi, sincos4		;; Load sin/cos pointer
	mov	eax, 6*256+4		;; 6 iters of 4 iters of 4 of 2*2*clm
c4a:	xtouch	[rcx+(clm-1)*128]	;; Preload TLB
c4b:	x4cl_four_complex_unfft rsi, 64, 4*clmblkdst8, 8*clmblkdst8, rdi
	xloop2	c4b, clmblkdst		;; 2 * 2*clm iterations
	add	eax, 80000000h/2	;; Test middle loop counter
	jnc	c4a			;; Iterate if necessary
	bump	rsi, -4*2*clmblkdst+clmblkdst8;; Next source pointer
	sub	al, 1			;; Test middle loop counter
	jnz	c4a			;; Iterate if necessary
	bump	rsi, -4*clmblkdst8+16*clmblkdst8;; Next source pointer
	bump	rdi, XMM_SCD		;; Next sine/cosine pointer
	sub	eax, 256		;; Test outer loop counter
	mov	al, 4			;; Load middle loop counter
	jnz	c4a			;; Iterate if necessary
	bump	rsi, -6*16*clmblkdst8	;; Restore source pointer
	end_timer 19

;; Do inverse FFT levels 4
;; On input the 64-byte cache lines hold these data values:
;;	0K	256K	512K	768K
;;	2K	...
;;	...
;;	254K
;;	1024K	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0K	512K	1024K	1536K
;;	2K	...
;;	...
;;	510K
;;	2048K	3072K	4096K	5120K
;;	2050K
;;	...
;;	3070K

;; Do 768 two_two_complex_unfft macros
;;	distance between fft data elements is 256K
;;	do 384 macros each processing 16 data values

	start_timer 19
	mov	rdi, sincos5		;; Load sin/cos pointer
	mov	eax, 3*256+16		;; 3 iters of 16 iters of 8*2*clm
c3a:	xtouch	[rcx+(clm-1)*128]	;; Preload TLB
c3b:	x2cl_two_complex_unfft rsi, 64, 16*clmblkdst8
	xloop8	c3b, clmblkdst		;; 8*2*clm iterations
	bump	rsi, -8*clmblkdst+clmblkdst8;; Next source pointer
	sub	al, 1			;; Test middle loop counter
	jnz	c3a
	bump	rsi, -16*clmblkdst8+32*clmblkdst8;; Next source pointer
	bump	rdi, XMM_SCD		;; Next sine/cosine pointer
	sub	eax, 256		;; Test outer loop counter
	mov	al, 16			;; Load middle loop counter
	jnz	c3a			;; Iterate if necessary
	bump	rsi, -3*32*clmblkdst8	;; Restore source pointer
	end_timer 19

;; Do inverse FFT levels 1,2,3
;; On input the 64-byte cache lines hold these data values:
;;	0K	512K	1024K	1536K
;;	2K	...
;;	...
;;	510K
;;	2048K	3072K	4096K	5120K
;;	2050K
;;	...
;;	3070K
;; On output the 64-byte cache lines hold these data values:
;;	0K	1536K	3072K	4608K
;;	...
;;	1534K

;; Do 1024 three_complex_last_unfft macros
;;	distance between fft data elements is 1024K
;;	do 256 macros each processing 24 data values

	start_timer 20
	mov	rdi, plus1_premults	;; Address of premultiplier table
	mov	eax, 32*256+2		;; 32 iters of 2 iters of 4*2*clm
c1a:	xtouch	[rcx+(clm-1)*128]	;; Preload TLB
c1b:	x3cl_three_complex_last_unfft rsi, 64, 32*clmblkdst8
	xloop4edi c1b, clmblkdst, 192 ;; 2 * 2*clm iterations
	sub	al, 1			;; Test loop counter
	jnz	c1a			;; Iterate if necessaryxn
	bump	rsi, -4*2*clmblkdst+clmblkdst8;; Next source pointer
	sub	ah, 1			;; Test outer loop counter
	mov	al, 2			;; Load middle loop counter
	jnz	c1a
	end_timer 20

;; Normalize these values

	pass1_normalize 1, yes_fft

;; Copy data from the scratch area back to the FFT data area

	copy_scratch_data_to_fft 768
	jmp	endlp			;; Skip FFT code if flag not set

;; Fire up auxiliary threads to do pass 1 forward FFT

pass1:	pass1_forward_fft_setup b0b

;; Copy data from the FFT data area back to the scratch area

b0b:	copy_fft_data_to_scratch 768

;; Do FFT levels 1,2,3
;;
;; On input the 64-byte cache lines hold these data values:
;;	0K	1536K	3072K	4608K
;;	...
;;	1534K
;; On output the 64-byte cache lines hold these data values:
;;	0K	512K	1024K	1536K
;;	2K	...
;;	...
;;	510K
;;	2048K	3072K	4096K	5120K
;;	2050K
;;	...
;;	3070K

;; Do 1024 three_complex_first_fft macros (FFT levels 1,2,3)
;;	distance between fft data elements is 1024K
;;	do 256 iterations each processing 24 data values

yes_fft:start_timer 22
	mov	rsi, scratch_area	;; Get address of scratch area
	mov	rdi, plus1_premults	;; Address of premultiplier table
	mov	rcx, DATA_ADDR		;; Load source address for prefetching
pfing	mov	rbp, DATA_PREFETCH	;; Set prefetch distance
pfing	sub	rbp, rcx
pfing	bump	rcx, 400*blkdst		;; Create prefetching pointer
	mov	eax, 32*65536		;; 32 iters of 2 iters of 4*2*clm
	sub	rdx, rdx
b1a:	xtouch	[rcx+(clm-1)*128]	;; Preload TLB
b1b:	x3cl_three_complex_fft rsi, 64, 32*clmblkdst8
	xloop8edi b1b, clmblkdst, 192	;; 4 * 2*clm iterations
	bump	rsi, -8*clmblkdst+clmblkdst8
	sub	eax, 65536		;; Test loop counter
	jnz	b1a			;; Iterate if necessary
	bump	rsi, -32*clmblkdst8	;; Restore source pointer
	end_timer 22

;; Do FFT levels 4
;;
;; On input the 64-byte cache lines hold these data values:
;;	0K	512K	1024K	1536K
;;	2K	...
;;	...
;;	510K
;;	2048K	3072K	4096K	5120K
;;	2050K
;;	...
;;	3070K
;; On output the 64-byte cache lines hold these data values:
;;	0K	512K	1024K	1536K
;;	2K	...
;;	...
;;	510K	...
;;	2048K	...
;;	...

;; Do 768 two_two_complex_fft macros
;;	distance between fft data elements is 256K
;;	do 384 macros each processing 16 data values

	start_timer 23
	mov	rdi, sincos5		;; Load sin/cos pointer
	mov	eax, 16*65536		;; 16 iters of 2 of 4*2*clm
b2a:	xtouch	[rcx+(clm-1)*128]	;; Preload TLB
b2b:	x2cl_two_complex_fft_in_place rsi, 64, 16*clmblkdst8, rdi
	xloop8	b2b, clmblkdst		;; 4 * 2*clm iterations
	bump	rsi, -8*clmblkdst+clmblkdst8
	sub	eax, 65536		;; Test outer loop counter
	jnz	b2a			;; Iterate if necessary
	bump	rsi, -16*clmblkdst8+32*clmblkdst8;; Next source pointer
	bump	rdi, XMM_SCD		;; Next sine/cosine pointer
	mov	eax, 32*65536		;; 32 iters of 2 of 4*2*clm
b3a:	xtouch	[rcx+(clm-1)*128]	;; Preload TLB
b3b:	x2cl_two_complex_fft rsi, 64, 32*clmblkdst8, rdi
	xloop8	b3b, clmblkdst		;; 4 * 2*clm iterations
	bump	rsi, -8*clmblkdst+clmblkdst8
	sub	eax, 65536		;; Test outer loop counter
	jnz	b3a			;; Iterate if necessary
	bump	rsi, -32*clmblkdst8-32*clmblkdst8;; Restore source pointer
	end_timer 23

;; Do FFT levels 5,6
;;
;; On input the 64-byte cache lines hold these data values:
;;	0K	512K	1024K	1536K
;;	2K	...
;;	...
;;	510K	...
;;	2048K	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0K	128K	256K	384K
;;	2K	...
;;	...
;;	126K	...
;;	512K	...
;;	...

;; Do 768 four_complex_fft macros
;;	distance between fft data elements is 128K
;;	do 192 macros each processing 32 data values

	start_timer 24
	mov	rdi, sincos4		;; Load sin/cos pointer
	mov	eax, 3*256+8		;; 3 iters of 8 iters of 4 of 2*2*clm
b4a:	xtouch	[rcx+(clm-1)*128]	;; Preload TLB
b4b:	x4cl_four_complex_fft rsi, 64, 8*clmblkdst8, 16*clmblkdst8, rdi
	xloop2	b4b, clmblkdst		;; 2 * 2*clm iterations
	add	eax, 80000000h/2	;; 4 iterations
	jnc	b4a
	bump	rsi, -4*2*clmblkdst+clmblkdst8
	sub	al, 1			;; Test middle loop counter
	jnz	b4a			;; Iterate if necessary
	bump	rsi, -8*clmblkdst8+32*clmblkdst8;; Next source pointer
	bump	rdi, 2*XMM_SCD		;; Next sine/cosine pointer
	sub	eax, 256		;; Test outer loop counter
	mov	al, 8			;; Load middle loop counter
	jnz	b4a			;; Iterate if necessary
	bump	rsi, -3*32*clmblkdst8	;; Restore source pointer
	end_timer 24

;; Do FFT levels 7,8
;;
;; On input the 64-byte cache lines hold these data values:
;;	0K	128K	256K	384K
;;	2K	...
;;	...
;;	126K	...
;;	512K	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0K	32K	64K	96K
;;	2K	...
;;	...
;;	30K	...
;;	128K	...
;;	...

;; Do 768 four_complex_fft macros
;;	distance between fft data elements is 32K
;;	do 192 macros each processing 32 data values

	start_timer 24
	mov	rdi, sincos3		;; Load sin/cos pointer
	mov	eax, 12*256+2		;; 12 iters of 2 iters of 4 of 2*2*clm
b5a:	xtouch	[rcx+(clm-1)*128]	;; Preload TLB
b5b:	x4cl_four_complex_fft rsi, 64, 2*clmblkdst8, 4*clmblkdst8, rdi
	xloop2	b5b, clmblkdst		;; 2 * 2*clm iterations
	add	eax, 80000000h/2	;; 4 iterations
	jnc	b5a
	bump	rsi, -4*2*clmblkdst+clmblkdst8
	sub	al, 1			;; Test middle loop counter
	jnz	b5a			;; Iterate if necessary
	bump	rsi, -2*clmblkdst8+8*clmblkdst8;; Next source pointer
	bump	rdi, 2*XMM_SCD		;; Next sine/cosine pointer
	sub	eax, 256		;; Test outer loop counter
	mov	al, 2			;; Load middle loop counter
	jnz	b5a			;; Iterate if necessary
	bump	rsi, -12*8*clmblkdst8	;; Restore source pointer
	end_timer 24

;; Do FFT levels 9,10
;;
;; On input the 64-byte cache lines hold these data values:
;;	0K	32K	64K	96K
;;	2K	...
;;	...
;;	30K	...
;;	128K	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0K	8K	16K	24K
;;	2K	...
;;	...
;;	6K	...
;;	32K	...
;;	...

;; Do 768 four_complex_fft macros
;;	distance between fft data elements is 8K
;;	do 192 macros each processing 32 data values

	start_timer 25
	mov	rdi, sincos2		;; Load sin/cos pointer
	mov	eax, 48*65536+2		;; 48 iterations of 2 iters of 2*2*clm
b7a:	xtouch	[rcx+(clm-1)*128]	;; Preload TLB
b7b:	x4cl_four_complex_fft rsi, 64, 4*clmblkdst, clmblkdst8, rdi
	xloop2	b7b, clmblkdst		;; 2*2*clm iterations
	sub	al, 1			;; Test middle loop counter
	jnz	b7a			;; Iterate if necessary
	bump	rsi, -2*2*clmblkdst+2*clmblkdst8;; Next source pointer
	bump	rdi, 2*XMM_SCD		;; Next sine/cosine pointer
	sub	eax, 65536		;; Test outer loop counter
	mov	al, 2			;; Load middle loop counter
	jnz	b7a			;; Iterate if necessary
	bump	rsi, -48*2*clmblkdst8	;; Restore source pointer
pfing	bump	rcx, -768*blkdst	;; Restore prefetch pointer
	end_timer 25

;; Do FFT levels 11,12
;;
;; On input the 64-byte cache lines hold these data values:
;;	0K	8K	16K	24K
;;	2K	...
;;	...
;;	6K	...
;;	32K	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0K	2K	4K	6K
;;	8K	...
;;	...

;; Do 768 four_complex_fft macros
;;	distance between fft data elements is 2K
;;	do 192 macros each processing 32 data values

	start_timer 26
	mov	rdi, sincos1		;; Load sin/cos pointer
	mov	eax, 96*65536+2		;; 96 iterations of 2 iters of 2*clm
b9b:	g4cl_four_complex_fft rsi, 64, clmblkdst, 2*clmblkdst, rcx, 64, blkdst, 2*blkdst
	L2prefetch128 [rcx-64][rbp]
	L2prefetch128 [rcx-64+blkdst][rbp]
	bump	rbp, -64+2*blkdst
	add	al, 256/2		;; Test inner loop counter
	jnc	b9b			;; Iterate if necessary
	bump	rbp, 128-4*blkdst
	IF clm NE 1
	add	dl, 256/clm
	jnc	b9b
	ENDIF
	bump	rsi, -2*clm*64+4*clmblkdst;; Next source pointer
	bump	rcx, -2*clm*64+4*blkdst	;; Next dest pointer
	bump	rdi, 2*XMM_SCD		;; Next sine/cosine pointer
	sub	al, 1			;; Test outer loop counter
	jnz	b9b
	bump	rsi, -8*clmblkdst+clmblkdst8
	sub	eax, 65536		;; Test outer loop counter
	mov	al, 2
	jnz	b9b			;; Iterate if necessary
	end_timer 26

;; Work on next 4 sets of 768 values

endlp:	xpass1_get_next_block pass2, c0b, b0b ;; Get next block and jump
	done:	xfft_footer
	ENDM


;; Perform a 7168K-element FFT.  This is done in two passes.  Pass 1
;; does 12 levels, pass 2 uses common code to perform the last 11 levels.

hg_pass1sclevels12pfa7 MACRO fftname, pass2_levels, clmarg
	LOCAL	b0b, b1a, b1b, b2b, b3a, b3b, b4b, b4c
	LOCAL	b5b, b5c, b5d, b6b, b7a, b7b, b8b, b9b
	LOCAL	c0b, c1b, c2a, c2b, c3b, c4b, c4d, c5b, c5c, c5d
	LOCAL	c6b, c7a, c7b, c8b, c9b
	LOCAL	pass1, pass2, endlp, done, yes_fft

	set_FFT_constants fftname, 0, 0, pass2_levels, clmarg
	xfft_header pass1

;; Do the last 11 FFT levels
;;	do 1 pass2_11_levels_real and 223 pass2_11_levels_complex

pass2:	start_timer 0
	call	xpass2name
	end_timer 0
	cmp	ffttype, 1		;; We're done if FFTing only
	je	done

;; Do the inverse FFT

	pass1_inverse_fft_setup c0b

;; Do inverse FFT levels 11,12
;; On input the 64-byte cache lines hold these data values:
;;	0K	2K	4K	6K
;;	8K	10K	12K	14K
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0K	4K	8K	12K
;;	2K	...
;;	16K	...
;;	...

;; Do 1 eight_reals_unfft_2 macros
;; Do 1 nop_two_two_complex_unfft_2 macros
;;	distance between fft data elements is 2K
;;	do 1 macros each processing 16 data values

c0b:	start_timer 16
	mov	rsi, DATA_ADDR		;; Get source address
	mov	rcx, scratch_area	;; Get address of scratch area
	sub	rax, rax
	sub	rdx, rdx
pfing	mov	rbp, DATA_PREFETCH	;; Set prefetch distance
pfing	sub	rbp, rsi
c8b:	g2cl_eight_reals_unfft_2 rsi, 64, blkdst, rcx, 64, clmblkdst
	add	al, 256/2/clm		;; Test loop counter
	jnc	c8b			;; Iterate if necessary

;; Do 894 four_complex_unfft macros
;;	distance between fft data elements is 2K
;;	do 447 macros each processing 16 data values

	bump	rsi, -2*clm*64+2*blkdst	;; Next source pointer
	bump	rcx, -2*clm*64+2*clmblkdst;; Next dest pointer
	mov	rdi, sincos1		;; Load sin/cos pointer
	mov	eax, 112*65536+256/4	;; 447 iterations of 2*clm
c9b:	g2cl_four_complex_unfft rsi, 64, blkdst, rcx, 64, clmblkdst
	add	dl, 256/2/clm		;; Test loop counter
	jnc	c9b			;; Iterate if necessary
	bump	rsi, -2*clm*64+2*blkdst	;; Next source pointer
	bump	rcx, -2*clm*64+2*clmblkdst;; Next dest pointer
	bump	rdi, XMM_SCD		;; Next sine/cosine pointer
	add	al, 256/4		;; Test loop counter
	jnc	c9b
	bump	rcx, -4*2*clmblkdst+clmblkdst8
	sub	eax, 65536		;; Test outer loop counter
	jnz	c9b			;; Iterate if necessary
	lea	rcx, [rsi-448*2*blkdst]	;; Create prefetch pointer
	mov	rsi, scratch_area	;; Restore source pointer
	end_timer 16

;; Do inverse FFT levels 9,10
;; On input the 64-byte cache lines hold these data values:
;;	0K	4K	8K	12K
;;	2K	...
;;	16K	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0K	16K	32K	48K
;;	2K	...
;;	...
;;	14K
;;	64K	...
;;	...

;; Do 4 eight_reals_unfft_2 macros
;; Do 4 nop_two_two_complex_unfft_2 macros
;;	distance between fft data elements is 8K
;;	do 2 macros each processing 32 data values

	start_timer 17
 	mov	al, 2			;; 2 iterations of 2*clm
c6b:	x4cl_eight_reals_unfft_2 rsi, 64, 2*clmblkdst, 4*clmblkdst
	add	al, 256/2/clm		;; Test loop counter
	jnc	c6b
	bump	rsi, -clm*2*64+clmblkdst;; Next source pointer
	sub	al, 1			;; Test outer loop counter
	jnz	c6b			;; Iterate if necessary

;; Do 888 four_complex_unfft macros
;;	distance between fft data elements is 8K
;;	do 222 macros each processing 32 data values

	bump	rsi, -2*clmblkdst+clmblkdst8;; Next source pointer
	mov	rdi, sincos2		;; Load sin/cos pointer
	mov	eax, 111*65536		;; 111 iterations of 2*2*clm
c7a:	xtouch	[rcx+(clm-1)*128]	;; Preload TLB
c7b:	x4cl_four_complex_unfft rsi, 64, 2*clmblkdst, 4*clmblkdst, rdi
	xloop2	c7b, clmblkdst
	bump	rsi, -2*clmblkdst+clmblkdst8;; Next source pointer
	bump	rdi, XMM_SCD		;; Next sine/cosine pointer
	sub	eax, 65536		;; Test outer loop counter
	jnz	c7a			;; Iterate if necessary
	bump	rsi, -112*clmblkdst8	;; Restore source pointer
	end_timer 17

;; Do inverse FFT levels 7,8
;; On input the 64-byte cache lines hold these data values:
;;	0K	16K	32K	48K
;;	2K	...
;;	...
;;	14K
;;	64K	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0K	64K	128K	192K
;;	2K	...
;;	...
;;	62K
;;	256K	...
;;	...

;; Do 16 eight_reals_unfft_2 macros
;; Do 16 nop_two_two_complex_unfft_2 macros
;;	distance between fft data elements is 32K
;;	do 8 macros each processing 32 data values

	start_timer 18
	mov	al, 8			;; 8 iterations of 2*clm
c4d:	x4cl_eight_reals_unfft_2 rsi, 64, clmblkdst8, 2*clmblkdst8
	add	al, 256/2/clm		;; Test loop counter
	jnc	c4d
	bump	rsi, -clm*2*64+clmblkdst;; Next source pointer
	sub	al, 1			;; Test outer loop counter
	jnz	c4d			;; Iterate if necessary

;; Do 864 four_complex_unfft macros
;;	distance between fft data elements is 32K
;;	do 216 macros each processing 32 data values

	bump	rsi, -8*clmblkdst+4*clmblkdst8;; Next source pointer
	mov	rdi, sincos3		;; Load sin/cos pointer
	mov	eax, 27*65536+4		;; 27 iterations of 4 iters of 2*2*clm
c5c:	xtouch	[rcx+(clm-1)*128]	;; Preload TLB
c5d:	x4cl_four_complex_unfft rsi, 64, clmblkdst8, 2*clmblkdst8, rdi
	xloop2	c5d, clmblkdst
	sub	al, 1			;; Test middle loop counter
	jnz	c5c			;; Iterate if necessary
	bump	rsi, -4*2*clmblkdst+4*clmblkdst8;; Next source pointer
	bump	rdi, XMM_SCD		;; Next sine/cosine pointer
	sub	eax, 65536		;; Test outer loop counter
	mov	al, 4			;; Load middle loop counter
	jnz	c5c			;; Iterate if necessary
	bump	rsi, -28*4*clmblkdst8	;; Restore source pointer
	end_timer 18

;; Do inverse FFT levels 6
;; On input the 64-byte cache lines hold these data values:
;;	0K	64K	128K	192K
;;	2K	...
;;	...
;;	62K
;;	256K	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0K	128K	256K	384K
;;	2K	...
;;	...
;;	126K
;;	512K	...
;;	...

;; Do 64 eight_reals_unfft_1 macros
;;	distance between fft data elements is 128K
;;	do 32 macros each processing 16 data values
;;
;; Do 832 four_complex_unfft macros
;;	distance between fft data elements is 128K
;;	do 416 macros each processing 16 data values

;; Do inverse FFT levels 4,5
;; On input the 64-byte cache lines hold these data values:
;;	0K	128K	256K	384K
;;	2K	...
;;	...
;;	126K
;;	512K	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0K	256K	512K	768K
;;	2K	...
;;	...
;;	254K
;;	1024K	1536K	2048K	2560K
;;	...
;;
;; Do 128 eight_reals_unfft_2 macros
;;	distance between fft data elements is 256K
;;	do 64 macros each processing 16 data values
;;
;; Do 768 four_complex_unfft macros
;;	distance between fft data elements is 256K
;;	do 192 macros each processing 32 data values
;;
;; Do inverse FFT levels 1,2,3
;; On input the 64-byte cache lines hold these data values:
;;	0K	256K	512K	768K
;;	2K	...
;;	...
;;	254K
;;	1024K	1536K	2048K	2560K
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0K	256K	3584K	3840K
;;	2K	...
;;	...
;;	254K
;;	512K	1024K	4096K	4608K
;;	514K
;;	...
;;	1022K
;;	1536K	2560K	5120K	6144K
;;	1538K
;;	...
;;	2558K
;;
;; Do 1024 seven_reals_last_unfft macros
;;	distance between fft data elements is 1024K
;;	do 128 macros each processing 56 data values

	start_timer 19
	mov	rdi, sincos4		;; Load sin/cos pointer
	mov	rbx, sincos5		;; Load sin/cos pointer
	mov	eax, 4			;; 4 iterations
	xseven_reals_unfft_preload

c2a:	xtouchlines 7			;; Preload TLBs

c4b:	x2cl_eight_reals_unfft_1 rsi, 64, 4*clmblkdst8
	add	al, 256/2		;; Test loop counter
	jnc	c4b			;; Iterate if necessary

	bump	rsi, -2*64+8*clmblkdst8	;; Next source pointer
	mov	ah, 13			;; 13 iterations
c5b:	x2cl_two_complex_unfft rsi, 64, 4*clmblkdst8
	add	al, 256/2		;; Test loop counter
	jnc	c5b			;; Iterate if necessary
	bump	rsi, -2*64+8*clmblkdst8	;; Next source pointer
	bump	rdi, XMM_SCD		;; Next sine/cosine pointer
	sub	ah, 1			;; Test outer loop counter
	jnz	c5b			;; Iterate if necessary
	bump	rdi, -13*XMM_SCD	;; Restore sine/cosine pointer

	bump	rsi, -14*8*clmblkdst8	;; Restore source pointer
c2b:	x2cl_half_eight_reals_unfft_2 rsi, 64, 8*clmblkdst8
	add	al, 256/2		;; Test loop counter
	jnc	c2b			;; Iterate if necessary
	bump	rsi, -2*64+4*clmblkdst8
	add	ah, 256/2		;; Test loop counter
	jnc	c2b			;; Iterate if necessary

	bump	rsi, -2*4*clmblkdst8+16*clmblkdst8;; Next source pointer
	mov	ah, 3			;; 3 iterations
c3b:	x4cl_four_complex_unfft rsi, 64, 8*clmblkdst8, 16*clmblkdst8, rbx
	xloop2sp c3b, 4*clmblkdst8
	bump	rsi, -2*4*clmblkdst8+32*clmblkdst8
	bump	rbx, XMM_SCD		;; Next sine/cosine pointer
	sub	ah, 1			;; Test outer loop counter
	jnz	c3b			;; Iterate if necessary
	bump	rbx, -3*XMM_SCD		;; Restore sine/cosine pointer

	bump	rsi, -3*32*clmblkdst8-16*clmblkdst8;; Restore src pointer
c1b:	x7cl_seven_reals_last_unfft rsi, 64, 16*clmblkdst8
	xloop1sp c1b, 4*clmblkdst8
	add	ah, 256/4
	jnc	c1b

	IF clm GT 1
	bump	rsi, -4*4*clmblkdst8+2*64;; Next source pointer
	bump	rcx, -7*blkdst+128	;; Next prefetch pointer
	add	dl, 256/clm		;; clm iterations
	jnc	c4b
	bump	rsi, -clm*2*64+clmblkdst;; Next source pointer
	bump	rcx, -clm*128+7*blkdst	;; Next prefetch pointer
	ELSE
	bump	rsi, -4*4*clmblkdst8+clmblkdst;; Next source pointer
	ENDIF

	add	eax, 80000000h/4	;; Test loop counter
	jnc	c2a			;; Iterate if necessary
	bump	rsi, -8*clmblkdst+clmblkdst8;; Next source pointer
	sub	al, 1			;; Test loop counter
	jnz	c2a			;; Iterate if necessary
	end_timer 19

;; Normalize these values

	pass1_normalize 1, yes_fft

;; Copy data from the scratch area back to the FFT data area

	copy_scratch_data_to_fft 896
	jmp	endlp			;; Skip FFT code if flag not set

;; Fire up auxiliary threads to do pass 1 forward FFT

pass1:	pass1_forward_fft_setup b0b

;; Copy data from the FFT data area back to the scratch area

b0b:	copy_fft_data_to_scratch 896

;; Do FFT levels 1,2,3
;;
;; On input the 64-byte cache lines hold these data values:
;;	0K	256K	3584K	3840K
;;	2K	...
;;	...
;;	254K
;;	512K	1024K	4096K	4608K
;;	514K
;;	...
;;	1022K
;;	1536K	2560K	5120K	6144K
;;	1538K
;;	...
;;	2558K
;; On output the 64-byte cache lines hold these data values:
;;	0K	256K	512K	768K
;;	2K	...
;;	...
;;	254K
;;	1024K	1536K	2048K	2560K
;;	1026K
;;	...
;;	1534K
;;	3072K	4096K	5120K	6144K
;;	3074K
;;	...
;;	4094K
;;
;; Do 1024 seven_reals_first_fft macros (FFT levels 1,2,3)
;;	distance between fft data elements is 1024K
;;	do 128 iterations each processing 56 data values

;; Do FFT levels 4,5
;; Values 0K-1023K is real data, 1024K-2047K is semi-real data, 2048K-7167K is
;; complex data.
;;
;; On input the 64-byte cache lines hold these data values:
;;	0K	256K	512K	768K
;;	2K	...
;;	...
;;	254K
;;	1024K	1536K	2048K	2560K
;;	1026K
;;	...
;;	1534K
;;	3072K	4096K	5120K	6144K
;;	3074K
;;	...
;;	4094K
;; On output the 64-byte cache lines hold these data values:
;;	0K	256K	512K	768K
;;	2K	...
;;	...
;;	254K	...
;;	1024K	...
;;	...
;;
;; Do 128 eight_reals_fft_2 macros
;;	distance between fft data elements is 256K
;;	do 128 macros each processing 8 data values
;;
;; Do 768 four_complex_fft macros
;;	distance between fft data elements is 256K
;;	do 192 macros each processing 32 data values

;; Do FFT levels 6
;; Values 0K-255K is real data, 256K-511K is semi-real data, 512K-7167K is
;; complex data.
;;
;; On input the 64-byte cache lines hold these data values:
;;	0K	256K	512K	768K
;;	2K	...
;;	...
;;	254K	...
;;	1024K	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0K	128K	256K	384K
;;	2K	...
;;	...
;;	126K	...
;;	512K	...
;;	...
;;
;; Do 64 eight_reals_fft_1 macros
;; Do 64 two_two_complex_fft macros
;;	distance between fft data elements is 128K
;;	do 64 macros each processing 16 data values
;;
;; Do 768 two_two_complex_fft macros
;;	distance between fft data elements is 128K
;;	do 384 macros each processing 16 data values

yes_fft:start_timer 22
	mov	rsi, scratch_area	;; Get address of scratch area
	mov	rcx, DATA_ADDR		;; Load source address for prefetching
pfing	mov	rbp, DATA_PREFETCH	;; Set prefetch distance
pfing	sub	rbp, rcx
pfing	bump	rcx, 426*blkdst		;; Create prefetching pointer
	mov	rdi, sincos5		;; Load sin/cos pointer
	mov	rbx, sincos4		;; Load sin/cos pointer
	mov	eax, 8			;; 8 iterations
	sub	rdx, rdx
	xseven_reals_fft_preload

b1a:	xtouchlines 4			;; Preload TLBs

b1b:	x7cl_seven_reals_fft rsi, 64, 16*clmblkdst8
	xloop1sp b1b, 8*clmblkdst8
	add	ah, 256/2
	jnc	b1b

	bump	rsi, -2*8*clmblkdst8	;; Restore source pointer
b2b:	x1cl_half_eight_reals_fft_2 rsi, 64
	add	al, 256/2		;; Test inner loop counter
	jnc	b2b			;; Iterate if necessary
	bump	rsi, -2*64+8*clmblkdst8	;; Next source pointer
	add	ah, 256/2
	jnc	b2b

	bump	rsi, -2*8*clmblkdst8+16*clmblkdst8;; Next source pointer
b3a:	x2cl_four_complex_fft rsi, 64, 16*clmblkdst8
	add	al, 256/2		;; Test inner loop counter
	jnc	b3a			;; Iterate if necessary
	bump	rsi, -2*64+8*clmblkdst8	;; Next source pointer
	add	ah, 256/2
	jnc	b3a

	bump	rsi, -2*8*clmblkdst8+32*clmblkdst8;; Next source pointer
	bump	rdi, XMM_SCD		;; Next sine/cosine pointer
b3b:	x4cl_four_complex_fft rsi, 64, 16*clmblkdst8, 32*clmblkdst8, rdi
	xloop1sp b3b, 8*clmblkdst8
	add	ah, 256/2
	jnc	b3b
	bump	rdi, -XMM_SCD		;; Restore sine/cosine pointer

	bump	rsi, -2*8*clmblkdst8-48*clmblkdst8;; Restore src pointer
b4b:	x2cl_eight_reals_fft_1 rsi, 64, 8*clmblkdst8
	add	al, 256/2		;; Test inner loop counter
	jnc	b4b			;; Iterate if necessary

	bump	rsi, -2*64+16*clmblkdst8;; Next source pointer
	bump	rbx, XMM_SCD		;; Next sine/cosine pointer
	mov	ah, 6			;; 6 iterations
b5b:	x2cl_two_complex_fft rsi, 64, 8*clmblkdst8, rbx
	add	al, 256/2		;; Test inner loop counter
	jnc	b5b			;; Iterate if necessary
	bump	rsi, -2*64+16*clmblkdst8;; Next source pointer
	bump	rbx, 2*XMM_SCD		;; Next sine/cosine pointer
	sub	ah, 1			;; Test outer loop counter
	jnz	b5b			;; Iterate if necessary
	bump	rbx, -13*XMM_SCD	;; Restore sine/cosine pointer

	IF clm GT 1
	bump	rsi, -7*16*clmblkdst8+2*64;; Next source pointer
	bump	rcx, -4*blkdst+128	;; Next prefetch pointer
	add	dl, 256/clm		;; clm iterations
	jnc	b1b
	bump	rsi, -clm*2*64+clmblkdst;; Next source pointer
	bump	rcx, -clm*128+4*blkdst	;; Next prefetch pointer
	ELSE
	bump	rsi, -7*16*clmblkdst8+clmblkdst;; Next source pointer
	ENDIF

	add	eax, 80000000h/4	;; Test loop counter
	jnc	b1a			;; Iterate if necessary
	bump	rsi, -8*clmblkdst+clmblkdst8;; Next source pointer
	sub	al, 1			;; Test loop counter
	jnz	b1a			;; Iterate if necessary
	bump	rsi, -8*clmblkdst8	;; Restore source pointer
	end_timer 22

;; Do FFT levels 7,8
;; Values 0K-127K is real data, 128K-255K is semi-real data, 256K-7167K is
;; complex data.
;;
;; On input the 64-byte cache lines hold these data values:
;;	0K	128K	256K	384K
;;	2K	...
;;	...
;;	126K	...
;;	512K	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0K	32K	64K	96K
;;	2K	...
;;	...
;;	30K	...
;;	128K	...
;;	...

;; Do 32 eight_reals_fft_2 macros
;; Do 32 two_two_complex_fft macros
;;	distance between fft data elements is 32K
;;	do 16 macros each processing 32 data values

	start_timer 24
	mov	rdi, sincos3		;; Load sin/cos pointer
	mov	al, 2			;; 2 iterations of 8 iters of 2*clm
b4c:	x4cl_eight_reals_fft_2 rsi, 64, 2*clmblkdst8, 4*clmblkdst8, rdi
	add	al, 256/2/clm		;; Test inner loop counter
	jnc	b4c			;; Iterate if necessary
	bump	rsi, -2*clm*64+clmblkdst;; Next source pointer
	add	ah, 256/8		;; Test loop counter
	jnc	b4c			;; Iterate if necessary
	bump	rsi, -8*clmblkdst+clmblkdst8;; Next source pointer
	sub	al, 1			;; Test outer loop counter
	jnz	b4c			;; Iterate if necessary

;; Do 832 four_complex_fft macros
;;	distance between fft data elements is 32K
;;	do 208 macros each processing 32 data values

	bump	rsi, -2*clmblkdst8+8*clmblkdst8;; Load source pointer
	bump	rdi, XMM_SCD		;; Next sine/cosine pointer
	mov	eax, 13*256+2		;; 13 iters of 2*4 iters of 2*2*clm
b5c:	xtouch	[rcx+(clm-1)*128]	;; Preload TLB
b5d:	x4cl_four_complex_fft rsi, 64, 2*clmblkdst8, 4*clmblkdst8, rdi
	xloop2	b5d, clmblkdst
	add	eax, 80000000h/2	;; 4 iterations
	jnc	b5c
	bump	rsi, -8*clmblkdst+clmblkdst8;; Next source pointer
	sub	al, 1			;; Test middle loop counter
	jnz	b5c			;; Iterate if necessary
	bump	rsi, -2*clmblkdst8+8*clmblkdst8;; Next source pointer
	bump	rdi, 2*XMM_SCD		;; Next sine/cosine pointer
	sub	eax, 256		;; Test outer loop counter
	mov	al, 2			;; Load middle loop counter
	jnz	b5c			;; Iterate if necessary
	bump	rsi, -14*8*clmblkdst8	;; Restore source pointer
	end_timer 24

;; Do FFT levels 9,10
;; Values 0K-31K is real data, 32K-63K is semi-real data, 64K-7167K is
;; complex data.
;;
;; On input the 64-byte cache lines hold these data values:
;;	0K	32K	64K	96K
;;	2K	...
;;	...
;;	30K	...
;;	128K	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0K	8K	16K	24K
;;	2K	...
;;	...
;;	6K	...
;;	32K	...
;;	...

;; Do 8 eight_reals_fft_2 macros
;; Do 8 two_two_complex_fft macros
;;	distance between fft data elements is 8K
;;	do 4 macros each processing 32 data values

	start_timer 25
	mov	rdi, sincos2		;; Load sin/cos pointer
	mov	al, 4			;; 4 iterations of 2*clm
b6b:	x4cl_eight_reals_fft_2 rsi, 64, 4*clmblkdst, clmblkdst8, rdi
	add	al, 256/2/clm		;; Test loop counter
	jnc	b6b
	bump	rsi, -2*clm*64+clmblkdst;; Next source pointer
	sub	al, 1			;; Test outer loop counter
	jnz	b6b			;; Iterate if necessary

;; Do 880 four_complex_fft macros
;;	distance between fft data elements is 8K
;;	do 220 macros each processing 32 data values

	bump	rsi, -4*clmblkdst+2*clmblkdst8;; Load source pointer
	bump	rdi, XMM_SCD		;; Next sine/cosine pointer
	mov	eax, 55*256+2		;; 55 iterations of 4 iters of 2*clm
b7a:	xtouch	[rcx+(clm-1)*128]	;; Preload TLB
b7b:	x4cl_four_complex_fft rsi, 64, 4*clmblkdst, clmblkdst8, rdi
	xloop2	b7b, clmblkdst
	sub	al, 1			;; Test middle loop counter
	jnz	b7a			;; Iterate if necessary
	bump	rsi, -4*clmblkdst+2*clmblkdst8;; Next source pointer
	bump	rdi, 2*XMM_SCD		;; Next sine/cosine pointer
	sub	eax, 256		;; Test outer loop counter
	mov	al, 2			;; Load middle loop counter
	jnz	b7a			;; Iterate if necessary
	bump	rsi, -56*2*clmblkdst8	;; Restore source pointer
pfing	bump	rcx, -896*blkdst	;; Restore prefetch pointer
	end_timer 25

;; Do FFT levels 11,12
;; Values 0K-7K is real data, 8K-15K is semi-real data, 16K-7167K is
;; complex data.
;;
;; On input the 64-byte cache lines hold these data values:
;;	0K	8K	16K	24K
;;	2K	...
;;	...
;;	6K	...
;;	32K	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0K	2K	4K	6K
;;	8K	...
;;	...

;; Do 1 eight_reals_fft_2 macros
;; Do 1 nop_two_two_complex_fft_2 macros
;; Do 2 four_complex_fft macros
;;	distance between fft data elements is 2K
;;	do 1 macros each processing 32 data values

	start_timer 26
	mov	rdi, sincos1		;; Load sin/cos pointer
b8b:	g4cl_eight_reals_fft_2 rsi, 64, clmblkdst, 2*clmblkdst, rcx, 64, blkdst, 2*blkdst
	add	al, 256/2/clm		;; Test inner loop counter
	jnc	b8b			;; Iterate if necessary

;; Do 892 four_complex_fft macros
;;	distance between fft data elements is 2K
;;	do 223 macros each processing 32 data values

	bump	rsi, -2*clm*64+4*clmblkdst;; Next source pointer
	bump	rcx, -2*clm*64+4*blkdst	;; Next dest pointer
	bump	rdi, XMM_SCD		;; Next sine/cosine pointer
	mov	eax, 112*65536+1	;; 223 iterations of 2*clm
b9b:	g4cl_four_complex_fft rsi, 64, clmblkdst, 2*clmblkdst, rcx, 64, blkdst, 2*blkdst
	L2prefetch128 [rcx-64][rbp]
	L2prefetch128 [rcx-64+blkdst][rbp]
	bump	rbp, -64+2*blkdst
	add	al, 256/2		;; Test inner loop counter
	jnc	b9b			;; Iterate if necessary
	bump	rbp, 128-4*blkdst
	IF clm NE 1
	add	dl, 256/clm
	jnc	b9b
	ENDIF
	bump	rsi, -2*clm*64+4*clmblkdst;; Next source pointer
	bump	rcx, -2*clm*64+4*blkdst	;; Next dest pointer
	bump	rdi, 2*XMM_SCD		;; Next sine/cosine pointer
	sub	al, 1
	jnz	b9b
	bump	rsi, -8*clmblkdst+clmblkdst8
	sub	eax, 65536		;; Test outer loop counter
	mov	al, 2
	jnz	b9b			;; Iterate if necessary
	end_timer 26

;; Work on next 4 sets of 896 values

endlp:	xpass1_get_next_block pass2, c0b, b0b ;; Get next block and jump
	done:	xfft_footer
	ENDM


;; Perform a 8M-element FFT.  This is done in two passes.  Pass 1
;; does 12 levels, pass 2 uses common code to perform the last 11 levels.

hg_pass1sclevels12pfa8 MACRO fftname, pass2_levels, clmarg
	LOCAL	b0b, b1a, b1b, b2b, b3b, b4b, b4c
	LOCAL	b5b, b5c, b5d, b6b, b7a, b7b, b8b, b9b
	LOCAL	c0b, c1b, c2a, c2b, c3b, c4b, c4c
	LOCAL	c5b, c5c, c5d, c6b, c7a, c7b, c8b, c9b
	LOCAL	pass1, pass2, endlp, done, yes_fft

	set_FFT_constants fftname, 0, 0, pass2_levels, clmarg
	xfft_header pass1

;; Do the last 11 FFT levels
;;	do 1 pass2_11_levels_real and 255 pass2_11_levels_complex

pass2:	start_timer 0
	call	xpass2name
	end_timer 0
	cmp	ffttype, 1		;; We're done if FFTing only
	je	done

;; Do the inverse FFT

	pass1_inverse_fft_setup c0b

;; Do inverse FFT levels 10,11
;; On input the 64-byte cache lines hold these data values:
;;	0K	2K	4K	6K
;;	8K	10K	12K	14K
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0K	4K	8K	12K
;;	2K	...
;;	16K	...
;;	...

;; Do 1 eight_reals_unfft_2 macros
;; Do 1 nop_two_two_complex_unfft_2 macros
;;	distance between fft data elements is 2K
;;	do 1 macros each processing 16 data values

c0b:	start_timer 16
	mov	rsi, DATA_ADDR		;; Get source address
	mov	rcx, scratch_area	;; Get address of scratch area
	sub	rax, rax
	sub	rdx, rdx
pfing	mov	rbp, DATA_PREFETCH	;; Set prefetch distance
pfing	sub	rbp, rsi
c8b:	g2cl_eight_reals_unfft_2 rsi, 64, blkdst, rcx, 64, clmblkdst
	add	al, 256/2/clm		;; Test loop counter
	jnc	c8b			;; Iterate if necessary

;; Do 1022 four_complex_unfft macros
;;	distance between fft data elements is 2K
;;	do 511 macros each processing 16 data values

	bump	rsi, -2*clm*64+2*blkdst	;; Next source pointer
	bump	rcx, -2*clm*64+2*clmblkdst;; Next dest pointer
	mov	rdi, sincos1		;; Load sin/cos pointer
	mov	eax, 128*65536+256/4	;; 511 iterations of 2*clm
c9b:	g2cl_four_complex_unfft rsi, 64, blkdst, rcx, 64, clmblkdst
	add	dl, 256/2/clm		;; Test loop counter
	jnc	c9b			;; Iterate if necessary
	bump	rsi, -2*clm*64+2*blkdst	;; Next source pointer
	bump	rcx, -2*clm*64+2*clmblkdst;; Next dest pointer
	bump	rdi, XMM_SCD		;; Next sine/cosine pointer
	add	al, 256/4
	jnc	c9b
	bump	rcx, -8*clmblkdst+clmblkdst8
	sub	eax, 65536		;; Test outer loop counter
	jnz	c9b			;; Iterate if necessary
	lea	rcx, [rsi-512*2*blkdst]	;; Create prefetch pointer
	mov	rsi, scratch_area	;; Restore source pointer
	end_timer 16

;; Do inverse FFT levels 9,10
;; On input the 64-byte cache lines hold these data values:
;;	0K	4K	8K	12K
;;	2K	...
;;	16K	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0K	16K	32K	48K
;;	2K	...
;;	...
;;	14K
;;	64K	...
;;	...

;; Do 4 eight_reals_unfft_2 macros
;; Do 4 nop_two_two_complex_unfft_2 macros
;;	distance between fft data elements is 8K
;;	do 2 macros each processing 32 data values

	start_timer 17
	mov	al, 2			;; 2 iterations of 2*clm
c6b:	x4cl_eight_reals_unfft_2 rsi, 64, 2*clmblkdst, 4*clmblkdst
	add	al, 256/2/clm		;; Test loop counter
	jnc	c6b			;; Iterate if necessary
	bump	rsi, -2*clm*64+clmblkdst;; Next source pointer
	sub	al, 1			;; Test outer loop counter
	jnz	c6b			;; Iterate if necessary

;; Do 1016 four_complex_unfft macros
;;	distance between fft data elements is 8K
;;	do 254 macros each processing 32 data values

	bump	rsi, -2*clmblkdst+clmblkdst8;; Next source pointer
	mov	rdi, sincos2		;; Load sin/cos pointer
	mov	eax, 127*65536		;; 127 iterations of 2*2*clm
c7a:	xtouch	[rcx+(clm-1)*128]	;; Preload TLB
c7b:	x4cl_four_complex_unfft rsi, 64, 2*clmblkdst, 4*clmblkdst, rdi
	xloop2	c7b, clmblkdst
	bump	rsi, -2*clmblkdst+clmblkdst8;; Next source pointer
	bump	rdi, XMM_SCD		;; Next sine/cosine pointer
	sub	eax, 65536		;; Test outer loop counter
	jnz	c7a			;; Iterate if necessary
	bump	rsi, -128*clmblkdst8	;; Restore source pointer
	end_timer 17

;; Do inverse FFT levels 7,8
;; On input the 64-byte cache lines hold these data values:
;;	0K	16K	32K	48K
;;	2K	...
;;	...
;;	14K
;;	64K	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0K	64K	128K	192K
;;	2K	...
;;	...
;;	62K
;;	256K	...
;;	...

;; Do 16 eight_reals_unfft_2 macros
;; Do 16 nop_two_two_complex_unfft_2 macros
;;	distance between fft data elements is 32K
;;	do 8 macros each processing 32 data values

	start_timer 18
	mov	al, 8			;; 8 iterations of 2*clm
c4c:	x4cl_eight_reals_unfft_2 rsi, 64, clmblkdst8, 2*clmblkdst8
	add	al, 256/2/clm		;; Test loop counter
	jnc	c4c			;; Iterate if necessary
	bump	rsi, -2*clm*64+clmblkdst;; Next source pointer
	sub	al, 1			;; Test outer loop counter
	jnz	c4c			;; Iterate if necessary

;; Do 992 four_complex_unfft macros
;;	distance between fft data elements is 32K
;;	do 248 macros each processing 32 data values

	bump	rsi, -8*clmblkdst+4*clmblkdst8;; Next source pointer
	mov	rdi, sincos3		;; Load sin/cos pointer
	mov	eax, 31*256+4		;; 31 iterations of 4 iters of 2*2*clm
c5c:	xtouch	[rcx+(clm-1)*128]	;; Preload TLB
c5d:	x4cl_four_complex_unfft rsi, 64, clmblkdst8, 2*clmblkdst8, rdi
	xloop2	c5d, clmblkdst
	sub	al, 1			;; Test middle loop counter
	jnz	c5c			;; Iterate if necessary
	bump	rsi, -4*2*clmblkdst+4*clmblkdst8;; Next source pointer
	bump	rdi, XMM_SCD		;; Next sine/cosine pointer
	sub	eax, 256		;; Test outer loop counter
	mov	al, 4			;; Load middle loop counter
	jnz	c5c			;; Iterate if necessary
	bump	rsi, -32*4*clmblkdst8	;; Restore source pointer
	end_timer 18

;; Do inverse FFT levels 6
;; On input the 64-byte cache lines hold these data values:
;;	0K	64K	128K	192K
;;	2K	...
;;	...
;;	62K
;;	256K	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0K	128K	256K	384K
;;	2K	...
;;	...
;;	126K
;;	512K	...
;;	...
;;
;; Do 64 eight_reals_unfft_1 macros
;;	distance between fft data elements is 128K
;;	do 32 macros each processing 16 data values
;;
;; Do 960 four_complex_unfft macros
;;	distance between fft data elements is 128K
;;	do 480 macros each processing 16 data values

;; Do inverse FFT levels 4,5
;; On input the 64-byte cache lines hold these data values:
;;	0K	128K	256K	384K
;;	2K	...
;;	...
;;	126K
;;	512K	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0K	512K	1024K	1536K
;;	2K	...
;;	...
;;	510K
;;	2048K	...
;;	...
;;
;; Do 128 eight_reals_unfft_2 macros
;; Do 128 nop_two_two_complex_unfft_2 macros
;;	distance between fft data elements is 256K
;;	do 64 macros each processing 32 data values
;;
;; Do 768 four_complex_unfft macros
;;	distance between fft data elements is 256K
;;	do 192 macros each processing 32 data values

;; Do inverse FFT levels 1,2,3
;; On input the 64-byte cache lines hold these data values:
;;	0K	512K	1024K	1536K
;;	2K	...
;;	...
;;	510K
;;	2048K	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0K	2048K	4096K	6144K
;;	2K	...
;;	...
;;	2046K
;;
;; Do 1024 eight_reals_last_unfft macros
;;	distance between fft data elements is 1024K
;;	do 256 macros each processing 32 data values

	start_timer 19
	mov	rdi, sincos4		;; Load sin/cos pointer
	mov	rbx, sincos5		;; Load sin/cos pointer
	mov	eax, 4			;; 4 iterations

c2a:	xtouchlines 9			;; Preload TLBs

c4b:	x2cl_eight_reals_unfft_1 rsi, 64, 4*clmblkdst8
	add	al, 256/2		;; Test loop counter
	jnc	c4b			;; Iterate if necessary

	bump	rsi, -2*64+8*clmblkdst8	;; Next source pointer
	mov	ah, 15			;; 15 iterations
c5b:	x2cl_two_complex_unfft rsi, 64, 4*clmblkdst8
	add	al, 256/2		;; Test loop counter
	jnc	c5b			;; Iterate if necessary
	bump	rsi, -2*64+8*clmblkdst8	;; Next source pointer
	bump	rdi, XMM_SCD		;; Next sine/cosine pointer
	sub	ah, 1			;; Test outer loop counter
	jnz	c5b			;; Iterate if necessary
	bump	rdi, -15*XMM_SCD	;; Restore sine/cosine pointer

	bump	rsi, -16*8*clmblkdst8	;; Restore source pointer
c2b:	x4cl_eight_reals_unfft_2 rsi, 64, 8*clmblkdst8, 16*clmblkdst8
	xloop1sp c2b, 4*clmblkdst8
	add	ah, 256/2		;; Test loop counter
	jnc	c2b			;; Iterate if necessary

	bump	rsi, -2*4*clmblkdst8+32*clmblkdst8;; Next source pointer
	mov	ah, 3			;; 3 iterations
c3b:	x4cl_four_complex_unfft rsi, 64, 8*clmblkdst8, 16*clmblkdst8, rbx
	xloop2sp c3b, 4*clmblkdst8
	bump	rsi, -2*4*clmblkdst8+32*clmblkdst8
	bump	rbx, XMM_SCD		;; Next sine/cosine pointer
	sub	ah, 1
	jnz	c3b
	bump	rbx, -3*XMM_SCD		;; Next sine/cosine pointer

	bump	rsi, -4*32*clmblkdst8	;; Restore source pointer
c1b:	x4cl_eight_reals_last_unfft rsi, 64, 32*clmblkdst8, 64*clmblkdst8
	xloop2sp c1b, 4*clmblkdst8
	add	ah, 256/4
	jnc	c1b

	IF clm GT 1
	bump	rsi, -8*4*clmblkdst8+2*64;; Next source pointer
	bump	rcx, -9*blkdst+128	;; Next prefetch pointer
	add	dl, 256/clm		;; clm iterations
	jnc	c4b
	bump	rsi, -clm*2*64+clmblkdst;; Next source pointer
	bump	rcx, -clm*128+9*blkdst	;; Next prefetch pointer
	ELSE
	bump	rsi, -8*4*clmblkdst8+clmblkdst;; Next source pointer
	ENDIF

	add	eax, 80000000h/4	;; Test loop counter
	jnc	c2a			;; Iterate if necessary
	bump	rsi, -8*clmblkdst+clmblkdst8;; Next source pointer
	sub	al, 1			;; Test loop counter
	jnz	c2a			;; Iterate if necessary
	end_timer 19

;; Normalize these values

	pass1_normalize 1, yes_fft

;; Copy data from the scratch area back to the FFT data area

	copy_scratch_data_to_fft 1024
	jmp	endlp			;; Skip FFT code if flag not set

;; Fire up auxiliary threads to do pass 1 forward FFT

pass1:	pass1_forward_fft_setup b0b

;; Copy data from the FFT data area back to the scratch area

b0b:	copy_fft_data_to_scratch 1024

;; Do FFT levels 1,2,3
;;
;; On input the 64-byte cache lines hold these data values:
;;	0K	2048K	4096K	6144K
;;	2K	...
;;	...
;;	2046K
;; On output the 64-byte cache lines hold these data values:
;;	0K	1024K	2048K	3072K
;;	2K	...
;;	...
;;	1022K	...
;;	4096K	...
;;	...
;;
;; Do 1024 eight_reals_first_fft macros (FFT levels 1,2,3)
;;	distance between fft data elements is 1024K
;;	do 256 iterations each processing 16 data values

;; Do FFT levels 4,5
;; Values 0K-1023K is real data, 1024K-2047K is semi-real data, 2048K-8191K is
;; complex data.
;;
;; On input the 64-byte cache lines hold these data values:
;;	0K	1024K	2048K	3072K
;;	2K	...
;;	...
;;	1022K	...
;;	4096K	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0K	256K	512K	768K
;;	2K	...
;;	...
;;	254K	...
;;	1024K	...
;;	...
;;
;; Do 128 eight_reals_fft_2 macros
;; Do 128 nop_two_two_complex_fft_2 macros
;; Do 256 four_complex_fft macros
;;	distance between fft data elements is 256K
;;	do 128 macros each processing 32 data values
;;
;; Do 512 four_complex_fft macros
;;	distance between fft data elements is 256K
;;	do 128 macros each processing 32 data values

;; Do FFT levels 6
;; Values 0K-255K is real data, 256K-511K is semi-real data, 512K-8191K is
;; complex data.
;;
;; On input the 64-byte cache lines hold these data values:
;;	0K	256K	512K	768K
;;	2K	...
;;	...
;;	254K	...
;;	1024K	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0K	128K	256K	384K
;;	2K	...
;;	...
;;	126K	...
;;	512K	...
;;	...
;;
;; Do 64 eight_reals_fft_1 macros
;; Do 64 two_two_complex_fft macros
;;	distance between fft data elements is 128K
;;	do 64 macros each processing 16 data values
;;
;; Do 896 two_two_complex_fft macros
;;	distance between fft data elements is 128K
;;	do 448 macros each processing 16 data values

yes_fft:start_timer 22
	mov	rsi, scratch_area	;; Get address of scratch area
	mov	rcx, DATA_ADDR		;; Load source address for prefetching
pfing	mov	rbp, DATA_PREFETCH	;; Set prefetch distance
pfing	sub	rbp, rcx
pfing	bump	rcx, 522*blkdst		;; Create prefetching pointer
	mov	rdi, sincos5		;; Load sin/cos pointer
	mov	rbx, sincos4		;; Load sin/cos pointer
	mov	eax, 8			;; 8 iterations
	sub	rdx, rdx

b1a:	xtouchlines 4			;; Preload TLBs

b1b:	x2cl_eight_reals_fft rsi, 64, 64*clmblkdst8
	add	al, 256/2		;; Test loop counter
	jnc	b1b			;; Iterate if necessary
	bump	rsi, -2*64+8*clmblkdst8	;; Next source pointer
	add	ah, 256/8
	jnc	b1b

	bump	rsi, -8*8*clmblkdst8	;; Restore source pointer
b2b:	x4cl_eight_reals_fft_2 rsi, 64, 16*clmblkdst8, 32*clmblkdst8, rdi
	xloop1sp b2b, 8*clmblkdst8
	add	ah, 256/2
	jnc	b2b

	bump	rsi, -2*8*clmblkdst8+64*clmblkdst8;; Next source pointer
	bump	rdi, XMM_SCD		;; Next sine/cosine pointer
b3b:	x4cl_four_complex_fft rsi, 64, 16*clmblkdst8, 32*clmblkdst8, rdi
	xloop1sp b3b, 8*clmblkdst8
	add	ah, 256/2
	jnc	b3b
	bump	rdi, -XMM_SCD		;; Restore sine/cosine pointer

	bump	rsi, -2*8*clmblkdst8-64*clmblkdst8;; Restore src pointer
b4b:	x2cl_eight_reals_fft_1 rsi, 64, 8*clmblkdst8
	add	al, 256/2		;; Test inner loop counter
	jnc	b4b			;; Iterate if necessary

	bump	rsi, -2*64+16*clmblkdst8;; Load source pointer
	bump	rbx, XMM_SCD		;; Next sine/cosine pointer
	mov	ah, 7			;; 7 iterations
b5b:	x2cl_two_complex_fft rsi, 64, 8*clmblkdst8, rbx
	add	al, 256/2		;; Test inner loop counter
	jnc	b5b			;; Iterate if necessary
	bump	rsi, -2*64+16*clmblkdst8;; Next source pointer
	bump	rbx, 2*XMM_SCD		;; Next sine/cosine pointer
	sub	ah, 1			;; Test outer loop counter
	jnz	b5b			;; Iterate if necessary
	bump	rbx, -15*XMM_SCD	;; Restore sine/cosine pointer

	IF clm GT 1
	bump	rsi, -8*16*clmblkdst8+2*64;; Next source pointer
	bump	rcx, -4*blkdst+128	;; Next prefetch pointer
	add	dl, 256/clm		;; clm iterations
	jnc	b1b
	bump	rsi, -clm*2*64+clmblkdst;; Next source pointer
	bump	rcx, -clm*128+4*blkdst	;; Next prefetch pointer
	ELSE
	bump	rsi, -8*16*clmblkdst8+clmblkdst;; Next source pointer
	ENDIF

	add	eax, 80000000h/4	;; Test loop counter
	jnc	b1a			;; Iterate if necessary
	bump	rsi, -8*clmblkdst+clmblkdst8;; Next source pointer
	sub	al, 1			;; Test loop counter
	jnz	b1a			;; Iterate if necessary
	bump	rsi, -8*clmblkdst8	;; Restore source pointer
	end_timer 22

;; Do FFT levels 7,8
;; Values 0K-127K is real data, 128K-255K is semi-real data, 256K-8191K is
;; complex data.
;;
;; On input the 64-byte cache lines hold these data values:
;;	0K	128K	256K	384K
;;	2K	...
;;	...
;;	126K	...
;;	512K	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0K	32K	64K	96K
;;	2K	...
;;	...
;;	30K	...
;;	128K	...
;;	...

;; Do 32 eight_reals_fft_2 macros
;; Do 32 two_two_complex_fft macros
;;	distance between fft data elements is 32K
;;	do 16 macros each processing 32 data values

	start_timer 24
	mov	rdi, sincos3		;; Load sin/cos pointer
	mov	al, 2			;; 2 iterations of 8 iters of 2*clm
b4c:	x4cl_eight_reals_fft_2 rsi, 64, 2*clmblkdst8, 4*clmblkdst8, rdi
	add	al, 256/2/clm		;; Test inner loop counter
	jnc	b4c			;; Iterate if necessary
	bump	rsi, -2*clm*64+clmblkdst;; Next source pointer
	add	ah, 256/8		;; Test loop counter
	jnc	b4c			;; Iterate if necessary
	bump	rsi, -8*clmblkdst+clmblkdst8;; Next source pointer
	sub	al, 1			;; Test outer loop counter
	jnz	b4c			;; Iterate if necessary

;; Do 960 four_complex_fft macros
;;	distance between fft data elements is 32K
;;	do 240 macros each processing 32 data values

	bump	rsi, -2*clmblkdst8+8*clmblkdst8;; Load source pointer
	bump	rdi, XMM_SCD		;; Next sine/cosine pointer
	mov	eax, 15*256+2		;; 15 iters of 2*4 iters of 2*2*clm
b5c:	xtouch	[rcx+(clm-1)*128]	;; Preload TLB
b5d:	x4cl_four_complex_fft rsi, 64, 2*clmblkdst8, 4*clmblkdst8, rdi
	xloop2	b5d, clmblkdst
	add	eax, 80000000h/2	;; 4 iterations
	jnc	b5c
	bump	rsi, -8*clmblkdst+clmblkdst8;; Next source pointer
	sub	al, 1			;; Test middle loop counter
	jnz	b5c			;; Iterate if necessary
	bump	rsi, -2*clmblkdst8+8*clmblkdst8;; Next source pointer
	bump	rdi, 2*XMM_SCD		;; Next sine/cosine pointer
	sub	eax, 256		;; Test outer loop counter
	mov	al, 2			;; Load middle loop counter
	jnz	b5c			;; Iterate if necessary
	bump	rsi, -16*8*clmblkdst8	;; Restore source pointer
	end_timer 24

;; Do FFT levels 9,10
;; Values 0K-31K is real data, 32K-63K is semi-real data, 64K-8191K is
;; complex data.
;;
;; On input the 64-byte cache lines hold these data values:
;;	0K	32K	64K	96K
;;	2K	...
;;	...
;;	30K	...
;;	128K	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0K	8K	16K	24K
;;	2K	...
;;	...
;;	6K	...
;;	32K	...
;;	...

;; Do 8 eight_reals_fft_2 macros
;; Do 8 two_two_complex_fft macros
;;	distance between fft data elements is 8K
;;	do 4 macros each processing 32 data values

	start_timer 25
	mov	rdi, sincos2		;; Load sin/cos pointer
	mov	al, 4			;; 4 iterations of 2*clm
b6b:	x4cl_eight_reals_fft_2 rsi, 64, 4*clmblkdst, clmblkdst8, rdi
	add	al, 256/2/clm		;; Test loop counter
	jnc	b6b			;; Iterate if necessary
	bump	rsi, -2*clm*64+clmblkdst;; Next source pointer
	sub	al, 1			;; Test outer loop counter
	jnz	b6b			;; Iterate if necessary

;; Do 1008 four_complex_fft macros
;;	distance between fft data elements is 8K
;;	do 252 macros each processing 32 data values

	bump	rsi, -4*clmblkdst+2*clmblkdst8;; Load source pointer
	bump	rdi, XMM_SCD		;; Next sine/cosine pointer
	mov	eax, 63*65536+2		;; 63 iterations of 2 iters of 2*2*clm
b7a:	xtouch	[rcx+(clm-1)*128]	;; Preload TLB
b7b:	x4cl_four_complex_fft rsi, 64, 4*clmblkdst, clmblkdst8, rdi
	xloop2	b7b, clmblkdst
	sub	al, 1			;; Test middle loop counter
	jnz	b7a			;; Iterate if necessary
	bump	rsi, -2*2*clmblkdst+2*clmblkdst8;; Next source pointer
	bump	rdi, 2*XMM_SCD		;; Next sine/cosine pointer
	sub	eax, 65536		;; Test outer loop counter
	mov	al, 2			;; Load middle loop counter
	jnz	b7a			;; Iterate if necessary
	bump	rsi, -64*2*clmblkdst8	;; Restore source pointer
pfing	bump	rcx, -1024*blkdst	;; Restore prefetch pointer
	end_timer 25

;; Do FFT levels 9,10
;; Values 0K-7K is real data, 8K-15K is semi-real data, 16K-8191K is
;; complex data.
;;
;; On input the 64-byte cache lines hold these data values:
;;	0K	8K	16K	24K
;;	2K	...
;;	...
;;	6K	...
;;	32K	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0K	2K	4K	6K
;;	8K	...
;;	...

;; Do 1 eight_reals_fft_2 macros
;; Do 1 nop_two_two_complex_fft_2 macros
;; Do 2 four_complex_fft macros
;;	distance between fft data elements is 2K
;;	do 1 macros each processing 32 data values

	start_timer 26
	mov	rdi, sincos1		;; Load sin/cos pointer
b8b:	g4cl_eight_reals_fft_2 rsi, 64, clmblkdst, 2*clmblkdst, rcx, 64, blkdst, 2*blkdst
	add	al, 256/2/clm		;; Test inner loop counter
	jnc	b8b			;; Iterate if necessary

;; Do 1020 four_complex_fft macros
;;	distance between fft data elements is 2K
;;	do 255 macros each processing 32 data values

	bump	rsi, -2*clm*64+4*clmblkdst;; Next source pointer
	bump	rcx, -2*clm*64+4*blkdst	;; Next dest pointer
	bump	rdi, XMM_SCD		;; Next sine/cosine pointer
	mov	eax, 128*65536+1	;; 255 iterations of 2*clm
b9b:	g4cl_four_complex_fft rsi, 64, clmblkdst, 2*clmblkdst, rcx, 64, blkdst, 2*blkdst
	L2prefetch128 [rcx-64][rbp]
	L2prefetch128 [rcx-64+blkdst][rbp]
	bump	rbp, -64+2*blkdst
	add	al, 256/2		;; Test inner loop counter
	jnc	b9b			;; Iterate if necessary
	bump	rbp, 128-4*blkdst
	IF clm NE 1
	add	dl, 256/clm
	jnc	b9b
	ENDIF
	bump	rsi, -2*clm*64+4*clmblkdst;; Next source pointer
	bump	rcx, -2*clm*64+4*blkdst	;; Next dest pointer
	bump	rdi, 2*XMM_SCD		;; Next sine/cosine pointer
	sub	al, 1
	jnz	b9b
	bump	rsi, -8*clmblkdst+clmblkdst8
	sub	eax, 65536		;; Test outer loop counter
	mov	al, 2
	jnz	b9b			;; Iterate if necessary
	end_timer 26

;; Work on next 4 sets of 1024 values

endlp:	xpass1_get_next_block pass2, c0b, b0b ;; Get next block and jump
	done:	xfft_footer
	ENDM


;; Perform a 8M-element negacyclic FFT.  This is done in two passes.  Pass 1
;; does 12 levels, pass 2 uses common code to perform the last 11 levels.

hg_pass1sclevels12complex4 MACRO fftname, pass2_levels, clmarg
	LOCAL	b0b, b1a, b1b, b3a, b3b, b4a, b4b, b5a, b5b, b7a, b7b, b9b
	LOCAL	c0b, c1a, c1b, c3a, c3b, c5a, c5b, c6a, c6b, c7a, c7b, c9b
	LOCAL	pass1, pass2, endlp, done, yes_fft

	set_FFT_constants fftname, 0, 1, pass2_levels, clmarg
	xfft_header pass1

;; Do the last 11 FFT levels
;;	do 256 pass2_11_levels_complex

pass2:	start_timer 0
	call	xpass2name
	end_timer 0
	cmp	ffttype, 1		;; We're done if FFTing only
	je	done

;; Do the inverse FFT

	pass1_inverse_fft_setup c0b

;; Do inverse FFT levels 11,12
;; On input the 64-byte cache lines hold these data values:
;;	0K	2K	4K	6K
;;	8K	10K	12K	14K
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0K	4K	8K	12K
;;	2K	...
;;	16K	...
;;	...

;; Do 1024 four_complex_unfft macros
;;	distance between fft data elements is 2K
;;	do 512 macros each processing 16 data values

c0b:	start_timer 15
	mov	rsi, DATA_ADDR		;; Get source address
	mov	rcx, scratch_area	;; Get address of scratch area
	mov	rdi, sincos1		;; Load sin/cos pointer
	mov	eax, 128*65536		;; 128 iterations
	sub	rdx, rdx
pfing	mov	rbp, DATA_PREFETCH	;; Set prefetch distance
pfing	sub	rbp, rsi
c9b:	g2cl_four_complex_unfft rsi, 64, blkdst, rcx, 64, clmblkdst
	add	dl, 256/2/clm		;; Test loop counter
	jnc	c9b			;; Iterate if necessary
	bump	rsi, -2*clm*64+2*blkdst	;; Next source pointer
	bump	rcx, -2*clm*64+2*clmblkdst;; Next dest pointer
	bump	rdi, XMM_SCD		;; Next sine/cosine pointer
	add	al, 256/4
	jnc	c9b
	bump	rcx, -8*clmblkdst+clmblkdst8
	sub	eax, 65536		;; Test outer loop counter
	jnz	c9b			;; Iterate if necessary
	lea	rcx, [rsi-512*2*blkdst]	;; Create prefetch pointer
	mov	rsi, scratch_area	;; Restore source pointer
	end_timer 15

;; Do inverse FFT levels 9,10
;; On input the 64-byte cache lines hold these data values:
;;	0K	4K	8K	12K
;;	2K	...
;;	16K	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0K	16K	32K	48K
;;	2K	...
;;	...
;;	14K
;;	64K	...
;;	...

;; Do 1024 four_complex_unfft macros
;;	distance between fft data elements is 8K
;;	do 256 macros each processing 32 data values

	start_timer 16
	mov	rdi, sincos2		;; Load sin/cos pointer
	mov	eax, 128*65536		;; 128 iterations of 2*2*clm
c7a:	xtouch	[rcx+(clm-1)*128]	;; Preload TLB
c7b:	x4cl_four_complex_unfft rsi, 64, 2*clmblkdst, 4*clmblkdst, rdi
	xloop2	c7b, clmblkdst		;; 2 * 2*clm iterations
	bump	rsi, -2*clmblkdst+clmblkdst8;; Next source pointer
	bump	rdi, XMM_SCD		;; Next sine/cosine pointer
	sub	eax, 65536		;; Test outer loop counter
	jnz	c7a			;; Iterate if necessary
	bump	rsi, -128*clmblkdst8	;; Restore source pointer
	end_timer 16

;; Do inverse FFT levels 7,8
;; On input the 64-byte cache lines hold these data values:
;;	0K	16K	32K	48K
;;	2K	...
;;	...
;;	14K
;;	64K	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0K	64K	128K	192K
;;	2K	...
;;	...
;;	62K
;;	256K	...
;;	...

;; Do 1024 four_complex_unfft macros
;;	distance between fft data elements is 32K
;;	do 256 macros each processing 32 data values

	start_timer 17
	mov	rdi, sincos3		;; Load sin/cos pointer
	mov	eax, 32*256+4		;; 24 iterations of 4 iters of 2*2*clm
c6a:	xtouch	[rcx+(clm-1)*128]	;; Preload TLB
c6b:	x4cl_four_complex_unfft rsi, 64, clmblkdst8, 2*clmblkdst8, rdi
	xloop2	c6b, clmblkdst		;; 2*2*clm iterations
	sub	al, 1			;; Test middle loop counter
	jnz	c6a			;; Iterate if necessary
	bump	rsi, -4*2*clmblkdst+4*clmblkdst8;; Next source pointer
	bump	rdi, XMM_SCD		;; Next sine/cosine pointer
	sub	eax, 256		;; Test outer loop counter
	mov	al, 4			;; Load middle loop counter
	jnz	c6a			;; Iterate if necessary
	bump	rsi, -32*4*clmblkdst8	;; Restore source pointer
	end_timer 17

;; Do inverse FFT levels 5,6
;; On input the 64-byte cache lines hold these data values:
;;	0K	64K	128K	192K
;;	2K	...
;;	...
;;	62K
;;	256K	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0K	256K	512K	768K
;;	2K	...
;;	...
;;	254K
;;	1024K	...
;;	...

;; Do 1024 four_complex_unfft macros
;;	distance between fft data elements is 128K
;;	do 256 macros each processing 32 data values

	start_timer 18
	mov	rdi, sincos4		;; Load sin/cos pointer
	mov	eax, 8*256+4		;; 8 iterations of 4 iters of 2*2*clm
c5a:	xtouch	[rcx+(clm-1)*128]	;; Preload TLB
c5b:	x4cl_four_complex_unfft rsi, 64, 4*clmblkdst8, 8*clmblkdst8, rdi
	xloop4	c5b, clmblkdst		;; 4 * 2*clm iterations
	add	eax, 80000000h		;; Test middle loop counter
	jnc	c5a			;; Iterate if necessary
	bump	rsi, -2*4*clmblkdst+clmblkdst8;; Next source pointer
	sub	al, 1			;; Test middle loop counter
	jnz	c5a			;; Iterate if necessary
	bump	rsi, -4*clmblkdst8+16*clmblkdst8;; Next source pointer
	bump	rdi, XMM_SCD		;; Next sine/cosine pointer
	sub	eax, 256		;; Test outer loop counter
	mov	al, 4			;; Load middle loop counter
	jnz	c5a			;; Iterate if necessary
	bump	rsi, -8*16*clmblkdst8	;; Restore source pointer
	end_timer 18

;; Do inverse FFT levels 4
;; On input the 64-byte cache lines hold these data values:
;;	0K	256K	512K	768K
;;	2K	...
;;	...
;;	254K
;;	1024K	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0K	512K	1024K	1536K
;;	2K	...
;;	...
;;	510K
;;	2048K	...
;;	...

;; Do 1024 two_two_complex_unfft macros
;;	distance between fft data elements is 256K
;;	do 512 macros each processing 16 data values

	start_timer 19
	mov	rdi, sincos5		;; Load sin/cos pointer
	mov	eax, 4*256+16		;; 4 iters of 16 iters 2 of 4*2*clm
c3a:	xtouch	[rcx+(clm-1)*128]	;; Preload TLB
c3b:	x2cl_two_complex_unfft rsi, 64, 16*clmblkdst8
	xloop8	c3b, clmblkdst		;; 8 * 2*clm iterations
	bump	rsi, -8*clmblkdst+clmblkdst8;; Next source pointer
	sub	al, 1			;; Test middle loop counter
	jnz	c3a
	bump	rsi, -16*clmblkdst8+32*clmblkdst8;; Next source pointer
	bump	rdi, XMM_SCD		;; Next sine/cosine pointer
	sub	eax, 256		;; Test outer loop counter
	mov	al, 16			;; Load middle loop counter
	jnz	c3a			;; Iterate if necessary
	bump	rsi, -4*32*clmblkdst8	;; Restore source pointer
	end_timer 19

;; Do inverse FFT levels 1,2,3
;; On input the 64-byte cache lines hold these data values:
;;	0K	512K	1024K	1536K
;;	2K	...
;;	...
;;	510K
;;	2048K	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0K	2048K	4096K	6144K
;;	2K	...
;;	...
;;	2046K

;; Do 1024 four_complex_unfft macros
;;	distance between fft data elements is 1024K
;;	do 256 macros each processing 32 data values

	start_timer 20
	mov	rdi, plus1_premults	;; Address of premultiplier table
	mov	eax, 32*65536+4		;; 8 iters of 4 iters of 2*2*clm
c1a:	xtouch	[rcx+(clm-1)*128]	;; Preload TLB
c1b:	x4cl_four_complex_last_unfft rsi, 64, 32*clmblkdst8, 64*clmblkdst8, 256*128
	xloop2edi c1b, clmblkdst, 128	;; 2 * 2*clm iterations
	sub	al, 1			;; Test loop counter
	jnz	c1a			;; Iterate if necessaryxn
	bump	rsi, -4*2*clmblkdst+clmblkdst8;; Next source pointer
	sub	eax, 65536		;; Test outer loop counter
	mov	al, 4			;; Load middle loop counter
	jnz	c1a
	end_timer 20

;; Normalize these values

	pass1_normalize 1, yes_fft

;; Copy data from the scratch area back to the FFT data area

	copy_scratch_data_to_fft 1024
	jmp	endlp			;; Skip FFT code if flag not set

;; Fire up auxiliary threads to do pass 1 forward FFT

pass1:	pass1_forward_fft_setup b0b

;; Copy data from the FFT data area back to the scratch area

b0b:	copy_fft_data_to_scratch 1024

;; Do FFT levels 1,2,3
;;
;; On input the 64-byte cache lines hold these data values:
;;	0K	2048K	4096K	6144K
;;	2K	...
;;	...
;;	2046K
;; On output the 64-byte cache lines hold these data values:
;;	0K	1024K	2048K	3072K
;;	2K	...
;;	...
;;	1022K	...
;;	4096K	...
;;	...

;; Do 1024 four_complex_first_fft macros (FFT levels 1,2,3)
;;	distance between fft data elements is 1024K
;;	do 512 iterations each processing 16 data values

yes_fft:start_timer 22
	mov	rsi, scratch_area	;; Get address of scratch area
	mov	rdi, plus1_premults	;; Address of premultiplier table
	mov	rcx, DATA_ADDR		;; Load source address for prefetching
pfing	mov	rbp, DATA_PREFETCH	;; Set prefetch distance
pfing	sub	rbp, rcx
pfing	bump	rcx, 512*blkdst		;; Create prefetching pointer
	mov	eax, 64*65536		;; 64 iters of 2 iters of 4*2*clm
	sub	rdx, rdx
b1a:	xtouch	[rcx+(clm-1)*128]	;; Preload TLB
b1b:	x2cl_four_complex_first_fft_scratch rsi, 64, 64*clmblkdst8
	xloop4edi b1b, clmblkdst, 128	;; 4 * 2*clm iterations
	add	eax, 80000000h		;; 2 iterations
	jnc	b1a
	bump	rsi, -2*4*clmblkdst+clmblkdst8
	sub	eax, 65536		;; Test loop counter
	jnz	b1a			;; Iterate if necessary
	bump	rsi, -64*clmblkdst8	;; Restore source pointer
	end_timer 22

;; Do FFT levels 4
;;
;; On input the 64-byte cache lines hold these data values:
;;	0K	1024K	2048K	3072K
;;	2K	...
;;	...
;;	1022K	...
;;	4096K	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0K	512K	1024K	1536K
;;	2K	...
;;	...
;;	510K	...
;;	2048K	...
;;	...

;; Do 1024 two_two_complex_fft macros
;;	distance between fft data elements is 256K
;;	do 512 macros each processing 16 data values

	start_timer 23
	mov	rdi, sincos5		;; Load sin/cos pointer
	mov	eax, 2			;; 2 iters of 32 iters of 2 of 4*2*clm
b3a:	xtouch	[rcx+(clm-1)*128]	;; Preload TLB
b3b:	x2cl_two_complex_fft rsi, 64, 32*clmblkdst8, rdi
	xloop8	b3b, clmblkdst		;; 8 * 2*clm iterations
	bump	rsi, -8*clmblkdst+clmblkdst8
	add	eax, 80000000h/16	;; 32 iterations
	jnc	b3a
	bump	rsi, -32*clmblkdst8+64*clmblkdst8;; Next source pointer
	bump	rdi, 2*XMM_SCD		;; Next sine/cosine pointer
	sub	al, 1
	jnz	b3a
	bump	rsi, -2*64*clmblkdst8	;; Restore source pointer
	end_timer 23

;; Do FFT levels 5,6
;;
;; On input the 64-byte cache lines hold these data values:
;;	0K	512K	1024K	1536K
;;	2K	...
;;	...
;;	510K	...
;;	2048K	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0K	128K	256K	384K
;;	2K	...
;;	...
;;	126K	...
;;	512K	...
;;	...

;; Do 1024 four_complex_fft macros
;;	distance between fft data elements is 128K
;;	do 256 macros each processing 32 data values

	start_timer 24
	mov	rdi, sincos4		;; Load sin/cos pointer
	mov	eax, 4*256+8		;; 4 iters of 8 iters of 4 of 2*2*clm
b4a:	xtouch	[rcx+(clm-1)*128]	;; Preload TLB
b4b:	x4cl_four_complex_fft rsi, 64, 8*clmblkdst8, 16*clmblkdst8, rdi
	xloop2	b4b, clmblkdst		;; 2 * 2*clm iterations
	add	eax, 80000000h/2	;; 4 iterations
	jnc	b4a
	bump	rsi, -4*2*clmblkdst+clmblkdst8
	sub	al, 1			;; Test middle loop counter
	jnz	b4a			;; Iterate if necessary
	bump	rsi, -8*clmblkdst8+32*clmblkdst8;; Next source pointer
	bump	rdi, 2*XMM_SCD		;; Next sine/cosine pointer
	sub	eax, 256		;; Test outer loop counter
	mov	al, 8			;; Load middle loop counter
	jnz	b4a			;; Iterate if necessary
	bump	rsi, -4*32*clmblkdst8	;; Restore source pointer
	end_timer 24

;; Do FFT levels 7,8
;;
;; On input the 64-byte cache lines hold these data values:
;;	0K	128K	256K	384K
;;	2K	...
;;	...
;;	126K	...
;;	512K	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0K	32K	64K	96K
;;	2K	...
;;	...
;;	30K	...
;;	128K	...
;;	...

;; Do 1024 four_complex_fft macros
;;	distance between fft data elements is 32K
;;	do 256 macros each processing 32 data values

	start_timer 24
	mov	rdi, sincos3		;; Load sin/cos pointer
	mov	eax, 16*256+2		;; 4 iters of 2 iters of 4 of 2*2*clm
b5a:	xtouch	[rcx+(clm-1)*128]	;; Preload TLB
b5b:	x4cl_four_complex_fft rsi, 64, 2*clmblkdst8, 4*clmblkdst8, rdi
	xloop2	b5b, clmblkdst		;; 2 * 2*clm iterations
	add	eax, 80000000h/2	;; 4 iterations
	jnc	b5a
	bump	rsi, -4*2*clmblkdst+clmblkdst8
	sub	al, 1			;; Test middle loop counter
	jnz	b5a			;; Iterate if necessary
	bump	rsi, -2*clmblkdst8+8*clmblkdst8;; Next source pointer
	bump	rdi, 2*XMM_SCD		;; Next sine/cosine pointer
	sub	eax, 256		;; Test outer loop counter
	mov	al, 2			;; Load middle loop counter
	jnz	b5a			;; Iterate if necessary
	bump	rsi, -16*8*clmblkdst8	;; Restore source pointer
	end_timer 24

;; Do FFT levels 9,10
;;
;; On input the 64-byte cache lines hold these data values:
;;	0K	32K	64K	96K
;;	2K	...
;;	...
;;	30K	...
;;	128K	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0K	8K	16K	24K
;;	2K	...
;;	...
;;	6K	...
;;	32K	...
;;	...

;; Do 1024 four_complex_fft macros
;;	distance between fft data elements is 8K
;;	do 256 macros each processing 32 data values

	start_timer 25
	mov	rdi, sincos2		;; Load sin/cos pointer
	mov	eax, 64*65536		;; 64 iterations of 4*2*clm
b7a:	xtouch	[rcx+(clm-1)*128]	;; Preload TLB
b7b:	x4cl_four_complex_fft rsi, 64, 4*clmblkdst, clmblkdst8, rdi
	xloop4	b7b, clmblkdst		;; 2 * 2*clm iterations
	bump	rsi, -4*clmblkdst+2*clmblkdst8;; Next source pointer
	bump	rdi, 2*XMM_SCD		;; Next sine/cosine pointer
	sub	eax, 65536		;; Test outer loop counter
	jnz	b7a			;; Iterate if necessary
	bump	rsi, -64*2*clmblkdst8	;; Restore source pointer
pfing	bump	rcx, -1024*blkdst	;; Restore prefetch pointer
	end_timer 25

;; Do FFT levels 11,12
;;
;; On input the 64-byte cache lines hold these data values:
;;	0K	8K	16K	24K
;;	2K	...
;;	...
;;	6K	...
;;	32K	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0K	2K	4K	6K
;;	8K	...
;;	...

;; Do 1024 four_complex_fft macros
;;	distance between fft data elements is 2K
;;	do 256 macros each processing 32 data values

	start_timer 26
	mov	rdi, sincos1		;; Load sin/cos pointer
	mov	eax, 128*65536+2	;; 128 iters of 2 of 2*clm
b9b:	g4cl_four_complex_fft rsi, 64, clmblkdst, 2*clmblkdst, rcx, 64, blkdst, 2*blkdst
	L2prefetch128 [rcx-64][rbp]
	L2prefetch128 [rcx-64+blkdst][rbp]
	bump	rbp, -64+2*blkdst
	add	al, 256/2		;; Test inner loop counter
	jnc	b9b			;; Iterate if necessary
	bump	rbp, 128-4*blkdst
	IF clm NE 1
	add	dl, 256/clm
	jnc	b9b
	ENDIF
	bump	rsi, -2*clm*64+4*clmblkdst;; Next source pointer
	bump	rcx, -2*clm*64+4*blkdst	;; Next dest pointer
	bump	rdi, 2*XMM_SCD		;; Next sine/cosine pointer
	sub	al, 1
	jnz	b9b
	bump	rsi, -8*clmblkdst+clmblkdst8
	sub	eax, 65536		;; Test outer loop counter
	mov	al, 2
	jnz	b9b			;; Iterate if necessary
	end_timer 26

;; Work on next 4 sets of 1024 values

endlp:	xpass1_get_next_block pass2, c0b, b0b ;; Get next block and jump
	done:	xfft_footer
	ENDM

;;
;; Utility macros
;;

;; Macros that help in looping.  They are called right after a building block
;; macro.

;; This macro then loops 2*clm times. If prefetching, a prefetch is done
;; every 2 times the building block macro is called.
xloop1	MACRO	label2, incr
	IF clm EQ 1
	add	al, 256/2/clm		;; Test inner loop counter
	jnc	label2			;; Iterate if necessary
	bump	rsi, -2*clm*64+incr	;; Next source pointer
	L2prefetch128 [rcx][rbp]
pfing	bump	rcx, blkdst		;; Next prefetch pointer
	ENDIF

	IF clm GE 2
	add	al, 256/2		;; Test inner loop counter
	jnc	label2			;; Iterate if necessary
	L2prefetch128 [rcx][rbp]
	bump	rcx, 128
	add	dl, 256/clm
	jnc	label2
	bump	rsi, -2*clm*64+incr	;; Next source pointer
	bump	rcx, -clm*128+blkdst	;; Next prefetch pointer
	ENDIF
	ENDM

;; This macro then loops 2*clm times. If prefetching, two prefetches are done
;; every 2 times the building block macro is called.
xloop1prefetch2	MACRO label2, incr
	IF clm EQ 1
	L2prefetch128 [rcx][rbp]
pfing	bump	rcx, blkdst		;; Next prefetch pointer
	add	al, 256/2/clm		;; Test inner loop counter
	jnc	label2			;; Iterate if necessary
	bump	rsi, -2*clm*64+incr	;; Next source pointer
	ENDIF

	IF clm GE 2
	L2prefetch128 [rcx][rbp]
	bump	rcx, blkdst		;; Next prefetch pointer
	add	al, 256/2		;; Test inner loop counter
	jnc	label2			;; Iterate if necessary
	bump	rcx, -2*blkdst+128	;; Next prefetch pointer
	add	dl, 256/clm
	jnc	label2
	bump	rsi, -2*clm*64+incr	;; Next source pointer
	bump	rcx, -clm*128+2*blkdst	;; Next prefetch pointer
	ENDIF
	ENDM

;; This macro then loops 4*clm times. If prefetching, a prefetch is done
;; every 4 times the building block macro is called.
xloop2	MACRO	label2, incr
	xloop2edi label2, incr, 0
	ENDM
xloop2edi MACRO label2, incr, incr_edi
	IF clm EQ 1
	IF incr NE 2*clm*64 OR incr_edi NE 0
	add	al, 256/2		;; Test inner loop counter
	jnc	label2			;; Iterate if necessary
	bump	rsi, -2*clm*64+incr	;; Next source pointer
	bump	rdi, incr_edi		;; Next premultiplier pointer
	add	ah, 256/2		;; Test middle loop counter
	jnc	label2			;; Iterate if necessary
	ELSE
	add	al, 256/2/2		;; Test inner loop counter
	jnc	label2			;; Iterate if necessary
	ENDIF
	L2prefetch128 [rcx][rbp]
pfing	bump	rcx, blkdst		;; Next prefetch pointer
	ENDIF

	IF clm EQ 2
	add	al, 256/2/clm		;; Test inner loop counter
	jnc	label2			;; Iterate if necessary
	L2prefetch128 [rcx][rbp]
	bump	rcx, 128
	bump	rsi, -2*clm*64+incr	;; Next source pointer
	bump	rdi, incr_edi		;; Next premultiplier pointer
	add	ah, 256/2		;; Test middle loop counter
	jnc	label2			;; Iterate if necessary
	bump	rcx, -clm*128+blkdst	;; Next prefetch pointer
	ENDIF

	IF clm GE 4
	add	al, 256/2/2		;; Test inner loop counter
	jnc	label2			;; Iterate if necessary
	L2prefetch128 [rcx][rbp]
	bump	rcx, 128
	add	dl, 256/(clm/2)
	jnc	label2
	bump	rsi, -2*clm*64+incr	;; Next source pointer
	bump	rdi, incr_edi		;; Next premultiplier pointer
	add	ah, 256/2		;; Test middle loop counter
	jnc	label2			;; Iterate if necessary
	bump	rcx, -clm*128+blkdst	;; Next prefetch pointer
	ENDIF
	ENDM

;; This macro then loops 8*clm times. If prefetching, a prefetch is done
;; every 8 times the building block macro is called.
xloop4	MACRO	label2, incr
	xloop4edi label2, incr, 0
	ENDM
xloop4edi	MACRO	label2, incr, incr_edi
	IF clm EQ 1
	add	al, 256/2/clm		;; Test inner loop counter
	jnc	label2			;; Iterate if necessary
	bump	rsi, -2*clm*64+incr	;; Next source pointer
	bump	rdi, incr_edi		;; Next premultiplier pointer
	add	dl, 256/4		;; Test outer loop counter
	jnc	label2			;; Iterate if necessary
	L2prefetch128 [rcx][rbp]
pfing	bump	rcx, blkdst		;; Next prefetch pointer
	ENDIF

	IF clm EQ 2
	add	al, 256/2/clm		;; Test inner loop counter
	jnc	label2			;; Iterate if necessary
	bump	rsi, -2*clm*64+incr	;; Next source pointer
	bump	rdi, incr_edi		;; Next premultiplier pointer
	add	dl, 256/2		;; Test middle loop counter
	jnc	label2			;; Iterate if necessary
	L2prefetch128 [rcx][rbp]
	bump	rcx, 128
	add	ah, 256/2		;; Test outer loop counter
	jnc	label2			;; Iterate if necessary
	bump	rcx, -clm*128+blkdst	;; Next prefetch pointer
	ENDIF

	IF clm EQ 4
	add	al, 256/2/clm		;; Test inner loop counter
	jnc	label2			;; Iterate if necessary
	bump	rsi, -2*clm*64+incr	;; Next source pointer
	bump	rdi, incr_edi		;; Next premultiplier pointer
	L2prefetch128 [rcx][rbp]
	bump	rcx, 128
	add	dl, 256/4		;; Test outer loop counter
	jnc	label2			;; Iterate if necessary
	bump	rcx, -clm*128+blkdst	;; Next prefetch pointer
	ENDIF

	IF clm GT 4
	add	al, 256/2/4		;; Test inner loop counter
	jnc	label2			;; Iterate if necessary
	L2prefetch128 [rcx][rbp]
	bump	rcx, 128
	add	dl, 256/(clm/4)		;; Test loop counter
	jnc	label2			;; Iterate if necessary
	bump	rsi, -2*clm*64+incr	;; Next source pointer
	bump	rdi, incr_edi		;; Next premultiplier pointer
	add	ah, 256/4		;; Test outer loop counter
	jnc	label2			;; Iterate if necessary
	bump	rcx, -clm*128+blkdst	;; Next prefetch pointer
	ENDIF
	ENDM

;; This macro then loops 8*2*clm times. If prefetching, a prefetch is done
;; every 16 times the building block macro is called.
xloop8	MACRO	label2, incr
	xloop8edi label2, incr, 0
	ENDM
xloop8edi MACRO label2, incr, incr_edi
	IF clm EQ 1
	add	al, 256/2/clm		;; Test inner loop counter
	jnc	label2			;; Iterate if necessary
	bump	rsi, -2*clm*64+incr	;; Next source pointer
	bump	rdi, incr_edi		;; Next premultiplier pointer
	add	dl, 256/8		;; Test outer loop counter
	jnc	label2			;; Iterate if necessary
	L2prefetch128 [rcx][rbp]
pfing	bump	rcx, blkdst		;; Next prefetch pointer
	ENDIF

	IF clm EQ 2
	add	al, 256/2/clm		;; Test inner loop counter
	jnc	label2			;; Iterate if necessary
	bump	rsi, -2*clm*64+incr	;; Next source pointer
	bump	rdi, incr_edi		;; Next premultiplier pointer
	add	dl, 256/4		;; Test middle loop counter
	jnc	label2			;; Iterate if necessary
	L2prefetch128 [rcx][rbp]
	bump	rcx, 128
	add	ah, 256/2		;; Test outer loop counter
	jnc	label2			;; Iterate if necessary
	bump	rcx, -clm*128+blkdst	;; Next prefetch pointer
	ENDIF

	IF clm EQ 4
	add	al, 256/2/clm		;; Test inner loop counter
	jnc	label2			;; Iterate if necessary
	bump	rsi, -2*clm*64+incr	;; Next source pointer
	bump	rdi, incr_edi		;; Next premultiplier pointer
	add	dl, 256/2		;; Test middle loop counter
	jnc	label2			;; Iterate if necessary
	L2prefetch128 [rcx][rbp]
	bump	rcx, 128
	add	ah, 256/4		;; Test outer loop counter
	jnc	label2			;; Iterate if necessary
	bump	rcx, -clm*128+blkdst	;; Next prefetch pointer
	ENDIF

	IF clm EQ 8
	add	al, 256/2/clm		;; Test inner loop counter
	jnc	label2			;; Iterate if necessary
	bump	rsi, -2*clm*64+incr	;; Next source pointer
	bump	rdi, incr_edi		;; Next premultiplier pointer
	L2prefetch128 [rcx][rbp]
	bump	rcx, 128
	add	dl, 256/8		;; Test outer loop counter
	jnc	label2			;; Iterate if necessary
	bump	rcx, -clm*128+blkdst	;; Next prefetch pointer
	ENDIF
	ENDM


xtouchlines MACRO num
	LOCAL	lab
	IF PREFETCHING NE 0
	IF TLB_PRIMING NE 0
	IF num EQ 1
	xtouch	[rcx+(clm-1)*128]	;; Preload TLB
	ENDIF
	IF num EQ 2
	xtouch	[rcx+(clm-1)*128]	;; Preload TLB
	xtouch	[rcx+(clm-1)*128+blkdst]
	ENDIF
	IF num EQ 3
	xtouch	[rcx+(clm-1)*128]	;; Preload TLB
	xtouch	[rcx+(clm-1)*128+blkdst]
	xtouch	[rcx+(clm-1)*128+2*blkdst]
	ENDIF
	IF num GE 4
	mov	ah, num
lab:	xtouch	[rcx+(clm-1)*128]	;; Preload TLB
	bump	rcx, blkdst
	sub	ah, 1
	jnz	short lab
	bump	rcx, -num*blkdst
	ENDIF
	ENDIF
	ENDIF
	ENDM

xloop1sp MACRO	label2, incr
	add	al, 256/2		;; Test inner loop counter
	jnc	label2			;; Iterate if necessary
	bump	rsi, -2*64+incr		;; Next source pointer
	L2prefetch128 [rcx][rbp]
pfing	bump	rcx, blkdst		;; Next prefetch pointer
	ENDM

xloop1prefetch2sp MACRO label2, incr
	L2prefetch128 [rcx][rbp]
pfing	bump	rcx, blkdst		;; Next prefetch pointer
	add	al, 256/2		;; Test inner loop counter
	jnc	label2			;; Iterate if necessary
	bump	rsi, -2*64+incr		;; Next source pointer
	ENDM

xloop2sp MACRO	label2, incr
	add	al, 256/2+128/2/2	;; Test loop counters
	jnc	label2			;; Iterate if necessary
	lea	rsi, [rsi-2*64+incr]	;; Next source pointer
	jns	label2			;; Iterate if necessary
	sub	al, 128			;; Restore loop counters
	L2prefetch128 [rcx][rbp]
pfing	bump	rcx, blkdst		;; Next prefetch pointer
	ENDM

xloop4sp MACRO	label2, incr
	add	al, 256/2+128/2/4	;; Test loop counters
	jnc	label2			;; Iterate if necessary
	lea	rsi, [rsi-2*64+incr]	;; Next source pointer
	jns	label2			;; Iterate if necessary
	sub	al, 128			;; Restore loop counters
	L2prefetch128 [rcx][rbp]
pfing	bump	rcx, blkdst		;; Next prefetch pointer
	ENDM

