; Copyright 1998-2024 - Mersenne Research, Inc.  All rights reserved
; Author:  George Woltman
; Email: woltman@alum.mit.edu
;
; These macros do the first pass of two-pass FFTs.  These macros are
; the same as in pass1.mac except that FFT data is copied to and from
; a scratch area.
;

; ********************************************************
; ********************************************************
; *******************  PASS 1 MACROS  ********************
; ********************************************************
; ********************************************************

;;***********************************************************************
;; Pass 1 macros
;;
;; p2cl = number of cache lines in a pass 2 block
;; clm = cache line multiplier (num cache lines processed each iteration)
;;***********************************************************************

;; Perform a 10K-element FFT.  This is done in two passes.  Pass 1
;; does 6 levels, pass 2 uses common code to perform the last 8 levels.

pass1levels6pfa5 MACRO pass2_macro, p2cl, clmarg
	LOCAL	b0b, b1b, b2b, b3b, b4b, b5b, c0b, c1b, c2b, c3b, c4b, c5b
	LOCAL	pass2, copyback, yes_fft, no_fft, more, fftdn, fftdn1

	set_FFT_constants clmarg

;; Call FFT code unless the FFT has already been started

	clear_timers
	cmp	DWORD PTR [esi-28][ebx], 0 ;; Test FFT-started flag
	jne	pass2			;; Jump is FFT already started
	mov	norm_ptr1, 0		;; Set FFT-ing flag
	mov	edx, p2cl/128*65536+128/clm*256;; Load loop counter
pfing	mov	ebp, clm*32		;; Set prefetch distance
	jmp	b0b			;; Do the pass 1 fft

;; Do the last 8 FFT levels
;;	do 1 pass2_8_levels_real and 19 pass2_8_levels_complex

pass2:	start_timer 0
	CALLP	pass2_macro
	end_timer 0
	cmp	ffttype, 1		;; We're done if FFTing only
	je	gw_finish_fft

;; Do the inverse FFT

	start_timer 1
	fldz				;; Init SUMOUT
	fstp	SUMOUT
	mov	edx, norm_biglit_array	;; Addr of the big/little flags array
	mov	norm_ptr1, edx		;; Save ptr
	mov	edx, norm_col_mults	;; Addr of the column multipliers
	mov	norm_ptr2, edx		;; Save ptr
	mov	edx, p2cl/128*65536+128/clm*256;; Load loop counter
pfing	mov	ebp, clm*32		;; Set prefetch distance

;; Do inverse FFT levels 5,6
;; On input the 32-byte cache lines hold these data values:
;;	0	256	1	257
;;	512	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	1K	1	1K+1
;;	256	...
;;	...
;;	2K	...
;;	...

;; Do 1 four_real_four_semireal_unfft macros
;;	distance between fft data elements is 1

c0b:	start_timer 17
	mov	ecx, scratch_area	;; Get address of scratch area
c2b:	cp_disp four_real_four_semireal_unfft, 8, blkdst, 2*blkdst, 8, clmblkdst, 2*clmblkdst
	lea	esi, [esi+dist1]	;; Next source pointer
	lea	ecx, [ecx+dist1]	;; Next dest pointer
	add	dl, 256/2/clm		;; Test inner loop counter
	jnc	c2b			;; Iterate if necessary
	lea	esi, [esi-clm*2*dist1+4*blkdst];; Next source pointer
	lea	ecx, [ecx-clm*2*dist1+4*clmblkdst];; Next dest pointer

;; Do 4 four_complex_unfft macros
;;	distance between fft data elements is 1

	mov	edi, sincos1		;; Load sin/cos pointer
	mov	al, 4			;; 4 iterations
c3b:	cp_disp four_complex_unfft, 8, blkdst, 2*blkdst, 8, clmblkdst, 2*clmblkdst
	lea	esi, [esi+dist1]	;; Next source pointer
	lea	ecx, [ecx+dist1]	;; Next dest pointer
	add	dl, 256/2/clm		;; Test loop counter
	jnc	c3b			;; Iterate if necessary
	lea	esi, [esi-clm*2*dist1+4*blkdst];; Next source pointer
	lea	ecx, [ecx-clm*2*dist1+4*clmblkdst];; Next dest pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al			;; Test outer loop counter
	jnz	c3b			;; Iterate if necessary
	lea	ecx, [esi-5*4*blkdst]	;; Create prefetch pointer
	mov	esi, scratch_area	;; Restore source pointer
	end_timer 17

;; Do inverse FFT levels 4
;; On input the 32-byte cache lines hold these data values:
;;	0	1K	1	1K+1
;;	256	...
;;	...
;;	2K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	2K	1	2K+1
;;	256	...
;;	...
;;	4K	...
;;	...

;; Do 1 eight_reals_unfft_1 macros
;;	distance between fft data elements is 4

	start_timer 18
c4b:	disp eight_reals_unfft_1, clmblkdst, 2*clmblkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2/clm		;; Test inner loop counter
	jnc	c4b			;; Iterate if necessary
	lea	esi, [esi-clm*2*dist1+4*clmblkdst];; Next source pointer

;; Do 4 four_complex_unfft_1 macros
;;	distance between fft data elements is 4

	mov	edi, sincos2		;; Load sin/cos pointer
	mov	eax, 2			;; 2 iterations of 2
c5b:	disp four_complex_unfft_1, 2*clmblkdst, 8, 4*clmblkdst
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2/clm		;; Test inner loop counter
	jnc	c5b			;; Iterate if necessary
	lea	esi, [esi-clm*2*dist1+clmblkdst];; Next source pointer
	add	ah, 256/2		;; Test middle loop counter
	jnc	c5b			;; Iterate if necessary
	lea	esi, [esi-2*clmblkdst+8*clmblkdst];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al
	jnz	c5b
	lea	esi, [esi-2*8*clmblkdst-4*clmblkdst];; Restore source pointer
	end_timer 18

;; Do inverse FFT levels 1,2,3
;; On input the 32-byte cache lines hold these data values:
;;	0	2K	1	2K+1
;;	256	...
;;	...
;;	4K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	5K	1	5K+1
;;	256
;;	...
;;	5K-256

;; Do 8 five_reals_last_unfft macros
;;	distance between fft data elements is 8

	start_timer 19
	mov	al, 4			;; Load loop counter
c1b:	two_five_reals_last_unfft 8, 4*clmblkdst
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2/clm		;; Test loop counter
	jnc	c1b			;; Iterate if necessary
	lea	esi, [esi-clm*2*dist1+clmblkdst];; Next source pointer
	dec	al			;; Test loop counter
	jnz	c1b			;; Iterate if necessary
	lea	esi, [esi-4*clmblkdst]	;; Restore source pointer
	end_timer 19

;; Normalize these values

	start_timer 20
	mov	eax, NORMRTN		;; Addr of normalization routine
	call	eax
	sub	eax, eax
	end_timer 20

;; Have we been given permission to start the FFT on the result?
;; If so, do so now while the data is in the L2 cache.

	cmp	POSTFFT, 0		;; Test flag
	je	short copyback		;; Copy data, skip FFT if flag not set
	copy_3_words clm, 1
	cmp	edx, p2cl/128*65536+(128/clm-4/clm)*256;; Delay fft on 8 sets
	jle	yes_fft			;; No delay, do FFT now

;; Copy data from the scratch area back to the FFT data area

copyback:
	copy_scratch_data_to_fft 20
	jmp	no_fft			;; Skip forward FFT code

;; Copy data from the FFT data area back to the scratch area

b0b:	copy_fft_data_to_scratch 20

;; Do FFT levels 1,2,3
;;
;; On input the 32-byte cache lines hold these data values:
;;	0	5K	1	5K+1
;;	256
;;	...
;;	5K-256
;; On output the 32-byte cache lines hold these data values:
;;	0	2K	1	2K+1
;;	256	...
;;	...
;;	4K	...
;;	...

;; Do 8 five_reals_fft macros
;;	distance between fft data elements is 8

yes_fft:start_timer 21
	mov	eax, 4			;; 4 iterations
b1b:	two_five_reals_fft 4*clmblkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	add	dl, 256/2/clm		;; Test loop counter
	jnc	b1b			;; Iterate if necessary
	lea	esi, [esi-clm*2*dist1+clmblkdst];; Next source pointer
	dec	al			;; Test loop counter
	jnz	b1b			;; Iterate if necessary
	lea	esi, [esi-4*clmblkdst]	;; Restore source pointer
	end_timer 21

;; Do FFT levels 4
;;
;; On input the 32-byte cache lines hold these data values:
;;	0	2K	1	2K+1
;;	256	...
;;	...
;;	4K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	1K	1	1K+1
;;	256	...
;;	...
;;	2K	...
;;	...

;; Do 1 eight_reals_fft_1 macros
;;	distance between fft data elements is 4

	start_timer 22
b4b:	disp eight_reals_fft_1, clmblkdst, 2*clmblkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2/clm		;; Test inner loop counter
	jnc	b4b			;; Iterate if necessary
	lea	esi, [esi-clm*2*dist1+4*clmblkdst];; Next source pointer

;; Do 4 four_complex_fft_1 macros
;;	distance between fft data elements is 4

	mov	edi, sincos2		;; Load sin/cos pointer
	mov	eax, 2			;; 2 iterations of 2
b5b:	disp four_complex_fft_1, 2*clmblkdst, 4*clmblkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2/clm		;; Test inner loop counter
	jnc	b5b			;; Iterate if necessary
	lea	esi, [esi-clm*2*dist1+clmblkdst];; Next source pointer
	add	ah, 256/2		;; Test middle loop counter
	jnc	b5b			;; Iterate if necessary
	lea	esi, [esi-2*clmblkdst+8*clmblkdst];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al
	jnz	b5b
	lea	esi, [esi-2*8*clmblkdst-4*clmblkdst];; Restore source pointer
	end_timer 22

;; Do FFT levels 5,6
;;
;; On input the 32-byte cache lines hold these data values:
;;	0	1K	1	1K+1
;;	256	...
;;	...
;;	2K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	256	1	257
;;	512	...
;;	...

;; Do 1 four_real_four_semireal_fft macros
;;	distance between fft data elements is 1

	start_timer 23
;;	lea	ecx, [ecx-1111*blkdst]	;; Create dest ptr from prefetch ptr
b2b:	cp_disp four_real_four_semireal_fft, clmblkdst, 2*clmblkdst, 8, blkdst, 2*blkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	lea	ecx, [ecx+dist1]	;; Next dest pointer
	add	al, 256/2/clm		;; Test inner loop counter
	jnc	b2b			;; Iterate if necessary
	lea	esi, [esi-clm*2*dist1+4*clmblkdst];; Next source pointer
	lea	ecx, [ecx-clm*2*dist1+4*blkdst];; Next dest pointer

;; Do 4 four_complex_fft macros
;;	distance between fft data elements is 1

	mov	edi, sincos1		;; Load sin/cos pointer
	mov	al, 4			;; 4 iterations
b3b:	cp_disp four_complex_fft, clmblkdst, 2*clmblkdst, 8, blkdst, 2*blkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	lea	ecx, [ecx+dist1]	;; Next dest pointer
	add	al, 256/2/clm		;; Test inner loop counter
	jnc	b3b			;; Iterate if necessary
	lea	esi, [esi-clm*2*dist1+4*clmblkdst];; Next source pointer
	lea	ecx, [ecx-clm*2*dist1+4*blkdst];; Next dest pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al
	jnz	b3b
	lea	esi, [ecx-5*4*blkdst]	;; Restore source pointer
	end_timer 23

;; Work on next 4 sets of 32 values

no_fft:	lea	esi, [esi+clm*2*dist1]	;; Next source pointer
	dec	dh			;; Test loop counter
	jnz	short more		;; Do more FFTs
	lea	esi, [esi+64]		;; Skip pad every 128 cache lines (4KB)
pfing	mov	ebp, clm*32		;; Set prefetch distance
	sub	edx, 65536		;; Test loop counter
	jz	short fftdn		;; Jump if done looping
	mov	dh, 128/clm		;; Restore cache line counter
more:	set_prefetch_gap
	cmp	norm_ptr1, 0		;; What was the loop start point?
	jne	c0b			;; Do inverse-FFT/norm/FFT iteration
	jmp	b0b			;; Do another FFT-only iteration
fftdn:	lea	esi, [esi-p2cl*2*dist1-p2cl/128*64];; Restore source pointer
	cmp	esi, DESTARG		;; If esi was not restored, then we 
	jne	short fftdn1		;; just finished skipped postfft data
	sub	ebx, ebx
	mov	DIST_TO_FFTSRCARG, ebx
	cmp	norm_ptr1, 0		;; What was the loop start point?
	je	pass2			;; FFT-only, go do pass 2

;; Split the carries into high and low carries (so they do not exceed
;; the desired number of bits per FFT word.  Then add the carries back to the
;; FFT data.

	start_timer 28
	call	gw_carries
	end_timer 28

;; Have we been given permission to start the FFT on the result?
;; If so, set loop to FFT the first few cache lines now that the carries
;; have been added in.

	cmp	POSTFFT, 0		;; Test flag
	je	short fftdn1		;; Skip FFT code if flag not set
	mov	esi, DESTARG		;; Load FFT data address
	mov	edx, 65536+4/clm*256	;; Do 8 previously skipped cache lines
	mov	norm_ptr1, 0		;; Set flag so only forward FFT is done
	copy_4_words
	jmp	b0b

;; Now add in the FFT'ed carries, cleanup and return

fftdn1:	end_timer 1
	jmp	gw_finish_mult
	ENDM


;; Perform a 12K-element FFT.  This is done in two passes.  Pass 1
;; does 6 levels, pass 2 uses common code to perform the last 8 levels.

pass1levels6pfa6 MACRO pass2_macro, p2cl, clmarg
	LOCAL	b0b, b1b, b2b, b3b, b4b, b5b, c0b, c1b, c2b, c3b, c4b, c5b
	LOCAL	pass2, copyback, yes_fft, no_fft, more, fftdn, fftdn1

	set_FFT_constants clmarg

;; Call FFT code unless the FFT has already been started

	clear_timers
	cmp	DWORD PTR [esi-28][ebx], 0 ;; Test FFT-started flag
	jne	pass2			;; Jump is FFT already started
	mov	norm_ptr1, 0		;; Set FFT-ing flag
	mov	edx, p2cl/128*65536+128/clm*256;; Load loop counter
pfing	mov	ebp, clm*32		;; Set prefetch distance
	jmp	b0b			;; Do the pass 1 fft

;; Do the last 8 FFT levels
;;	do 1 pass2_8_levels_real and 23 pass2_8_levels_complex

pass2:	start_timer 0
	CALLP	pass2_macro
	end_timer 0
	cmp	ffttype, 1		;; We're done if FFTing only
	je	gw_finish_fft

;; Do the inverse FFT

	start_timer 1
	fldz				;; Init SUMOUT
	fstp	SUMOUT
	mov	edx, norm_biglit_array	;; Addr of the big/little flags array
	mov	norm_ptr1, edx		;; Save ptr
	mov	edx, norm_col_mults	;; Addr of the column multipliers
	mov	norm_ptr2, edx		;; Save ptr
	mov	edx, p2cl/128*65536+128/clm*256;; Load loop counter
pfing	mov	ebp, clm*32		;; Set prefetch distance

;; Do inverse FFT levels 5,6
;; On input the 32-byte cache lines hold these data values:
;;	0	256	1	257
;;	512	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	1K	1	1K+1
;;	256	...
;;	...
;;	2K	...
;;	...

;; Do 1 four_real_four_semireal_unfft macros
;;	distance between fft data elements is 1

c0b:	start_timer 17
	mov	ecx, scratch_area	;; Get address of scratch area
c2b:	cp_disp four_real_four_semireal_unfft, 8, blkdst, 2*blkdst, 8, clmblkdst, 2*clmblkdst
	lea	esi, [esi+dist1]	;; Next source pointer
	lea	ecx, [ecx+dist1]	;; Next dest pointer
	add	dl, 256/2/clm		;; Test inner loop counter
	jnc	c2b			;; Iterate if necessary
	lea	esi, [esi-clm*2*dist1+4*blkdst];; Next source pointer
	lea	ecx, [ecx-clm*2*dist1+4*clmblkdst];; Next dest pointer

;; Do 5 four_complex_unfft macros
;;	distance between fft data elements is 1

	mov	edi, sincos1		;; Load sin/cos pointer
	mov	al, 5			;; 5 iterations
c3b:	cp_disp four_complex_unfft, 8, blkdst, 2*blkdst, 8, clmblkdst, 2*clmblkdst
	lea	esi, [esi+dist1]	;; Next source pointer
	lea	ecx, [ecx+dist1]	;; Next dest pointer
	add	dl, 256/2/clm		;; Test loop counter
	jnc	c3b			;; Iterate if necessary
	lea	esi, [esi-clm*2*dist1+4*blkdst];; Next source pointer
	lea	ecx, [ecx-clm*2*dist1+4*clmblkdst];; Next dest pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al			;; Test outer loop counter
	jnz	c3b			;; Iterate if necessary
	lea	ecx, [esi-6*4*blkdst]	;; Create prefetch pointer
	mov	esi, scratch_area	;; Restore source pointer
	end_timer 17

;; Do inverse FFT levels 4
;; On input the 32-byte cache lines hold these data values:
;;	0	1K	1	1K+1
;;	256	...
;;	...
;;	2K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	2K	1	2K+1
;;	256	...
;;	...
;;	4K	...
;;	...

;; Do 2 four_real_four_semireal_unfft_1 macros
;;	distance between fft data elements is 4

	start_timer 18
	mov	al, 2			;; Load loop counter
c4b:	disp four_real_four_semireal_unfft_1, 2*clmblkdst, 8, 4*clmblkdst
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2/clm		;; Test inner loop counter
	jnc	c4b			;; Iterate if necessary
	lea	esi, [esi-clm*2*dist1+clmblkdst];; Next source pointer
	dec	al
	jnz	c4b
	lea	esi, [esi-2*clmblkdst+8*clmblkdst];; Next source pointer

;; Do 4 four_complex_unfft_1 macros
;;	distance between fft data elements is 4

	mov	edi, sincos2		;; Load sin/cos pointer
	mov	eax, 2			;; 2 iterations of 2
c5b:	disp four_complex_unfft_1, 2*clmblkdst, 8, 4*clmblkdst
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2/clm		;; Test inner loop counter
	jnc	c5b			;; Iterate if necessary
	lea	esi, [esi-clm*2*dist1+clmblkdst];; Next source pointer
	add	ah, 256/2		;; Test middle loop counter
	jnc	c5b			;; Iterate if necessary
	lea	esi, [esi-2*clmblkdst+8*clmblkdst];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al
	jnz	c5b
	lea	esi, [esi-3*8*clmblkdst]	;; Restore source pointer
	end_timer 18

;; Do inverse FFT levels 1,2,3
;; On input the 32-byte cache lines hold these data values:
;;	0	2K	1	2K+1
;;	256	...
;;	...
;;	4K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	6K	1	6K+1
;;	256
;;	...
;;	6K-256

;; Do 8 six_reals_last_unfft macros
;;	distance between fft data elements is 8

	start_timer 19
	mov	al, 8			;; Load loop counter
c1b:	disp six_reals_last_unfft, 8, 8*clmblkdst, 16*clmblkdst
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2/clm		;; Test loop counter
	jnc	c1b			;; Iterate if necessary
	lea	esi, [esi-clm*2*dist1+clmblkdst];; Next source pointer
	dec	al			;; Test loop counter
	jnz	c1b			;; Iterate if necessary
	lea	esi, [esi-8*clmblkdst]	;; Restore source pointer
	end_timer 19

;; Normalize these values

	start_timer 20
	mov	eax, NORMRTN		;; Addr of normalization routine
	call	eax
	sub	eax, eax
	end_timer 20

;; Have we been given permission to start the FFT on the result?
;; If so, do so now while the data is in the L2 cache.

	cmp	POSTFFT, 0		;; Test flag
	je	short copyback		;; Copy data, skip FFT if flag not set
	copy_3_words clm, 1
	cmp	edx, p2cl/128*65536+(128/clm-4/clm)*256;; Delay fft on 8 sets
	jle	yes_fft			;; No delay, do FFT now

;; Copy data from the scratch area back to the FFT data area

copyback:
	copy_scratch_data_to_fft 24
	jmp	no_fft			;; Skip forward FFT code

;; Copy data from the FFT data area back to the scratch area

b0b:	copy_fft_data_to_scratch 24

;; Do FFT levels 1,2,3
;;
;; On input the 32-byte cache lines hold these data values:
;;	0	6K	1	6K+1
;;	256
;;	...
;;	6K-256
;; On output the 32-byte cache lines hold these data values:
;;	0	2K	1	2K+1
;;	256	...
;;	...
;;	4K	...
;;	...

;; Do 8 six_reals_fft macros
;;	distance between fft data elements is 8

yes_fft:start_timer 21
	mov	eax, 8			;; 8 iterations
b1b:	disp six_reals_fft, 8*clmblkdst, 16*clmblkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	add	dl, 256/2/clm		;; Test loop counter
	jnc	b1b			;; Iterate if necessary
	lea	esi, [esi-clm*2*dist1+clmblkdst];; Next source pointer
	dec	al			;; Test loop counter
	jnz	b1b			;; Iterate if necessary
	lea	esi, [esi-8*clmblkdst]	;; Restore source pointer
	end_timer 21

;; Do FFT levels 4
;;
;; On input the 32-byte cache lines hold these data values:
;;	0	2K	1	2K+1
;;	256	...
;;	...
;;	4K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	1K	1	1K+1
;;	256	...
;;	...
;;	2K	...
;;	...

;; Do 2 four_real_four_semireal_fft_1 macros
;;	distance between fft data elements is 4

	start_timer 22
	mov	al, 2			;; Load loop counter
b4b:	disp four_real_four_semireal_fft_1, 2*clmblkdst, 4*clmblkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2/clm		;; Test inner loop counter
	jnc	b4b			;; Iterate if necessary
	lea	esi, [esi-clm*2*dist1+clmblkdst];; Next source pointer
	dec	al
	jnz	b4b
	lea	esi, [esi-2*clmblkdst+8*clmblkdst];; Next source pointer

;; Do 4 four_complex_fft_1 macros
;;	distance between fft data elements is 4

	mov	edi, sincos2		;; Load sin/cos pointer
	mov	eax, 2			;; 2 iterations of 2
b5b:	disp four_complex_fft_1, 2*clmblkdst, 4*clmblkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2/clm		;; Test inner loop counter
	jnc	b5b			;; Iterate if necessary
	lea	esi, [esi-clm*2*dist1+clmblkdst];; Next source pointer
	add	ah, 256/2		;; Test middle loop counter
	jnc	b5b			;; Iterate if necessary
	lea	esi, [esi-2*clmblkdst+8*clmblkdst];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al
	jnz	b5b
	lea	esi, [esi-3*8*clmblkdst]	;; Restore source pointer
	end_timer 22

;; Do FFT levels 5,6
;;
;; On input the 32-byte cache lines hold these data values:
;;	0	1K	1	1K+1
;;	256	...
;;	...
;;	2K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	256	1	257
;;	512	...
;;	...

;; Do 1 four_real_four_semireal_fft macros
;;	distance between fft data elements is 1

	start_timer 23
;	lea	ecx, [ecx-1111*blkdst]	;; Create dest ptr from prefetch ptr
b2b:	cp_disp four_real_four_semireal_fft, clmblkdst, 2*clmblkdst, 8, blkdst, 2*blkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	lea	ecx, [ecx+dist1]	;; Next dest pointer
	add	al, 256/2/clm		;; Test inner loop counter
	jnc	b2b			;; Iterate if necessary
	lea	esi, [esi-clm*2*dist1+4*clmblkdst];; Next source pointer
	lea	ecx, [ecx-clm*2*dist1+4*blkdst];; Next dest pointer

;; Do 5 four_complex_fft macros
;;	distance between fft data elements is 1

	mov	edi, sincos1		;; Load sin/cos pointer
	mov	al, 5			;; 5 iterations
b3b:	cp_disp four_complex_fft, clmblkdst, 2*clmblkdst, 8, blkdst, 2*blkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	lea	ecx, [ecx+dist1]	;; Next dest pointer
	add	al, 256/2/clm		;; Test inner loop counter
	jnc	b3b			;; Iterate if necessary
	lea	esi, [esi-clm*2*dist1+4*clmblkdst];; Next source pointer
	lea	ecx, [ecx-clm*2*dist1+4*blkdst];; Next dest pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al
	jnz	b3b
	lea	esi, [ecx-6*4*blkdst]	;; Restore source pointer
	end_timer 23

;; Work on next 4 sets of 32 values

no_fft:	lea	esi, [esi+clm*2*dist1]	;; Next source pointer
	dec	dh			;; Test loop counter
	jnz	short more		;; Do more FFTs
	lea	esi, [esi+64]		;; Skip pad every 128 cache lines (4KB)
pfing	mov	ebp, clm*32		;; Set prefetch distance
	sub	edx, 65536		;; Test loop counter
	jz	short fftdn		;; Jump if done looping
	mov	dh, 128/clm		;; Restore cache line counter
more:	set_prefetch_gap
	cmp	norm_ptr1, 0		;; What was the loop start point?
	jne	c0b			;; Do inverse-FFT/norm/FFT iteration
	jmp	b0b			;; Do another FFT-only iteration
fftdn:	lea	esi, [esi-p2cl*2*dist1-p2cl/128*64];; Restore source pointer
	cmp	esi, DESTARG		;; If esi was not restored, then we 
	jne	short fftdn1		;; just finished skipped postfft data
	sub	ebx, ebx
	mov	DIST_TO_FFTSRCARG, ebx
	cmp	norm_ptr1, 0		;; What was the loop start point?
	je	pass2			;; FFT-only, go do pass 2

;; Split the carries into high and low carries (so they do not exceed
;; the desired number of bits per FFT word.  Then add the carries back to the
;; FFT data.

	start_timer 28
	call	gw_carries
	end_timer 28

;; Have we been given permission to start the FFT on the result?
;; If so, set loop to FFT the first few cache lines now that the carries
;; have been added in.

	cmp	POSTFFT, 0		;; Test flag
	je	short fftdn1		;; Skip FFT code if flag not set
	mov	esi, DESTARG		;; Load FFT data address
	mov	edx, 65536+4/clm*256	;; Do 8 previously skipped cache lines
	mov	norm_ptr1, 0		;; Set flag so only forward FFT is done
	copy_4_words
	jmp	b0b

;; Now add in the FFT'ed carries, cleanup and return

fftdn1:	end_timer 1
	jmp	gw_finish_mult
	ENDM


;; Perform a 12K-element negacyclic FFT.  This is done in two passes.  Pass 1
;; does 6 levels, pass 2 uses common code to perform the last 8 levels.

pass1levels6complex3 MACRO pass2_macro, p2cl, clmarg
	LOCAL	b0b, b1b, b3b, b5b, c0b, c1b, c3b, c5b
	LOCAL	pass2, copyback, yes_fft, no_fft, more, fftdn, fftdn1

	set_FFT_constants clmarg

;; Call FFT code unless the FFT has already been started

	clear_timers
	cmp	DWORD PTR [esi-28][ebx], 0 ;; Test FFT-started flag
	jne	pass2			;; Jump is FFT already started
	mov	norm_ptr1, 0		;; Set FFT-ing flag
	mov	edx, p2cl/128*65536+128/clm*256;; Load loop counter
pfing	mov	ebp, clm*32		;; Set prefetch distance
	jmp	b0b			;; Do the pass 1 fft

;; Do the last 8 FFT levels
;;	do 24 pass2_8_levels_complex

pass2:	start_timer 0
	CALLP	pass2_macro
	end_timer 0
	cmp	ffttype, 1		;; We're done if FFTing only
	je	gw_finish_fft

;; Do the inverse FFT

	start_timer 1
	fldz				;; Init SUMOUT
	fstp	SUMOUT
	mov	edx, norm_biglit_array	;; Addr of the big/little flags array
	mov	norm_ptr1, edx		;; Save ptr
	mov	edx, norm_col_mults	;; Addr of the column multipliers
	mov	norm_ptr2, edx		;; Save ptr
	mov	edx, p2cl/128*65536+128/clm*256;; Load loop counter
pfing	mov	ebp, clm*32		;; Set prefetch distance

;; Do inverse FFT levels 5,6
;; On input the 32-byte cache lines hold these data values:
;;	0	256	1	257
;;	512	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	1K	1	1K+1
;;	256	...
;;	...
;;	2K	...
;;	...

;; Do 8 four_complex_unfft macros
;;	distance between fft data elements is 1

c0b:	start_timer 17
	mov	ecx, scratch_area	;; Get address of scratch area
	mov	edi, sincos1		;; Load sin/cos pointer
	mov	al, 6			;; 6 iterations
c3b:	cp_disp four_complex_unfft, 8, blkdst, 2*blkdst, 8, clmblkdst, 2*clmblkdst
	lea	esi, [esi+dist1]	;; Next source pointer
	lea	ecx, [ecx+dist1]	;; Next dest pointer
	add	dl, 256/2/clm		;; Test loop counter
	jnc	c3b			;; Iterate if necessary
	lea	esi, [esi-clm*2*dist1+4*blkdst];; Next source pointer
	lea	ecx, [ecx-clm*2*dist1+4*clmblkdst];; Next dest pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al			;; Test outer loop counter
	jnz	c3b			;; Iterate if necessary
	lea	ecx, [esi-6*4*blkdst]	;; Create prefetch pointer
	mov	esi, scratch_area	;; Restore source pointer
	end_timer 17

;; Do inverse FFT levels 4
;; On input the 32-byte cache lines hold these data values:
;;	0	1K	1	1K+1
;;	256	...
;;	...
;;	2K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	2K	1	2K+1
;;	256	...
;;	...
;;	4K	...
;;	...

;; Do 6 four_complex_unfft_1 macros
;;	distance between fft data elements is 4

	start_timer 18
	mov	edi, sincos2		;; Load sin/cos pointer
	mov	eax, 3			;; 3 iterations of 2
c5b:	disp four_complex_unfft_1, 2*clmblkdst, 8, 4*clmblkdst
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2/clm		;; Test inner loop counter
	jnc	c5b			;; Iterate if necessary
	lea	esi, [esi-clm*2*dist1+clmblkdst];; Next source pointer
	add	ah, 256/2		;; Test middle loop counter
	jnc	c5b			;; Iterate if necessary
	lea	esi, [esi-2*clmblkdst+8*clmblkdst];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al
	jnz	c5b
	lea	esi, [esi-3*8*clmblkdst]	;; Restore source pointer
	end_timer 18

;; Do inverse FFT levels 1,2,3
;; On input the 32-byte cache lines hold these data values:
;;	0	2K	1	2K+1
;;	256	...
;;	...
;;	4K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	6K	1	6K+1
;;	256
;;	...
;;	6K-256

;; Do 8 three_complex_last_unfft macros
;;	distance between fft data elements is 8

	start_timer 19
	mov	edi, plus1_premults	;; Address of premultiplier table
	mov	al, 8			;; Load loop counter
c1b:	three_complex_last_unfft 8, 8*clmblkdst, 16*clmblkdst
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2/clm		;; Test loop counter
	jnc	c1b			;; Iterate if necessary
	lea	esi, [esi-clm*2*dist1+clmblkdst];; Next source pointer
	lea	edi, [edi+48]		;; Next premultiplier pointer
	dec	al			;; Test loop counter
	jnz	c1b			;; Iterate if necessary
	lea	esi, [esi-8*clmblkdst]	;; Restore source pointer
	end_timer 19

;; Normalize these values

	start_timer 20
	mov	eax, NORMRTN		;; Addr of normalization routine
	call	eax
	sub	eax, eax
	end_timer 20

;; Have we been given permission to start the FFT on the result?
;; If so, do so now while the data is in the L2 cache.

	cmp	POSTFFT, 0		;; Test flag
	je	short copyback		;; Copy data, skip FFT if flag not set
	copy_3_words clm, 1
	cmp	edx, p2cl/128*65536+(128/clm-4/clm)*256;; Delay fft on 8 sets
	jle	yes_fft			;; No delay, do FFT now

;; Copy data from the scratch area back to the FFT data area

copyback:
	copy_scratch_data_to_fft 24
	jmp	no_fft			;; Skip forward FFT code

;; Copy data from the FFT data area back to the scratch area

b0b:	copy_fft_data_to_scratch 24

;; Do FFT levels 1,2,3
;;
;; On input the 32-byte cache lines hold these data values:
;;	0	6K	1	6K+1
;;	256
;;	...
;;	6K-256
;; On output the 32-byte cache lines hold these data values:
;;	0	2K	1	2K+1
;;	256	...
;;	...
;;	4K	...
;;	...

;; Do 8 three_complex_fft_premult macros
;;	distance between fft data elements is 8

yes_fft:start_timer 21
	mov	edi, plus1_premults	;; Address of premultiplier table
	mov	eax, 8			;; 8 iterations
b1b:	three_complex_fft_premult 8*clmblkdst, 16*clmblkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	add	dl, 256/2/clm		;; Test loop counter
	jnc	b1b			;; Iterate if necessary
	lea	esi, [esi-clm*2*dist1+clmblkdst];; Next source pointer
	lea	edi, [edi+48]		;; Next premultiplier pointer
	dec	al			;; Test loop counter
	jnz	b1b			;; Iterate if necessary
	lea	esi, [esi-8*clmblkdst]	;; Restore source pointer
	end_timer 21

;; Do FFT levels 4
;;
;; On input the 32-byte cache lines hold these data values:
;;	0	2K	1	2K+1
;;	256	...
;;	...
;;	4K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	1K	1	1K+1
;;	256	...
;;	...
;;	2K	...
;;	...

;; Do 6 four_complex_fft_1 macros
;;	distance between fft data elements is 4

	start_timer 22
	mov	edi, sincos2		;; Load sin/cos pointer
	mov	eax, 3			;; 3 iterations of 2
b5b:	disp four_complex_fft_1, 2*clmblkdst, 4*clmblkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2/clm		;; Test inner loop counter
	jnc	b5b			;; Iterate if necessary
	lea	esi, [esi-clm*2*dist1+clmblkdst];; Next source pointer
	add	ah, 256/2		;; Test middle loop counter
	jnc	b5b			;; Iterate if necessary
	lea	esi, [esi-2*clmblkdst+8*clmblkdst];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al
	jnz	b5b
	lea	esi, [esi-3*8*clmblkdst]	;; Restore source pointer
	end_timer 22

;; Do FFT levels 5,6
;;
;; On input the 32-byte cache lines hold these data values:
;;	0	1K	1	1K+1
;;	256	...
;;	...
;;	2K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	256	1	257
;;	512	...
;;	...

;; Do 6 four_complex_fft macros
;;	distance between fft data elements is 1

	start_timer 23
;	lea	ecx, [ecx-1111*blkdst]	;; Create dest ptr from prefetch ptr
	mov	edi, sincos1		;; Load sin/cos pointer
	mov	al, 6			;; 6 iterations
b3b:	cp_disp four_complex_fft, clmblkdst, 2*clmblkdst, 8, blkdst, 2*blkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	lea	ecx, [ecx+dist1]	;; Next dest pointer
	add	al, 256/2/clm		;; Test inner loop counter
	jnc	b3b			;; Iterate if necessary
	lea	esi, [esi-clm*2*dist1+4*clmblkdst];; Next source pointer
	lea	ecx, [ecx-clm*2*dist1+4*blkdst];; Next dest pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al
	jnz	b3b
	lea	esi, [ecx-6*4*blkdst]	;; Restore source pointer
	end_timer 23

;; Work on next 4 sets of 32 values

no_fft:	lea	esi, [esi+clm*2*dist1]	;; Next source pointer
	dec	dh			;; Test loop counter
	jnz	short more		;; Do more FFTs
	lea	esi, [esi+64]		;; Skip pad every 128 cache lines (4KB)
pfing	mov	ebp, clm*32		;; Set prefetch distance
	sub	edx, 65536		;; Test loop counter
	jz	short fftdn		;; Jump if done looping
	mov	dh, 128/clm		;; Restore cache line counter
more:	set_prefetch_gap
	cmp	norm_ptr1, 0		;; What was the loop start point?
	jne	c0b			;; Do inverse-FFT/norm/FFT iteration
	jmp	b0b			;; Do another FFT-only iteration
fftdn:	lea	esi, [esi-p2cl*2*dist1-p2cl/128*64];; Restore source pointer
	cmp	esi, DESTARG		;; If esi was not restored, then we 
	jne	short fftdn1		;; just finished skipped postfft data
	sub	ebx, ebx
	mov	DIST_TO_FFTSRCARG, ebx
	cmp	norm_ptr1, 0		;; What was the loop start point?
	je	pass2			;; FFT-only, go do pass 2

;; Split the carries into high and low carries (so they do not exceed
;; the desired number of bits per FFT word.  Then add the carries back to the
;; FFT data.

	start_timer 28
	call	gw_carries
	end_timer 28

;; Have we been given permission to start the FFT on the result?
;; If so, set loop to FFT the first few cache lines now that the carries
;; have been added in.

	cmp	POSTFFT, 0		;; Test flag
	je	short fftdn1		;; Skip FFT code if flag not set
	mov	esi, DESTARG		;; Load FFT data address
	mov	edx, 65536+4/clm*256	;; Do 8 previously skipped cache lines
	mov	norm_ptr1, 0		;; Set flag so only forward FFT is done
	copy_4_words
	jmp	b0b

;; Now add in the FFT'ed carries, cleanup and return

fftdn1:	end_timer 1
	jmp	gw_finish_mult
	ENDM


;; Perform a 14K-element FFT.  This is done in two passes.  Pass 1
;; does 6 levels, pass 2 uses common code to perform the last 8 levels.

pass1levels6pfa7 MACRO pass2_macro, p2cl, clmarg
	LOCAL	b0b, b1b, b2b, b3b, b4b, b5b, c0b, c1b, c2b, c3b, c4b, c5b
	LOCAL	pass2, copyback, yes_fft, no_fft, more, fftdn, fftdn1

	set_FFT_constants clmarg

;; Call FFT code unless the FFT has already been started

	clear_timers
	cmp	DWORD PTR [esi-28][ebx], 0 ;; Test FFT-started flag
	jne	pass2			;; Jump is FFT already started
	mov	norm_ptr1, 0		;; Set FFT-ing flag
	mov	edx, p2cl/128*65536+128/clm*256;; Load loop counter
pfing	mov	ebp, clm*32		;; Set prefetch distance
	jmp	b0b			;; Do the pass 1 fft

;; Do the last 8 FFT levels
;;	do 1 pass2_8_levels_real and 27 pass2_8_levels_complex

pass2:	start_timer 0
	CALLP	pass2_macro
	end_timer 0
	cmp	ffttype, 1		;; We're done if FFTing only
	je	gw_finish_fft

;; Do the inverse FFT

	start_timer 1
	fldz				;; Init SUMOUT
	fstp	SUMOUT
	mov	edx, norm_biglit_array	;; Addr of the big/little flags array
	mov	norm_ptr1, edx		;; Save ptr
	mov	edx, norm_col_mults	;; Addr of the column multipliers
	mov	norm_ptr2, edx		;; Save ptr
	mov	edx, p2cl/128*65536+128/clm*256;; Load loop counter
pfing	mov	ebp, clm*32		;; Set prefetch distance

;; Do inverse FFT levels 5,6
;; On input the 32-byte cache lines hold these data values:
;;	0	256	1	257
;;	512	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	1K	1	1K+1
;;	256	...
;;	...
;;	2K	...
;;	...

;; Do 1 four_real_four_semireal_unfft macros
;;	distance between fft data elements is 1

c0b:	start_timer 17
	mov	ecx, scratch_area	;; Get address of scratch area
c2b:	cp_disp four_real_four_semireal_unfft, 8, blkdst, 2*blkdst, 8, clmblkdst, 2*clmblkdst
	lea	esi, [esi+dist1]	;; Next source pointer
	lea	ecx, [ecx+dist1]	;; Next dest pointer
	add	dl, 256/2/clm		;; Test inner loop counter
	jnc	c2b			;; Iterate if necessary
	lea	esi, [esi-clm*2*dist1+4*blkdst];; Next source pointer
	lea	ecx, [ecx-clm*2*dist1+4*clmblkdst];; Next dest pointer

;; Do 6 four_complex_unfft macros
;;	distance between fft data elements is 1

	mov	edi, sincos1		;; Load sin/cos pointer
	mov	al, 6			;; 6 iterations
c3b:	cp_disp four_complex_unfft, 8, blkdst, 2*blkdst, 8, clmblkdst, 2*clmblkdst
	lea	esi, [esi+dist1]	;; Next source pointer
	lea	ecx, [ecx+dist1]	;; Next dest pointer
	add	dl, 256/2/clm		;; Test loop counter
	jnc	c3b			;; Iterate if necessary
	lea	esi, [esi-clm*2*dist1+4*blkdst];; Next source pointer
	lea	ecx, [ecx-clm*2*dist1+4*clmblkdst];; Next dest pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al			;; Test outer loop counter
	jnz	c3b			;; Iterate if necessary
	lea	ecx, [esi-7*4*blkdst]	;; Create prefetch pointer
	mov	esi, scratch_area	;; Restore source pointer
	end_timer 17

;; Do inverse FFT levels 4
;; On input the 32-byte cache lines hold these data values:
;;	0	1K	1	1K+1
;;	256	...
;;	...
;;	2K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	2K	1	2K+1
;;	256	...
;;	...
;;	4K	...
;;	...

;; Do 1 eight_reals_unfft_1 macros
;;	distance between fft data elements is 4

	start_timer 18
c4b:	disp eight_reals_unfft_1, clmblkdst, 2*clmblkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2/clm		;; Test inner loop counter
	jnc	c4b			;; Iterate if necessary
	lea	esi, [esi-clm*2*dist1+4*clmblkdst];; Next source pointer

;; Do 6 four_complex_unfft_1 macros
;;	distance between fft data elements is 4

	mov	edi, sincos2		;; Load sin/cos pointer
	mov	eax, 3			;; 3 iterations of 2
c5b:	disp four_complex_unfft_1, 2*clmblkdst, 8, 4*clmblkdst
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2/clm		;; Test inner loop counter
	jnc	c5b			;; Iterate if necessary
	lea	esi, [esi-clm*2*dist1+clmblkdst];; Next source pointer
	add	ah, 256/2		;; Test middle loop counter
	jnc	c5b			;; Iterate if necessary
	lea	esi, [esi-2*clmblkdst+8*clmblkdst];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al
	jnz	c5b
	lea	esi, [esi-3*8*clmblkdst-4*clmblkdst];; Restore source pointer
	end_timer 18

;; Do inverse FFT levels 1,2,3
;; On input the 32-byte cache lines hold these data values:
;;	0	2K	1	2K+1
;;	256	...
;;	...
;;	4K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	7K	1	7K+1
;;	256
;;	...
;;	7K-256

;; Do 8 seven_reals_last_unfft macros
;;	distance between fft data elements is 8

	start_timer 19
	mov	al, 4			;; Load loop counter
c1b:	two_seven_reals_last_unfft 8, 4*clmblkdst
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2/clm		;; Test loop counter
	jnc	c1b			;; Iterate if necessary
	lea	esi, [esi-clm*2*dist1+clmblkdst];; Next source pointer
	dec	al			;; Test loop counter
	jnz	c1b			;; Iterate if necessary
	lea	esi, [esi-4*clmblkdst]	;; Restore source pointer
	end_timer 19

;; Normalize these values

	start_timer 20
	mov	eax, NORMRTN		;; Addr of normalization routine
	call	eax
	sub	eax, eax
	end_timer 20

;; Have we been given permission to start the FFT on the result?
;; If so, do so now while the data is in the L2 cache.

	cmp	POSTFFT, 0		;; Test flag
	je	short copyback		;; Copy data, skip FFT if flag not set
	copy_3_words clm, 1
	cmp	edx, p2cl/128*65536+(128/clm-4/clm)*256;; Delay fft on 8 sets
	jle	yes_fft			;; No delay, do FFT now

;; Copy data from the scratch area back to the FFT data area

copyback:
	copy_scratch_data_to_fft 28
	jmp	no_fft			;; Skip forward FFT code

;; Copy data from the FFT data area back to the scratch area

b0b:	copy_fft_data_to_scratch 28

;; Do FFT levels 1,2,3
;;
;; On input the 32-byte cache lines hold these data values:
;;	0	7K	1	7K+1
;;	256
;;	...
;;	7K-256
;; On output the 32-byte cache lines hold these data values:
;;	0	2K	1	2K+1
;;	256	...
;;	...
;;	4K	...
;;	...

;; Do 8 seven_reals_fft macros
;;	distance between fft data elements is 8

yes_fft:start_timer 21
	mov	eax, 4			;; 4 iterations
b1b:	two_seven_reals_fft 4*clmblkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	add	dl, 256/2/clm		;; Test loop counter
	jnc	b1b			;; Iterate if necessary
	lea	esi, [esi-clm*2*dist1+clmblkdst];; Next source pointer
	dec	al			;; Test loop counter
	jnz	b1b			;; Iterate if necessary
	lea	esi, [esi-4*clmblkdst]	;; Restore source pointer
	end_timer 21

;; Do FFT levels 4
;;
;; On input the 32-byte cache lines hold these data values:
;;	0	2K	1	2K+1
;;	256	...
;;	...
;;	4K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	1K	1	1K+1
;;	256	...
;;	...
;;	2K	...
;;	...

;; Do 1 eight_reals_fft_1 macros
;;	distance between fft data elements is 4

	start_timer 22
b4b:	disp eight_reals_fft_1, clmblkdst, 2*clmblkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2/clm		;; Test inner loop counter
	jnc	b4b			;; Iterate if necessary
	lea	esi, [esi-clm*2*dist1+4*clmblkdst];; Next source pointer

;; Do 6 four_complex_fft_1 macros
;;	distance between fft data elements is 4

	mov	edi, sincos2		;; Load sin/cos pointer
	mov	eax, 3			;; 3 iterations of 2
b5b:	disp four_complex_fft_1, 2*clmblkdst, 4*clmblkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2/clm		;; Test inner loop counter
	jnc	b5b			;; Iterate if necessary
	lea	esi, [esi-clm*2*dist1+clmblkdst];; Next source pointer
	add	ah, 256/2		;; Test middle loop counter
	jnc	b5b			;; Iterate if necessary
	lea	esi, [esi-2*clmblkdst+8*clmblkdst];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al
	jnz	b5b
	lea	esi, [esi-3*8*clmblkdst-4*clmblkdst];; Restore source pointer
	end_timer 22

;; Do FFT levels 5,6
;;
;; On input the 32-byte cache lines hold these data values:
;;	0	1K	1	1K+1
;;	256	...
;;	...
;;	2K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	256	1	257
;;	512	...
;;	...

;; Do 1 four_real_four_semireal_fft macros
;;	distance between fft data elements is 1

	start_timer 23
;	lea	ecx, [ecx-1111*blkdst]	;; Create dest ptr from prefetch ptr
b2b:	cp_disp four_real_four_semireal_fft, clmblkdst, 2*clmblkdst, 8, blkdst, 2*blkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	lea	ecx, [ecx+dist1]	;; Next dest pointer
	add	al, 256/2/clm		;; Test inner loop counter
	jnc	b2b			;; Iterate if necessary
	lea	esi, [esi-clm*2*dist1+4*clmblkdst];; Next source pointer
	lea	ecx, [ecx-clm*2*dist1+4*blkdst];; Next dest pointer

;; Do 6 four_complex_fft macros
;;	distance between fft data elements is 1

	mov	edi, sincos1		;; Load sin/cos pointer
	mov	al, 6			;; 6 iterations
b3b:	cp_disp four_complex_fft, clmblkdst, 2*clmblkdst, 8, blkdst, 2*blkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	lea	ecx, [ecx+dist1]	;; Next dest pointer
	add	al, 256/2/clm		;; Test inner loop counter
	jnc	b3b			;; Iterate if necessary
	lea	esi, [esi-clm*2*dist1+4*clmblkdst];; Next source pointer
	lea	ecx, [ecx-clm*2*dist1+4*blkdst];; Next dest pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al
	jnz	b3b
	lea	esi, [ecx-7*4*blkdst]	;; Restore source pointer
	end_timer 23

;; Work on next 4 sets of 32 values

no_fft:	lea	esi, [esi+clm*2*dist1]	;; Next source pointer
	dec	dh			;; Test loop counter
	jnz	short more		;; Do more FFTs
	lea	esi, [esi+64]		;; Skip pad every 128 cache lines (4KB)
pfing	mov	ebp, clm*32		;; Set prefetch distance
	sub	edx, 65536		;; Test loop counter
	jz	short fftdn		;; Jump if done looping
	mov	dh, 128/clm		;; Restore cache line counter
more:	set_prefetch_gap
	cmp	norm_ptr1, 0		;; What was the loop start point?
	jne	c0b			;; Do inverse-FFT/norm/FFT iteration
	jmp	b0b			;; Do another FFT-only iteration
fftdn:	lea	esi, [esi-p2cl*2*dist1-p2cl/128*64];; Restore source pointer
	cmp	esi, DESTARG		;; If esi was not restored, then we 
	jne	short fftdn1		;; just finished skipped postfft data
	sub	ebx, ebx
	mov	DIST_TO_FFTSRCARG, ebx
	cmp	norm_ptr1, 0		;; What was the loop start point?
	je	pass2			;; FFT-only, go do pass 2

;; Split the carries into high and low carries (so they do not exceed
;; the desired number of bits per FFT word.  Then add the carries back to the
;; FFT data.

	start_timer 28
	call	gw_carries
	end_timer 28

;; Have we been given permission to start the FFT on the result?
;; If so, set loop to FFT the first few cache lines now that the carries
;; have been added in.

	cmp	POSTFFT, 0		;; Test flag
	je	short fftdn1		;; Skip FFT code if flag not set
	mov	esi, DESTARG		;; Load FFT data address
	mov	edx, 65536+4/clm*256	;; Do 8 previously skipped cache lines
	mov	norm_ptr1, 0		;; Set flag so only forward FFT is done
	copy_4_words
	jmp	b0b

;; Now add in the FFT'ed carries, cleanup and return

fftdn1:	end_timer 1
	jmp	gw_finish_mult
	ENDM


;; Perform a 16K-element FFT.  This is done in two passes.  Pass 1
;; does 6 levels, pass 2 uses common code to perform the last 8 levels.

pass1levels6pfa8 MACRO pass2_macro, p2cl, clmarg
	LOCAL	b0b, b1b, b2b, b3b, b4b, b5b, c0b, c1b, c2b, c3b, c4b, c5b
	LOCAL	pass2, copyback, yes_fft, no_fft, more, fftdn, fftdn1

	set_FFT_constants clmarg

;; Call FFT code unless the FFT has already been started

	clear_timers
	cmp	DWORD PTR [esi-28][ebx], 0 ;; Test FFT-started flag
	jne	pass2			;; Jump is FFT already started
	mov	norm_ptr1, 0		;; Set FFT-ing flag
	mov	edx, p2cl/128*65536+128/clm*256;; Load loop counter
pfing	mov	ebp, clm*32		;; Set prefetch distance
	jmp	b0b			;; Do the pass 1 fft

;; Do the last 8 FFT levels
;;	do 1 pass2_8_levels_real and 31 pass2_8_levels_complex

pass2:	start_timer 0
	CALLP	pass2_macro
	end_timer 0
	cmp	ffttype, 1		;; We're done if FFTing only
	je	gw_finish_fft

;; Do the inverse FFT

	start_timer 1
	fldz				;; Init SUMOUT
	fstp	SUMOUT
	mov	edx, norm_biglit_array	;; Addr of the big/little flags array
	mov	norm_ptr1, edx		;; Save ptr
	mov	edx, norm_col_mults	;; Addr of the column multipliers
	mov	norm_ptr2, edx		;; Save ptr
	mov	edx, p2cl/128*65536+128/clm*256;; Load loop counter
pfing	mov	ebp, clm*32		;; Set prefetch distance

;; Do inverse FFT levels 5,6
;; On input the 32-byte cache lines hold these data values:
;;	0	256	1	257
;;	512	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	1K	1	1K+1
;;	256	...
;;	...
;;	2K	...
;;	...

;; Do 1 four_real_four_semireal_unfft macros
;;	distance between fft data elements is 1

c0b:	start_timer 17
	mov	ecx, scratch_area	;; Get address of scratch area
c2b:	cp_disp four_real_four_semireal_unfft, 8, blkdst, 2*blkdst, 8, clmblkdst, 2*clmblkdst
	lea	esi, [esi+dist1]	;; Next source pointer
	lea	ecx, [ecx+dist1]	;; Next dest pointer
	add	dl, 256/2/clm		;; Test inner loop counter
	jnc	c2b			;; Iterate if necessary
	lea	esi, [esi-clm*2*dist1+4*blkdst];; Next source pointer
	lea	ecx, [ecx-clm*2*dist1+4*clmblkdst];; Next dest pointer

;; Do 7 four_complex_unfft macros
;;	distance between fft data elements is 1

	mov	edi, sincos1		;; Load sin/cos pointer
	mov	al, 7			;; 7 iterations
c3b:	cp_disp four_complex_unfft, 8, blkdst, 2*blkdst, 8, clmblkdst, 2*clmblkdst
	lea	esi, [esi+dist1]	;; Next source pointer
	lea	ecx, [ecx+dist1]	;; Next dest pointer
	add	dl, 256/2/clm		;; Test loop counter
	jnc	c3b			;; Iterate if necessary
	lea	esi, [esi-clm*2*dist1+4*blkdst];; Next source pointer
	lea	ecx, [ecx-clm*2*dist1+4*clmblkdst];; Next dest pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al			;; Test outer loop counter
	jnz	c3b			;; Iterate if necessary
	lea	ecx, [esi-8*4*blkdst]	;; Create prefetch pointer
	mov	esi, scratch_area	;; Restore source pointer
	end_timer 17

;; Do inverse FFT levels 4
;; On input the 32-byte cache lines hold these data values:
;;	0	1K	1	1K+1
;;	256	...
;;	...
;;	2K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	2K	1	2K+1
;;	256	...
;;	...
;;	4K	...
;;	...

;; Do 2 four_real_four_semireal_unfft_1 macros
;;	distance between fft data elements is 4

	start_timer 18
	mov	al, 2			;; Load loop counter
c4b:	disp four_real_four_semireal_unfft_1, 2*clmblkdst, 8, 4*clmblkdst
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2/clm		;; Test inner loop counter
	jnc	c4b			;; Iterate if necessary
	lea	esi, [esi-clm*2*dist1+clmblkdst];; Next source pointer
	dec	al
	jnz	c4b
	lea	esi, [esi-2*clmblkdst+8*clmblkdst];; Next source pointer

;; Do 6 four_complex_unfft_1 macros
;;	distance between fft data elements is 4

	mov	edi, sincos2		;; Load sin/cos pointer
	mov	eax, 3			;; 3 iterations of 2
c5b:	disp four_complex_unfft_1, 2*clmblkdst, 8, 4*clmblkdst
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2/clm		;; Test inner loop counter
	jnc	c5b			;; Iterate if necessary
	lea	esi, [esi-clm*2*dist1+clmblkdst];; Next source pointer
	add	ah, 256/2		;; Test middle loop counter
	jnc	c5b			;; Iterate if necessary
	lea	esi, [esi-2*clmblkdst+8*clmblkdst];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al
	jnz	c5b
	lea	esi, [esi-4*8*clmblkdst]	;; Restore source pointer
	end_timer 18

;; Do inverse FFT levels 1,2,3
;; On input the 32-byte cache lines hold these data values:
;;	0	2K	1	2K+1
;;	256	...
;;	...
;;	4K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	8K	1	8K+1
;;	256
;;	...
;;	8K-256

;; Do 8 eight_reals_last_unfft macros
;;	distance between fft data elements is 8

	start_timer 19
	mov	al, 8			;; Load loop counter
c1b:	disp eight_reals_last_unfft, 8, 8*clmblkdst, 16*clmblkdst
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2/clm		;; Test loop counter
	jnc	c1b			;; Iterate if necessary
	lea	esi, [esi-clm*2*dist1+clmblkdst];; Next source pointer
	dec	al			;; Test loop counter
	jnz	c1b			;; Iterate if necessary
	lea	esi, [esi-8*clmblkdst]	;; Restore source pointer
	end_timer 19

;; Normalize these values

	start_timer 20
	mov	eax, NORMRTN		;; Addr of normalization routine
	call	eax
	sub	eax, eax
	end_timer 20

;; Have we been given permission to start the FFT on the result?
;; If so, do so now while the data is in the L2 cache.

	cmp	POSTFFT, 0		;; Test flag
	je	short copyback		;; Copy data, skip FFT if flag not set
	copy_3_words clm, 1
	cmp	edx, p2cl/128*65536+(128/clm-4/clm)*256;; Delay fft on 8 sets
	jle	yes_fft			;; No delay, do FFT now

;; Copy data from the scratch area back to the FFT data area

copyback:
	copy_scratch_data_to_fft 32
	jmp	no_fft			;; Skip forward FFT code

;; Copy data from the FFT data area back to the scratch area

b0b:	copy_fft_data_to_scratch 32

;; Do FFT levels 1,2,3
;;
;; On input the 32-byte cache lines hold these data values:
;;	0	8K	1	8K+1
;;	256
;;	...
;;	8K-256
;; On output the 32-byte cache lines hold these data values:
;;	0	2K	1	2K+1
;;	256	...
;;	...
;;	4K	...
;;	...

;; Do 8 eight_reals_fft macros
;;	distance between fft data elements is 8

yes_fft:start_timer 21
	mov	eax, 8			;; 8 iterations
b1b:	disp eight_reals_fft, 8*clmblkdst, 16*clmblkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	add	dl, 256/2/clm		;; Test loop counter
	jnc	b1b			;; Iterate if necessary
	lea	esi, [esi-clm*2*dist1+clmblkdst];; Next source pointer
	dec	al			;; Test loop counter
	jnz	b1b			;; Iterate if necessary
	lea	esi, [esi-8*clmblkdst]	;; Restore source pointer
	end_timer 21

;; Do FFT levels 4
;;
;; On input the 32-byte cache lines hold these data values:
;;	0	2K	1	2K+1
;;	256	...
;;	...
;;	4K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	1K	1	1K+1
;;	256	...
;;	...
;;	2K	...
;;	...

;; Do 2 four_real_four_semireal_fft_1 macros
;;	distance between fft data elements is 4

	start_timer 22
	mov	al, 2			;; Load loop counter
b4b:	disp four_real_four_semireal_fft_1, 2*clmblkdst, 4*clmblkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2/clm		;; Test inner loop counter
	jnc	b4b			;; Iterate if necessary
	lea	esi, [esi-clm*2*dist1+clmblkdst];; Next source pointer
	dec	al
	jnz	b4b
	lea	esi, [esi-2*clmblkdst+8*clmblkdst];; Next source pointer

;; Do 6 four_complex_fft_1 macros
;;	distance between fft data elements is 4

	mov	edi, sincos2		;; Load sin/cos pointer
	mov	eax, 3			;; 3 iterations of 2
b5b:	disp four_complex_fft_1, 2*clmblkdst, 4*clmblkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2/clm		;; Test inner loop counter
	jnc	b5b			;; Iterate if necessary
	lea	esi, [esi-clm*2*dist1+clmblkdst];; Next source pointer
	add	ah, 256/2		;; Test middle loop counter
	jnc	b5b			;; Iterate if necessary
	lea	esi, [esi-2*clmblkdst+8*clmblkdst];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al
	jnz	b5b
	lea	esi, [esi-4*8*clmblkdst]	;; Restore source pointer
	end_timer 22

;; Do FFT levels 5,6
;;
;; On input the 32-byte cache lines hold these data values:
;;	0	1K	1	1K+1
;;	256	...
;;	...
;;	2K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	256	1	257
;;	512	...
;;	...

;; Do 1 four_real_four_semireal_fft macros
;;	distance between fft data elements is 1

	start_timer 23
;	lea	ecx, [ecx-1111*blkdst]	;; Create dest ptr from prefetch ptr
b2b:	cp_disp four_real_four_semireal_fft, clmblkdst, 2*clmblkdst, 8, blkdst, 2*blkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	lea	ecx, [ecx+dist1]	;; Next dest pointer
	add	al, 256/2/clm		;; Test inner loop counter
	jnc	b2b			;; Iterate if necessary
	lea	esi, [esi-clm*2*dist1+4*clmblkdst];; Next source pointer
	lea	ecx, [ecx-clm*2*dist1+4*blkdst];; Next dest pointer

;; Do 7 four_complex_fft macros
;;	distance between fft data elements is 1

	mov	edi, sincos1		;; Load sin/cos pointer
	mov	al, 7			;; 7 iterations
b3b:	cp_disp four_complex_fft, clmblkdst, 2*clmblkdst, 8, blkdst, 2*blkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	lea	ecx, [ecx+dist1]	;; Next dest pointer
	add	al, 256/2/clm		;; Test inner loop counter
	jnc	b3b			;; Iterate if necessary
	lea	esi, [esi-clm*2*dist1+4*clmblkdst];; Next source pointer
	lea	ecx, [ecx-clm*2*dist1+4*blkdst];; Next dest pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al
	jnz	b3b
	lea	esi, [ecx-8*4*blkdst]	;; Restore source pointer
	end_timer 23

;; Work on next 4 sets of 32 values

no_fft:	lea	esi, [esi+clm*2*dist1]	;; Next source pointer
	dec	dh			;; Test loop counter
	jnz	short more		;; Do more FFTs
	lea	esi, [esi+64]		;; Skip pad every 128 cache lines (4KB)
pfing	mov	ebp, clm*32		;; Set prefetch distance
	sub	edx, 65536		;; Test loop counter
	jz	short fftdn		;; Jump if done looping
	mov	dh, 128/clm		;; Restore cache line counter
more:	set_prefetch_gap
	cmp	norm_ptr1, 0		;; What was the loop start point?
	jne	c0b			;; Do inverse-FFT/norm/FFT iteration
	jmp	b0b			;; Do another FFT-only iteration
fftdn:	lea	esi, [esi-p2cl*2*dist1-p2cl/128*64];; Restore source pointer
	cmp	esi, DESTARG		;; If esi was not restored, then we 
	jne	short fftdn1		;; just finished skipped postfft data
	sub	ebx, ebx
	mov	DIST_TO_FFTSRCARG, ebx
	cmp	norm_ptr1, 0		;; What was the loop start point?
	je	pass2			;; FFT-only, go do pass 2

;; Split the carries into high and low carries (so they do not exceed
;; the desired number of bits per FFT word.  Then add the carries back to the
;; FFT data.

	start_timer 28
	call	gw_carries
	end_timer 28

;; Have we been given permission to start the FFT on the result?
;; If so, set loop to FFT the first few cache lines now that the carries
;; have been added in.

	cmp	POSTFFT, 0		;; Test flag
	je	short fftdn1		;; Skip FFT code if flag not set
	mov	esi, DESTARG		;; Load FFT data address
	mov	edx, 65536+4/clm*256	;; Do 8 previously skipped cache lines
	mov	norm_ptr1, 0		;; Set flag so only forward FFT is done
	copy_4_words
	jmp	b0b

;; Now add in the FFT'ed carries, cleanup and return

fftdn1:	end_timer 1
	jmp	gw_finish_mult
	ENDM


;; Perform a 16K-element negacyclic FFT.  This is done in two passes.  Pass 1
;; does 6 levels, pass 2 uses common code to perform the last 8 levels.

pass1levels6complex4 MACRO pass2_macro, p2cl, clmarg
	LOCAL	b0b, b1b, b3b, b5b, c0b, c1b, c3b, c5b
	LOCAL	pass2, copyback, yes_fft, no_fft, more, fftdn, fftdn1

	set_FFT_constants clmarg

;; Call FFT code unless the FFT has already been started

	clear_timers
	cmp	DWORD PTR [esi-28][ebx], 0 ;; Test FFT-started flag
	jne	pass2			;; Jump is FFT already started
	mov	norm_ptr1, 0		;; Set FFT-ing flag
	mov	edx, p2cl/128*65536+128/clm*256;; Load loop counter
pfing	mov	ebp, clm*32		;; Set prefetch distance
	jmp	b0b			;; Do the pass 1 fft

;; Do the last 8 FFT levels
;;	do 32 pass2_8_levels_complex

pass2:	start_timer 0
	CALLP	pass2_macro
	end_timer 0
	cmp	ffttype, 1		;; We're done if FFTing only
	je	gw_finish_fft

;; Do the inverse FFT

	start_timer 1
	fldz				;; Init SUMOUT
	fstp	SUMOUT
	mov	edx, norm_biglit_array	;; Addr of the big/little flags array
	mov	norm_ptr1, edx		;; Save ptr
	mov	edx, norm_col_mults	;; Addr of the column multipliers
	mov	norm_ptr2, edx		;; Save ptr
	mov	edx, p2cl/128*65536+128/clm*256;; Load loop counter
pfing	mov	ebp, clm*32		;; Set prefetch distance

;; Do inverse FFT levels 5,6
;; On input the 32-byte cache lines hold these data values:
;;	0	256	1	257
;;	512	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	1K	1	1K+1
;;	256	...
;;	...
;;	2K	...
;;	...

;; Do 8 four_complex_unfft macros
;;	distance between fft data elements is 1

c0b:	start_timer 17
	mov	ecx, scratch_area	;; Get address of scratch area
	mov	edi, sincos1		;; Load sin/cos pointer
	mov	al, 8			;; 8 iterations
c3b:	cp_disp four_complex_unfft, 8, blkdst, 2*blkdst, 8, clmblkdst, 2*clmblkdst
	lea	esi, [esi+dist1]	;; Next source pointer
	lea	ecx, [ecx+dist1]	;; Next dest pointer
	add	dl, 256/2/clm		;; Test loop counter
	jnc	c3b			;; Iterate if necessary
	lea	esi, [esi-clm*2*dist1+4*blkdst];; Next source pointer
	lea	ecx, [ecx-clm*2*dist1+4*clmblkdst];; Next dest pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al			;; Test outer loop counter
	jnz	c3b			;; Iterate if necessary
	lea	ecx, [esi-8*4*blkdst]	;; Create prefetch pointer
	mov	esi, scratch_area	;; Restore source pointer
	end_timer 17

;; Do inverse FFT levels 4
;; On input the 32-byte cache lines hold these data values:
;;	0	1K	1	1K+1
;;	256	...
;;	...
;;	2K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	2K	1	2K+1
;;	256	...
;;	...
;;	4K	...
;;	...

;; Do 8 four_complex_unfft_1 macros
;;	distance between fft data elements is 4

	start_timer 18
	mov	edi, sincos2		;; Load sin/cos pointer
	mov	eax, 4			;; 4 iterations of 2
c5b:	disp four_complex_unfft_1, 2*clmblkdst, 8, 4*clmblkdst
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2/clm		;; Test inner loop counter
	jnc	c5b			;; Iterate if necessary
	lea	esi, [esi-clm*2*dist1+clmblkdst];; Next source pointer
	add	ah, 256/2		;; Test middle loop counter
	jnc	c5b			;; Iterate if necessary
	lea	esi, [esi-2*clmblkdst+8*clmblkdst];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al
	jnz	c5b
	lea	esi, [esi-4*8*clmblkdst]	;; Restore source pointer
	end_timer 18

;; Do inverse FFT levels 1,2,3
;; On input the 32-byte cache lines hold these data values:
;;	0	2K	1	2K+1
;;	256	...
;;	...
;;	4K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	8K	1	8K+1
;;	256
;;	...
;;	8K-256

;; Do 8 four_complex_last_unfft macros
;;	distance between fft data elements is 8

	start_timer 19
	mov	edi, plus1_premults	;; Address of premultiplier table
	mov	al, 8			;; Load loop counter
c1b:	disp four_complex_last_unfft, 8, 8*clmblkdst, 16*clmblkdst
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2/clm		;; Test loop counter
	jnc	c1b			;; Iterate if necessary
	lea	esi, [esi-clm*2*dist1+clmblkdst];; Next source pointer
	lea	edi, [edi+64]		;; Next premultiplier pointer
	dec	al			;; Test loop counter
	jnz	c1b			;; Iterate if necessary
	lea	esi, [esi-8*clmblkdst]	;; Restore source pointer
	end_timer 19

;; Normalize these values

	start_timer 20
	mov	eax, NORMRTN		;; Addr of normalization routine
	call	eax
	sub	eax, eax
	end_timer 20

;; Have we been given permission to start the FFT on the result?
;; If so, do so now while the data is in the L2 cache.

	cmp	POSTFFT, 0		;; Test flag
	je	short copyback		;; Copy data, skip FFT if flag not set
	copy_3_words clm, 1
	cmp	edx, p2cl/128*65536+(128/clm-4/clm)*256;; Delay fft on 8 sets
	jle	yes_fft			;; No delay, do FFT now

;; Copy data from the scratch area back to the FFT data area

copyback:
	copy_scratch_data_to_fft 32
	jmp	no_fft			;; Skip forward FFT code

;; Copy data from the FFT data area back to the scratch area

b0b:	copy_fft_data_to_scratch 32

;; Do FFT levels 1,2,3
;;
;; On input the 32-byte cache lines hold these data values:
;;	0	8K	1	8K+1
;;	256
;;	...
;;	8K-256
;; On output the 32-byte cache lines hold these data values:
;;	0	2K	1	2K+1
;;	256	...
;;	...
;;	4K	...
;;	...

;; Do 8 four_complex_fft_premult macros
;;	distance between fft data elements is 8

yes_fft:start_timer 21
	mov	edi, plus1_premults	;; Address of premultiplier table
	mov	eax, 8			;; 8 iterations
b1b:	disp four_complex_fft_premult, 8*clmblkdst, 16*clmblkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	add	dl, 256/2/clm		;; Test loop counter
	jnc	b1b			;; Iterate if necessary
	lea	esi, [esi-clm*2*dist1+clmblkdst];; Next source pointer
	lea	edi, [edi+64]		;; Next premultiplier pointer
	dec	al			;; Test loop counter
	jnz	b1b			;; Iterate if necessary
	lea	esi, [esi-8*clmblkdst]	;; Restore source pointer
	end_timer 21

;; Do FFT levels 4
;;
;; On input the 32-byte cache lines hold these data values:
;;	0	2K	1	2K+1
;;	256	...
;;	...
;;	4K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	1K	1	1K+1
;;	256	...
;;	...
;;	2K	...
;;	...

;; Do 8 four_complex_fft_1 macros
;;	distance between fft data elements is 4

	start_timer 22
	mov	edi, sincos2		;; Load sin/cos pointer
	mov	eax, 4			;; 4 iterations of 2
b5b:	disp four_complex_fft_1, 2*clmblkdst, 4*clmblkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2/clm		;; Test inner loop counter
	jnc	b5b			;; Iterate if necessary
	lea	esi, [esi-clm*2*dist1+clmblkdst];; Next source pointer
	add	ah, 256/2		;; Test middle loop counter
	jnc	b5b			;; Iterate if necessary
	lea	esi, [esi-2*clmblkdst+8*clmblkdst];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al
	jnz	b5b
	lea	esi, [esi-4*8*clmblkdst]	;; Restore source pointer
	end_timer 22

;; Do FFT levels 5,6
;;
;; On input the 32-byte cache lines hold these data values:
;;	0	1K	1	1K+1
;;	256	...
;;	...
;;	2K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	256	1	257
;;	512	...
;;	...

;; Do 8 four_complex_fft macros
;;	distance between fft data elements is 1

	start_timer 23
;	lea	ecx, [ecx-1111*blkdst]	;; Create dest ptr from prefetch ptr
	mov	edi, sincos1		;; Load sin/cos pointer
	mov	al, 8			;; 8 iterations
b3b:	cp_disp four_complex_fft, clmblkdst, 2*clmblkdst, 8, blkdst, 2*blkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	lea	ecx, [ecx+dist1]	;; Next dest pointer
	add	al, 256/2/clm		;; Test inner loop counter
	jnc	b3b			;; Iterate if necessary
	lea	esi, [esi-clm*2*dist1+4*clmblkdst];; Next source pointer
	lea	ecx, [ecx-clm*2*dist1+4*blkdst];; Next dest pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al
	jnz	b3b
	lea	esi, [ecx-8*4*blkdst]	;; Restore source pointer
	end_timer 23

;; Work on next 4 sets of 32 values

no_fft:	lea	esi, [esi+clm*2*dist1]	;; Next source pointer
	dec	dh			;; Test loop counter
	jnz	short more		;; Do more FFTs
	lea	esi, [esi+64]		;; Skip pad every 128 cache lines (4KB)
pfing	mov	ebp, clm*32		;; Set prefetch distance
	sub	edx, 65536		;; Test loop counter
	jz	short fftdn		;; Jump if done looping
	mov	dh, 128/clm		;; Restore cache line counter
more:	set_prefetch_gap
	cmp	norm_ptr1, 0		;; What was the loop start point?
	jne	c0b			;; Do inverse-FFT/norm/FFT iteration
	jmp	b0b			;; Do another FFT-only iteration
fftdn:	lea	esi, [esi-p2cl*2*dist1-p2cl/128*64];; Restore source pointer
	cmp	esi, DESTARG		;; If esi was not restored, then we 
	jne	short fftdn1		;; just finished skipped postfft data
	sub	ebx, ebx
	mov	DIST_TO_FFTSRCARG, ebx
	cmp	norm_ptr1, 0		;; What was the loop start point?
	je	pass2			;; FFT-only, go do pass 2

;; Split the carries into high and low carries (so they do not exceed
;; the desired number of bits per FFT word.  Then add the carries back to the
;; FFT data.

	start_timer 28
	call	gw_carries
	end_timer 28

;; Have we been given permission to start the FFT on the result?
;; If so, set loop to FFT the first few cache lines now that the carries
;; have been added in.

	cmp	POSTFFT, 0		;; Test flag
	je	short fftdn1		;; Skip FFT code if flag not set
	mov	esi, DESTARG		;; Load FFT data address
	mov	edx, 65536+4/clm*256	;; Do 8 previously skipped cache lines
	mov	norm_ptr1, 0		;; Set flag so only forward FFT is done
	copy_4_words
	jmp	b0b

;; Now add in the FFT'ed carries, cleanup and return

fftdn1:	end_timer 1
	jmp	gw_finish_mult
	ENDM


;; Perform a 20K-element FFT.  This is done in two passes.  Pass 1
;; does 7 levels, pass 2 uses common code to perform the last 8 levels.

pass1levels7pfa5 MACRO pass2_macro, p2cl, clmarg
	LOCAL	b0b, b1b, b2b, b3b, b4b, b5b, c0b, c1b, c2b, c3b, c4b, c5b
	LOCAL	pass2, copyback, yes_fft, no_fft, more, fftdn, fftdn1

	set_FFT_constants clmarg

;; Call FFT code unless the FFT has already been started

	clear_timers
	cmp	DWORD PTR [esi-28][ebx], 0 ;; Test FFT-started flag
	jne	pass2			;; Jump is FFT already started
	mov	norm_ptr1, 0		;; Set FFT-ing flag
	mov	edx, p2cl/128*65536+128/clm*256;; Load loop counter
pfing	mov	ebp, clm*32		;; Set prefetch distance
	jmp	b0b			;; Do the pass 1 fft

;; Do the last 8 FFT levels
;;	do 1 pass2_8_levels_real and 39 pass2_8_levels_complex

pass2:	start_timer 0
	CALLP	pass2_macro
	end_timer 0
	cmp	ffttype, 1		;; We're done if FFTing only
	je	gw_finish_fft

;; Do the inverse FFT

	start_timer 1
	fldz				;; Init SUMOUT
	fstp	SUMOUT
	mov	edx, norm_biglit_array	;; Addr of the big/little flags array
	mov	norm_ptr1, edx		;; Save ptr
	mov	edx, norm_col_mults	;; Addr of the column multipliers
	mov	norm_ptr2, edx		;; Save ptr
	mov	edx, p2cl/128*65536+128/clm*256;; Load loop counter
pfing	mov	ebp, clm*32		;; Set prefetch distance

;; Do inverse FFT levels 6,7
;; On input the 32-byte cache lines hold these data values:
;;	0	256	1	257
;;	512	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	1K	1	1K+1
;;	256	...
;;	...
;;	2K	...
;;	...

;; Do 1 four_real_four_semireal_unfft macros
;;	distance between fft data elements is 1

c0b:	start_timer 17
	mov	ecx, scratch_area	;; Get address of scratch area
c2b:	cp_disp four_real_four_semireal_unfft, 8, blkdst, 2*blkdst, 8, clmblkdst, 2*clmblkdst
	lea	esi, [esi+dist1]	;; Next source pointer
	lea	ecx, [ecx+dist1]	;; Next dest pointer
	add	dl, 256/2/clm		;; Test inner loop counter
	jnc	c2b			;; Iterate if necessary
	lea	esi, [esi-clm*2*dist1+4*blkdst];; Next source pointer
	lea	ecx, [ecx-clm*2*dist1+4*clmblkdst];; Next dest pointer

;; Do 9 four_complex_unfft macros
;;	distance between fft data elements is 1

	mov	edi, sincos1		;; Load sin/cos pointer
	mov	al, 9			;; 9 iterations
c3b:	cp_disp four_complex_unfft, 8, blkdst, 2*blkdst, 8, clmblkdst, 2*clmblkdst
	lea	esi, [esi+dist1]	;; Next source pointer
	lea	ecx, [ecx+dist1]	;; Next dest pointer
	add	dl, 256/2/clm		;; Test loop counter
	jnc	c3b			;; Iterate if necessary
	lea	esi, [esi-clm*2*dist1+4*blkdst];; Next source pointer
	lea	ecx, [ecx-clm*2*dist1+4*clmblkdst];; Next dest pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al			;; Test outer loop counter
	jnz	c3b			;; Iterate if necessary
	lea	ecx, [esi-10*4*blkdst]	;; Create prefetch pointer
	mov	esi, scratch_area	;; Restore source pointer
	end_timer 17

;; Do inverse FFT levels 4,5
;; On input the 32-byte cache lines hold these data values:
;;	0	1K	1	1K+1
;;	256	...
;;	...
;;	2K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	4K	1	4K+1
;;	256	...
;;	...
;;	8K	...
;;	...

;; Do 2 eight_reals_unfft_2 macros
;;	distance between fft data elements is 4

	start_timer 18
	mov	al, 2			;; Load loop counter
c4b:	disp eight_reals_unfft_2, 2*clmblkdst, 8, 4*clmblkdst
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2/clm		;; Test inner loop counter
	jnc	c4b			;; Iterate if necessary
	lea	esi, [esi-clm*2*dist1+clmblkdst];; Next source pointer
	dec	al
	jnz	c4b
	lea	esi, [esi-2*clmblkdst+8*clmblkdst];; Next source pointer

;; Do 8 four_complex_unfft macros
;;	distance between fft data elements is 4

	mov	edi, sincos2		;; Load sin/cos pointer
	mov	eax, 2			;; 2 iterations of 4
c5b:	disp four_complex_unfft, 8, 4*clmblkdst, 8*clmblkdst
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2/clm		;; Test inner loop counter
	jnc	c5b			;; Iterate if necessary
	lea	esi, [esi-clm*2*dist1+clmblkdst];; Next source pointer
	add	ah, 256/4		;; Test middle loop counter
	jnc	c5b			;; Iterate if necessary
	lea	esi, [esi-4*clmblkdst+16*clmblkdst];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al
	jnz	c5b
	lea	esi, [esi-2*16*clmblkdst-8*clmblkdst];; Restore source pointer
	end_timer 18

;; Do inverse FFT levels 1,2,3
;; On input the 32-byte cache lines hold these data values:
;;	0	4K	1	4K+1
;;	256	...
;;	...
;;	8K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	10K	1	10K+1
;;	256
;;	...
;;	10K-256

;; Do 16 five_reals_last_unfft macros
;;	distance between fft data elements is 16

	start_timer 19
	mov	al, 8			;; Load loop counter
c1b:	two_five_reals_last_unfft 8, 8*clmblkdst
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2/clm		;; Test loop counter
	jnc	c1b			;; Iterate if necessary
	lea	esi, [esi-clm*2*dist1+clmblkdst];; Next source pointer
	dec	al			;; Test loop counter
	jnz	c1b			;; Iterate if necessary
	lea	esi, [esi-8*clmblkdst]	;; Restore source pointer
	end_timer 19

;; Normalize these values

	start_timer 20
	mov	eax, NORMRTN		;; Addr of normalization routine
	call	eax
	sub	eax, eax
	end_timer 20

;; Have we been given permission to start the FFT on the result?
;; If so, do so now while the data is in the L2 cache.

	cmp	POSTFFT, 0		;; Test flag
	je	short copyback		;; Copy data, skip FFT if flag not set
	copy_3_words clm, 1
	cmp	edx, p2cl/128*65536+(128/clm-4/clm)*256;; Delay fft on 8 sets
	jle	yes_fft			;; No delay, do FFT now

;; Copy data from the scratch area back to the FFT data area

copyback:
	copy_scratch_data_to_fft 40
	jmp	no_fft			;; Skip forward FFT code

;; Copy data from the FFT data area back to the scratch area

b0b:	copy_fft_data_to_scratch 40

;; Do FFT levels 1,2,3
;;
;; On input the 32-byte cache lines hold these data values:
;;	0	10K	1	10K+1
;;	256
;;	...
;;	10K-256
;; On output the 32-byte cache lines hold these data values:
;;	0	4K	1	4K+1
;;	256	...
;;	...
;;	8K	...
;;	...

;; Do 16 five_reals_fft macros
;;	distance between fft data elements is 16

yes_fft:start_timer 21
	mov	eax, 8			;; 8 iterations
b1b:	two_five_reals_fft 8*clmblkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	add	dl, 256/2/clm		;; Test loop counter
	jnc	b1b			;; Iterate if necessary
	lea	esi, [esi-clm*2*dist1+clmblkdst];; Next source pointer
	dec	al			;; Test loop counter
	jnz	b1b			;; Iterate if necessary
	lea	esi, [esi-8*clmblkdst]	;; Restore source pointer
	end_timer 21

;; Do FFT levels 4,5
;;
;; On input the 32-byte cache lines hold these data values:
;;	0	4K	1	4K+1
;;	256	...
;;	...
;;	8K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	1K	1	1K+1
;;	256	...
;;	...
;;	2K	...
;;	...

;; Do 2 eight_reals_fft_2 macros
;;	distance between fft data elements is 4

	start_timer 22
	mov	al, 2			;; Load loop counter
b4b:	disp eight_reals_fft_2, 2*clmblkdst, 4*clmblkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2/clm		;; Test inner loop counter
	jnc	b4b			;; Iterate if necessary
	lea	esi, [esi-clm*2*dist1+clmblkdst];; Next source pointer
	dec	al
	jnz	b4b
	lea	esi, [esi-2*clmblkdst+8*clmblkdst];; Next source pointer

;; Do 8 four_complex_fft macros
;;	distance between fft data elements is 4

	mov	edi, sincos2		;; Load sin/cos pointer
	mov	eax, 2			;; 2 iterations of 4
b5b:	disp four_complex_fft, 4*clmblkdst, 8*clmblkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2/clm		;; Test inner loop counter
	jnc	b5b			;; Iterate if necessary
	lea	esi, [esi-clm*2*dist1+clmblkdst];; Next source pointer
	add	ah, 256/4		;; Test middle loop counter
	jnc	b5b			;; Iterate if necessary
	lea	esi, [esi-4*clmblkdst+16*clmblkdst];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al
	jnz	b5b
	lea	esi, [esi-2*16*clmblkdst-8*clmblkdst];; Restore source pointer
	end_timer 22

;; Do FFT levels 6,7
;;
;; On input the 32-byte cache lines hold these data values:
;;	0	1K	1	1K+1
;;	256	...
;;	...
;;	2K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	256	1	257
;;	512	...
;;	...

;; Do 1 four_real_four_semireal_fft macros
;;	distance between fft data elements is 1

	start_timer 23
;	lea	ecx, [ecx-1111*blkdst]	;; Create dest ptr from prefetch ptr
b2b:	cp_disp four_real_four_semireal_fft, clmblkdst, 2*clmblkdst, 8, blkdst, 2*blkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	lea	ecx, [ecx+dist1]	;; Next dest pointer
	add	al, 256/2/clm		;; Test inner loop counter
	jnc	b2b			;; Iterate if necessary
	lea	esi, [esi-clm*2*dist1+4*clmblkdst];; Next source pointer
	lea	ecx, [ecx-clm*2*dist1+4*blkdst];; Next dest pointer

;; Do 9 four_complex_fft macros
;;	distance between fft data elements is 1

	mov	edi, sincos1		;; Load sin/cos pointer
	mov	al, 9			;; 9 iterations
b3b:	cp_disp four_complex_fft, clmblkdst, 2*clmblkdst, 8, blkdst, 2*blkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	lea	ecx, [ecx+dist1]	;; Next dest pointer
	add	al, 256/2/clm		;; Test inner loop counter
	jnc	b3b			;; Iterate if necessary
	lea	esi, [esi-clm*2*dist1+4*clmblkdst];; Next source pointer
	lea	ecx, [ecx-clm*2*dist1+4*blkdst];; Next dest pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al
	jnz	b3b
	lea	esi, [ecx-10*4*blkdst]	;; Restore source pointer
	end_timer 23

;; Work on next 4 sets of 32 values

no_fft:	lea	esi, [esi+clm*2*dist1]	;; Next source pointer
	dec	dh			;; Test loop counter
	jnz	short more		;; Do more FFTs
	lea	esi, [esi+64]		;; Skip pad every 128 cache lines (4KB)
pfing	mov	ebp, clm*32		;; Set prefetch distance
	sub	edx, 65536		;; Test loop counter
	jz	short fftdn		;; Jump if done looping
	mov	dh, 128/clm		;; Restore cache line counter
more:	set_prefetch_gap
	cmp	norm_ptr1, 0		;; What was the loop start point?
	jne	c0b			;; Do inverse-FFT/norm/FFT iteration
	jmp	b0b			;; Do another FFT-only iteration
fftdn:	lea	esi, [esi-p2cl*2*dist1-p2cl/128*64];; Restore source pointer
	cmp	esi, DESTARG		;; If esi was not restored, then we 
	jne	short fftdn1		;; just finished skipped postfft data
	sub	ebx, ebx
	mov	DIST_TO_FFTSRCARG, ebx
	cmp	norm_ptr1, 0		;; What was the loop start point?
	je	pass2			;; FFT-only, go do pass 2

;; Split the carries into high and low carries (so they do not exceed
;; the desired number of bits per FFT word.  Then add the carries back to the
;; FFT data.

	start_timer 28
	call	gw_carries
	end_timer 28

;; Have we been given permission to start the FFT on the result?
;; If so, set loop to FFT the first few cache lines now that the carries
;; have been added in.

	cmp	POSTFFT, 0		;; Test flag
	je	short fftdn1		;; Skip FFT code if flag not set
	mov	esi, DESTARG		;; Load FFT data address
	mov	edx, 65536+4/clm*256	;; Do 8 previously skipped cache lines
	mov	norm_ptr1, 0		;; Set flag so only forward FFT is done
	copy_4_words
	jmp	b0b

;; Now add in the FFT'ed carries, cleanup and return

fftdn1:	end_timer 1
	jmp	gw_finish_mult
	ENDM


;; Perform a 24K-element FFT.  This is done in two passes.  Pass 1
;; does 7 levels, pass 2 uses common code to perform the last 8 levels.

pass1levels7pfa6 MACRO pass2_macro, p2cl, clmarg
	LOCAL	b0b, b1b, b2b, b3b, b4b, b5b, c0b, c1b, c2b, c3b, c4b, c5b
	LOCAL	pass2, copyback, yes_fft, no_fft, more, fftdn, fftdn1

	set_FFT_constants clmarg

;; Call FFT code unless the FFT has already been started

	clear_timers
	cmp	DWORD PTR [esi-28][ebx], 0 ;; Test FFT-started flag
	jne	pass2			;; Jump is FFT already started
	mov	norm_ptr1, 0		;; Set FFT-ing flag
	mov	edx, p2cl/128*65536+128/clm*256;; Load loop counter
pfing	mov	ebp, clm*32		;; Set prefetch distance
	jmp	b0b			;; Do the pass 1 fft

;; Do the last 8 FFT levels
;;	do 1 pass2_8_levels_real and 63 pass2_8_levels_complex

pass2:	start_timer 0
	CALLP	pass2_macro
	end_timer 0
	cmp	ffttype, 1		;; We're done if FFTing only
	je	gw_finish_fft

;; Do the inverse FFT

	start_timer 1
	fldz				;; Init SUMOUT
	fstp	SUMOUT
	mov	edx, norm_biglit_array	;; Addr of the big/little flags array
	mov	norm_ptr1, edx		;; Save ptr
	mov	edx, norm_col_mults	;; Addr of the column multipliers
	mov	norm_ptr2, edx		;; Save ptr
	mov	edx, p2cl/128*65536+128/clm*256;; Load loop counter
pfing	mov	ebp, clm*32		;; Set prefetch distance

;; Do inverse FFT levels 6,7
;; On input the 32-byte cache lines hold these data values:
;;	0	256	1	257
;;	512	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	1K	1	1K+1
;;	256	...
;;	...
;;	2K	...
;;	...

;; Do 1 four_real_four_semireal_unfft macros
;;	distance between fft data elements is 1

c0b:	start_timer 17
	mov	ecx, scratch_area	;; Get address of scratch area
c2b:	cp_disp four_real_four_semireal_unfft, 8, blkdst, 2*blkdst, 8, clmblkdst, 2*clmblkdst
	lea	esi, [esi+dist1]	;; Next source pointer
	lea	ecx, [ecx+dist1]	;; Next dest pointer
	add	dl, 256/2/clm		;; Test inner loop counter
	jnc	c2b			;; Iterate if necessary
	lea	esi, [esi-clm*2*dist1+4*blkdst];; Next source pointer
	lea	ecx, [ecx-clm*2*dist1+4*clmblkdst];; Next dest pointer

;; Do 11 four_complex_unfft macros
;;	distance between fft data elements is 1

	mov	edi, sincos1		;; Load sin/cos pointer
	mov	al, 11			;; 11 iterations
c3b:	cp_disp four_complex_unfft, 8, blkdst, 2*blkdst, 8, clmblkdst, 2*clmblkdst
	lea	esi, [esi+dist1]	;; Next source pointer
	lea	ecx, [ecx+dist1]	;; Next dest pointer
	add	dl, 256/2/clm		;; Test loop counter
	jnc	c3b			;; Iterate if necessary
	lea	esi, [esi-clm*2*dist1+4*blkdst];; Next source pointer
	lea	ecx, [ecx-clm*2*dist1+4*clmblkdst];; Next dest pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al			;; Test outer loop counter
	jnz	c3b			;; Iterate if necessary
	lea	ecx, [esi-12*4*blkdst]	;; Create prefetch pointer
	mov	esi, scratch_area	;; Restore source pointer
	end_timer 17

;; Do inverse FFT levels 4,5
;; On input the 32-byte cache lines hold these data values:
;;	0	1K	1	1K+1
;;	256	...
;;	...
;;	2K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	4K	1	4K+1
;;	256	...
;;	...
;;	8K	...
;;	...

;; Do 4 four_real_four_semireal_unfft macros
;;	distance between fft data elements is 4

	start_timer 18
	mov	al, 4			;; Load loop counter
c4b:	disp four_real_four_semireal_unfft, 8, 4*clmblkdst, 8*clmblkdst
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2/clm		;; Test inner loop counter
	jnc	c4b			;; Iterate if necessary
	lea	esi, [esi-clm*2*dist1+clmblkdst];; Next source pointer
	dec	al
	jnz	c4b
	lea	esi, [esi-4*clmblkdst+16*clmblkdst];; Next source pointer

;; Do 8 four_complex_unfft macros
;;	distance between fft data elements is 4

	mov	edi, sincos2		;; Load sin/cos pointer
	mov	eax, 2			;; 2 iterations of 4
c5b:	disp four_complex_unfft, 8, 4*clmblkdst, 8*clmblkdst
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2/clm		;; Test inner loop counter
	jnc	c5b			;; Iterate if necessary
	lea	esi, [esi-clm*2*dist1+clmblkdst];; Next source pointer
	add	ah, 256/4		;; Test middle loop counter
	jnc	c5b			;; Iterate if necessary
	lea	esi, [esi-4*clmblkdst+16*clmblkdst];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al
	jnz	c5b
	lea	esi, [esi-3*16*clmblkdst]	;; Restore source pointer
	end_timer 18

;; Do inverse FFT levels 1,2,3
;; On input the 32-byte cache lines hold these data values:
;;	0	4K	1	4K+1
;;	256	...
;;	...
;;	8K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	12K	1	12K+1
;;	256
;;	...
;;	12K-256

;; Do 16 six_reals_last_unfft macros
;;	distance between fft data elements is 16

	start_timer 19
	mov	al, 16			;; Load loop counter
c1b:	disp six_reals_last_unfft, 8, 16*clmblkdst, 32*clmblkdst
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2/clm		;; Test loop counter
	jnc	c1b			;; Iterate if necessary
	lea	esi, [esi-clm*2*dist1+clmblkdst];; Next source pointer
	dec	al			;; Test loop counter
	jnz	c1b			;; Iterate if necessary
	lea	esi, [esi-16*clmblkdst]	;; Restore source pointer
	end_timer 19

;; Normalize these values

	start_timer 20
	mov	eax, NORMRTN		;; Addr of normalization routine
	call	eax
	sub	eax, eax
	end_timer 20

;; Have we been given permission to start the FFT on the result?
;; If so, do so now while the data is in the L2 cache.

	cmp	POSTFFT, 0		;; Test flag
	je	short copyback		;; Copy data, skip FFT if flag not set
	copy_3_words clm, 1
	cmp	edx, p2cl/128*65536+(128/clm-4/clm)*256;; Delay fft on 8 sets
	jle	yes_fft			;; No delay, do FFT now

;; Copy data from the scratch area back to the FFT data area

copyback:
	copy_scratch_data_to_fft 48
	jmp	no_fft			;; Skip forward FFT code

;; Copy data from the FFT data area back to the scratch area

b0b:	copy_fft_data_to_scratch 48

;; Do FFT levels 1,2,3
;;
;; On input the 32-byte cache lines hold these data values:
;;	0	12K	1	12K+1
;;	256
;;	...
;;	12K-256
;; On output the 32-byte cache lines hold these data values:
;;	0	4K	1	4K+1
;;	256	...
;;	...
;;	8K	...
;;	...

;; Do 16 six_reals_fft macros
;;	distance between fft data elements is 16

yes_fft:start_timer 21
	mov	eax, 16			;; 16 iterations
b1b:	disp six_reals_fft, 16*clmblkdst, 32*clmblkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	add	dl, 256/2/clm		;; Test loop counter
	jnc	b1b			;; Iterate if necessary
	lea	esi, [esi-clm*2*dist1+clmblkdst];; Next source pointer
	dec	al			;; Test loop counter
	jnz	b1b			;; Iterate if necessary
	lea	esi, [esi-16*clmblkdst]	;; Restore source pointer
	end_timer 21

;; Do FFT levels 4,5
;;
;; On input the 32-byte cache lines hold these data values:
;;	0	4K	1	4K+1
;;	256	...
;;	...
;;	8K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	1K	1	1K+1
;;	256	...
;;	...
;;	2K	...
;;	...

;; Do 4 four_real_four_semireal_fft macros
;;	distance between fft data elements is 4

	start_timer 22
	mov	al, 4			;; Load loop counter
b4b:	disp four_real_four_semireal_fft, 4*clmblkdst, 8*clmblkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2/clm		;; Test inner loop counter
	jnc	b4b			;; Iterate if necessary
	lea	esi, [esi-clm*2*dist1+clmblkdst];; Next source pointer
	dec	al
	jnz	b4b
	lea	esi, [esi-4*clmblkdst+16*clmblkdst];; Next source pointer

;; Do 8 four_complex_fft macros
;;	distance between fft data elements is 4

	mov	edi, sincos2		;; Load sin/cos pointer
	mov	eax, 2			;; 2 iterations of 4
b5b:	disp four_complex_fft, 4*clmblkdst, 8*clmblkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2/clm		;; Test inner loop counter
	jnc	b5b			;; Iterate if necessary
	lea	esi, [esi-clm*2*dist1+clmblkdst];; Next source pointer
	add	ah, 256/4		;; Test middle loop counter
	jnc	b5b			;; Iterate if necessary
	lea	esi, [esi-4*clmblkdst+16*clmblkdst];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al
	jnz	b5b
	lea	esi, [esi-3*16*clmblkdst]	;; Restore source pointer
	end_timer 22

;; Do FFT levels 6,7
;;
;; On input the 32-byte cache lines hold these data values:
;;	0	1K	1	1K+1
;;	256	...
;;	...
;;	2K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	256	1	257
;;	512	...
;;	...

;; Do 1 four_real_four_semireal_fft macros
;;	distance between fft data elements is 1

	start_timer 23
;	lea	ecx, [ecx-1111*blkdst]	;; Create dest ptr from prefetch ptr
b2b:	cp_disp four_real_four_semireal_fft, clmblkdst, 2*clmblkdst, 8, blkdst, 2*blkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	lea	ecx, [ecx+dist1]	;; Next dest pointer
	add	al, 256/2/clm		;; Test inner loop counter
	jnc	b2b			;; Iterate if necessary
	lea	esi, [esi-clm*2*dist1+4*clmblkdst];; Next source pointer
	lea	ecx, [ecx-clm*2*dist1+4*blkdst];; Next dest pointer

;; Do 11 four_complex_fft macros
;;	distance between fft data elements is 1

	mov	edi, sincos1		;; Load sin/cos pointer
	mov	al, 11			;; 11 iterations
b3b:	cp_disp four_complex_fft, clmblkdst, 2*clmblkdst, 8, blkdst, 2*blkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	lea	ecx, [ecx+dist1]	;; Next dest pointer
	add	al, 256/2/clm		;; Test inner loop counter
	jnc	b3b			;; Iterate if necessary
	lea	esi, [esi-clm*2*dist1+4*clmblkdst];; Next source pointer
	lea	ecx, [ecx-clm*2*dist1+4*blkdst];; Next dest pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al
	jnz	b3b
	lea	esi, [ecx-12*4*blkdst]	;; Restore source pointer
	end_timer 23

;; Work on next 4 sets of 32 values

no_fft:	lea	esi, [esi+clm*2*dist1]	;; Next source pointer
	dec	dh			;; Test loop counter
	jnz	short more		;; Do more FFTs
	lea	esi, [esi+64]		;; Skip pad every 128 cache lines (4KB)
pfing	mov	ebp, clm*32		;; Set prefetch distance
	sub	edx, 65536		;; Test loop counter
	jz	short fftdn		;; Jump if done looping
	mov	dh, 128/clm		;; Restore cache line counter
more:	set_prefetch_gap
	cmp	norm_ptr1, 0		;; What was the loop start point?
	jne	c0b			;; Do inverse-FFT/norm/FFT iteration
	jmp	b0b			;; Do another FFT-only iteration
fftdn:	lea	esi, [esi-p2cl*2*dist1-p2cl/128*64];; Restore source pointer
	cmp	esi, DESTARG		;; If esi was not restored, then we 
	jne	short fftdn1		;; just finished skipped postfft data
	sub	ebx, ebx
	mov	DIST_TO_FFTSRCARG, ebx
	cmp	norm_ptr1, 0		;; What was the loop start point?
	je	pass2			;; FFT-only, go do pass 2

;; Split the carries into high and low carries (so they do not exceed
;; the desired number of bits per FFT word.  Then add the carries back to the
;; FFT data.

	start_timer 28
	call	gw_carries
	end_timer 28

;; Have we been given permission to start the FFT on the result?
;; If so, set loop to FFT the first few cache lines now that the carries
;; have been added in.

	cmp	POSTFFT, 0		;; Test flag
	je	short fftdn1		;; Skip FFT code if flag not set
	mov	esi, DESTARG		;; Load FFT data address
	mov	edx, 65536+4/clm*256	;; Do 8 previously skipped cache lines
	mov	norm_ptr1, 0		;; Set flag so only forward FFT is done
	copy_4_words
	jmp	b0b

;; Now add in the FFT'ed carries, cleanup and return

fftdn1:	end_timer 1
	jmp	gw_finish_mult
	ENDM


;; Perform a 24K-element negacyclic FFT.  This is done in two passes.  Pass 1
;; does 7 levels, pass 2 uses common code to perform the last 8 levels.

pass1levels7complex3 MACRO pass2_macro, p2cl, clmarg
	LOCAL	b0b, b1b, b3b, b5b, c0b, c1b, c3b, c5b
	LOCAL	pass2, copyback, yes_fft, no_fft, more, fftdn, fftdn1

	set_FFT_constants clmarg

;; Call FFT code unless the FFT has already been started

	clear_timers
	cmp	DWORD PTR [esi-28][ebx], 0 ;; Test FFT-started flag
	jne	pass2			;; Jump is FFT already started
	mov	norm_ptr1, 0		;; Set FFT-ing flag
	mov	edx, p2cl/128*65536+128/clm*256;; Load loop counter
pfing	mov	ebp, clm*32		;; Set prefetch distance
	jmp	b0b			;; Do the pass 1 fft

;; Do the last 8 FFT levels
;;	do 48 pass2_8_levels_complex

pass2:	start_timer 0
	CALLP	pass2_macro
	end_timer 0
	cmp	ffttype, 1		;; We're done if FFTing only
	je	gw_finish_fft

;; Do the inverse FFT

	start_timer 1
	fldz				;; Init SUMOUT
	fstp	SUMOUT
	mov	edx, norm_biglit_array	;; Addr of the big/little flags array
	mov	norm_ptr1, edx		;; Save ptr
	mov	edx, norm_col_mults	;; Addr of the column multipliers
	mov	norm_ptr2, edx		;; Save ptr
	mov	edx, p2cl/128*65536+128/clm*256;; Load loop counter
pfing	mov	ebp, clm*32		;; Set prefetch distance

;; Do inverse FFT levels 6,7
;; On input the 32-byte cache lines hold these data values:
;;	0	256	1	257
;;	512	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	1K	1	1K+1
;;	256	...
;;	...
;;	2K	...
;;	...

;; Do 12 four_complex_unfft macros
;;	distance between fft data elements is 1

c0b:	start_timer 17
	mov	ecx, scratch_area	;; Get address of scratch area
	mov	edi, sincos1		;; Load sin/cos pointer
	mov	al, 12			;; 12 iterations
c3b:	cp_disp four_complex_unfft, 8, blkdst, 2*blkdst, 8, clmblkdst, 2*clmblkdst
	lea	esi, [esi+dist1]	;; Next source pointer
	lea	ecx, [ecx+dist1]	;; Next dest pointer
	add	dl, 256/2/clm		;; Test loop counter
	jnc	c3b			;; Iterate if necessary
	lea	esi, [esi-clm*2*dist1+4*blkdst];; Next source pointer
	lea	ecx, [ecx-clm*2*dist1+4*clmblkdst];; Next dest pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al			;; Test outer loop counter
	jnz	c3b			;; Iterate if necessary
	lea	ecx, [esi-12*4*blkdst]	;; Create prefetch pointer
	mov	esi, scratch_area	;; Restore source pointer
	end_timer 17

;; Do inverse FFT levels 4,5
;; On input the 32-byte cache lines hold these data values:
;;	0	1K	1	1K+1
;;	256	...
;;	...
;;	2K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	4K	1	4K+1
;;	256	...
;;	...
;;	8K	...
;;	...

;; Do 12 four_complex_unfft macros
;;	distance between fft data elements is 4

	start_timer 18
	mov	edi, sincos2		;; Load sin/cos pointer
	mov	eax, 3			;; 3 iterations of 4
c5b:	disp four_complex_unfft, 8, 4*clmblkdst, 8*clmblkdst
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2/clm		;; Test inner loop counter
	jnc	c5b			;; Iterate if necessary
	lea	esi, [esi-clm*2*dist1+clmblkdst];; Next source pointer
	add	ah, 256/4		;; Test middle loop counter
	jnc	c5b			;; Iterate if necessary
	lea	esi, [esi-4*clmblkdst+16*clmblkdst];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al
	jnz	c5b
	lea	esi, [esi-3*16*clmblkdst]	;; Restore source pointer
	end_timer 18

;; Do inverse FFT levels 1,2,3
;; On input the 32-byte cache lines hold these data values:
;;	0	4K	1	4K+1
;;	256	...
;;	...
;;	8K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	12K	1	12K+1
;;	256
;;	...
;;	12K-256

;; Do 16 three_complex_last_unfft macros
;;	distance between fft data elements is 16

	start_timer 19
	mov	edi, plus1_premults	;; Address of premultiplier table
	mov	al, 16			;; Load loop counter
c1b:	three_complex_last_unfft 8, 16*clmblkdst, 32*clmblkdst
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2/clm		;; Test loop counter
	jnc	c1b			;; Iterate if necessary
	lea	esi, [esi-clm*2*dist1+clmblkdst];; Next source pointer
	lea	edi, [edi+48]		;; Next premultiplier pointer
	dec	al			;; Test loop counter
	jnz	c1b			;; Iterate if necessary
	lea	esi, [esi-16*clmblkdst]	;; Restore source pointer
	end_timer 19

;; Normalize these values

	start_timer 20
	mov	eax, NORMRTN		;; Addr of normalization routine
	call	eax
	sub	eax, eax
	end_timer 20

;; Have we been given permission to start the FFT on the result?
;; If so, do so now while the data is in the L2 cache.

	cmp	POSTFFT, 0		;; Test flag
	je	short copyback		;; Copy data, skip FFT if flag not set
	copy_3_words clm, 1
	cmp	edx, p2cl/128*65536+(128/clm-4/clm)*256;; Delay fft on 8 sets
	jle	yes_fft			;; No delay, do FFT now

;; Copy data from the scratch area back to the FFT data area

copyback:
	copy_scratch_data_to_fft 48
	jmp	no_fft			;; Skip forward FFT code

;; Copy data from the FFT data area back to the scratch area

b0b:	copy_fft_data_to_scratch 48

;; Do FFT levels 1,2,3
;;
;; On input the 32-byte cache lines hold these data values:
;;	0	12K	1	12K+1
;;	256
;;	...
;;	12K-256
;; On output the 32-byte cache lines hold these data values:
;;	0	4K	1	4K+1
;;	256	...
;;	...
;;	8K	...
;;	...

;; Do 16 three_complex_fft_premult macros
;;	distance between fft data elements is 16

yes_fft:start_timer 21
	mov	edi, plus1_premults	;; Address of premultiplier table
	mov	eax, 16			;; 16 iterations
b1b:	three_complex_fft_premult 16*clmblkdst, 32*clmblkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	add	dl, 256/2/clm		;; Test loop counter
	jnc	b1b			;; Iterate if necessary
	lea	esi, [esi-clm*2*dist1+clmblkdst];; Next source pointer
	lea	edi, [edi+48]		;; Next premultiplier pointer
	dec	al			;; Test loop counter
	jnz	b1b			;; Iterate if necessary
	lea	esi, [esi-16*clmblkdst]	;; Restore source pointer
	end_timer 21

;; Do FFT levels 4,5
;;
;; On input the 32-byte cache lines hold these data values:
;;	0	4K	1	4K+1
;;	256	...
;;	...
;;	8K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	1K	1	1K+1
;;	256	...
;;	...
;;	2K	...
;;	...

;; Do 12 four_complex_fft macros
;;	distance between fft data elements is 4

	start_timer 22
	mov	edi, sincos2		;; Load sin/cos pointer
	mov	eax, 3			;; 3 iterations of 4
b5b:	disp four_complex_fft, 4*clmblkdst, 8*clmblkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2/clm		;; Test inner loop counter
	jnc	b5b			;; Iterate if necessary
	lea	esi, [esi-clm*2*dist1+clmblkdst];; Next source pointer
	add	ah, 256/4		;; Test middle loop counter
	jnc	b5b			;; Iterate if necessary
	lea	esi, [esi-4*clmblkdst+16*clmblkdst];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al
	jnz	b5b
	lea	esi, [esi-3*16*clmblkdst]	;; Restore source pointer
	end_timer 22

;; Do FFT levels 6,7
;;
;; On input the 32-byte cache lines hold these data values:
;;	0	1K	1	1K+1
;;	256	...
;;	...
;;	2K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	256	1	257
;;	512	...
;;	...

;; Do 12 four_complex_fft macros
;;	distance between fft data elements is 1

	start_timer 23
;	lea	ecx, [ecx-1111*blkdst]	;; Create dest ptr from prefetch ptr
	mov	edi, sincos1		;; Load sin/cos pointer
	mov	al, 12			;; 12 iterations
b3b:	cp_disp four_complex_fft, clmblkdst, 2*clmblkdst, 8, blkdst, 2*blkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	lea	ecx, [ecx+dist1]	;; Next dest pointer
	add	al, 256/2/clm		;; Test inner loop counter
	jnc	b3b			;; Iterate if necessary
	lea	esi, [esi-clm*2*dist1+4*clmblkdst];; Next source pointer
	lea	ecx, [ecx-clm*2*dist1+4*blkdst];; Next dest pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al
	jnz	b3b
	lea	esi, [ecx-12*4*blkdst]	;; Restore source pointer
	end_timer 23

;; Work on next 4 sets of 32 values

no_fft:	lea	esi, [esi+clm*2*dist1]	;; Next source pointer
	dec	dh			;; Test loop counter
	jnz	short more		;; Do more FFTs
	lea	esi, [esi+64]		;; Skip pad every 128 cache lines (4KB)
pfing	mov	ebp, clm*32		;; Set prefetch distance
	sub	edx, 65536		;; Test loop counter
	jz	short fftdn		;; Jump if done looping
	mov	dh, 128/clm		;; Restore cache line counter
more:	set_prefetch_gap
	cmp	norm_ptr1, 0		;; What was the loop start point?
	jne	c0b			;; Do inverse-FFT/norm/FFT iteration
	jmp	b0b			;; Do another FFT-only iteration
fftdn:	lea	esi, [esi-p2cl*2*dist1-p2cl/128*64];; Restore source pointer
	cmp	esi, DESTARG		;; If esi was not restored, then we 
	jne	short fftdn1		;; just finished skipped postfft data
	sub	ebx, ebx
	mov	DIST_TO_FFTSRCARG, ebx
	cmp	norm_ptr1, 0		;; What was the loop start point?
	je	pass2			;; FFT-only, go do pass 2

;; Split the carries into high and low carries (so they do not exceed
;; the desired number of bits per FFT word.  Then add the carries back to the
;; FFT data.

	start_timer 28
	call	gw_carries
	end_timer 28

;; Have we been given permission to start the FFT on the result?
;; If so, set loop to FFT the first few cache lines now that the carries
;; have been added in.

	cmp	POSTFFT, 0		;; Test flag
	je	short fftdn1		;; Skip FFT code if flag not set
	mov	esi, DESTARG		;; Load FFT data address
	mov	edx, 65536+4/clm*256	;; Do 8 previously skipped cache lines
	mov	norm_ptr1, 0		;; Set flag so only forward FFT is done
	copy_4_words
	jmp	b0b

;; Now add in the FFT'ed carries, cleanup and return

fftdn1:	end_timer 1
	jmp	gw_finish_mult
	ENDM


;; Perform a 28K-element FFT.  This is done in two passes.  Pass 1
;; does 7 levels, pass 2 uses common code to perform the last 8 levels.

pass1levels7pfa7 MACRO pass2_macro, p2cl, clmarg
	LOCAL	b0b, b1b, b2b, b3b, b4b, b5b, c0b, c1b, c2b, c3b, c4b, c5b
	LOCAL	pass2, copyback, yes_fft, no_fft, more, fftdn, fftdn1

	set_FFT_constants clmarg

;; Call FFT code unless the FFT has already been started

	clear_timers
	cmp	DWORD PTR [esi-28][ebx], 0 ;; Test FFT-started flag
	jne	pass2			;; Jump is FFT already started
	mov	norm_ptr1, 0		;; Set FFT-ing flag
	mov	edx, p2cl/128*65536+128/clm*256;; Load loop counter
pfing	mov	ebp, clm*32		;; Set prefetch distance
	jmp	b0b			;; Do the pass 1 fft

;; Do the last 8 FFT levels
;;	do 1 pass2_8_levels_real and 55 pass2_8_levels_complex

pass2:	start_timer 0
	CALLP	pass2_macro
	end_timer 0
	cmp	ffttype, 1		;; We're done if FFTing only
	je	gw_finish_fft

;; Do the inverse FFT

	start_timer 1
	fldz				;; Init SUMOUT
	fstp	SUMOUT
	mov	edx, norm_biglit_array	;; Addr of the big/little flags array
	mov	norm_ptr1, edx		;; Save ptr
	mov	edx, norm_col_mults	;; Addr of the column multipliers
	mov	norm_ptr2, edx		;; Save ptr
	mov	edx, p2cl/128*65536+128/clm*256;; Load loop counter
pfing	mov	ebp, clm*32		;; Set prefetch distance

;; Do inverse FFT levels 6,7
;; On input the 32-byte cache lines hold these data values:
;;	0	256	1	257
;;	512	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	1K	1	1K+1
;;	256	...
;;	...
;;	2K	...
;;	...

;; Do 1 four_real_four_semireal_unfft macros
;;	distance between fft data elements is 1

c0b:	start_timer 17
	mov	ecx, scratch_area	;; Get address of scratch area
c2b:	cp_disp four_real_four_semireal_unfft, 8, blkdst, 2*blkdst, 8, clmblkdst, 2*clmblkdst
	lea	esi, [esi+dist1]	;; Next source pointer
	lea	ecx, [ecx+dist1]	;; Next dest pointer
	add	dl, 256/2/clm		;; Test inner loop counter
	jnc	c2b			;; Iterate if necessary
	lea	esi, [esi-clm*2*dist1+4*blkdst];; Next source pointer
	lea	ecx, [ecx-clm*2*dist1+4*clmblkdst];; Next dest pointer

;; Do 13 four_complex_unfft macros
;;	distance between fft data elements is 1

	mov	edi, sincos1		;; Load sin/cos pointer
	mov	al, 13			;; 13 iterations
c3b:	cp_disp four_complex_unfft, 8, blkdst, 2*blkdst, 8, clmblkdst, 2*clmblkdst
	lea	esi, [esi+dist1]	;; Next source pointer
	lea	ecx, [ecx+dist1]	;; Next dest pointer
	add	dl, 256/2/clm		;; Test loop counter
	jnc	c3b			;; Iterate if necessary
	lea	esi, [esi-clm*2*dist1+4*blkdst];; Next source pointer
	lea	ecx, [ecx-clm*2*dist1+4*clmblkdst];; Next dest pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al			;; Test outer loop counter
	jnz	c3b			;; Iterate if necessary
	lea	ecx, [esi-14*4*blkdst]	;; Create prefetch pointer
	mov	esi, scratch_area	;; Restore source pointer
	end_timer 17

;; Do inverse FFT levels 4,5
;; On input the 32-byte cache lines hold these data values:
;;	0	1K	1	1K+1
;;	256	...
;;	...
;;	2K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	4K	1	4K+1
;;	256	...
;;	...
;;	8K	...
;;	...

;; Do 2 eight_reals_unfft_2 macros
;;	distance between fft data elements is 4

	start_timer 18
	mov	al, 2			;; Load loop counter
c4b:	disp eight_reals_unfft_2, 2*clmblkdst, 8, 4*clmblkdst
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2/clm		;; Test inner loop counter
	jnc	c4b			;; Iterate if necessary
	lea	esi, [esi-clm*2*dist1+clmblkdst];; Next source pointer
	dec	al
	jnz	c4b
	lea	esi, [esi-2*clmblkdst+8*clmblkdst];; Next source pointer

;; Do 12 four_complex_unfft macros
;;	distance between fft data elements is 4

	mov	edi, sincos2		;; Load sin/cos pointer
	mov	eax, 3			;; 3 iterations of 4
c5b:	disp four_complex_unfft, 8, 4*clmblkdst, 8*clmblkdst
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2/clm		;; Test inner loop counter
	jnc	c5b			;; Iterate if necessary
	lea	esi, [esi-clm*2*dist1+clmblkdst];; Next source pointer
	add	ah, 256/4		;; Test middle loop counter
	jnc	c5b			;; Iterate if necessary
	lea	esi, [esi-4*clmblkdst+16*clmblkdst];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al
	jnz	c5b
	lea	esi, [esi-3*16*clmblkdst-8*clmblkdst];; Restore source pointer
	end_timer 18

;; Do inverse FFT levels 1,2,3
;; On input the 32-byte cache lines hold these data values:
;;	0	4K	1	4K+1
;;	256	...
;;	...
;;	8K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	14K	1	14K+1
;;	256
;;	...
;;	14K-256

;; Do 16 seven_reals_last_unfft macros
;;	distance between fft data elements is 16

	start_timer 19
	mov	al, 8			;; Load loop counter
c1b:	two_seven_reals_last_unfft 8, 8*clmblkdst
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2/clm		;; Test loop counter
	jnc	c1b			;; Iterate if necessary
	lea	esi, [esi-clm*2*dist1+clmblkdst];; Next source pointer
	dec	al			;; Test loop counter
	jnz	c1b			;; Iterate if necessary
	lea	esi, [esi-8*clmblkdst]	;; Restore source pointer
	end_timer 19

;; Normalize these values

	start_timer 20
	mov	eax, NORMRTN		;; Addr of normalization routine
	call	eax
	sub	eax, eax
	end_timer 20

;; Have we been given permission to start the FFT on the result?
;; If so, do so now while the data is in the L2 cache.

	cmp	POSTFFT, 0		;; Test flag
	je	short copyback		;; Copy data, skip FFT if flag not set
	copy_3_words clm, 1
	cmp	edx, p2cl/128*65536+(128/clm-4/clm)*256;; Delay fft on 8 sets
	jle	yes_fft			;; No delay, do FFT now

;; Copy data from the scratch area back to the FFT data area

copyback:
	copy_scratch_data_to_fft 56
	jmp	no_fft			;; Skip forward FFT code

;; Copy data from the FFT data area back to the scratch area

b0b:	copy_fft_data_to_scratch 56

;; Do FFT levels 1,2,3
;;
;; On input the 32-byte cache lines hold these data values:
;;	0	14K	1	14K+1
;;	256
;;	...
;;	14K-256
;; On output the 32-byte cache lines hold these data values:
;;	0	4K	1	4K+1
;;	256	...
;;	...
;;	8K	...
;;	...

;; Do 16 seven_reals_fft macros
;;	distance between fft data elements is 16

yes_fft:start_timer 21
	mov	eax, 8			;; 8 iterations
b1b:	two_seven_reals_fft 8*clmblkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	add	dl, 256/2/clm		;; Test loop counter
	jnc	b1b			;; Iterate if necessary
	lea	esi, [esi-clm*2*dist1+clmblkdst];; Next source pointer
	dec	al			;; Test loop counter
	jnz	b1b			;; Iterate if necessary
	lea	esi, [esi-8*clmblkdst]	;; Restore source pointer
	end_timer 21

;; Do FFT levels 4,5
;;
;; On input the 32-byte cache lines hold these data values:
;;	0	4K	1	4K+1
;;	256	...
;;	...
;;	8K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	1K	1	1K+1
;;	256	...
;;	...
;;	2K	...
;;	...

;; Do 2 eight_reals_fft_2 macros
;;	distance between fft data elements is 4

	start_timer 22
	mov	al, 2			;; Load loop counter
b4b:	disp eight_reals_fft_2, 2*clmblkdst, 4*clmblkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2/clm		;; Test inner loop counter
	jnc	b4b			;; Iterate if necessary
	lea	esi, [esi-clm*2*dist1+clmblkdst];; Next source pointer
	dec	al
	jnz	b4b
	lea	esi, [esi-2*clmblkdst+8*clmblkdst];; Next source pointer

;; Do 12 four_complex_fft macros
;;	distance between fft data elements is 4

	mov	edi, sincos2		;; Load sin/cos pointer
	mov	eax, 3			;; 3 iterations of 4
b5b:	disp four_complex_fft, 4*clmblkdst, 8*clmblkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2/clm		;; Test inner loop counter
	jnc	b5b			;; Iterate if necessary
	lea	esi, [esi-clm*2*dist1+clmblkdst];; Next source pointer
	add	ah, 256/4		;; Test middle loop counter
	jnc	b5b			;; Iterate if necessary
	lea	esi, [esi-4*clmblkdst+16*clmblkdst];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al
	jnz	b5b
	lea	esi, [esi-3*16*clmblkdst-8*clmblkdst];; Restore source pointer
	end_timer 22

;; Do FFT levels 6,7
;;
;; On input the 32-byte cache lines hold these data values:
;;	0	1K	1	1K+1
;;	256	...
;;	...
;;	2K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	256	1	257
;;	512	...
;;	...

;; Do 1 four_real_four_semireal_fft macros
;;	distance between fft data elements is 1

	start_timer 23
;	lea	ecx, [ecx-1111*blkdst]	;; Create dest ptr from prefetch ptr
b2b:	cp_disp four_real_four_semireal_fft, clmblkdst, 2*clmblkdst, 8, blkdst, 2*blkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	lea	ecx, [ecx+dist1]	;; Next dest pointer
	add	al, 256/2/clm		;; Test inner loop counter
	jnc	b2b			;; Iterate if necessary
	lea	esi, [esi-clm*2*dist1+4*clmblkdst];; Next source pointer
	lea	ecx, [ecx-clm*2*dist1+4*blkdst];; Next dest pointer

;; Do 13 four_complex_fft macros
;;	distance between fft data elements is 1

	mov	edi, sincos1		;; Load sin/cos pointer
	mov	al, 13			;; 13 iterations
b3b:	cp_disp four_complex_fft, clmblkdst, 2*clmblkdst, 8, blkdst, 2*blkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	lea	ecx, [ecx+dist1]	;; Next dest pointer
	add	al, 256/2/clm		;; Test inner loop counter
	jnc	b3b			;; Iterate if necessary
	lea	esi, [esi-clm*2*dist1+4*clmblkdst];; Next source pointer
	lea	ecx, [ecx-clm*2*dist1+4*blkdst];; Next dest pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al
	jnz	b3b
	lea	esi, [ecx-14*4*blkdst]	;; Restore source pointer
	end_timer 23

;; Work on next 4 sets of 32 values

no_fft:	lea	esi, [esi+clm*2*dist1]	;; Next source pointer
	dec	dh			;; Test loop counter
	jnz	short more		;; Do more FFTs
	lea	esi, [esi+64]		;; Skip pad every 128 cache lines (4KB)
pfing	mov	ebp, clm*32		;; Set prefetch distance
	sub	edx, 65536		;; Test loop counter
	jz	short fftdn		;; Jump if done looping
	mov	dh, 128/clm		;; Restore cache line counter
more:	set_prefetch_gap
	cmp	norm_ptr1, 0		;; What was the loop start point?
	jne	c0b			;; Do inverse-FFT/norm/FFT iteration
	jmp	b0b			;; Do another FFT-only iteration
fftdn:	lea	esi, [esi-p2cl*2*dist1-p2cl/128*64];; Restore source pointer
	cmp	esi, DESTARG		;; If esi was not restored, then we 
	jne	short fftdn1		;; just finished skipped postfft data
	sub	ebx, ebx
	mov	DIST_TO_FFTSRCARG, ebx
	cmp	norm_ptr1, 0		;; What was the loop start point?
	je	pass2			;; FFT-only, go do pass 2

;; Split the carries into high and low carries (so they do not exceed
;; the desired number of bits per FFT word.  Then add the carries back to the
;; FFT data.

	start_timer 28
	call	gw_carries
	end_timer 28

;; Have we been given permission to start the FFT on the result?
;; If so, set loop to FFT the first few cache lines now that the carries
;; have been added in.

	cmp	POSTFFT, 0		;; Test flag
	je	short fftdn1		;; Skip FFT code if flag not set
	mov	esi, DESTARG		;; Load FFT data address
	mov	edx, 65536+4/clm*256	;; Do 8 previously skipped cache lines
	mov	norm_ptr1, 0		;; Set flag so only forward FFT is done
	copy_4_words
	jmp	b0b

;; Now add in the FFT'ed carries, cleanup and return

fftdn1:	end_timer 1
	jmp	gw_finish_mult
	ENDM


;; Perform a 32K-element FFT.  This is done in two passes.  Pass 1
;; does 7 levels, pass 2 uses common code to perform the last 8 levels.

pass1levels7pfa8 MACRO pass2_macro, p2cl, clmarg
	LOCAL	b0b, b1b, b2b, b3b, b4b, b5b, c0b, c1b, c2b, c3b, c4b, c5b
	LOCAL	pass2, copyback, yes_fft, no_fft, more, fftdn, fftdn1

	set_FFT_constants clmarg

;; Call FFT code unless the FFT has already been started

	clear_timers
	cmp	DWORD PTR [esi-28][ebx], 0 ;; Test FFT-started flag
	jne	pass2			;; Jump is FFT already started
	mov	norm_ptr1, 0		;; Set FFT-ing flag
	mov	edx, p2cl/128*65536+128/clm*256;; Load loop counter
pfing	mov	ebp, clm*32		;; Set prefetch distance
	jmp	b0b			;; Do the pass 1 fft

;; Do the last 8 FFT levels
;;	do 1 pass2_8_levels_real and 63 pass2_8_levels_complex

pass2:	start_timer 0
	CALLP	pass2_macro
	end_timer 0
	cmp	ffttype, 1		;; We're done if FFTing only
	je	gw_finish_fft

;; Do the inverse FFT

	start_timer 1
	fldz				;; Init SUMOUT
	fstp	SUMOUT
	mov	edx, norm_biglit_array	;; Addr of the big/little flags array
	mov	norm_ptr1, edx		;; Save ptr
	mov	edx, norm_col_mults	;; Addr of the column multipliers
	mov	norm_ptr2, edx		;; Save ptr
	mov	edx, p2cl/128*65536+128/clm*256;; Load loop counter
pfing	mov	ebp, clm*32		;; Set prefetch distance

;; Do inverse FFT levels 6,7
;; On input the 32-byte cache lines hold these data values:
;;	0	256	1	257
;;	512	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	1K	1	1K+1
;;	256	...
;;	...
;;	2K	...
;;	...

;; Do 1 four_real_four_semireal_unfft macros
;;	distance between fft data elements is 1

c0b:	start_timer 17
	mov	ecx, scratch_area	;; Get address of scratch area
c2b:	cp_disp four_real_four_semireal_unfft, 8, blkdst, 2*blkdst, 8, clmblkdst, 2*clmblkdst
	lea	esi, [esi+dist1]	;; Next source pointer
	lea	ecx, [ecx+dist1]	;; Next dest pointer
	add	dl, 256/2/clm		;; Test inner loop counter
	jnc	c2b			;; Iterate if necessary
	lea	esi, [esi-clm*2*dist1+4*blkdst];; Next source pointer
	lea	ecx, [ecx-clm*2*dist1+4*clmblkdst];; Next dest pointer

;; Do 15 four_complex_unfft macros
;;	distance between fft data elements is 1

	mov	edi, sincos1		;; Load sin/cos pointer
	mov	al, 15			;; 15 iterations
c3b:	cp_disp four_complex_unfft, 8, blkdst, 2*blkdst, 8, clmblkdst, 2*clmblkdst
	lea	esi, [esi+dist1]	;; Next source pointer
	lea	ecx, [ecx+dist1]	;; Next dest pointer
	add	dl, 256/2/clm		;; Test loop counter
	jnc	c3b			;; Iterate if necessary
	lea	esi, [esi-clm*2*dist1+4*blkdst];; Next source pointer
	lea	ecx, [ecx-clm*2*dist1+4*clmblkdst];; Next dest pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al			;; Test outer loop counter
	jnz	c3b			;; Iterate if necessary
	lea	ecx, [esi-16*4*blkdst]	;; Create prefetch pointer
	mov	esi, scratch_area	;; Restore source pointer
	end_timer 17

;; Do inverse FFT levels 4,5
;; On input the 32-byte cache lines hold these data values:
;;	0	1K	1	1K+1
;;	256	...
;;	...
;;	2K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	4K	1	4K+1
;;	256	...
;;	...
;;	8K	...
;;	...

;; Do 4 four_real_four_semireal_unfft macros
;;	distance between fft data elements is 4

	start_timer 18
	mov	al, 4			;; Load loop counter
c4b:	disp four_real_four_semireal_unfft, 8, 4*clmblkdst, 8*clmblkdst
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2/clm		;; Test inner loop counter
	jnc	c4b			;; Iterate if necessary
	lea	esi, [esi-clm*2*dist1+clmblkdst];; Next source pointer
	dec	al
	jnz	c4b
	lea	esi, [esi-4*clmblkdst+16*clmblkdst];; Next source pointer

;; Do 12 four_complex_unfft macros
;;	distance between fft data elements is 4

	mov	edi, sincos2		;; Load sin/cos pointer
	mov	eax, 3			;; 3 iterations of 4
c5b:	disp four_complex_unfft, 8, 4*clmblkdst, 8*clmblkdst
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2/clm		;; Test inner loop counter
	jnc	c5b			;; Iterate if necessary
	lea	esi, [esi-clm*2*dist1+clmblkdst];; Next source pointer
	add	ah, 256/4		;; Test middle loop counter
	jnc	c5b			;; Iterate if necessary
	lea	esi, [esi-4*clmblkdst+16*clmblkdst];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al
	jnz	c5b
	lea	esi, [esi-4*16*clmblkdst]	;; Restore source pointer
	end_timer 18

;; Do inverse FFT levels 1,2,3
;; On input the 32-byte cache lines hold these data values:
;;	0	4K	1	4K+1
;;	256	...
;;	...
;;	8K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	16K	1	16K+1
;;	256
;;	...
;;	16K-256

;; Do 16 eight_reals_last_unfft macros
;;	distance between fft data elements is 16

	start_timer 19
	mov	al, 16			;; Load loop counter
c1b:	disp eight_reals_last_unfft, 8, 16*clmblkdst, 32*clmblkdst
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2/clm		;; Test loop counter
	jnc	c1b			;; Iterate if necessary
	lea	esi, [esi-clm*2*dist1+clmblkdst];; Next source pointer
	dec	al			;; Test loop counter
	jnz	c1b			;; Iterate if necessary
	lea	esi, [esi-16*clmblkdst]	;; Restore source pointer
	end_timer 19

;; Normalize these values

	start_timer 20
	mov	eax, NORMRTN		;; Addr of normalization routine
	call	eax
	sub	eax, eax
	end_timer 20

;; Have we been given permission to start the FFT on the result?
;; If so, do so now while the data is in the L2 cache.

	cmp	POSTFFT, 0		;; Test flag
	je	short copyback		;; Copy data, skip FFT if flag not set
	copy_3_words clm, 1
	cmp	edx, p2cl/128*65536+(128/clm-4/clm)*256;; Delay fft on 8 sets
	jle	yes_fft			;; No delay, do FFT now

;; Copy data from the scratch area back to the FFT data area

copyback:
	copy_scratch_data_to_fft 64
	jmp	no_fft			;; Skip forward FFT code

;; Copy data from the FFT data area back to the scratch area

b0b:	copy_fft_data_to_scratch 64

;; Do FFT levels 1,2,3
;;
;; On input the 32-byte cache lines hold these data values:
;;	0	16K	1	16K+1
;;	256
;;	...
;;	16K-256
;; On output the 32-byte cache lines hold these data values:
;;	0	4K	1	4K+1
;;	256	...
;;	...
;;	8K	...
;;	...

;; Do 16 eight_reals_fft macros
;;	distance between fft data elements is 16

yes_fft:start_timer 21
	mov	eax, 16			;; 16 iterations
b1b:	disp eight_reals_fft, 16*clmblkdst, 32*clmblkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	add	dl, 256/2/clm		;; Test loop counter
	jnc	b1b			;; Iterate if necessary
	lea	esi, [esi-clm*2*dist1+clmblkdst];; Next source pointer
	dec	al			;; Test loop counter
	jnz	b1b			;; Iterate if necessary
	lea	esi, [esi-16*clmblkdst]	;; Restore source pointer
	end_timer 21

;; Do FFT levels 4,5
;;
;; On input the 32-byte cache lines hold these data values:
;;	0	4K	1	4K+1
;;	256	...
;;	...
;;	8K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	1K	1	1K+1
;;	256	...
;;	...
;;	2K	...
;;	...

;; Do 4 four_real_four_semireal_fft macros
;;	distance between fft data elements is 4

	start_timer 22
	mov	al, 4			;; Load loop counter
b4b:	disp four_real_four_semireal_fft, 4*clmblkdst, 8*clmblkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2/clm		;; Test inner loop counter
	jnc	b4b			;; Iterate if necessary
	lea	esi, [esi-clm*2*dist1+clmblkdst];; Next source pointer
	dec	al
	jnz	b4b
	lea	esi, [esi-4*clmblkdst+16*clmblkdst];; Next source pointer

;; Do 12 four_complex_fft macros
;;	distance between fft data elements is 4

	mov	edi, sincos2		;; Load sin/cos pointer
	mov	eax, 3			;; 3 iterations of 4
b5b:	disp four_complex_fft, 4*clmblkdst, 8*clmblkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2/clm		;; Test inner loop counter
	jnc	b5b			;; Iterate if necessary
	lea	esi, [esi-clm*2*dist1+clmblkdst];; Next source pointer
	add	ah, 256/4		;; Test middle loop counter
	jnc	b5b			;; Iterate if necessary
	lea	esi, [esi-4*clmblkdst+16*clmblkdst];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al
	jnz	b5b
	lea	esi, [esi-4*16*clmblkdst]	;; Restore source pointer
	end_timer 22

;; Do FFT levels 6,7
;;
;; On input the 32-byte cache lines hold these data values:
;;	0	1K	1	1K+1
;;	256	...
;;	...
;;	2K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	256	1	257
;;	512	...
;;	...

;; Do 1 four_real_four_semireal_fft macros
;;	distance between fft data elements is 1

	start_timer 23
;	lea	ecx, [ecx-1111*blkdst]	;; Create dest ptr from prefetch ptr
b2b:	cp_disp four_real_four_semireal_fft, clmblkdst, 2*clmblkdst, 8, blkdst, 2*blkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	lea	ecx, [ecx+dist1]	;; Next dest pointer
	add	al, 256/2/clm		;; Test inner loop counter
	jnc	b2b			;; Iterate if necessary
	lea	esi, [esi-clm*2*dist1+4*clmblkdst];; Next source pointer
	lea	ecx, [ecx-clm*2*dist1+4*blkdst];; Next dest pointer

;; Do 15 four_complex_fft macros
;;	distance between fft data elements is 1

	mov	edi, sincos1		;; Load sin/cos pointer
	mov	al, 15			;; 15 iterations
b3b:	cp_disp four_complex_fft, clmblkdst, 2*clmblkdst, 8, blkdst, 2*blkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	lea	ecx, [ecx+dist1]	;; Next dest pointer
	add	al, 256/2/clm		;; Test inner loop counter
	jnc	b3b			;; Iterate if necessary
	lea	esi, [esi-clm*2*dist1+4*clmblkdst];; Next source pointer
	lea	ecx, [ecx-clm*2*dist1+4*blkdst];; Next dest pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al
	jnz	b3b
	lea	esi, [ecx-16*4*blkdst]	;; Restore source pointer
	end_timer 23

;; Work on next 4 sets of 32 values

no_fft:	lea	esi, [esi+clm*2*dist1]	;; Next source pointer
	dec	dh			;; Test loop counter
	jnz	short more		;; Do more FFTs
	lea	esi, [esi+64]		;; Skip pad every 128 cache lines (4KB)
pfing	mov	ebp, clm*32		;; Set prefetch distance
	sub	edx, 65536		;; Test loop counter
	jz	short fftdn		;; Jump if done looping
	mov	dh, 128/clm		;; Restore cache line counter
more:	set_prefetch_gap
	cmp	norm_ptr1, 0		;; What was the loop start point?
	jne	c0b			;; Do inverse-FFT/norm/FFT iteration
	jmp	b0b			;; Do another FFT-only iteration
fftdn:	lea	esi, [esi-p2cl*2*dist1-p2cl/128*64];; Restore source pointer
	cmp	esi, DESTARG		;; If esi was not restored, then we 
	jne	short fftdn1		;; just finished skipped postfft data
	sub	ebx, ebx
	mov	DIST_TO_FFTSRCARG, ebx
	cmp	norm_ptr1, 0		;; What was the loop start point?
	je	pass2			;; FFT-only, go do pass 2

;; Split the carries into high and low carries (so they do not exceed
;; the desired number of bits per FFT word.  Then add the carries back to the
;; FFT data.

	start_timer 28
	call	gw_carries
	end_timer 28

;; Have we been given permission to start the FFT on the result?
;; If so, set loop to FFT the first few cache lines now that the carries
;; have been added in.

	cmp	POSTFFT, 0		;; Test flag
	je	short fftdn1		;; Skip FFT code if flag not set
	mov	esi, DESTARG		;; Load FFT data address
	mov	edx, 65536+4/clm*256	;; Do 8 previously skipped cache lines
	mov	norm_ptr1, 0		;; Set flag so only forward FFT is done
	copy_4_words
	jmp	b0b

;; Now add in the FFT'ed carries, cleanup and return

fftdn1:	end_timer 1
	jmp	gw_finish_mult
	ENDM


;; Perform a 32K-element negacyclic FFT.  This is done in two passes.  Pass 1
;; does 7 levels, pass 2 uses common code to perform the last 8 levels.

pass1levels7complex4 MACRO pass2_macro, p2cl, clmarg
	LOCAL	b0b, b1b, b3b, b5b, c0b, c1b, c3b, c5b
	LOCAL	pass2, copyback, yes_fft, no_fft, more, fftdn, fftdn1

	set_FFT_constants clmarg

;; Call FFT code unless the FFT has already been started

	clear_timers
	cmp	DWORD PTR [esi-28][ebx], 0 ;; Test FFT-started flag
	jne	pass2			;; Jump is FFT already started
	mov	norm_ptr1, 0		;; Set FFT-ing flag
	mov	edx, p2cl/128*65536+128/clm*256;; Load loop counter
pfing	mov	ebp, clm*32		;; Set prefetch distance
	jmp	b0b			;; Do the pass 1 fft

;; Do the last 8 FFT levels
;;	do 64 pass2_8_levels_complex

pass2:	start_timer 0
	CALLP	pass2_macro
	end_timer 0
	cmp	ffttype, 1		;; We're done if FFTing only
	je	gw_finish_fft

;; Do the inverse FFT

	start_timer 1
	fldz				;; Init SUMOUT
	fstp	SUMOUT
	mov	edx, norm_biglit_array	;; Addr of the big/little flags array
	mov	norm_ptr1, edx		;; Save ptr
	mov	edx, norm_col_mults	;; Addr of the column multipliers
	mov	norm_ptr2, edx		;; Save ptr
	mov	edx, p2cl/128*65536+128/clm*256;; Load loop counter
pfing	mov	ebp, clm*32		;; Set prefetch distance

;; Do inverse FFT levels 6,7
;; On input the 32-byte cache lines hold these data values:
;;	0	256	1	257
;;	512	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	1K	1	1K+1
;;	256	...
;;	...
;;	2K	...
;;	...

;; Do 16 four_complex_unfft macros
;;	distance between fft data elements is 1

c0b:	start_timer 17
	mov	ecx, scratch_area	;; Get address of scratch area
	mov	edi, sincos1		;; Load sin/cos pointer
	mov	al, 16			;; 16 iterations
c3b:	cp_disp four_complex_unfft, 8, blkdst, 2*blkdst, 8, clmblkdst, 2*clmblkdst
	lea	esi, [esi+dist1]	;; Next source pointer
	lea	ecx, [ecx+dist1]	;; Next dest pointer
	add	dl, 256/2/clm		;; Test loop counter
	jnc	c3b			;; Iterate if necessary
	lea	esi, [esi-clm*2*dist1+4*blkdst];; Next source pointer
	lea	ecx, [ecx-clm*2*dist1+4*clmblkdst];; Next dest pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al			;; Test outer loop counter
	jnz	c3b			;; Iterate if necessary
	lea	ecx, [esi-16*4*blkdst]	;; Create prefetch pointer
	mov	esi, scratch_area	;; Restore source pointer
	end_timer 17

;; Do inverse FFT levels 4,5
;; On input the 32-byte cache lines hold these data values:
;;	0	1K	1	1K+1
;;	256	...
;;	...
;;	2K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	4K	1	4K+1
;;	256	...
;;	...
;;	8K	...
;;	...

;; Do 16 four_complex_unfft macros
;;	distance between fft data elements is 4

	start_timer 18
	mov	edi, sincos2		;; Load sin/cos pointer
	mov	eax, 4			;; 4 iterations of 4
c5b:	disp four_complex_unfft, 8, 4*clmblkdst, 8*clmblkdst
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2/clm		;; Test inner loop counter
	jnc	c5b			;; Iterate if necessary
	lea	esi, [esi-clm*2*dist1+clmblkdst];; Next source pointer
	add	ah, 256/4		;; Test middle loop counter
	jnc	c5b			;; Iterate if necessary
	lea	esi, [esi-4*clmblkdst+16*clmblkdst];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al
	jnz	c5b
	lea	esi, [esi-4*16*clmblkdst]	;; Restore source pointer
	end_timer 18

;; Do inverse FFT levels 1,2,3
;; On input the 32-byte cache lines hold these data values:
;;	0	4K	1	4K+1
;;	256	...
;;	...
;;	8K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	16K	1	16K+1
;;	256
;;	...
;;	16K-256

;; Do 16 four_complex_last_unfft macros
;;	distance between fft data elements is 16

	start_timer 19
	mov	edi, plus1_premults	;; Address of premultiplier table
	mov	al, 16			;; Load loop counter
c1b:	disp four_complex_last_unfft, 8, 16*clmblkdst, 32*clmblkdst
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2/clm		;; Test loop counter
	jnc	c1b			;; Iterate if necessary
	lea	esi, [esi-clm*2*dist1+clmblkdst];; Next source pointer
	lea	edi, [edi+64]		;; Next premultiplier pointer
	dec	al			;; Test loop counter
	jnz	c1b			;; Iterate if necessary
	lea	esi, [esi-16*clmblkdst]	;; Restore source pointer
	end_timer 19

;; Normalize these values

	start_timer 20
	mov	eax, NORMRTN		;; Addr of normalization routine
	call	eax
	sub	eax, eax
	end_timer 20

;; Have we been given permission to start the FFT on the result?
;; If so, do so now while the data is in the L2 cache.

	cmp	POSTFFT, 0		;; Test flag
	je	short copyback		;; Copy data, skip FFT if flag not set
	copy_3_words clm, 1
	cmp	edx, p2cl/128*65536+(128/clm-4/clm)*256;; Delay fft on 8 sets
	jle	yes_fft			;; No delay, do FFT now

;; Copy data from the scratch area back to the FFT data area

copyback:
	copy_scratch_data_to_fft 64
	jmp	no_fft			;; Skip forward FFT code

;; Copy data from the FFT data area back to the scratch area

b0b:	copy_fft_data_to_scratch 64

;; Do FFT levels 1,2,3
;;
;; On input the 32-byte cache lines hold these data values:
;;	0	16K	1	16K+1
;;	256
;;	...
;;	16K-256
;; On output the 32-byte cache lines hold these data values:
;;	0	4K	1	4K+1
;;	256	...
;;	...
;;	8K	...
;;	...

;; Do 16 four_complex_fft_premult macros
;;	distance between fft data elements is 16

yes_fft:start_timer 21
	mov	edi, plus1_premults	;; Address of premultiplier table
	mov	eax, 16			;; 16 iterations
b1b:	disp four_complex_fft_premult, 16*clmblkdst, 32*clmblkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	add	dl, 256/2/clm		;; Test loop counter
	jnc	b1b			;; Iterate if necessary
	lea	esi, [esi-clm*2*dist1+clmblkdst];; Next source pointer
	lea	edi, [edi+64]		;; Next premultiplier pointer
	dec	al			;; Test loop counter
	jnz	b1b			;; Iterate if necessary
	lea	esi, [esi-16*clmblkdst]	;; Restore source pointer
	end_timer 21

;; Do FFT levels 4,5
;;
;; On input the 32-byte cache lines hold these data values:
;;	0	4K	1	4K+1
;;	256	...
;;	...
;;	8K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	1K	1	1K+1
;;	256	...
;;	...
;;	2K	...
;;	...

;; Do 16 four_complex_fft macros
;;	distance between fft data elements is 4

	start_timer 22
	mov	edi, sincos2		;; Load sin/cos pointer
	mov	eax, 4			;; 4 iterations of 4
b5b:	disp four_complex_fft, 4*clmblkdst, 8*clmblkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2/clm		;; Test inner loop counter
	jnc	b5b			;; Iterate if necessary
	lea	esi, [esi-clm*2*dist1+clmblkdst];; Next source pointer
	add	ah, 256/4		;; Test middle loop counter
	jnc	b5b			;; Iterate if necessary
	lea	esi, [esi-4*clmblkdst+16*clmblkdst];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al
	jnz	b5b
	lea	esi, [esi-4*16*clmblkdst]	;; Restore source pointer
	end_timer 22

;; Do FFT levels 6,7
;;
;; On input the 32-byte cache lines hold these data values:
;;	0	1K	1	1K+1
;;	256	...
;;	...
;;	2K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	256	1	257
;;	512	...
;;	...

;; Do 16 four_complex_fft macros
;;	distance between fft data elements is 1

	start_timer 23
;	lea	ecx, [ecx-1111*blkdst]	;; Create dest ptr from prefetch ptr
	mov	edi, sincos1		;; Load sin/cos pointer
	mov	al, 16			;; 16 iterations
b3b:	cp_disp four_complex_fft, clmblkdst, 2*clmblkdst, 8, blkdst, 2*blkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	lea	ecx, [ecx+dist1]	;; Next dest pointer
	add	al, 256/2/clm		;; Test inner loop counter
	jnc	b3b			;; Iterate if necessary
	lea	esi, [esi-clm*2*dist1+4*clmblkdst];; Next source pointer
	lea	ecx, [ecx-clm*2*dist1+4*blkdst];; Next dest pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al
	jnz	b3b
	lea	esi, [ecx-16*4*blkdst]	;; Restore source pointer
	end_timer 23

;; Work on next 4 sets of 32 values

no_fft:	lea	esi, [esi+clm*2*dist1]	;; Next source pointer
	dec	dh			;; Test loop counter
	jnz	short more		;; Do more FFTs
	lea	esi, [esi+64]		;; Skip pad every 128 cache lines (4KB)
pfing	mov	ebp, clm*32		;; Set prefetch distance
	sub	edx, 65536		;; Test loop counter
	jz	short fftdn		;; Jump if done looping
	mov	dh, 128/clm		;; Restore cache line counter
more:	set_prefetch_gap
	cmp	norm_ptr1, 0		;; What was the loop start point?
	jne	c0b			;; Do inverse-FFT/norm/FFT iteration
	jmp	b0b			;; Do another FFT-only iteration
fftdn:	lea	esi, [esi-p2cl*2*dist1-p2cl/128*64];; Restore source pointer
	cmp	esi, DESTARG		;; If esi was not restored, then we 
	jne	short fftdn1		;; just finished skipped postfft data
	sub	ebx, ebx
	mov	DIST_TO_FFTSRCARG, ebx
	cmp	norm_ptr1, 0		;; What was the loop start point?
	je	pass2			;; FFT-only, go do pass 2

;; Split the carries into high and low carries (so they do not exceed
;; the desired number of bits per FFT word.  Then add the carries back to the
;; FFT data.

	start_timer 28
	call	gw_carries
	end_timer 28

;; Have we been given permission to start the FFT on the result?
;; If so, set loop to FFT the first few cache lines now that the carries
;; have been added in.

	cmp	POSTFFT, 0		;; Test flag
	je	short fftdn1		;; Skip FFT code if flag not set
	mov	esi, DESTARG		;; Load FFT data address
	mov	edx, 65536+4/clm*256	;; Do 8 previously skipped cache lines
	mov	norm_ptr1, 0		;; Set flag so only forward FFT is done
	copy_4_words
	jmp	b0b

;; Now add in the FFT'ed carries, cleanup and return

fftdn1:	end_timer 1
	jmp	gw_finish_mult
	ENDM


;; Perform a 40K-element FFT.  This is done in two passes.  Pass 1
;; does 8 levels, pass 2 uses common code to perform the last 8 levels.

pass1levels8pfa5 MACRO pass2_macro, p2cl, clmarg
	LOCAL	b0b, b1a, b1b, b2b, b3b, b4a, b4b, b5a, b5b, b6b, b7a, b7b
	LOCAL	c0b, c1a, c1b, c2b, c3b, c4a, c4b, c5a, c5b, c6b, c7a, c7b
	LOCAL	pass2, copyback, yes_fft, no_fft, more, fftdn, fftdn1

	set_FFT_constants clmarg

;; Call FFT code unless the FFT has already been started

	clear_timers
	cmp	DWORD PTR [esi-28][ebx], 0 ;; Test FFT-started flag
	jne	pass2			;; Jump is FFT already started
	mov	norm_ptr1, 0		;; Set FFT-ing flag
	mov	edx, p2cl/128*65536+128/clm*256;; Load loop counter
pfing	mov	ebp, clm*32		;; Set prefetch distance
	jmp	b0b			;; Do the pass 1 fft

;; Do the last 8 FFT levels
;;	do 1 pass2_8_levels_real and 79 pass2_8_levels_complex

pass2:	start_timer 0
	CALLP	pass2_macro
	end_timer 0
	cmp	ffttype, 1		;; We're done if FFTing only
	je	gw_finish_fft

;; Do the inverse FFT

	start_timer 1
	fldz				;; Init SUMOUT
	fstp	SUMOUT
	mov	edx, norm_biglit_array	;; Addr of the big/little flags array
	mov	norm_ptr1, edx		;; Save ptr
	mov	edx, norm_col_mults	;; Addr of the column multipliers
	mov	norm_ptr2, edx		;; Save ptr
	mov	edx, p2cl/128*65536+128/clm*256;; Load loop counter
pfing	mov	ebp, clm*32		;; Set prefetch distance

;; Do inverse FFT levels 7,8
;; On input the 32-byte cache lines hold these data values:
;;	0	256	1	257
;;	512	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	1K	1	1K+1
;;	256	...
;;	...
;;	2K	...
;;	...

;; Do 1 four_real_four_semireal_unfft macros
;;	distance between fft data elements is 1

c0b:	start_timer 16
	mov	ecx, scratch_area	;; Get address of scratch area
c2b:	cp_disp four_real_four_semireal_unfft, 8, blkdst, 2*blkdst, 8, clmblkdst, 2*clmblkdst
	lea	esi, [esi+dist1]	;; Next source pointer
	lea	ecx, [ecx+dist1]	;; Next dest pointer
	add	dl, 256/2/clm		;; Test inner loop counter
	jnc	c2b			;; Iterate if necessary
	lea	esi, [esi-clm*2*dist1+4*blkdst];; Next source pointer
	lea	ecx, [ecx-clm*2*dist1+4*clmblkdst];; Next dest pointer

;; Do 19 four_complex_unfft macros
;;	distance between fft data elements is 1

	mov	edi, sincos1		;; Load sin/cos pointer
	mov	al, 19			;; 19 iterations
c3b:	cp_disp four_complex_unfft, 8, blkdst, 2*blkdst, 8, clmblkdst, 2*clmblkdst
	lea	esi, [esi+dist1]	;; Next source pointer
	lea	ecx, [ecx+dist1]	;; Next dest pointer
	add	dl, 256/2/clm		;; Test loop counter
	jnc	c3b			;; Iterate if necessary
	lea	esi, [esi-clm*2*dist1+4*blkdst];; Next source pointer
	lea	ecx, [ecx-clm*2*dist1+4*clmblkdst];; Next dest pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al			;; Test outer loop counter
	jnz	c3b			;; Iterate if necessary
	lea	ecx, [esi-20*4*blkdst]	;; Create prefetch pointer
	mov	esi, scratch_area	;; Restore source pointer
	end_timer 16

;; Do inverse FFT levels 5,6
;; On input the 32-byte cache lines hold these data values:
;;	0	1K	1	1K+1
;;	256	...
;;	...
;;	2K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	4K	1	4K+1
;;	256	...
;;	...
;;	8K	...
;;	...

;; Do 4 four_real_four_semireal_unfft macros
;;	distance between fft data elements is 4

	start_timer 17
	mov	eax, 2			;; Load loop counter
c4a:	touch				;; Preload TLB
c4b:	disp four_real_four_semireal_unfft, 8, 4*clmblkdst, 8*clmblkdst
	loop2	c4b, clmblkdst		;; 2 * 2*clm iterations
	dec	al
	jnz	c4a
	lea	esi, [esi-4*clmblkdst+16*clmblkdst];; Next source pointer

;; Do 16 four_complex_unfft macros
;;	distance between fft data elements is 4

	mov	edi, sincos2		;; Load sin/cos pointer
	mov	eax, 4			;; 4 iterations of 4
c5a:	touch				;; Preload TLB
c5b:	disp four_complex_unfft, 8, 4*clmblkdst, 8*clmblkdst
	loop2	c5b, clmblkdst		;; 2 * 2*clm iterations
	add	eax, 80000000h		;; Test middle loop counter (2 iters)
	jnc	c5a			;; Iterate if necessary
	lea	esi, [esi-4*clmblkdst+16*clmblkdst];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al
	jnz	c5a
	lea	esi, [esi-5*16*clmblkdst]	;; Restore source pointer
	end_timer 17

;; Do inverse FFT levels 4
;; On input the 32-byte cache lines hold these data values:
;;	0	4K	1	4K+1
;;	256	...
;;	...
;;	8K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	8K	1	8K+1
;;	256	...
;;	...
;;	16K	...
;;	...

;; Do 4 eight_reals_unfft_1 macros
;;	distance between fft data elements is 16

	start_timer 18
	touch				;; Preload TLB
c6b:	disp eight_reals_unfft_1, 4*clmblkdst, 8*clmblkdst, 8
	loop4	c6b, clmblkdst		;; 4 * 2*clm iterations
	lea	esi, [esi-4*clmblkdst+16*clmblkdst];; Next source pointer

;; Do 16 four_complex_unfft_1 macros
;;	distance between fft data elements is 16

	mov	edi, sincos3		;; Load sin/cos pointer
	mov	eax, 2			;; 2 iterations of 8
c7a:	touch				;; Preload TLB
c7b:	disp four_complex_unfft_1, 8*clmblkdst, 8, 16*clmblkdst
	loop2	c7b, clmblkdst		;; 2 * 2*clm iterations
	add	eax, 80000000h/2	;; Test middle loop counter (4 iters)
	jnc	c7a			;; Iterate if necessary
	lea	esi, [esi-8*clmblkdst+32*clmblkdst];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al
	jnz	c7a
	lea	esi, [esi-2*32*clmblkdst-16*clmblkdst];; Restore source pointer
	end_timer 18

;; Do inverse FFT levels 1,2,3
;; On input the 32-byte cache lines hold these data values:
;;	0	8K	1	8K+1
;;	256	...
;;	...
;;	16K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	20K	1	20K+1
;;	256
;;	...
;;	20K-256

;; Do 32 five_reals_last_unfft macros
;;	distance between fft data elements is 32

	start_timer 19
	mov	al, 16			;; Load loop counter
c1a:	touch				;; Preload TLB
c1b:	two_five_reals_last_unfft 8, 16*clmblkdst
	loop1	c1b, clmblkdst		;; 2*clm iterations
	dec	al			;; Test loop counter
	jnz	c1a			;; Iterate if necessary
	lea	esi, [esi-16*clmblkdst]	;; Restore source pointer
pfing	lea	ecx, [ecx-35*blkdst]	;; Restore dest ptr
	end_timer 19

;; Normalize these values

	start_timer 20
	mov	eax, NORMRTN		;; Addr of normalization routine
	call	eax
	sub	eax, eax
	end_timer 20

;; Have we been given permission to start the FFT on the result?
;; If so, do so now while the data is in the L2 cache.

	cmp	POSTFFT, 0		;; Test flag
	je	short copyback		;; Copy data, skip FFT if flag not set
	copy_3_words clm, 1
	cmp	edx, p2cl/128*65536+(128/clm-4/clm)*256;; Delay fft on 8 sets
	jle	yes_fft			;; No delay, do FFT now

;; Copy data from the scratch area back to the FFT data area

copyback:
	copy_scratch_data_to_fft 80
	jmp	no_fft			;; Skip forward FFT code

;; Copy data from the FFT data area back to the scratch area

b0b:	copy_fft_data_to_scratch 80

;; Do FFT levels 1,2,3
;;
;; On input the 32-byte cache lines hold these data values:
;;	0	20K	1	20K+1
;;	256
;;	...
;;	20K-256
;; On output the 32-byte cache lines hold these data values:
;;	0	8K	1	8K+1
;;	256	...
;;	...
;;	16K	...
;;	...

;; Do 32 five_reals_fft macros
;;	distance between fft data elements is 32

yes_fft:start_timer 21
	mov	eax, 16			;; 16 iterations
pfing	lea	ecx, [ecx+35*blkdst]	;; Create prefetch ptr
b1a:	touch				;; Preload TLB
b1b:	two_five_reals_fft 16*clmblkdst, 8
	loop1	b1b, clmblkdst		;; 2*clm iterations
	dec	al			;; Test loop counter
	jnz	b1a			;; Iterate if necessary
	lea	esi, [esi-16*clmblkdst]	;; Restore source pointer
	end_timer 21

;; Do FFT levels 4
;;
;; On input the 32-byte cache lines hold these data values:
;;	0	8K	1	8K+1
;;	256	...
;;	...
;;	16K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	4K	1	4K+1
;;	256	...
;;	...
;;	8K	...
;;	...

;; Do 4 eight_reals_fft_1 macros
;;	distance between fft data elements is 16

	start_timer 22
	touch				;; Preload TLB
b6b:	disp eight_reals_fft_1, 4*clmblkdst, 8*clmblkdst, 8
	loop4	b6b, clmblkdst		;; 4 * 2*clm iterations
	lea	esi, [esi-4*clmblkdst+16*clmblkdst];; Next source pointer

;; Do 16 four_complex_fft_1 macros
;;	distance between fft data elements is 16

	mov	edi, sincos3		;; Load sin/cos pointer
	mov	eax, 2			;; 2 iterations of 8
b7a:	touch				;; Preload TLB
b7b:	disp four_complex_fft_1, 8*clmblkdst, 16*clmblkdst, 8
	loop2	b7b, clmblkdst		;; 2 * 2*clm iterations
	add	eax, 80000000h/2	;; Test middle loop counter (4 iters)
	jnc	b7a			;; Iterate if necessary
	lea	esi, [esi-8*clmblkdst+32*clmblkdst];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al
	jnz	b7a
	lea	esi, [esi-2*32*clmblkdst-16*clmblkdst];; Restore source pointer
	end_timer 22

;; Do FFT levels 5,6
;;
;; On input the 32-byte cache lines hold these data values:
;;	0	4K	1	4K+1
;;	256	...
;;	...
;;	8K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	1K	1	1K+1
;;	256	...
;;	...
;;	2K	...
;;	...

;; Do 4 four_real_four_semireal_fft macros
;;	distance between fft data elements is 4

	start_timer 23
	mov	al, 4			;; Load loop counter
b4a:	touch				;; Preload TLB
b4b:	disp four_real_four_semireal_fft, 4*clmblkdst, 8*clmblkdst, 8
	loop1	b4b, clmblkdst		;; 2*clm iterations
	dec	al
	jnz	b4a
	lea	esi, [esi-4*clmblkdst+16*clmblkdst];; Next source pointer

;; Do 16 four_complex_fft macros
;;	distance between fft data elements is 4

	mov	edi, sincos2		;; Load sin/cos pointer
	mov	eax, 4			;; 4 iterations of 4
b5a:	touch				;; Preload TLB
b5b:	disp four_complex_fft, 4*clmblkdst, 8*clmblkdst, 8
	loop1	b5b, clmblkdst		;; 2*clm iterations
	add	eax, 80000000h/2	;; Test middle loop counter (4 iters)
	jnc	b5a			;; Iterate if necessary
	lea	esi, [esi-4*clmblkdst+16*clmblkdst];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al
	jnz	b5a
	lea	esi, [esi-5*16*clmblkdst]	;; Restore source pointer
	end_timer 23

;; Do FFT levels 7,8
;;
;; On input the 32-byte cache lines hold these data values:
;;	0	1K	1	1K+1
;;	256	...
;;	...
;;	2K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	256	1	257
;;	512	...
;;	...

;; Do 1 four_real_four_semireal_fft macros
;;	distance between fft data elements is 1

	start_timer 24
pfing	lea	ecx, [ecx-80*blkdst]	;; Create dest ptr from prefetch ptr
b2b:	cp_disp four_real_four_semireal_fft, clmblkdst, 2*clmblkdst, 8, blkdst, 2*blkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	lea	ecx, [ecx+dist1]	;; Next dest pointer
	add	dl, 256/2/clm		;; Test inner loop counter
	jnc	b2b			;; Iterate if necessary
	lea	esi, [esi-clm*2*dist1+4*clmblkdst];; Next source pointer
	lea	ecx, [ecx-clm*2*dist1+4*blkdst];; Next dest pointer

;; Do 19 four_complex_fft macros
;;	distance between fft data elements is 1

	mov	edi, sincos1		;; Load sin/cos pointer
	mov	al, 19			;; 19 iterations
b3b:	cp_disp four_complex_fft, clmblkdst, 2*clmblkdst, 8, blkdst, 2*blkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	lea	ecx, [ecx+dist1]	;; Next dest pointer
	add	dl, 256/2/clm		;; Test inner loop counter
	jnc	b3b			;; Iterate if necessary
	lea	esi, [esi-clm*2*dist1+4*clmblkdst];; Next source pointer
	lea	ecx, [ecx-clm*2*dist1+4*blkdst];; Next dest pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al
	jnz	b3b
	lea	esi, [ecx-20*4*blkdst]	;; Restore source pointer
	end_timer 24

;; Work on next 4 sets of 32 values

no_fft:	lea	esi, [esi+clm*2*dist1]	;; Next source pointer
	dec	dh			;; Test loop counter
	jnz	short more		;; Do more FFTs
	lea	esi, [esi+64]		;; Skip pad every 128 cache lines (4KB)
pfing	mov	ebp, clm*32		;; Set prefetch distance
	sub	edx, 65536		;; Test loop counter
	jz	short fftdn		;; Jump if done looping
	mov	dh, 128/clm		;; Restore cache line counter
more:	set_prefetch_gap
	cmp	norm_ptr1, 0		;; What was the loop start point?
	jne	c0b			;; Do inverse-FFT/norm/FFT iteration
	jmp	b0b			;; Do another FFT-only iteration
fftdn:	lea	esi, [esi-p2cl*2*dist1-p2cl/128*64];; Restore source pointer
	cmp	esi, DESTARG		;; If esi was not restored, then we 
	jne	short fftdn1		;; just finished skipped postfft data
	sub	ebx, ebx
	mov	DIST_TO_FFTSRCARG, ebx
	cmp	norm_ptr1, 0		;; What was the loop start point?
	je	pass2			;; FFT-only, go do pass 2

;; Split the carries into high and low carries (so they do not exceed
;; the desired number of bits per FFT word.  Then add the carries back to the
;; FFT data.

	start_timer 28
	call	gw_carries
	end_timer 28

;; Have we been given permission to start the FFT on the result?
;; If so, set loop to FFT the first few cache lines now that the carries
;; have been added in.

	cmp	POSTFFT, 0		;; Test flag
	je	short fftdn1		;; Skip FFT code if flag not set
	mov	esi, DESTARG		;; Load FFT data address
	mov	edx, 65536+4/clm*256	;; Do 8 previously skipped cache lines
	mov	norm_ptr1, 0		;; Set flag so only forward FFT is done
	copy_4_words
	jmp	b0b

;; Now add in the FFT'ed carries, cleanup and return

fftdn1:	end_timer 1
	jmp	gw_finish_mult
	ENDM


;; Perform a 48K-element FFT.  This is done in two passes.  Pass 1
;; does 8 levels, pass 2 uses common code to perform the last 8 levels.

pass1levels8pfa6 MACRO pass2_macro, p2cl, clmarg
	LOCAL	b0b, b1a, b1b, b2b, b3b, b4a, b4b, b5a, b5b, b6a, b6b, b7a, b7b
	LOCAL	c0b, c1a, c1b, c2b, c3b, c4a, c4b, c5a, c5b, c6a, c6b, c7a, c7b
	LOCAL	pass2, copyback, yes_fft, no_fft, more, fftdn, fftdn1

	set_FFT_constants clmarg

;; Call FFT code unless the FFT has already been started

	clear_timers
	cmp	DWORD PTR [esi-28][ebx], 0 ;; Test FFT-started flag
	jne	pass2			;; Jump is FFT already started
	mov	norm_ptr1, 0		;; Set FFT-ing flag
	mov	edx, p2cl/128*65536+128/clm*256;; Load loop counter
pfing	mov	ebp, clm*32		;; Set prefetch distance
	jmp	b0b			;; Do the pass 1 fft

;; Do the last 8 FFT levels
;;	do 1 pass2_8_levels_real and 95 pass2_8_levels_complex

pass2:	start_timer 0
	CALLP	pass2_macro
	end_timer 0
	cmp	ffttype, 1		;; We're done if FFTing only
	je	gw_finish_fft

;; Do the inverse FFT

	start_timer 1
	fldz				;; Init SUMOUT
	fstp	SUMOUT
	mov	edx, norm_biglit_array	;; Addr of the big/little flags array
	mov	norm_ptr1, edx		;; Save ptr
	mov	edx, norm_col_mults	;; Addr of the column multipliers
	mov	norm_ptr2, edx		;; Save ptr
	mov	edx, p2cl/128*65536+128/clm*256;; Load loop counter
pfing	mov	ebp, clm*32		;; Set prefetch distance

;; Do inverse FFT levels 7,8
;; On input the 32-byte cache lines hold these data values:
;;	0	256	1	257
;;	512	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	1K	1	1K+1
;;	256	...
;;	...
;;	2K	...
;;	...

;; Do 1 four_real_four_semireal_unfft macros
;;	distance between fft data elements is 1

c0b:	start_timer 16
	mov	ecx, scratch_area	;; Get address of scratch area
c2b:	cp_disp four_real_four_semireal_unfft, 8, blkdst, 2*blkdst, 8, clmblkdst, 2*clmblkdst
	lea	esi, [esi+dist1]	;; Next source pointer
	lea	ecx, [ecx+dist1]	;; Next dest pointer
	add	dl, 256/2/clm		;; Test inner loop counter
	jnc	c2b			;; Iterate if necessary
	lea	esi, [esi-clm*2*dist1+4*blkdst];; Next source pointer
	lea	ecx, [ecx-clm*2*dist1+4*clmblkdst];; Next dest pointer

;; Do 23 four_complex_unfft macros
;;	distance between fft data elements is 1

	mov	edi, sincos1		;; Load sin/cos pointer
	mov	al, 23			;; 23 iterations
c3b:	cp_disp four_complex_unfft, 8, blkdst, 2*blkdst, 8, clmblkdst, 2*clmblkdst
	lea	esi, [esi+dist1]	;; Next source pointer
	lea	ecx, [ecx+dist1]	;; Next dest pointer
	add	dl, 256/2/clm		;; Test loop counter
	jnc	c3b			;; Iterate if necessary
	lea	esi, [esi-clm*2*dist1+4*blkdst];; Next source pointer
	lea	ecx, [ecx-clm*2*dist1+4*clmblkdst];; Next dest pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al			;; Test outer loop counter
	jnz	c3b			;; Iterate if necessary
	lea	ecx, [esi-24*4*blkdst]	;; Create prefetch pointer
	mov	esi, scratch_area	;; Restore source pointer
	end_timer 16

;; Do inverse FFT levels 5,6
;; On input the 32-byte cache lines hold these data values:
;;	0	1K	1	1K+1
;;	256	...
;;	...
;;	2K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	4K	1	4K+1
;;	256	...
;;	...
;;	8K	...
;;	...

;; Do 4 four_real_four_semireal_unfft macros
;;	distance between fft data elements is 4

	start_timer 17
	mov	eax, 2			;; Load loop counter
c4a:	touch				;; Preload TLB
c4b:	disp four_real_four_semireal_unfft, 8, 4*clmblkdst, 8*clmblkdst
	loop2	c4b, clmblkdst		;; 2 * 2*clm iterations
	dec	al
	jnz	c4a
	lea	esi, [esi-4*clmblkdst+16*clmblkdst];; Next source pointer

;; Do 20 four_complex_unfft macros
;;	distance between fft data elements is 4

	mov	edi, sincos2		;; Load sin/cos pointer
	mov	eax, 5			;; 5 iterations of 4
c5a:	touch				;; Preload TLB
c5b:	disp four_complex_unfft, 8, 4*clmblkdst, 8*clmblkdst
	loop1	c5b, clmblkdst		;; 2*clm iterations
	add	ah, 256/4		;; Test middle loop counter
	jnc	c5a			;; Iterate if necessary
	lea	esi, [esi-4*clmblkdst+16*clmblkdst];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al
	jnz	c5a
	lea	esi, [esi-6*16*clmblkdst];; Restore source pointer
	end_timer 17

;; Do inverse FFT levels 4
;; On input the 32-byte cache lines hold these data values:
;;	0	4K	1	4K+1
;;	256	...
;;	...
;;	8K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	8K	1	8K+1
;;	256	...
;;	...
;;	16K	...
;;	...

;; Do 8 four_real_four_semireal_unfft_1 macros
;;	distance between fft data elements is 16

	start_timer 18
	mov	al, 2			;; Load loop counter
c6a:	touch				;; Preload TLB
c6b:	disp four_real_four_semireal_unfft_1, 8*clmblkdst, 8, 16*clmblkdst
	loop4	c6b, clmblkdst		;; 4 * 2*clm iterations
	dec	al
	jnz	c6a
	lea	esi, [esi-8*clmblkdst+32*clmblkdst];; Next source pointer

;; Do 16 four_complex_unfft_1 macros
;;	distance between fft data elements is 16

	mov	edi, sincos3		;; Load sin/cos pointer
	mov	eax, 2			;; 2 iterations of 8
c7a:	touch				;; Preload TLB
c7b:	disp four_complex_unfft_1, 8*clmblkdst, 8, 16*clmblkdst
	loop2	c7b, clmblkdst		;; 2 * 2*clm iterations
	add	eax, 80000000h/2	;; Test middle loop counter (4 iters)
	jnc	c7a			;; Iterate if necessary
	lea	esi, [esi-8*clmblkdst+32*clmblkdst];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al
	jnz	c7a
	lea	esi, [esi-3*32*clmblkdst]	;; Restore source pointer
	end_timer 18

;; Do inverse FFT levels 1,2,3
;; On input the 32-byte cache lines hold these data values:
;;	0	8K	1	8K+1
;;	256	...
;;	...
;;	16K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	24K	1	24K+1
;;	256
;;	...
;;	24K-256

;; Do 32 six_reals_last_unfft macros
;;	distance between fft data elements is 32

	start_timer 19
	mov	al, 16			;; Load loop counter
c1a:	touch				;; Preload TLB
c1b:	disp six_reals_last_unfft, 8, 32*clmblkdst, 64*clmblkdst
	loop2	c1b, clmblkdst		;; 2 * 2*clm iterations
	dec	al			;; Test loop counter
	jnz	c1a			;; Iterate if necessary
	lea	esi, [esi-32*clmblkdst]	;; Restore source pointer
pfing	lea	ecx, [ecx-48*blkdst]	;; Restore dest ptr
	end_timer 19

;; Normalize these values

	start_timer 20
	mov	eax, NORMRTN		;; Addr of normalization routine
	call	eax
	sub	eax, eax
	end_timer 20

;; Have we been given permission to start the FFT on the result?
;; If so, do so now while the data is in the L2 cache.

	cmp	POSTFFT, 0		;; Test flag
	je	short copyback		;; Copy data, skip FFT if flag not set
	copy_3_words clm, 1
	cmp	edx, p2cl/128*65536+(128/clm-4/clm)*256;; Delay fft on 8 sets
	jle	yes_fft			;; No delay, do FFT now

;; Copy data from the scratch area back to the FFT data area

copyback:
	copy_scratch_data_to_fft 96
	jmp	no_fft			;; Skip forward FFT code

;; Copy data from the FFT data area back to the scratch area

b0b:	copy_fft_data_to_scratch 96

;; Do FFT levels 1,2,3
;;
;; On input the 32-byte cache lines hold these data values:
;;	0	24K	1	24K+1
;;	256
;;	...
;;	24K-256
;; On output the 32-byte cache lines hold these data values:
;;	0	8K	1	8K+1
;;	256	...
;;	...
;;	16K	...
;;	...

;; Do 32 six_reals_fft macros
;;	distance between fft data elements is 32

yes_fft:start_timer 21
	mov	eax, 16			;; 32 iterations
pfing	lea	ecx, [ecx+48*blkdst]	;; Create prefetch ptr
b1a:	touch				;; Preload TLB
b1b:	disp six_reals_fft, 32*clmblkdst, 64*clmblkdst, 8
	loop2	b1b, clmblkdst		;; 2 * 2*clm iterations
	dec	al			;; Test loop counter
	jnz	b1a			;; Iterate if necessary
	lea	esi, [esi-32*clmblkdst]	;; Restore source pointer
	end_timer 21

;; Do FFT levels 4
;;
;; On input the 32-byte cache lines hold these data values:
;;	0	8K	1	8K+1
;;	256	...
;;	...
;;	16K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	4K	1	4K+1
;;	256	...
;;	...
;;	8K	...
;;	...

;; Do 8 four_real_four_semireal_fft_1 macros
;;	distance between fft data elements is 16

	start_timer 22
	mov	eax, 2			;; Load loop counter
b6a:	touch				;; Preload TLB
b6b:	disp four_real_four_semireal_fft_1, 8*clmblkdst, 16*clmblkdst, 8
	loop4	b6b, clmblkdst		;; 4 * 2*clm iterations
	dec	al
	jnz	b6a
	lea	esi, [esi-8*clmblkdst+32*clmblkdst];; Next source pointer

;; Do 16 four_complex_fft_1 macros
;;	distance between fft data elements is 16

	mov	edi, sincos3		;; Load sin/cos pointer
	mov	eax, 2			;; 2 iterations of 8
b7a:	touch				;; Preload TLB
b7b:	disp four_complex_fft_1, 8*clmblkdst, 16*clmblkdst, 8
	loop2	b7b, clmblkdst		;; 2 * 2*clm iterations
	add	eax, 80000000h/2	;; Test middle loop counter (4 iters)
	jnc	b7a			;; Iterate if necessary
	lea	esi, [esi-8*clmblkdst+32*clmblkdst];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al
	jnz	b7a
	lea	esi, [esi-3*32*clmblkdst]	;; Restore source pointer
	end_timer 22

;; Do FFT levels 5,6
;;
;; On input the 32-byte cache lines hold these data values:
;;	0	4K	1	4K+1
;;	256	...
;;	...
;;	8K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	1K	1	1K+1
;;	256	...
;;	...
;;	2K	...
;;	...

;; Do 4 four_real_four_semireal_fft macros
;;	distance between fft data elements is 4

	start_timer 23
	mov	al, 2			;; Load loop counter
b4a:	touch				;; Preload TLB
b4b:	disp four_real_four_semireal_fft, 4*clmblkdst, 8*clmblkdst, 8
	loop2	b4b, clmblkdst		;; 2 * 2*clm iterations
	dec	al
	jnz	b4a
	lea	esi, [esi-4*clmblkdst+16*clmblkdst];; Next source pointer

;; Do 20 four_complex_fft macros
;;	distance between fft data elements is 4

	mov	edi, sincos2		;; Load sin/cos pointer
	mov	eax, 5			;; 5 iterations of 4
b5a:	touch				;; Preload TLB
b5b:	disp four_complex_fft, 4*clmblkdst, 8*clmblkdst, 8
	loop1	b5b, clmblkdst		;; 2*clm iterations
	add	ah, 256/4		;; Test middle loop counter
	jnc	b5a			;; Iterate if necessary
	lea	esi, [esi-4*clmblkdst+16*clmblkdst];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al
	jnz	b5a
	lea	esi, [esi-6*16*clmblkdst]	;; Restore source pointer
	end_timer 23

;; Do FFT levels 7,8
;;
;; On input the 32-byte cache lines hold these data values:
;;	0	1K	1	1K+1
;;	256	...
;;	...
;;	2K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	256	1	257
;;	512	...
;;	...

;; Do 1 four_real_four_semireal_fft macros
;;	distance between fft data elements is 1

	start_timer 24
pfing	lea	ecx, [ecx-96*blkdst]	;; Create dest ptr from prefetch ptr
b2b:	cp_disp four_real_four_semireal_fft, clmblkdst, 2*clmblkdst, 8, blkdst, 2*blkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	lea	ecx, [ecx+dist1]	;; Next dest pointer
	add	dl, 256/2/clm		;; Test inner loop counter
	jnc	b2b			;; Iterate if necessary
	lea	esi, [esi-clm*2*dist1+4*clmblkdst];; Next source pointer
	lea	ecx, [ecx-clm*2*dist1+4*blkdst];; Next dest pointer

;; Do 23 four_complex_fft macros
;;	distance between fft data elements is 1

	mov	edi, sincos1		;; Load sin/cos pointer
	mov	al, 23			;; 23 iterations
b3b:	cp_disp four_complex_fft, clmblkdst, 2*clmblkdst, 8, blkdst, 2*blkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	lea	ecx, [ecx+dist1]	;; Next dest pointer
	add	dl, 256/2/clm		;; Test inner loop counter
	jnc	b3b			;; Iterate if necessary
	lea	esi, [esi-clm*2*dist1+4*clmblkdst];; Next source pointer
	lea	ecx, [ecx-clm*2*dist1+4*blkdst];; Next dest pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al
	jnz	b3b
	lea	esi, [ecx-24*4*blkdst]	;; Restore source pointer
	end_timer 24

;; Work on next 4 sets of 32 values

no_fft:	lea	esi, [esi+clm*2*dist1]	;; Next source pointer
	dec	dh			;; Test loop counter
	jnz	short more		;; Do more FFTs
	lea	esi, [esi+64]		;; Skip pad every 128 cache lines (4KB)
pfing	mov	ebp, clm*32		;; Set prefetch distance
	sub	edx, 65536		;; Test loop counter
	jz	short fftdn		;; Jump if done looping
	mov	dh, 128/clm		;; Restore cache line counter
more:	set_prefetch_gap
	cmp	norm_ptr1, 0		;; What was the loop start point?
	jne	c0b			;; Do inverse-FFT/norm/FFT iteration
	jmp	b0b			;; Do another FFT-only iteration
fftdn:	lea	esi, [esi-p2cl*2*dist1-p2cl/128*64];; Restore source pointer
	cmp	esi, DESTARG		;; If esi was not restored, then we 
	jne	short fftdn1		;; just finished skipped postfft data
	sub	ebx, ebx
	mov	DIST_TO_FFTSRCARG, ebx
	cmp	norm_ptr1, 0		;; What was the loop start point?
	je	pass2			;; FFT-only, go do pass 2

;; Split the carries into high and low carries (so they do not exceed
;; the desired number of bits per FFT word.  Then add the carries back to the
;; FFT data.

	start_timer 28
	call	gw_carries
	end_timer 28

;; Have we been given permission to start the FFT on the result?
;; If so, set loop to FFT the first few cache lines now that the carries
;; have been added in.

	cmp	POSTFFT, 0		;; Test flag
	je	short fftdn1		;; Skip FFT code if flag not set
	mov	esi, DESTARG		;; Load FFT data address
	mov	edx, 65536+4/clm*256	;; Do 8 previously skipped cache lines
	mov	norm_ptr1, 0		;; Set flag so only forward FFT is done
	copy_4_words
	jmp	b0b

;; Now add in the FFT'ed carries, cleanup and return

fftdn1:	end_timer 1
	jmp	gw_finish_mult
	ENDM


;; Perform a 48K-element negacyclic FFT.  This is done in two passes.  Pass 1
;; does 8 levels, pass 2 uses common code to perform the last 8 levels.

pass1levels8complex3 MACRO pass2_macro, p2cl, clmarg
	LOCAL	b0b, b1a, b1b, b3b, b5a, b5b, b7a, b7b
	LOCAL	c0b, c1a, c1b, c3b, c5a, c5b, c7a, c7b
	LOCAL	pass2, copyback, yes_fft, no_fft, more, fftdn, fftdn1

	set_FFT_constants clmarg

;; Call FFT code unless the FFT has already been started

	clear_timers
	cmp	DWORD PTR [esi-28][ebx], 0 ;; Test FFT-started flag
	jne	pass2			;; Jump is FFT already started
	mov	norm_ptr1, 0		;; Set FFT-ing flag
	mov	edx, p2cl/128*65536+128/clm*256;; Load loop counter
pfing	mov	ebp, clm*32		;; Set prefetch distance
	jmp	b0b			;; Do the pass 1 fft

;; Do the last 8 FFT levels
;;	do 96 pass2_8_levels_complex

pass2:	start_timer 0
	CALLP	pass2_macro
	end_timer 0
	cmp	ffttype, 1		;; We're done if FFTing only
	je	gw_finish_fft

;; Do the inverse FFT

	start_timer 1
	fldz				;; Init SUMOUT
	fstp	SUMOUT
	mov	edx, norm_biglit_array	;; Addr of the big/little flags array
	mov	norm_ptr1, edx		;; Save ptr
	mov	edx, norm_col_mults	;; Addr of the column multipliers
	mov	norm_ptr2, edx		;; Save ptr
	mov	edx, p2cl/128*65536+128/clm*256;; Load loop counter
pfing	mov	ebp, clm*32		;; Set prefetch distance

;; Do inverse FFT levels 7,8
;; On input the 32-byte cache lines hold these data values:
;;	0	256	1	257
;;	512	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	1K	1	1K+1
;;	256	...
;;	...
;;	2K	...
;;	...

;; Do 24 four_complex_unfft macros
;;	distance between fft data elements is 1

c0b:	start_timer 16
	mov	ecx, scratch_area	;; Get address of scratch area
	mov	edi, sincos1		;; Load sin/cos pointer
	mov	al, 24			;; 24 iterations
c3b:	cp_disp four_complex_unfft, 8, blkdst, 2*blkdst, 8, clmblkdst, 2*clmblkdst
	lea	esi, [esi+dist1]	;; Next source pointer
	lea	ecx, [ecx+dist1]	;; Next dest pointer
	add	dl, 256/2/clm		;; Test loop counter
	jnc	c3b			;; Iterate if necessary
	lea	esi, [esi-clm*2*dist1+4*blkdst];; Next source pointer
	lea	ecx, [ecx-clm*2*dist1+4*clmblkdst];; Next dest pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al			;; Test outer loop counter
	jnz	c3b			;; Iterate if necessary
	lea	ecx, [esi-24*4*blkdst]	;; Create prefetch pointer
	mov	esi, scratch_area	;; Restore source pointer
	end_timer 16

;; Do inverse FFT levels 5,6
;; On input the 32-byte cache lines hold these data values:
;;	0	1K	1	1K+1
;;	256	...
;;	...
;;	2K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	4K	1	4K+1
;;	256	...
;;	...
;;	8K	...
;;	...

;; Do 24 four_complex_unfft macros
;;	distance between fft data elements is 4

	start_timer 17
	mov	edi, sincos2		;; Load sin/cos pointer
	mov	eax, 6			;; 6 iterations of 4
c5a:	touch				;; Preload TLB
c5b:	disp four_complex_unfft, 8, 4*clmblkdst, 8*clmblkdst
	loop1	c5b, clmblkdst		;; 2*clm iterations
	add	ah, 256/4		;; Test middle loop counter
	jnc	c5a			;; Iterate if necessary
	lea	esi, [esi-4*clmblkdst+16*clmblkdst];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al
	jnz	c5a
	lea	esi, [esi-6*16*clmblkdst]	;; Restore source pointer
	end_timer 17

;; Do inverse FFT levels 4
;; On input the 32-byte cache lines hold these data values:
;;	0	4K	1	4K+1
;;	256	...
;;	...
;;	8K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	8K	1	8K+1
;;	256	...
;;	...
;;	16K	...
;;	...

;; Do 24 four_complex_unfft_1 macros
;;	distance between fft data elements is 16

	start_timer 18
	mov	edi, sincos3		;; Load sin/cos pointer
	mov	eax, 3			;; 3 iterations of 8
c7a:	touch				;; Preload TLB
c7b:	disp four_complex_unfft_1, 8*clmblkdst, 8, 16*clmblkdst
	loop4	c7b, clmblkdst		;; 4 * 2*clm iterations
	add	eax, 80000000h		;; Test middle loop counter (2 iters)
	jnc	c7a			;; Iterate if necessary
	lea	esi, [esi-8*clmblkdst+32*clmblkdst];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al
	jnz	c7a
	lea	esi, [esi-3*32*clmblkdst]	;; Restore source pointer
	end_timer 18

;; Do inverse FFT levels 1,2,3
;; On input the 32-byte cache lines hold these data values:
;;	0	8K	1	8K+1
;;	256	...
;;	...
;;	16K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	24K	1	24K+1
;;	256
;;	...
;;	24K-256

;; Do 32 three_complex_last_unfft macros
;;	distance between fft data elements is 32

	start_timer 19
	mov	edi, plus1_premults	;; Address of premultiplier table
	mov	al, 16			;; Load loop counter
c1a:	touch				;; Preload TLB
c1b:	three_complex_last_unfft 8, 32*clmblkdst, 64*clmblkdst
	loop2edi c1b, clmblkdst, 48	;; 2 * 2*clm iterations bumping edi
	dec	al			;; Test loop counter
	jnz	c1a			;; Iterate if necessary
	lea	esi, [esi-32*clmblkdst]	;; Restore source pointer
pfing	lea	ecx, [ecx-46*blkdst]	;; Restore dest ptr
	end_timer 19

;; Normalize these values

	start_timer 20
	mov	eax, NORMRTN		;; Addr of normalization routine
	call	eax
	sub	eax, eax
	end_timer 20

;; Have we been given permission to start the FFT on the result?
;; If so, do so now while the data is in the L2 cache.

	cmp	POSTFFT, 0		;; Test flag
	je	short copyback		;; Copy data, skip FFT if flag not set
	copy_3_words clm, 1
	cmp	edx, p2cl/128*65536+(128/clm-4/clm)*256;; Delay fft on 8 sets
	jle	yes_fft			;; No delay, do FFT now

;; Copy data from the scratch area back to the FFT data area

copyback:
	copy_scratch_data_to_fft 96
	jmp	no_fft			;; Skip forward FFT code

;; Copy data from the FFT data area back to the scratch area

b0b:	copy_fft_data_to_scratch 96

;; Do FFT levels 1,2,3
;;
;; On input the 32-byte cache lines hold these data values:
;;	0	24K	1	24K+1
;;	256
;;	...
;;	24K-256
;; On output the 32-byte cache lines hold these data values:
;;	0	8K	1	8K+1
;;	256	...
;;	...
;;	16K	...
;;	...

;; Do 32 three_complex_fft_premult macros
;;	distance between fft data elements is 32

yes_fft:start_timer 21
	mov	edi, plus1_premults	;; Address of premultiplier table
	mov	eax, 16			;; 32 iterations
pfing	lea	ecx, [ecx+44*blkdst]	;; Create prefetch ptr
b1a:	touch				;; Preload TLB
b1b:	three_complex_fft_premult 32*clmblkdst, 64*clmblkdst, 8
	loop2edi b1b, clmblkdst, 48	;; 2 * 2*clm iterations bumping edi
	dec	al			;; Test loop counter
	jnz	b1a			;; Iterate if necessary
	lea	esi, [esi-32*clmblkdst]	;; Restore source pointer
	end_timer 21

;; Do FFT levels 4
;;
;; On input the 32-byte cache lines hold these data values:
;;	0	8K	1	8K+1
;;	256	...
;;	...
;;	16K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	4K	1	4K+1
;;	256	...
;;	...
;;	8K	...
;;	...

;; Do 24 four_complex_fft_1 macros
;;	distance between fft data elements is 16

	start_timer 22
	mov	edi, sincos3		;; Load sin/cos pointer
	mov	eax, 3			;; 3 iterations of 8
b7a:	touch				;; Preload TLB
b7b:	disp four_complex_fft_1, 8*clmblkdst, 16*clmblkdst, 8
	loop2	b7b, clmblkdst		;; 2 * 2*clm iterations
	add	eax, 80000000h/2	;; Test middle loop counter (4 iters)
	jnc	b7a			;; Iterate if necessary
	lea	esi, [esi-8*clmblkdst+32*clmblkdst];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al
	jnz	b7a
	lea	esi, [esi-3*32*clmblkdst]	;; Restore source pointer
	end_timer 22

;; Do FFT levels 5,6
;;
;; On input the 32-byte cache lines hold these data values:
;;	0	4K	1	4K+1
;;	256	...
;;	...
;;	8K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	1K	1	1K+1
;;	256	...
;;	...
;;	2K	...
;;	...

;; Do 24 four_complex_fft macros
;;	distance between fft data elements is 4

	start_timer 23
	mov	edi, sincos2		;; Load sin/cos pointer
	mov	eax, 6			;; 6 iterations of 4
b5a:	touch				;; Preload TLB
b5b:	disp four_complex_fft, 4*clmblkdst, 8*clmblkdst, 8
	loop1	b5b, clmblkdst		;; 2*clm iterations
	add	ah, 256/4		;; Test middle loop counter
	jnc	b5a			;; Iterate if necessary
	lea	esi, [esi-4*clmblkdst+16*clmblkdst];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al
	jnz	b5a
	lea	esi, [esi-6*16*clmblkdst]	;; Restore source pointer
	end_timer 23

;; Do FFT levels 7,8
;;
;; On input the 32-byte cache lines hold these data values:
;;	0	1K	1	1K+1
;;	256	...
;;	...
;;	2K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	256	1	257
;;	512	...
;;	...

;; Do 24 four_complex_fft macros
;;	distance between fft data elements is 1

	start_timer 24
	mov	edi, sincos1		;; Load sin/cos pointer
pfing	lea	ecx, [ecx-96*blkdst]	;; Create dest ptr from prefetch ptr
	mov	al, 24			;; 24 iterations
b3b:	cp_disp four_complex_fft, clmblkdst, 2*clmblkdst, 8, blkdst, 2*blkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	lea	ecx, [ecx+dist1]	;; Next dest pointer
	add	dl, 256/2/clm		;; Test inner loop counter
	jnc	b3b			;; Iterate if necessary
	lea	esi, [esi-clm*2*dist1+4*clmblkdst];; Next source pointer
	lea	ecx, [ecx-clm*2*dist1+4*blkdst];; Next dest pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al
	jnz	b3b
	lea	esi, [ecx-24*4*blkdst]	;; Restore source pointer
	end_timer 24

;; Work on next 4 sets of 32 values

no_fft:	lea	esi, [esi+clm*2*dist1]	;; Next source pointer
	dec	dh			;; Test loop counter
	jnz	short more		;; Do more FFTs
	lea	esi, [esi+64]		;; Skip pad every 128 cache lines (4KB)
pfing	mov	ebp, clm*32		;; Set prefetch distance
	sub	edx, 65536		;; Test loop counter
	jz	short fftdn		;; Jump if done looping
	mov	dh, 128/clm		;; Restore cache line counter
more:	set_prefetch_gap
	cmp	norm_ptr1, 0		;; What was the loop start point?
	jne	c0b			;; Do inverse-FFT/norm/FFT iteration
	jmp	b0b			;; Do another FFT-only iteration
fftdn:	lea	esi, [esi-p2cl*2*dist1-p2cl/128*64];; Restore source pointer
	cmp	esi, DESTARG		;; If esi was not restored, then we 
	jne	short fftdn1		;; just finished skipped postfft data
	sub	ebx, ebx
	mov	DIST_TO_FFTSRCARG, ebx
	cmp	norm_ptr1, 0		;; What was the loop start point?
	je	pass2			;; FFT-only, go do pass 2

;; Split the carries into high and low carries (so they do not exceed
;; the desired number of bits per FFT word.  Then add the carries back to the
;; FFT data.

	start_timer 28
	call	gw_carries
	end_timer 28

;; Have we been given permission to start the FFT on the result?
;; If so, set loop to FFT the first few cache lines now that the carries
;; have been added in.

	cmp	POSTFFT, 0		;; Test flag
	je	short fftdn1		;; Skip FFT code if flag not set
	mov	esi, DESTARG		;; Load FFT data address
	mov	edx, 65536+4/clm*256	;; Do 8 previously skipped cache lines
	mov	norm_ptr1, 0		;; Set flag so only forward FFT is done
	copy_4_words
	jmp	b0b

;; Now add in the FFT'ed carries, cleanup and return

fftdn1:	end_timer 1
	jmp	gw_finish_mult
	ENDM


;; Perform a 56K-element FFT.  This is done in two passes.  Pass 1
;; does 8 levels, pass 2 uses common code to perform the last 8 levels.

pass1levels8pfa7 MACRO pass2_macro, p2cl, clmarg
	LOCAL	b0b, b1a, b1b, b2b, b3b, b4a, b4b, b5a, b5b, b6b, b7a, b7b
	LOCAL	c0b, c1a, c1b, c2b, c3b, c4a, c4b, c5a, c5b, c6b, c7a, c7b
	LOCAL	pass2, copyback, yes_fft, no_fft, more, fftdn, fftdn1

	set_FFT_constants clmarg

;; Call FFT code unless the FFT has already been started

	clear_timers
	cmp	DWORD PTR [esi-28][ebx], 0 ;; Test FFT-started flag
	jne	pass2			;; Jump is FFT already started
	mov	norm_ptr1, 0		;; Set FFT-ing flag
	mov	edx, p2cl/128*65536+128/clm*256;; Load loop counter
pfing	mov	ebp, clm*32		;; Set prefetch distance
	jmp	b0b			;; Do the pass 1 fft

;; Do the last 8 FFT levels
;;	do 1 pass2_8_levels_real and 111 pass2_8_levels_complex

pass2:	start_timer 0
	CALLP	pass2_macro
	end_timer 0
	cmp	ffttype, 1		;; We're done if FFTing only
	je	gw_finish_fft

;; Do the inverse FFT

	start_timer 1
	fldz				;; Init SUMOUT
	fstp	SUMOUT
	mov	edx, norm_biglit_array	;; Addr of the big/little flags array
	mov	norm_ptr1, edx		;; Save ptr
	mov	edx, norm_col_mults	;; Addr of the column multipliers
	mov	norm_ptr2, edx		;; Save ptr
	mov	edx, p2cl/128*65536+128/clm*256;; Load loop counter
pfing	mov	ebp, clm*32		;; Set prefetch distance

;; Do inverse FFT levels 7,8
;; On input the 32-byte cache lines hold these data values:
;;	0	256	1	257
;;	512	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	1K	1	1K+1
;;	256	...
;;	...
;;	2K	...
;;	...

;; Do 1 four_real_four_semireal_unfft macros
;;	distance between fft data elements is 1

c0b:	start_timer 16
	mov	ecx, scratch_area	;; Get address of scratch area
c2b:	cp_disp four_real_four_semireal_unfft, 8, blkdst, 2*blkdst, 8, clmblkdst, 2*clmblkdst
	lea	esi, [esi+dist1]	;; Next source pointer
	lea	ecx, [ecx+dist1]	;; Next dest pointer
	add	dl, 256/2/clm		;; Test inner loop counter
	jnc	c2b			;; Iterate if necessary
	lea	esi, [esi-clm*2*dist1+4*blkdst];; Next source pointer
	lea	ecx, [ecx-clm*2*dist1+4*clmblkdst];; Next dest pointer

;; Do 27 four_complex_unfft macros
;;	distance between fft data elements is 1

	mov	edi, sincos1		;; Load sin/cos pointer
	mov	al, 27			;; 27 iterations
c3b:	cp_disp four_complex_unfft, 8, blkdst, 2*blkdst, 8, clmblkdst, 2*clmblkdst
	lea	esi, [esi+dist1]	;; Next source pointer
	lea	ecx, [ecx+dist1]	;; Next dest pointer
	add	dl, 256/2/clm		;; Test loop counter
	jnc	c3b			;; Iterate if necessary
	lea	esi, [esi-clm*2*dist1+4*blkdst];; Next source pointer
	lea	ecx, [ecx-clm*2*dist1+4*clmblkdst];; Next dest pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al			;; Test outer loop counter
	jnz	c3b			;; Iterate if necessary
	lea	ecx, [esi-28*4*blkdst]	;; Create prefetch pointer
	mov	esi, scratch_area	;; Restore source pointer
	end_timer 16

;; Do inverse FFT levels 5,6
;; On input the 32-byte cache lines hold these data values:
;;	0	1K	1	1K+1
;;	256	...
;;	...
;;	2K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	4K	1	4K+1
;;	256	...
;;	...
;;	8K	...
;;	...

;; Do 4 four_real_four_semireal_unfft macros
;;	distance between fft data elements is 4

	start_timer 17
	mov	al, 2			;; Load loop counter
c4a:	touch				;; Preload TLB
c4b:	disp four_real_four_semireal_unfft, 8, 4*clmblkdst, 8*clmblkdst
	loop2	c4b, clmblkdst		;; 2 * 2*clm iterations
	dec	al
	jnz	c4a
	lea	esi, [esi-4*clmblkdst+16*clmblkdst];; Next source pointer

;; Do 24 four_complex_unfft macros
;;	distance between fft data elements is 4

	mov	edi, sincos2		;; Load sin/cos pointer
	mov	eax, 6			;; 6 iterations of 4
c5a:	touch				;; Preload TLB
c5b:	disp four_complex_unfft, 8, 4*clmblkdst, 8*clmblkdst
	loop1	c5b, clmblkdst		;; 2*clm iterations
	add	ah, 256/4		;; Test middle loop counter
	jnc	c5a			;; Iterate if necessary
	lea	esi, [esi-4*clmblkdst+16*clmblkdst];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al
	jnz	c5a
	lea	esi, [esi-7*16*clmblkdst]	;; Restore source pointer
	end_timer 17

;; Do inverse FFT levels 4
;; On input the 32-byte cache lines hold these data values:
;;	0	4K	1	4K+1
;;	256	...
;;	...
;;	8K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	8K	1	8K+1
;;	256	...
;;	...
;;	16K	...
;;	...

;; Do 4 eight_reals_unfft_1 macros
;;	distance between fft data elements is 16

	start_timer 18
	touch				;; Preload TLB
c6b:	disp eight_reals_unfft_1, 4*clmblkdst, 8*clmblkdst, 8
	loop4	c6b, clmblkdst		;; 4 * 2*clm iterations
	lea	esi, [esi-4*clmblkdst+16*clmblkdst];; Next source pointer

;; Do 24 four_complex_unfft_1 macros
;;	distance between fft data elements is 16

	mov	edi, sincos3		;; Load sin/cos pointer
	mov	eax, 3			;; 3 iterations of 8
c7a:	touch				;; Preload TLB
c7b:	disp four_complex_unfft_1, 8*clmblkdst, 8, 16*clmblkdst
	loop2	c7b, clmblkdst		;; 2 * 2*clm iterations
	add	eax, 80000000h/2	;; Test middle loop counter (4 iters)
	jnc	c7a			;; Iterate if necessary
	lea	esi, [esi-8*clmblkdst+32*clmblkdst];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al
	jnz	c7a
	lea	esi, [esi-3*32*clmblkdst-16*clmblkdst];; Restore source pointer
	end_timer 18

;; Do inverse FFT levels 1,2,3
;; On input the 32-byte cache lines hold these data values:
;;	0	8K	1	8K+1
;;	256	...
;;	...
;;	16K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	28K	1	28K+1
;;	256
;;	...
;;	28K-256

;; Do 32 seven_reals_last_unfft macros
;;	distance between fft data elements is 32

	start_timer 19
	mov	al, 16			;; Load loop counter
c1a:	touch				;; Preload TLB
c1b:	two_seven_reals_last_unfft 8, 16*clmblkdst
	loop1	c1b, clmblkdst		;; 2*clm iterations
	dec	al			;; Test loop counter
	jnz	c1a			;; Iterate if necessary
	lea	esi, [esi-16*clmblkdst]	;; Restore source pointer
pfing	lea	ecx, [ecx-55*blkdst]	;; Restore dest ptr
	end_timer 19

;; Normalize these values

	start_timer 20
	mov	eax, NORMRTN		;; Addr of normalization routine
	call	eax
	sub	eax, eax
	end_timer 20

;; Have we been given permission to start the FFT on the result?
;; If so, do so now while the data is in the L2 cache.

	cmp	POSTFFT, 0		;; Test flag
	je	short copyback		;; Copy data, skip FFT if flag not set
	copy_3_words clm, 1
	cmp	edx, p2cl/128*65536+(128/clm-4/clm)*256;; Delay fft on 8 sets
	jle	yes_fft			;; No delay, do FFT now

;; Copy data from the scratch area back to the FFT data area

copyback:
	copy_scratch_data_to_fft 112
	jmp	no_fft			;; Skip forward FFT code

;; Copy data from the FFT data area back to the scratch area

b0b:	copy_fft_data_to_scratch 112

;; Do FFT levels 1,2,3
;;
;; On input the 32-byte cache lines hold these data values:
;;	0	28K	1	28K+1
;;	256
;;	...
;;	28K-256
;; On output the 32-byte cache lines hold these data values:
;;	0	8K	1	8K+1
;;	256	...
;;	...
;;	16K	...
;;	...

;; Do 32 seven_reals_fft macros
;;	distance between fft data elements is 32

yes_fft:start_timer 21
	mov	eax, 16			;; 16 iterations
pfing	lea	ecx, [ecx+55*blkdst]	;; Create prefetch ptr
b1a:	touch				;; Preload TLB
b1b:	two_seven_reals_fft 16*clmblkdst, 8
	loop1	b1b, clmblkdst		;; 2*clm iterations
	dec	al			;; Test loop counter
	jnz	b1a			;; Iterate if necessary
	lea	esi, [esi-16*clmblkdst]	;; Restore source pointer
	end_timer 21

;; Do FFT levels 4
;;
;; On input the 32-byte cache lines hold these data values:
;;	0	8K	1	8K+1
;;	256	...
;;	...
;;	16K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	4K	1	4K+1
;;	256	...
;;	...
;;	8K	...
;;	...

;; Do 4 eight_reals_fft_1 macros
;;	distance between fft data elements is 16

	start_timer 22
	touch				;; Preload TLB
b6b:	disp eight_reals_fft_1, 4*clmblkdst, 8*clmblkdst, 8
	loop4	b6b, clmblkdst		;; 4 * 2*clm iterations
	lea	esi, [esi-4*clmblkdst+16*clmblkdst];; Next source pointer

;; Do 24 four_complex_fft_1 macros
;;	distance between fft data elements is 16

	mov	edi, sincos3		;; Load sin/cos pointer
	mov	eax, 3			;; 3 iterations of 8
b7a:	touch				;; Preload TLB
b7b:	disp four_complex_fft_1, 8*clmblkdst, 16*clmblkdst, 8
	loop2	b7b, clmblkdst		;; 2 * 2*clm iterations
	add	eax, 80000000h/2	;; Test middle loop counter (4 iters)
	jnc	b7a			;; Iterate if necessary
	lea	esi, [esi-8*clmblkdst+32*clmblkdst];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al
	jnz	b7a
	lea	esi, [esi-3*32*clmblkdst-16*clmblkdst];; Restore source pointer
	end_timer 22

;; Do FFT levels 5,6
;;
;; On input the 32-byte cache lines hold these data values:
;;	0	4K	1	4K+1
;;	256	...
;;	...
;;	8K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	1K	1	1K+1
;;	256	...
;;	...
;;	2K	...
;;	...

;; Do 4 four_real_four_semireal_fft macros
;;	distance between fft data elements is 4

	start_timer 23
	mov	al, 4			;; Load loop counter
b4a:	touch				;; Preload TLB
b4b:	disp four_real_four_semireal_fft, 4*clmblkdst, 8*clmblkdst, 8
	loop1	b4b, clmblkdst		;; 2*clm iterations
	dec	al
	jnz	b4a
	lea	esi, [esi-4*clmblkdst+16*clmblkdst];; Next source pointer

;; Do 24 four_complex_fft macros
;;	distance between fft data elements is 4

	mov	edi, sincos2		;; Load sin/cos pointer
	mov	eax, 6			;; 6 iterations of 4
b5a:	touch				;; Preload TLB
b5b:	disp four_complex_fft, 4*clmblkdst, 8*clmblkdst, 8
	loop1	b5b, clmblkdst		;; 2*clm iterations
	add	ah, 256/4		;; Test middle loop counter
	jnc	b5a			;; Iterate if necessary
	lea	esi, [esi-4*clmblkdst+16*clmblkdst];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al
	jnz	b5a
	lea	esi, [esi-7*16*clmblkdst]	;; Restore source pointer
	end_timer 23

;; Do FFT levels 7,8
;;
;; On input the 32-byte cache lines hold these data values:
;;	0	1K	1	1K+1
;;	256	...
;;	...
;;	2K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	256	1	257
;;	512	...
;;	...

;; Do 1 four_real_four_semireal_fft macros
;;	distance between fft data elements is 1

	start_timer 24
pfing	lea	ecx, [ecx-112*blkdst]	;; Create dest ptr from prefetch ptr
b2b:	cp_disp four_real_four_semireal_fft, clmblkdst, 2*clmblkdst, 8, blkdst, 2*blkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	lea	ecx, [ecx+dist1]	;; Next dest pointer
	add	dl, 256/2/clm		;; Test inner loop counter
	jnc	b2b			;; Iterate if necessary
	lea	esi, [esi-clm*2*dist1+4*clmblkdst];; Next source pointer
	lea	ecx, [ecx-clm*2*dist1+4*blkdst];; Next dest pointer

;; Do 27 four_complex_fft macros
;;	distance between fft data elements is 1

	mov	edi, sincos1		;; Load sin/cos pointer
	mov	al, 27			;; 27 iterations
b3b:	cp_disp four_complex_fft, clmblkdst, 2*clmblkdst, 8, blkdst, 2*blkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	lea	ecx, [ecx+dist1]	;; Next dest pointer
	add	dl, 256/2/clm		;; Test inner loop counter
	jnc	b3b			;; Iterate if necessary
	lea	esi, [esi-clm*2*dist1+4*clmblkdst];; Next source pointer
	lea	ecx, [ecx-clm*2*dist1+4*blkdst];; Next dest pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al
	jnz	b3b
	lea	esi, [ecx-28*4*blkdst]	;; Restore source pointer
	end_timer 24

;; Work on next 4 sets of 32 values

no_fft:	lea	esi, [esi+clm*2*dist1]	;; Next source pointer
	dec	dh			;; Test loop counter
	jnz	short more		;; Do more FFTs
	lea	esi, [esi+64]		;; Skip pad every 128 cache lines (4KB)
pfing	mov	ebp, clm*32		;; Set prefetch distance
	sub	edx, 65536		;; Test loop counter
	jz	short fftdn		;; Jump if done looping
	mov	dh, 128/clm		;; Restore cache line counter
more:	set_prefetch_gap
	cmp	norm_ptr1, 0		;; What was the loop start point?
	jne	c0b			;; Do inverse-FFT/norm/FFT iteration
	jmp	b0b			;; Do another FFT-only iteration
fftdn:	lea	esi, [esi-p2cl*2*dist1-p2cl/128*64];; Restore source pointer
	cmp	esi, DESTARG		;; If esi was not restored, then we 
	jne	short fftdn1		;; just finished skipped postfft data
	sub	ebx, ebx
	mov	DIST_TO_FFTSRCARG, ebx
	cmp	norm_ptr1, 0		;; What was the loop start point?
	je	pass2			;; FFT-only, go do pass 2

;; Split the carries into high and low carries (so they do not exceed
;; the desired number of bits per FFT word.  Then add the carries back to the
;; FFT data.

	start_timer 28
	call	gw_carries
	end_timer 28

;; Have we been given permission to start the FFT on the result?
;; If so, set loop to FFT the first few cache lines now that the carries
;; have been added in.

	cmp	POSTFFT, 0		;; Test flag
	je	short fftdn1		;; Skip FFT code if flag not set
	mov	esi, DESTARG		;; Load FFT data address
	mov	edx, 65536+4/clm*256	;; Do 8 previously skipped cache lines
	mov	norm_ptr1, 0		;; Set flag so only forward FFT is done
	copy_4_words
	jmp	b0b

;; Now add in the FFT'ed carries, cleanup and return

fftdn1:	end_timer 1
	jmp	gw_finish_mult
	ENDM


;; Perform a 64K-element FFT.  This is done in two passes.  Pass 1
;; does 8 levels, pass 2 uses common code to perform the last 8 levels.

pass1levels8pfa8 MACRO pass2_macro, p2cl, clmarg
	LOCAL	b0b, b1a, b1b, b2b, b3b, b4a, b4b, b5a, b5b, b6a, b6b, b7a, b7b
	LOCAL	c0b, c1a, c1b, c2b, c3b, c4a, c4b, c5a, c5b, c6a, c6b, c7a, c7b
	LOCAL	pass2, copyback, yes_fft, no_fft, more, fftdn, fftdn1

	set_FFT_constants clmarg

;; Call FFT code unless the FFT has already been started

	clear_timers
	cmp	DWORD PTR [esi-28][ebx], 0 ;; Test FFT-started flag
	jne	pass2			;; Jump is FFT already started
	mov	norm_ptr1, 0		;; Set FFT-ing flag
	mov	edx, p2cl/128*65536+128/clm*256;; Load loop counter
pfing	mov	ebp, clm*32		;; Set prefetch distance
	jmp	b0b			;; Do the pass 1 fft

;; Do the last 8 FFT levels
;;	do 1 pass2_8_levels_real and 127 pass2_8_levels_complex

pass2:	start_timer 0
	CALLP	pass2_macro
	end_timer 0
	cmp	ffttype, 1		;; We're done if FFTing only
	je	gw_finish_fft

;; Do the inverse FFT

	start_timer 1
	fldz				;; Init SUMOUT
	fstp	SUMOUT
	mov	edx, norm_biglit_array	;; Addr of the big/little flags array
	mov	norm_ptr1, edx		;; Save ptr
	mov	edx, norm_col_mults	;; Addr of the column multipliers
	mov	norm_ptr2, edx		;; Save ptr
	mov	edx, p2cl/128*65536+128/clm*256;; Load loop counter
pfing	mov	ebp, clm*32		;; Set prefetch distance

;; Do inverse FFT levels 7,8
;; On input the 32-byte cache lines hold these data values:
;;	0	256	1	257
;;	512	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	1K	1	1K+1
;;	256	...
;;	...
;;	2K	...
;;	...

;; Do 1 four_real_four_semireal_unfft macros
;;	distance between fft data elements is 1

c0b:	start_timer 16
	mov	ecx, scratch_area	;; Get address of scratch area
c2b:	cp_disp four_real_four_semireal_unfft, 8, blkdst, 2*blkdst, 8, clmblkdst, 2*clmblkdst
	lea	esi, [esi+dist1]	;; Next source pointer
	lea	ecx, [ecx+dist1]	;; Next dest pointer
	add	dl, 256/2/clm		;; Test inner loop counter
	jnc	c2b			;; Iterate if necessary
	lea	esi, [esi-clm*2*dist1+4*blkdst];; Next source pointer
	lea	ecx, [ecx-clm*2*dist1+4*clmblkdst];; Next dest pointer

;; Do 31 four_complex_unfft macros
;;	distance between fft data elements is 1

	mov	edi, sincos1		;; Load sin/cos pointer
	mov	eax, 256/8*256+4	;; 4 iterations of 7 or 8
c3b:	cp_disp four_complex_unfft, 8, blkdst, 2*blkdst, 8, clmblkdst, 2*clmblkdst
	lea	esi, [esi+dist1]	;; Next source pointer
	lea	ecx, [ecx+dist1]	;; Next dest pointer
	add	dl, 256/2/clm		;; Test loop counter
	jnc	c3b			;; Iterate if necessary
	lea	esi, [esi-clm*2*dist1+4*blkdst];; Next source pointer
	lea	ecx, [ecx-clm*2*dist1+4*clmblkdst];; Next dest pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	add	ah, 256/8		;; Test loop counter
	jnc	c3b			;; Iterate if necessary
	lea	ecx, [ecx-8*4*clmblkdst+clmblkdst32];; Next dest pointer
	dec	al			;; Test outer loop counter
	jnz	c3b			;; Iterate if necessary
	lea	ecx, [esi-32*4*blkdst]	;; Create prefetch pointer
	mov	esi, scratch_area	;; Restore source pointer
	end_timer 16

;; Do inverse FFT levels 5,6
;; On input the 32-byte cache lines hold these data values:
;;	0	1K	1	1K+1
;;	256	...
;;	...
;;	2K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	4K	1	4K+1
;;	256	...
;;	...
;;	8K	...
;;	...

;; Do 4 four_real_four_semireal_unfft macros
;;	distance between fft data elements is 4

	start_timer 17
	mov	al, 4			;; Load loop counter
c4a:	touch				;; Preload TLB
c4b:	disp four_real_four_semireal_unfft, 8, 4*clmblkdst, 8*clmblkdst
	loop1	c4b, clmblkdst		;; 2*clm iterations
	dec	al
	jnz	c4a
	lea	esi, [esi-4*clmblkdst+16*clmblkdst];; Next source pointer

;; Do 28 four_complex_unfft macros
;;	distance between fft data elements is 4

	mov	edi, sincos2		;; Load sin/cos pointer
	mov	eax, 80000000h+4	;; 4 iters of 1 or 2 iters of 4
c5a:	touch				;; Preload TLB
c5b:	disp four_complex_unfft, 8, 4*clmblkdst, 8*clmblkdst
	loop1	c5b, clmblkdst		;; 2*clm iterations
	add	ah, 256/4		;; Test middle loop counter
	jnc	c5a			;; Iterate if necessary
	lea	esi, [esi-4*clmblkdst+16*clmblkdst];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	add	eax, 80000000h		;; Test loop counter
	jnc	c5a			;; Iterate if necessary
	lea	esi, [esi-2*16*clmblkdst+clmblkdst32];; Next source pointer
	dec	al
	jnz	c5a
	lea	esi, [esi-4*clmblkdst32];; Restore source pointer
	end_timer 17

;; Do inverse FFT levels 4
;; On input the 32-byte cache lines hold these data values:
;;	0	4K	1	4K+1
;;	256	...
;;	...
;;	8K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	8K	1	8K+1
;;	256	...
;;	...
;;	16K	...
;;	...

;; Do 8 four_real_four_semireal_unfft_1 macros
;;	distance between fft data elements is 16

	start_timer 18
	mov	al, 4			;; Load loop counter
c6a:	touch				;; Preload TLB
c6b:	disp four_real_four_semireal_unfft_1, 8*clmblkdst, 8, 16*clmblkdst
	loop2	c6b, clmblkdst		;; 2 * 2*clm iterations
	dec	al
	jnz	c6a
	lea	esi, [esi-8*clmblkdst+clmblkdst32];; Next source pointer

;; Do 24 four_complex_unfft_1 macros
;;	distance between fft data elements is 16

	mov	edi, sincos3		;; Load sin/cos pointer
	mov	eax, 3			;; 3 iterations of 8
c7a:	touch				;; Preload TLB
c7b:	disp four_complex_unfft_1, 8*clmblkdst, 8, 16*clmblkdst
	loop2	c7b, clmblkdst		;; 2 * 2*clm iterations
	add	eax, 80000000h/2	;; Test middle loop counter (4 iters)
	jnc	c7a			;; Iterate if necessary
	lea	esi, [esi-8*clmblkdst+clmblkdst32];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al
	jnz	c7a
	lea	esi, [esi-4*clmblkdst32];; Restore source pointer
	end_timer 18

;; Do inverse FFT levels 1,2,3
;; On input the 32-byte cache lines hold these data values:
;;	0	8K	1	8K+1
;;	256	...
;;	...
;;	16K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	32K	1	32K+1
;;	256
;;	...
;;	32K-256

;; Do 32 eight_reals_last_unfft macros
;;	distance between fft data elements is 32

	start_timer 19
	mov	al, 16			;; Load loop counter
c1a:	touch				;; Preload TLB
c1b:	disp eight_reals_last_unfft, 8, clmblkdst32, 2*clmblkdst32
	loop2	c1b, clmblkdst		;; 2 * 2*clm iterations
	dec	al			;; Test loop counter
	jnz	c1a			;; Iterate if necessary
	lea	esi, [esi-32*clmblkdst]	;; Restore source pointer
pfing	lea	ecx, [ecx-64*blkdst]	;; Restore dest ptr
	end_timer 19

;; Normalize these values

	start_timer 20
	mov	eax, NORMRTN		;; Addr of normalization routine
	call	eax
	sub	eax, eax
	end_timer 20

;; Have we been given permission to start the FFT on the result?
;; If so, do so now while the data is in the L2 cache.

	cmp	POSTFFT, 0		;; Test flag
	je	short copyback		;; Copy data, skip FFT if flag not set
	copy_3_words clm, 1
	cmp	edx, p2cl/128*65536+(128/clm-4/clm)*256;; Delay fft on 8 sets
	jle	yes_fft			;; No delay, do FFT now

;; Copy data from the scratch area back to the FFT data area

copyback:
	copy_scratch_data_to_fft 4, 32, clmblkdst32
	jmp	no_fft			;; Skip forward FFT code

;; Copy data from the FFT data area back to the scratch area

b0b:	copy_fft_data_to_scratch 4, 32, clmblkdst32

;; Do FFT levels 1,2,3
;;
;; On input the 32-byte cache lines hold these data values:
;;	0	32K	1	32K+1
;;	256
;;	...
;;	32K-256
;; On output the 32-byte cache lines hold these data values:
;;	0	8K	1	8K+1
;;	256	...
;;	...
;;	16K	...
;;	...

;; Do 32 eight_reals_fft macros
;;	distance between fft data elements is 32

yes_fft:start_timer 21
	mov	eax, 16			;; 32 iterations
pfing	lea	ecx, [ecx+64*blkdst]	;; Create prefetch ptr
b1a:	touch				;; Preload TLB
b1b:	disp eight_reals_fft, clmblkdst32, 2*clmblkdst32, 8
	loop2	b1b, clmblkdst		;; 2 * 2*clm iterations
	dec	al			;; Test loop counter
	jnz	b1a			;; Iterate if necessary
	lea	esi, [esi-32*clmblkdst]	;; Restore source pointer
	end_timer 21

;; Do FFT levels 4
;;
;; On input the 32-byte cache lines hold these data values:
;;	0	8K	1	8K+1
;;	256	...
;;	...
;;	16K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	4K	1	4K+1
;;	256	...
;;	...
;;	8K	...
;;	...

;; Do 8 four_real_four_semireal_fft_1 macros
;;	distance between fft data elements is 16

	start_timer 22
	mov	al, 4			;; Load loop counter
b6a:	touch				;; Preload TLB
b6b:	disp four_real_four_semireal_fft_1, 8*clmblkdst, 16*clmblkdst, 8
	loop2	b6b, clmblkdst		;; 2 * 2*clm iterations
	dec	al
	jnz	b6a
	lea	esi, [esi-8*clmblkdst+clmblkdst32];; Next source pointer

;; Do 24 four_complex_fft_1 macros
;;	distance between fft data elements is 16

	mov	edi, sincos3		;; Load sin/cos pointer
	mov	eax, 3			;; 3 iterations of 8
b7a:	touch				;; Preload TLB
b7b:	disp four_complex_fft_1, 8*clmblkdst, 16*clmblkdst, 8
	loop2	b7b, clmblkdst		;; 2 * 2*clm iterations
	add	eax, 80000000h/2	;; Test middle loop counter (4 iters)
	jnc	b7a			;; Iterate if necessary
	lea	esi, [esi-8*clmblkdst+clmblkdst32];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al
	jnz	b7a
	lea	esi, [esi-4*clmblkdst32];; Restore source pointer
	end_timer 22

;; Do FFT levels 5,6
;;
;; On input the 32-byte cache lines hold these data values:
;;	0	4K	1	4K+1
;;	256	...
;;	...
;;	8K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	1K	1	1K+1
;;	256	...
;;	...
;;	2K	...
;;	...

;; Do 4 four_real_four_semireal_fft macros
;;	distance between fft data elements is 4

	start_timer 23
	mov	al, 4			;; Load loop counter
b4a:	touch				;; Preload TLB
b4b:	disp four_real_four_semireal_fft, 4*clmblkdst, 8*clmblkdst, 8
	loop1	b4b, clmblkdst		;; 2*clm iterations
	dec	al
	jnz	b4a
	lea	esi, [esi-4*clmblkdst+16*clmblkdst];; Next source pointer

;; Do 28 four_complex_fft macros
;;	distance between fft data elements is 4

	mov	edi, sincos2		;; Load sin/cos pointer
	mov	eax, 80000000h+4	;; 4 iters of 1 or 2 iters of 4
b5a:	touch				;; Preload TLB
b5b:	disp four_complex_fft, 4*clmblkdst, 8*clmblkdst, 8
	loop1	b5b, clmblkdst		;; 2*clm iterations
	add	ah, 256/4		;; Test middle loop counter
	jnc	b5a			;; Iterate if necessary
	lea	esi, [esi-4*clmblkdst+16*clmblkdst];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	add	eax, 80000000h		;; Test middle loop counter (2 iters)
	jnc	b5a			;; Iterate if necessary
	lea	esi, [esi-2*16*clmblkdst+clmblkdst32];; Next source pointer
	dec	al
	jnz	b5a
	lea	esi, [esi-4*clmblkdst32];; Restore source pointer
	end_timer 23

;; Do FFT levels 7,8
;;
;; On input the 32-byte cache lines hold these data values:
;;	0	1K	1	1K+1
;;	256	...
;;	...
;;	2K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	256	1	257
;;	512	...
;;	...

;; Do 1 four_real_four_semireal_fft macros
;;	distance between fft data elements is 1

	start_timer 24
pfing	lea	ecx, [ecx-128*blkdst]	;; Create dest ptr from prefetch ptr
b2b:	cp_disp four_real_four_semireal_fft, clmblkdst, 2*clmblkdst, 8, blkdst, 2*blkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	lea	ecx, [ecx+dist1]	;; Next dest pointer
	add	dl, 256/2/clm		;; Test inner loop counter
	jnc	b2b			;; Iterate if necessary
	lea	esi, [esi-clm*2*dist1+4*clmblkdst];; Next source pointer
	lea	ecx, [ecx-clm*2*dist1+4*blkdst];; Next dest pointer

;; Do 31 four_complex_fft macros
;;	distance between fft data elements is 1

	mov	edi, sincos1		;; Load sin/cos pointer
	mov	eax, 256/8*256+4	;; 4 iters of 7 or 8 iters
b3b:	cp_disp four_complex_fft, clmblkdst, 2*clmblkdst, 8, blkdst, 2*blkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	lea	ecx, [ecx+dist1]	;; Next dest pointer
	add	dl, 256/2/clm		;; Test inner loop counter
	jnc	b3b			;; Iterate if necessary
	lea	esi, [esi-clm*2*dist1+4*clmblkdst];; Next source pointer
	lea	ecx, [ecx-clm*2*dist1+4*blkdst];; Next dest pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	add	ah, 256/8		;; Test loop counter
	jnc	b3b			;; Iterate if necessary
	lea	esi, [esi-8*4*clmblkdst+clmblkdst32];; Next source pointer
	dec	al
	jnz	b3b
	lea	esi, [ecx-32*4*blkdst]	;; Restore source pointer
	end_timer 24

;; Work on next 4 sets of 32 values

no_fft:	lea	esi, [esi+clm*2*dist1]	;; Next source pointer
	dec	dh			;; Test loop counter
	jnz	short more		;; Do more FFTs
	lea	esi, [esi+64]		;; Skip pad every 128 cache lines (4KB)
pfing	mov	ebp, clm*32		;; Set prefetch distance
	sub	edx, 65536		;; Test loop counter
	jz	short fftdn		;; Jump if done looping
	mov	dh, 128/clm		;; Restore cache line counter
more:	set_prefetch_gap
	cmp	norm_ptr1, 0		;; What was the loop start point?
	jne	c0b			;; Do inverse-FFT/norm/FFT iteration
	jmp	b0b			;; Do another FFT-only iteration
fftdn:	lea	esi, [esi-p2cl*2*dist1-p2cl/128*64];; Restore source pointer
	cmp	esi, DESTARG		;; If esi was not restored, then we 
	jne	short fftdn1		;; just finished skipped postfft data
	sub	ebx, ebx
	mov	DIST_TO_FFTSRCARG, ebx
	cmp	norm_ptr1, 0		;; What was the loop start point?
	je	pass2			;; FFT-only, go do pass 2

;; Split the carries into high and low carries (so they do not exceed
;; the desired number of bits per FFT word.  Then add the carries back to the
;; FFT data.

	start_timer 28
	call	gw_carries
	end_timer 28

;; Have we been given permission to start the FFT on the result?
;; If so, set loop to FFT the first few cache lines now that the carries
;; have been added in.

	cmp	POSTFFT, 0		;; Test flag
	je	short fftdn1		;; Skip FFT code if flag not set
	mov	esi, DESTARG		;; Load FFT data address
	mov	edx, 65536+4/clm*256	;; Do 8 previously skipped cache lines
	mov	norm_ptr1, 0		;; Set flag so only forward FFT is done
	copy_4_words
	jmp	b0b

;; Now add in the FFT'ed carries, cleanup and return

fftdn1:	end_timer 1
	jmp	gw_finish_mult
	ENDM


;; Perform a 64K-element negacyclic FFT.  This is done in two passes.  Pass 1
;; does 8 levels, pass 2 uses common code to perform the last 8 levels.

pass1levels8complex4 MACRO pass2_macro, p2cl, clmarg
	LOCAL	b0b, b1a, b1b, b3b, b5a, b5b, b7a, b7b
	LOCAL	c0b, c1a, c1b, c3b, c5a, c5b, c7a, c7b
	LOCAL	pass2, copyback, yes_fft, no_fft, more, fftdn, fftdn1

	set_FFT_constants clmarg

;; Call FFT code unless the FFT has already been started

	clear_timers
	cmp	DWORD PTR [esi-28][ebx], 0 ;; Test FFT-started flag
	jne	pass2			;; Jump is FFT already started
	mov	norm_ptr1, 0		;; Set FFT-ing flag
	mov	edx, p2cl/128*65536+128/clm*256;; Load loop counter
pfing	mov	ebp, clm*32		;; Set prefetch distance
	jmp	b0b			;; Do the pass 1 fft

;; Do the last 8 FFT levels
;;	do 128 pass2_8_levels_complex

pass2:	start_timer 0
	CALLP	pass2_macro
	end_timer 0
	cmp	ffttype, 1		;; We're done if FFTing only
	je	gw_finish_fft

;; Do the inverse FFT

	start_timer 1
	fldz				;; Init SUMOUT
	fstp	SUMOUT
	mov	edx, norm_biglit_array	;; Addr of the big/little flags array
	mov	norm_ptr1, edx		;; Save ptr
	mov	edx, norm_col_mults	;; Addr of the column multipliers
	mov	norm_ptr2, edx		;; Save ptr
	mov	edx, p2cl/128*65536+128/clm*256;; Load loop counter
pfing	mov	ebp, clm*32		;; Set prefetch distance

;; Do inverse FFT levels 7,8
;; On input the 32-byte cache lines hold these data values:
;;	0	256	1	257
;;	512	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	1K	1	1K+1
;;	256	...
;;	...
;;	2K	...
;;	...

;; Do 32 four_complex_unfft macros
;;	distance between fft data elements is 1

c0b:	start_timer 16
	mov	ecx, scratch_area	;; Get address of scratch area
	mov	edi, sincos1		;; Load sin/cos pointer
	mov	eax, 4			;; 4 iterations 8
c3b:	cp_disp four_complex_unfft, 8, blkdst, 2*blkdst, 8, clmblkdst, 2*clmblkdst
	lea	esi, [esi+dist1]	;; Next source pointer
	lea	ecx, [ecx+dist1]	;; Next dest pointer
	add	dl, 256/2/clm		;; Test loop counter
	jnc	c3b			;; Iterate if necessary
	lea	esi, [esi-clm*2*dist1+4*blkdst];; Next source pointer
	lea	ecx, [ecx-clm*2*dist1+4*clmblkdst];; Next dest pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	add	ah, 256/8		;; Test loop counter
	jnc	c3b			;; Iterate if necessary
	lea	ecx, [ecx-8*4*clmblkdst+clmblkdst32];; Next dest pointer
	dec	al			;; Test outer loop counter
	jnz	c3b			;; Iterate if necessary
	lea	ecx, [esi-32*4*blkdst]	;; Create prefetch pointer
	mov	esi, scratch_area	;; Restore source pointer
	end_timer 16

;; Do inverse FFT levels 5,6
;; On input the 32-byte cache lines hold these data values:
;;	0	1K	1	1K+1
;;	256	...
;;	...
;;	2K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	4K	1	4K+1
;;	256	...
;;	...
;;	8K	...
;;	...

;; Do 32 four_complex_unfft macros
;;	distance between fft data elements is 4

	start_timer 17
	mov	edi, sincos2		;; Load sin/cos pointer
	mov	eax, 4			;; 4 iters of 2 iters of 4
c5a:	touch				;; Preload TLB
c5b:	disp four_complex_unfft, 8, 4*clmblkdst, 8*clmblkdst
	loop1	c5b, clmblkdst		;; 2*clm iterations
	add	ah, 256/4		;; Test middle loop counter
	jnc	c5a			;; Iterate if necessary
	lea	esi, [esi-4*clmblkdst+16*clmblkdst];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	add	eax, 80000000h		;; Test loop counter
	jnc	c5a			;; Iterate if necessary
	lea	esi, [esi-2*16*clmblkdst+clmblkdst32];; Next source pointer
	dec	al
	jnz	c5a
	lea	esi, [esi-4*clmblkdst32];; Restore source pointer
	end_timer 17

;; Do inverse FFT levels 4
;; On input the 32-byte cache lines hold these data values:
;;	0	4K	1	4K+1
;;	256	...
;;	...
;;	8K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	8K	1	8K+1
;;	256	...
;;	...
;;	16K	...
;;	...

;; Do 32 four_complex_unfft_1 macros
;;	distance between fft data elements is 16

	start_timer 18
	mov	edi, sincos3		;; Load sin/cos pointer
	mov	eax, 4			;; 4 iterations of 8
c7a:	touch				;; Preload TLB
c7b:	disp four_complex_unfft_1, 8*clmblkdst, 8, 16*clmblkdst
	loop2	c7b, clmblkdst		;; 2 * 2*clm iterations
	add	eax, 80000000h/2	;; Test middle loop counter (4 iters)
	jnc	c7a			;; Iterate if necessary
	lea	esi, [esi-8*clmblkdst+clmblkdst32];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al
	jnz	c7a
	lea	esi, [esi-4*clmblkdst32];; Restore source pointer
	end_timer 18

;; Do inverse FFT levels 1,2,3
;; On input the 32-byte cache lines hold these data values:
;;	0	8K	1	8K+1
;;	256	...
;;	...
;;	16K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	32K	1	32K+1
;;	256
;;	...
;;	32K-256

;; Do 32 four_complex_last_unfft macros
;;	distance between fft data elements is 32

	start_timer 19
	mov	edi, plus1_premults	;; Address of premultiplier table
	mov	al, 16			;; Load loop counter
c1a:	touch				;; Preload TLB
c1b:	disp four_complex_last_unfft, 8, clmblkdst32, 2*clmblkdst32
	loop2edi c1b, clmblkdst, 64	;; 2 * 2*clm iterations bumping edi
	dec	al			;; Test loop counter
	jnz	c1a			;; Iterate if necessary
	lea	esi, [esi-32*clmblkdst]	;; Restore source pointer
pfing	lea	ecx, [ecx-64*blkdst]	;; Restore dest ptr
	end_timer 19

;; Normalize these values

	start_timer 20
	mov	eax, NORMRTN		;; Addr of normalization routine
	call	eax
	sub	eax, eax
	end_timer 20

;; Have we been given permission to start the FFT on the result?
;; If so, do so now while the data is in the L2 cache.

	cmp	POSTFFT, 0		;; Test flag
	je	short copyback		;; Copy data, skip FFT if flag not set
	copy_3_words clm, 1
	cmp	edx, p2cl/128*65536+(128/clm-4/clm)*256;; Delay fft on 8 sets
	jle	yes_fft			;; No delay, do FFT now

;; Copy data from the scratch area back to the FFT data area

copyback:
	copy_scratch_data_to_fft 4, 32, clmblkdst32
	jmp	no_fft			;; Skip forward FFT code

;; Copy data from the FFT data area back to the scratch area

b0b:	copy_fft_data_to_scratch 4, 32, clmblkdst32

;; Do FFT levels 1,2,3
;;
;; On input the 32-byte cache lines hold these data values:
;;	0	32K	1	32K+1
;;	256
;;	...
;;	32K-256
;; On output the 32-byte cache lines hold these data values:
;;	0	8K	1	8K+1
;;	256	...
;;	...
;;	16K	...
;;	...

;; Do 32 four_complex_fft_premult macros
;;	distance between fft data elements is 32

yes_fft:start_timer 21
	mov	edi, plus1_premults	;; Address of premultiplier table
	mov	eax, 16			;; 32 iterations
pfing	lea	ecx, [ecx+64*blkdst]	;; Create prefetch ptr
b1a:	touch				;; Preload TLB
b1b:	disp four_complex_fft_premult, clmblkdst32, 2*clmblkdst32, 8
	loop2edi b1b, clmblkdst, 64	;; 2 * 2*clm iterations bumping edi
	dec	al			;; Test loop counter
	jnz	b1a			;; Iterate if necessary
	lea	esi, [esi-32*clmblkdst]	;; Restore source pointer
	end_timer 21

;; Do FFT levels 4
;;
;; On input the 32-byte cache lines hold these data values:
;;	0	8K	1	8K+1
;;	256	...
;;	...
;;	16K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	4K	1	4K+1
;;	256	...
;;	...
;;	8K	...
;;	...

;; Do 32 four_complex_fft_1 macros
;;	distance between fft data elements is 16

	start_timer 22
	mov	edi, sincos3		;; Load sin/cos pointer
	mov	eax, 4			;; 4 iterations of 8
b7a:	touch				;; Preload TLB
b7b:	disp four_complex_fft_1, 8*clmblkdst, 16*clmblkdst, 8
	loop2	b7b, clmblkdst		;; 2 * 2*clm iterations
	add	eax, 80000000h/2	;; Test middle loop counter (4 iters)
	jnc	b7a			;; Iterate if necessary
	lea	esi, [esi-8*clmblkdst+clmblkdst32];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al
	jnz	b7a
	lea	esi, [esi-4*clmblkdst32];; Restore source pointer
	end_timer 22

;; Do FFT levels 5,6
;;
;; On input the 32-byte cache lines hold these data values:
;;	0	4K	1	4K+1
;;	256	...
;;	...
;;	8K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	1K	1	1K+1
;;	256	...
;;	...
;;	2K	...
;;	...

;; Do 32 four_complex_fft macros
;;	distance between fft data elements is 4

	start_timer 23
	mov	edi, sincos2		;; Load sin/cos pointer
	mov	eax, 4			;; 4 iters of 2 iters of 4
b5a:	touch				;; Preload TLB
b5b:	disp four_complex_fft, 4*clmblkdst, 8*clmblkdst, 8
	loop1	b5b, clmblkdst		;; 2*clm iterations
	add	ah, 256/4		;; Test middle loop counter
	jnc	b5a			;; Iterate if necessary
	lea	esi, [esi-4*clmblkdst+16*clmblkdst];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	add	eax, 80000000h		;; Test middle loop counter (2 iters)
	jnc	b5a			;; Iterate if necessary
	lea	esi, [esi-2*16*clmblkdst+clmblkdst32];; Next source pointer
	dec	al
	jnz	b5a
	lea	esi, [esi-4*clmblkdst32];; Restore source pointer
	end_timer 23

;; Do FFT levels 7,8
;;
;; On input the 32-byte cache lines hold these data values:
;;	0	1K	1	1K+1
;;	256	...
;;	...
;;	2K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	256	1	257
;;	512	...
;;	...

;; Do 32 four_complex_fft macros
;;	distance between fft data elements is 1

	start_timer 24
	mov	edi, sincos1		;; Load sin/cos pointer
pfing	lea	ecx, [ecx-128*blkdst]	;; Create dest ptr from prefetch ptr
	mov	eax, 4			;; 4 iters of 8 iters
b3b:	cp_disp four_complex_fft, clmblkdst, 2*clmblkdst, 8, blkdst, 2*blkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	lea	ecx, [ecx+dist1]	;; Next dest pointer
	add	dl, 256/2/clm		;; Test inner loop counter
	jnc	b3b			;; Iterate if necessary
	lea	esi, [esi-clm*2*dist1+4*clmblkdst];; Next source pointer
	lea	ecx, [ecx-clm*2*dist1+4*blkdst];; Next dest pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	add	ah, 256/8		;; Test loop counter
	jnc	b3b			;; Iterate if necessary
	lea	esi, [esi-8*4*clmblkdst+clmblkdst32];; Next source pointer
	dec	al
	jnz	b3b
	lea	esi, [ecx-32*4*blkdst]	;; Restore source pointer
	end_timer 24

;; Work on next 4 sets of 32 values

no_fft:	lea	esi, [esi+clm*2*dist1]	;; Next source pointer
	dec	dh			;; Test loop counter
	jnz	short more		;; Do more FFTs
	lea	esi, [esi+64]		;; Skip pad every 128 cache lines (4KB)
pfing	mov	ebp, clm*32		;; Set prefetch distance
	sub	edx, 65536		;; Test loop counter
	jz	short fftdn		;; Jump if done looping
	mov	dh, 128/clm		;; Restore cache line counter
more:	set_prefetch_gap
	cmp	norm_ptr1, 0		;; What was the loop start point?
	jne	c0b			;; Do inverse-FFT/norm/FFT iteration
	jmp	b0b			;; Do another FFT-only iteration
fftdn:	lea	esi, [esi-p2cl*2*dist1-p2cl/128*64];; Restore source pointer
	cmp	esi, DESTARG		;; If esi was not restored, then we 
	jne	short fftdn1		;; just finished skipped postfft data
	sub	ebx, ebx
	mov	DIST_TO_FFTSRCARG, ebx
	cmp	norm_ptr1, 0		;; What was the loop start point?
	je	pass2			;; FFT-only, go do pass 2

;; Split the carries into high and low carries (so they do not exceed
;; the desired number of bits per FFT word.  Then add the carries back to the
;; FFT data.

	start_timer 28
	call	gw_carries
	end_timer 28

;; Have we been given permission to start the FFT on the result?
;; If so, set loop to FFT the first few cache lines now that the carries
;; have been added in.

	cmp	POSTFFT, 0		;; Test flag
	je	short fftdn1		;; Skip FFT code if flag not set
	mov	esi, DESTARG		;; Load FFT data address
	mov	edx, 65536+4/clm*256	;; Do 8 previously skipped cache lines
	mov	norm_ptr1, 0		;; Set flag so only forward FFT is done
	copy_4_words
	jmp	b0b

;; Now add in the FFT'ed carries, cleanup and return

fftdn1:	end_timer 1
	jmp	gw_finish_mult
	ENDM


;; Perform a 80K-element FFT.  This is done in two passes.  Pass 1
;; does 9 levels, pass 2 uses common code to perform the last 8 levels.

pass1levels9pfa5 MACRO pass2_macro, p2cl, clmarg
	LOCAL	b0b, b1a, b1b, b2b, b3b, b4b, b5a, b5b, b6a, b6b, b7a, b7b
	LOCAL	c0b, c1a, c1b, c2b, c3b, c4b, c5a, c5b, c6a, c6b, c7a, c7b
	LOCAL	pass2, copyback, yes_fft, no_fft, more, fftdn, fftdn1

	set_FFT_constants clmarg

;; Call FFT code unless the FFT has already been started

	clear_timers
	cmp	DWORD PTR [esi-28][ebx], 0 ;; Test FFT-started flag
	jne	pass2			;; Jump is FFT already started
	mov	norm_ptr1, 0		;; Set FFT-ing flag
	mov	edx, p2cl/128*65536+128/clm*256;; Load loop counter
pfing	mov	ebp, clm*32		;; Set prefetch distance
	jmp	b0b			;; Do the pass 1 fft

;; Do the last 8 FFT levels
;;	do 1 pass2_8_levels_real and 159 pass2_8_levels_complex

pass2:	start_timer 0
	CALLP	pass2_macro
	end_timer 0
	cmp	ffttype, 1		;; We're done if FFTing only
	je	gw_finish_fft

;; Do the inverse FFT

	start_timer 1
	fldz				;; Init SUMOUT
	fstp	SUMOUT
	mov	edx, norm_biglit_array	;; Addr of the big/little flags array
	mov	norm_ptr1, edx		;; Save ptr
	mov	edx, norm_col_mults	;; Addr of the column multipliers
	mov	norm_ptr2, edx		;; Save ptr
	mov	edx, p2cl/128*65536+128/clm*256;; Load loop counter
pfing	mov	ebp, clm*32		;; Set prefetch distance

;; Do inverse FFT levels 8,9
;; On input the 32-byte cache lines hold these data values:
;;	0	256	1	257
;;	512	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	1K	1	1K+1
;;	256	...
;;	...
;;	2K	...
;;	...

;; Do 1 four_real_four_semireal_unfft macros
;;	distance between fft data elements is 1

c0b:	start_timer 16
	mov	ecx, scratch_area	;; Get address of scratch area
c2b:	cp_disp four_real_four_semireal_unfft, 8, blkdst, 2*blkdst, 8, clmblkdst, 2*clmblkdst
	lea	esi, [esi+dist1]	;; Next source pointer
	lea	ecx, [ecx+dist1]	;; Next dest pointer
	add	dl, 256/2/clm		;; Test inner loop counter
	jnc	c2b			;; Iterate if necessary
	lea	esi, [esi-clm*2*dist1+4*blkdst];; Next source pointer
	lea	ecx, [ecx-clm*2*dist1+4*clmblkdst];; Next dest pointer

;; Do 39 four_complex_unfft macros
;;	distance between fft data elements is 1

	mov	edi, sincos1		;; Load sin/cos pointer
	mov	eax, 256/8*256+5	;; 5 iterations of 7 or 8
c3b:	cp_disp four_complex_unfft, 8, blkdst, 2*blkdst, 8, clmblkdst, 2*clmblkdst
	lea	esi, [esi+dist1]	;; Next source pointer
	lea	ecx, [ecx+dist1]	;; Next dest pointer
	add	dl, 256/2/clm		;; Test loop counter
	jnc	c3b			;; Iterate if necessary
	lea	esi, [esi-clm*2*dist1+4*blkdst];; Next source pointer
	lea	ecx, [ecx-clm*2*dist1+4*clmblkdst];; Next dest pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	add	ah, 256/8		;; Test loop counter
	jnc	c3b			;; Iterate if necessary
	lea	ecx, [ecx-8*4*clmblkdst+clmblkdst32];; Next dest pointer
	dec	al			;; Test outer loop counter
	jnz	c3b			;; Iterate if necessary
	lea	ecx, [esi-40*4*blkdst]	;; Create prefetch pointer
	mov	esi, scratch_area	;; Restore source pointer
	end_timer 16

;; Do inverse FFT levels 6,7
;; On input the 32-byte cache lines hold these data values:
;;	0	1K	1	1K+1
;;	256	...
;;	...
;;	2K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	4K	1	4K+1
;;	256	...
;;	...
;;	8K	...
;;	...

;; Do 4 four_real_four_semireal_unfft macros
;;	distance between fft data elements is 4

	start_timer 17
	touch				;; Preload TLB
c4b:	disp four_real_four_semireal_unfft, 8, 4*clmblkdst, 8*clmblkdst
	loop4	c4b, clmblkdst		;; 4 * 2*clm iterations
	lea	esi, [esi-4*clmblkdst+16*clmblkdst];; Next source pointer

;; Do 36 four_complex_unfft macros
;;	distance between fft data elements is 4

	mov	edi, sincos2		;; Load sin/cos pointer
	mov	eax, 80000000h+5	;; 5 iters of 1 or 2 iters of 4
c5a:	touch				;; Preload TLB
c5b:	disp four_complex_unfft, 8, 4*clmblkdst, 8*clmblkdst
	loop2	c5b, clmblkdst		;; 2 * 2*clm iterations
	add	edx, 80000000h		;; Test middle loop counter (2 iters)
	jnc	c5a			;; Iterate if necessary
	lea	esi, [esi-4*clmblkdst+16*clmblkdst];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	add	eax, 80000000h		;; Test loop counter
	jnc	c5a			;; Iterate if necessary
	lea	esi, [esi-2*16*clmblkdst+clmblkdst32];; Next source pointer
	dec	al
	jnz	c5a
	lea	esi, [esi-5*clmblkdst32];; Restore source pointer
	end_timer 17

;; Do inverse FFT levels 4,5
;; On input the 32-byte cache lines hold these data values:
;;	0	4K	1	4K+1
;;	256	...
;;	...
;;	8K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	16K	1	16K+1
;;	256	...
;;	...
;;	32K	...
;;	...

;; Do 8 eight_reals_unfft_2 macros
;;	distance between fft data elements is 16

	start_timer 18
	mov	al, 4			;; Load loop counter
c6a:	touch				;; Preload TLB
c6b:	disp eight_reals_unfft_2, 8*clmblkdst, 8, 16*clmblkdst
	loop2	c6b, clmblkdst		;; 2 * 2*clm iterations
	dec	al
	jnz	c6a
	lea	esi, [esi-8*clmblkdst+clmblkdst32];; Next source pointer

;; Do 32 four_complex_unfft macros
;;	distance between fft data elements is 16

	mov	edi, sincos3		;; Load sin/cos pointer
	mov	eax, 2			;; 2 iterations of 16
c7a:	touch				;; Preload TLB
c7b:	disp four_complex_unfft, 8, 16*clmblkdst, clmblkdst32
	loop2	c7b, clmblkdst		;; 2 * 2*clm iterations
	add	eax, 80000000h/4	;; Test middle loop counter (8 iters)
	jnc	c7a			;; Iterate if necessary
	lea	esi, [esi-16*clmblkdst+2*clmblkdst32];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al
	jnz	c7a
	lea	esi, [esi-2*2*clmblkdst32-clmblkdst32];; Restore source pointer
	end_timer 18

;; Do inverse FFT levels 1,2,3
;; On input the 32-byte cache lines hold these data values:
;;	0	16K	1	16K+1
;;	256	...
;;	...
;;	32K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	40K	1	40K+1
;;	256
;;	...
;;	40K-256

;; Do 64 five_reals_last_unfft macros
;;	distance between fft data elements is 64

	start_timer 19
	mov	al, 32			;; Load loop counter
c1a:	touch				;; Preload TLB
c1b:	two_five_reals_last_unfft 8, clmblkdst32
	loop1	c1b, clmblkdst		;; 2*clm iterations
	dec	al			;; Test loop counter
	jnz	c1a			;; Iterate if necessary
	lea	esi, [esi-32*clmblkdst]	;; Restore source pointer
pfing	lea	ecx, [ecx-71*blkdst]	;; Restore dest ptr
	end_timer 19

;; Normalize these values

	start_timer 20
	mov	eax, NORMRTN		;; Addr of normalization routine
	call	eax
	sub	eax, eax
	end_timer 20

;; Have we been given permission to start the FFT on the result?
;; If so, do so now while the data is in the L2 cache.

	cmp	POSTFFT, 0		;; Test flag
	je	short copyback		;; Copy data, skip FFT if flag not set
	copy_3_words clm, 1
	cmp	edx, p2cl/128*65536+(128/clm-4/clm)*256;; Delay fft on 8 sets
	jle	yes_fft			;; No delay, do FFT now

;; Copy data from the scratch area back to the FFT data area

copyback:
	copy_scratch_data_to_fft 5, 32, clmblkdst32
	jmp	no_fft			;; Skip forward FFT code

;; Copy data from the FFT data area back to the scratch area

b0b:	copy_fft_data_to_scratch 5, 32, clmblkdst32

;; Do FFT levels 1,2,3
;;
;; On input the 32-byte cache lines hold these data values:
;;	0	40K	1	40K+1
;;	256
;;	...
;;	40K-256
;; On output the 32-byte cache lines hold these data values:
;;	0	16K	1	16K+1
;;	256	...
;;	...
;;	32K	...
;;	...

;; Do 64 five_reals_fft macros
;;	distance between fft data elements is 64

yes_fft:start_timer 21
	mov	eax, 32			;; 32 iterations
pfing	lea	ecx, [ecx+71*blkdst]	;; Create prefetch ptr
b1a:	touch				;; Preload TLB
b1b:	two_five_reals_fft clmblkdst32, 8
	loop1	b1b, clmblkdst		;; 2*clm iterations
	dec	al			;; Test loop counter
	jnz	b1a			;; Iterate if necessary
	lea	esi, [esi-32*clmblkdst]	;; Restore source pointer
	end_timer 21

;; Do FFT levels 4,5
;;
;; On input the 32-byte cache lines hold these data values:
;;	0	16K	1	16K+1
;;	256	...
;;	...
;;	32K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	4K	1	4K+1
;;	256	...
;;	...
;;	8K	...
;;	...

;; Do 8 eight_reals_fft_2 macros
;;	distance between fft data elements is 16

	start_timer 22
	mov	al, 4			;; Load loop counter
b6a:	touch				;; Preload TLB
b6b:	disp eight_reals_fft_2, 8*clmblkdst, 16*clmblkdst, 8
	loop2	b6b, clmblkdst		;; 2 * 2*clm iterations
	dec	al
	jnz	b6a
	lea	esi, [esi-8*clmblkdst+clmblkdst32];; Next source pointer

;; Do 32 four_complex_fft macros
;;	distance between fft data elements is 16

	mov	edi, sincos3		;; Load sin/cos pointer
	mov	eax, 2			;; 2 iterations of 16
b7a:	touch				;; Preload TLB
b7b:	disp four_complex_fft, 16*clmblkdst, clmblkdst32, 8
	loop2	b7b, clmblkdst		;; 2 * 2*clm iterations
	add	eax, 80000000h/4	;; Test middle loop counter (8 iters)
	jnc	b7a			;; Iterate if necessary
	lea	esi, [esi-16*clmblkdst+2*clmblkdst32];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al
	jnz	b7a
	lea	esi, [esi-2*2*clmblkdst32-clmblkdst32];; Restore source pointer
	end_timer 22

;; Do FFT levels 6,7
;;
;; On input the 32-byte cache lines hold these data values:
;;	0	4K	1	4K+1
;;	256	...
;;	...
;;	8K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	1K	1	1K+1
;;	256	...
;;	...
;;	2K	...
;;	...

;; Do 4 four_real_four_semireal_fft macros
;;	distance between fft data elements is 4

	start_timer 23
	touch				;; Preload TLB
b4b:	disp four_real_four_semireal_fft, 4*clmblkdst, 8*clmblkdst, 8
	loop4	b4b, clmblkdst		;; 4 * 2*clm iterations
	lea	esi, [esi-4*clmblkdst+16*clmblkdst];; Next source pointer

;; Do 36 four_complex_fft macros
;;	distance between fft data elements is 4

	mov	edi, sincos2		;; Load sin/cos pointer
	mov	eax, 80000000h+5	;; 5 iters of 1 or 2 iters of 4
b5a:	touch				;; Preload TLB
b5b:	disp four_complex_fft, 4*clmblkdst, 8*clmblkdst, 8
	loop1	b5b, clmblkdst		;; 2*clm iterations
	add	ah, 256/4		;; Test middle loop counter
	jnc	b5a			;; Iterate if necessary
	lea	esi, [esi-4*clmblkdst+16*clmblkdst];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	add	eax, 80000000h		;; Test middle loop counter (2 iters)
	jnc	b5a			;; Iterate if necessary
	lea	esi, [esi-2*16*clmblkdst+clmblkdst32];; Next source pointer
	dec	al
	jnz	b5a
	lea	esi, [esi-5*clmblkdst32];; Restore source pointer
	end_timer 23

;; Do FFT levels 8,9
;;
;; On input the 32-byte cache lines hold these data values:
;;	0	1K	1	1K+1
;;	256	...
;;	...
;;	2K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	256	1	257
;;	512	...
;;	...

;; Do 1 four_real_four_semireal_fft macros
;;	distance between fft data elements is 1

	start_timer 24
pfing	lea	ecx, [ecx-160*blkdst]	;; Create dest ptr from prefetch ptr
b2b:	cp_disp four_real_four_semireal_fft, clmblkdst, 2*clmblkdst, 8, blkdst, 2*blkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	lea	ecx, [ecx+dist1]	;; Next dest pointer
	add	dl, 256/2/clm		;; Test inner loop counter
	jnc	b2b			;; Iterate if necessary
	lea	esi, [esi-clm*2*dist1+4*clmblkdst];; Next source pointer
	lea	ecx, [ecx-clm*2*dist1+4*blkdst];; Next dest pointer

;; Do 39 four_complex_fft macros
;;	distance between fft data elements is 1

	mov	edi, sincos1		;; Load sin/cos pointer
	mov	eax, 256/8*256+5	;; 5 iters of 7 or 8 iters
b3b:	cp_disp four_complex_fft, clmblkdst, 2*clmblkdst, 8, blkdst, 2*blkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	lea	ecx, [ecx+dist1]	;; Next dest pointer
	add	dl, 256/2/clm		;; Test inner loop counter
	jnc	b3b			;; Iterate if necessary
	lea	esi, [esi-clm*2*dist1+4*clmblkdst];; Next source pointer
	lea	ecx, [ecx-clm*2*dist1+4*blkdst];; Next dest pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	add	ah, 256/8		;; Test loop counter
	jnc	b3b			;; Iterate if necessary
	lea	esi, [esi-8*4*clmblkdst+clmblkdst32];; Next source pointer
	dec	al
	jnz	b3b
	lea	esi, [ecx-40*4*blkdst]	;; Restore source pointer
	end_timer 24

;; Work on next 4 sets of 32 values

no_fft:	lea	esi, [esi+clm*2*dist1]	;; Next source pointer
	dec	dh			;; Test loop counter
	jnz	short more		;; Do more FFTs
	lea	esi, [esi+64]		;; Skip pad every 128 cache lines (4KB)
pfing	mov	ebp, clm*32		;; Set prefetch distance
	sub	edx, 65536		;; Test loop counter
	jz	short fftdn		;; Jump if done looping
	mov	dh, 128/clm		;; Restore cache line counter
more:	set_prefetch_gap
	cmp	norm_ptr1, 0		;; What was the loop start point?
	jne	c0b			;; Do inverse-FFT/norm/FFT iteration
	jmp	b0b			;; Do another FFT-only iteration
fftdn:	lea	esi, [esi-p2cl*2*dist1-p2cl/128*64];; Restore source pointer
	cmp	esi, DESTARG		;; If esi was not restored, then we 
	jne	short fftdn1		;; just finished skipped postfft data
	sub	ebx, ebx
	mov	DIST_TO_FFTSRCARG, ebx
	cmp	norm_ptr1, 0		;; What was the loop start point?
	je	pass2			;; FFT-only, go do pass 2

;; Split the carries into high and low carries (so they do not exceed
;; the desired number of bits per FFT word.  Then add the carries back to the
;; FFT data.

	start_timer 28
	call	gw_carries
	end_timer 28

;; Have we been given permission to start the FFT on the result?
;; If so, set loop to FFT the first few cache lines now that the carries
;; have been added in.

	cmp	POSTFFT, 0		;; Test flag
	je	short fftdn1		;; Skip FFT code if flag not set
	mov	esi, DESTARG		;; Load FFT data address
	mov	edx, 65536+4/clm*256	;; Do 8 previously skipped cache lines
	mov	norm_ptr1, 0		;; Set flag so only forward FFT is done
	copy_4_words
	jmp	b0b

;; Now add in the FFT'ed carries, cleanup and return

fftdn1:	end_timer 1
	jmp	gw_finish_mult
	ENDM


;; Perform a 96K-element FFT.  This is done in two passes.  Pass 1
;; does 9 levels, pass 2 uses common code to perform the last 8 levels.

pass1levels9pfa6 MACRO pass2_macro, p2cl, clmarg
	LOCAL	b0b, b1a, b1b, b2b, b3b, b4b, b5a, b5b, b6a, b6b, b7a, b7b
	LOCAL	c0b, c1a, c1b, c2b, c3b, c4b, c5a, c5b, c6a, c6b, c7a, c7b
	LOCAL	pass2, copyback, yes_fft, no_fft, more, fftdn, fftdn1

	set_FFT_constants clmarg

;; Call FFT code unless the FFT has already been started

	clear_timers
	cmp	DWORD PTR [esi-28][ebx], 0 ;; Test FFT-started flag
	jne	pass2			;; Jump is FFT already started
	mov	norm_ptr1, 0		;; Set FFT-ing flag
	mov	edx, p2cl/128*65536+128/clm*256;; Load loop counter
pfing	mov	ebp, clm*32		;; Set prefetch distance
	jmp	b0b			;; Do the pass 1 fft

;; Do the last 8 FFT levels
;;	do 1 pass2_8_levels_real and 191 pass2_8_levels_complex

pass2:	start_timer 0
	CALLP	pass2_macro
	end_timer 0
	cmp	ffttype, 1		;; We're done if FFTing only
	je	gw_finish_fft

;; Do the inverse FFT

	start_timer 1
	fldz				;; Init SUMOUT
	fstp	SUMOUT
	mov	edx, norm_biglit_array	;; Addr of the big/little flags array
	mov	norm_ptr1, edx		;; Save ptr
	mov	edx, norm_col_mults	;; Addr of the column multipliers
	mov	norm_ptr2, edx		;; Save ptr
	mov	edx, p2cl/128*65536+128/clm*256;; Load loop counter
pfing	mov	ebp, clm*32		;; Set prefetch distance

;; Do inverse FFT levels 8,9
;; On input the 32-byte cache lines hold these data values:
;;	0	256	1	257
;;	512	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	1K	1	1K+1
;;	256	...
;;	...
;;	2K	...
;;	...

;; Do 1 four_real_four_semireal_unfft macros
;;	distance between fft data elements is 1

c0b:	start_timer 16
	mov	ecx, scratch_area	;; Get address of scratch area
c2b:	cp_disp four_real_four_semireal_unfft, 8, blkdst, 2*blkdst, 8, clmblkdst, 2*clmblkdst
	lea	esi, [esi+dist1]	;; Next source pointer
	lea	ecx, [ecx+dist1]	;; Next dest pointer
	add	dl, 256/2/clm		;; Test inner loop counter
	jnc	c2b			;; Iterate if necessary
	lea	esi, [esi-clm*2*dist1+4*blkdst];; Next source pointer
	lea	ecx, [ecx-clm*2*dist1+4*clmblkdst];; Next dest pointer

;; Do 47 four_complex_unfft macros
;;	distance between fft data elements is 1

	mov	edi, sincos1		;; Load sin/cos pointer
	mov	eax, 256/8*256+6	;; 6 iterations of 7 or 8
c3b:	cp_disp four_complex_unfft, 8, blkdst, 2*blkdst, 8, clmblkdst, 2*clmblkdst
	lea	esi, [esi+dist1]	;; Next source pointer
	lea	ecx, [ecx+dist1]	;; Next dest pointer
	add	dl, 256/2/clm		;; Test loop counter
	jnc	c3b			;; Iterate if necessary
	lea	esi, [esi-clm*2*dist1+4*blkdst];; Next source pointer
	lea	ecx, [ecx-clm*2*dist1+4*clmblkdst];; Next dest pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	add	ah, 256/8		;; Test loop counter
	jnc	c3b			;; Iterate if necessary
	lea	ecx, [ecx-8*4*clmblkdst+clmblkdst32];; Next dest pointer
	dec	al			;; Test outer loop counter
	jnz	c3b			;; Iterate if necessary
	lea	ecx, [esi-48*4*blkdst]	;; Create prefetch pointer
	mov	esi, scratch_area	;; Restore source pointer
	end_timer 16

;; Do inverse FFT levels 6,7
;; On input the 32-byte cache lines hold these data values:
;;	0	1K	1	1K+1
;;	256	...
;;	...
;;	2K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	4K	1	4K+1
;;	256	...
;;	...
;;	8K	...
;;	...

;; Do 4 four_real_four_semireal_unfft macros
;;	distance between fft data elements is 4

	start_timer 17
	touch				;; Preload TLB
c4b:	disp four_real_four_semireal_unfft, 8, 4*clmblkdst, 8*clmblkdst
	loop4	c4b, clmblkdst		;; 4 * 2*clm iterations
	lea	esi, [esi-4*clmblkdst+16*clmblkdst];; Next source pointer

;; Do 44 four_complex_unfft macros
;;	distance between fft data elements is 4

	mov	edi, sincos2		;; Load sin/cos pointer
	mov	eax, 80000000h+6	;; 6 iters of 1 or 2 iters of 4
c5a:	touch				;; Preload TLB
c5b:	disp four_complex_unfft, 8, 4*clmblkdst, 8*clmblkdst
	loop2	c5b, clmblkdst		;; 2 * 2*clm iterations
	add	edx, 80000000h		;; Test middle loop counter (2 iters)
	jnc	c5a			;; Iterate if necessary
	lea	esi, [esi-4*clmblkdst+16*clmblkdst];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	add	eax, 80000000h		;; Test loop counter
	jnc	c5a			;; Iterate if necessary
	lea	esi, [esi-2*16*clmblkdst+clmblkdst32];; Next source pointer
	dec	al
	jnz	c5a
	lea	esi, [esi-6*clmblkdst32];; Restore source pointer
	end_timer 17

;; Do inverse FFT levels 4,5
;; On input the 32-byte cache lines hold these data values:
;;	0	4K	1	4K+1
;;	256	...
;;	...
;;	8K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	16K	1	16K+1
;;	256	...
;;	...
;;	32K	...
;;	...

;; Do 16 four_real_four_semireal_unfft macros
;;	distance between fft data elements is 16

	start_timer 18
	mov	al, 8			;; Load loop counter
c6a:	touch				;; Preload TLB
c6b:	disp four_real_four_semireal_unfft, 8, 16*clmblkdst, clmblkdst32
	loop2	c6b, clmblkdst		;; 2 * 2*clm iterations
	dec	al
	jnz	c6a
	lea	esi, [esi-16*clmblkdst+2*clmblkdst32];; Next source pointer

;; Do 32 four_complex_unfft macros
;;	distance between fft data elements is 16

	mov	edi, sincos3		;; Load sin/cos pointer
	mov	eax, 2			;; 2 iterations of 16
c7a:	touch				;; Preload TLB
c7b:	disp four_complex_unfft, 8, 16*clmblkdst, clmblkdst32
	loop2	c7b, clmblkdst		;; 2 * 2*clm iterations
	add	eax, 80000000h/4	;; Test middle loop counter (8 iters)
	jnc	c7a			;; Iterate if necessary
	lea	esi, [esi-16*clmblkdst+2*clmblkdst32];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al
	jnz	c7a
	lea	esi, [esi-3*2*clmblkdst32];; Restore source pointer
	end_timer 18

;; Do inverse FFT levels 1,2,3
;; On input the 32-byte cache lines hold these data values:
;;	0	16K	1	16K+1
;;	256	...
;;	...
;;	32K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	48K	1	48K+1
;;	256
;;	...
;;	48K-256

;; Do 64 six_reals_last_unfft macros
;;	distance between fft data elements is 64

	start_timer 19
c1a:	touch				;; Preload TLB
c1b:	disp six_reals_last_unfft, 8, 2*clmblkdst32, 4*clmblkdst32
	loop2	c1b, clmblkdst		;; 2 * 2*clm iterations
	add	eax, 80000000h/8	;; Test loop counter (16 iters)
	jnc	c1a			;; Iterate if necessary
	lea	esi, [esi-32*clmblkdst+clmblkdst32];; Next source pointer
	add	edx, 80000000h		;; Test loop counter (2 iters)
	jnc	c1a			;; Iterate if necessary
	lea	esi, [esi-2*clmblkdst32];; Restore source pointer
pfing	lea	ecx, [ecx-79*blkdst]	;; Restore dest ptr
	end_timer 19

;; Normalize these values

	start_timer 20
	mov	eax, NORMRTN		;; Addr of normalization routine
	call	eax
	sub	eax, eax
	end_timer 20

;; Have we been given permission to start the FFT on the result?
;; If so, do so now while the data is in the L2 cache.

	cmp	POSTFFT, 0		;; Test flag
	je	short copyback		;; Copy data, skip FFT if flag not set
	copy_3_words clm, 1
	cmp	edx, p2cl/128*65536+(128/clm-4/clm)*256;; Delay fft on 8 sets
	jle	yes_fft			;; No delay, do FFT now

;; Copy data from the scratch area back to the FFT data area

copyback:
	copy_scratch_data_to_fft 6, 32, clmblkdst32
	jmp	no_fft			;; Skip forward FFT code

;; Copy data from the FFT data area back to the scratch area

b0b:	copy_fft_data_to_scratch 6, 32, clmblkdst32

;; Do FFT levels 1,2,3
;;
;; On input the 32-byte cache lines hold these data values:
;;	0	48K	1	48K+1
;;	256
;;	...
;;	48K-256
;; On output the 32-byte cache lines hold these data values:
;;	0	16K	1	16K+1
;;	256	...
;;	...
;;	32K	...
;;	...

;; Do 64 six_reals_fft macros
;;	distance between fft data elements is 64

yes_fft:start_timer 21
	sub	eax, eax		;; 64 iterations
pfing	lea	ecx, [ecx+79*blkdst]	;; Create prefetch ptr
b1a:	touch				;; Preload TLB
b1b:	disp six_reals_fft, 2*clmblkdst32, 4*clmblkdst32, 8
	loop2	b1b, clmblkdst		;; 2 * 2*clm iterations
	add	eax, 80000000h/8	;; Test loop counter (16 iters)
	jnc	b1a			;; Iterate if necessary
	lea	esi, [esi-32*clmblkdst+clmblkdst32];; Next source pointer
	add	edx, 80000000h		;; Test loop counter (2 iters)
	jnc	b1a			;; Iterate if necessary
	lea	esi, [esi-2*clmblkdst32];; Restore source pointer
	end_timer 21

;; Do FFT levels 4,5
;;
;; On input the 32-byte cache lines hold these data values:
;;	0	16K	1	16K+1
;;	256	...
;;	...
;;	32K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	4K	1	4K+1
;;	256	...
;;	...
;;	8K	...
;;	...

;; Do 16 four_real_four_semireal_fft macros
;;	distance between fft data elements is 16

	start_timer 22
	mov	al, 4			;; Load loop counter
b6a:	touch				;; Preload TLB
b6b:	disp four_real_four_semireal_fft, 16*clmblkdst, clmblkdst32, 8
	loop4	b6b, clmblkdst		;; 4 * 2*clm iterations
	dec	al
	jnz	b6a
	lea	esi, [esi-16*clmblkdst+2*clmblkdst32];; Next source pointer

;; Do 32 four_complex_fft macros
;;	distance between fft data elements is 16

	mov	edi, sincos3		;; Load sin/cos pointer
	mov	eax, 2			;; 2 iterations of 16
b7a:	touch				;; Preload TLB
b7b:	disp four_complex_fft, 16*clmblkdst, clmblkdst32, 8
	loop1	b7b, clmblkdst		;; 2*clm iterations
	add	ah, 256/16		;; Test middle loop counter
	jnc	b7a			;; Iterate if necessary
	lea	esi, [esi-16*clmblkdst+2*clmblkdst32];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al
	jnz	b7a
	lea	esi, [esi-3*2*clmblkdst32];; Restore source pointer
	end_timer 22

;; Do FFT levels 6,7
;;
;; On input the 32-byte cache lines hold these data values:
;;	0	4K	1	4K+1
;;	256	...
;;	...
;;	8K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	1K	1	1K+1
;;	256	...
;;	...
;;	2K	...
;;	...

;; Do 4 four_real_four_semireal_fft macros
;;	distance between fft data elements is 4

	start_timer 23
	touch				;; Preload TLB
b4b:	disp four_real_four_semireal_fft, 4*clmblkdst, 8*clmblkdst, 8
	loop4	b4b, clmblkdst		;; 4 * 2*clm iterations
	lea	esi, [esi-4*clmblkdst+16*clmblkdst];; Next source pointer

;; Do 44 four_complex_fft macros
;;	distance between fft data elements is 4

	mov	edi, sincos2		;; Load sin/cos pointer
	mov	eax, 80000000h+6	;; 6 iters of 1 or 2 iters of 4
b5a:	touch				;; Preload TLB
b5b:	disp four_complex_fft, 4*clmblkdst, 8*clmblkdst, 8
	loop1	b5b, clmblkdst		;; 2*clm iterations
	add	ah, 256/4		;; Test middle loop counter
	jnc	b5a			;; Iterate if necessary
	lea	esi, [esi-4*clmblkdst+16*clmblkdst];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	add	eax, 80000000h		;; Test middle loop counter (2 iters)
	jnc	b5a			;; Iterate if necessary
	lea	esi, [esi-2*16*clmblkdst+clmblkdst32];; Next source pointer
	dec	al
	jnz	b5a
	lea	esi, [esi-6*clmblkdst32];; Restore source pointer
	end_timer 23

;; Do FFT levels 8,9
;;
;; On input the 32-byte cache lines hold these data values:
;;	0	1K	1	1K+1
;;	256	...
;;	...
;;	2K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	256	1	257
;;	512	...
;;	...

;; Do 1 four_real_four_semireal_fft macros
;;	distance between fft data elements is 1

	start_timer 24
pfing	lea	ecx, [ecx-192*blkdst]	;; Create dest ptr from prefetch ptr
b2b:	cp_disp four_real_four_semireal_fft, clmblkdst, 2*clmblkdst, 8, blkdst, 2*blkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	lea	ecx, [ecx+dist1]	;; Next dest pointer
	add	dl, 256/2/clm		;; Test inner loop counter
	jnc	b2b			;; Iterate if necessary
	lea	esi, [esi-clm*2*dist1+4*clmblkdst];; Next source pointer
	lea	ecx, [ecx-clm*2*dist1+4*blkdst];; Next dest pointer

;; Do 47 four_complex_fft macros
;;	distance between fft data elements is 1

	mov	edi, sincos1		;; Load sin/cos pointer
	mov	eax, 256/8*256+6	;; 6 iters of 7 or 8 iters
b3b:	cp_disp four_complex_fft, clmblkdst, 2*clmblkdst, 8, blkdst, 2*blkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	lea	ecx, [ecx+dist1]	;; Next dest pointer
	add	dl, 256/2/clm		;; Test inner loop counter
	jnc	b3b			;; Iterate if necessary
	lea	esi, [esi-clm*2*dist1+4*clmblkdst];; Next source pointer
	lea	ecx, [ecx-clm*2*dist1+4*blkdst];; Next dest pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	add	ah, 256/8		;; Test loop counter
	jnc	b3b			;; Iterate if necessary
	lea	esi, [esi-8*4*clmblkdst+clmblkdst32];; Next source pointer
	dec	al
	jnz	b3b
	lea	esi, [ecx-48*4*blkdst]	;; Restore source pointer
	end_timer 24

;; Work on next 4 sets of 32 values

no_fft:	lea	esi, [esi+clm*2*dist1]	;; Next source pointer
	dec	dh			;; Test loop counter
	jnz	short more		;; Do more FFTs
	lea	esi, [esi+64]		;; Skip pad every 128 cache lines (4KB)
pfing	mov	ebp, clm*32		;; Set prefetch distance
	sub	edx, 65536		;; Test loop counter
	jz	short fftdn		;; Jump if done looping
	mov	dh, 128/clm		;; Restore cache line counter
more:	set_prefetch_gap
	cmp	norm_ptr1, 0		;; What was the loop start point?
	jne	c0b			;; Do inverse-FFT/norm/FFT iteration
	jmp	b0b			;; Do another FFT-only iteration
fftdn:	lea	esi, [esi-p2cl*2*dist1-p2cl/128*64];; Restore source pointer
	cmp	esi, DESTARG		;; If esi was not restored, then we 
	jne	short fftdn1		;; just finished skipped postfft data
	sub	ebx, ebx
	mov	DIST_TO_FFTSRCARG, ebx
	cmp	norm_ptr1, 0		;; What was the loop start point?
	je	pass2			;; FFT-only, go do pass 2

;; Split the carries into high and low carries (so they do not exceed
;; the desired number of bits per FFT word.  Then add the carries back to the
;; FFT data.

	start_timer 28
	call	gw_carries
	end_timer 28

;; Have we been given permission to start the FFT on the result?
;; If so, set loop to FFT the first few cache lines now that the carries
;; have been added in.

	cmp	POSTFFT, 0		;; Test flag
	je	short fftdn1		;; Skip FFT code if flag not set
	mov	esi, DESTARG		;; Load FFT data address
	mov	edx, 65536+4/clm*256	;; Do 8 previously skipped cache lines
	mov	norm_ptr1, 0		;; Set flag so only forward FFT is done
	copy_4_words
	jmp	b0b

;; Now add in the FFT'ed carries, cleanup and return

fftdn1:	end_timer 1
	jmp	gw_finish_mult
	ENDM


;; Perform a 96K-element negacyclic FFT.  This is done in two passes.  Pass 1
;; does 9 levels, pass 2 uses common code to perform the last 8 levels.

pass1levels9complex3 MACRO pass2_macro, p2cl, clmarg
	LOCAL	b0b, b1a, b1b, b3b, b5a, b5b, b7a, b7b
	LOCAL	c0b, c1a, c1b, c3b, c5a, c5b, c7a, c7b
	LOCAL	pass2, copyback, yes_fft, no_fft, more, fftdn, fftdn1

	set_FFT_constants clmarg

;; Call FFT code unless the FFT has already been started

	clear_timers
	cmp	DWORD PTR [esi-28][ebx], 0 ;; Test FFT-started flag
	jne	pass2			;; Jump is FFT already started
	mov	norm_ptr1, 0		;; Set FFT-ing flag
	mov	edx, p2cl/128*65536+128/clm*256;; Load loop counter
pfing	mov	ebp, clm*32		;; Set prefetch distance
	jmp	b0b			;; Do the pass 1 fft

;; Do the last 8 FFT levels
;;	do 192 pass2_8_levels_complex

pass2:	start_timer 0
	CALLP	pass2_macro
	end_timer 0
	cmp	ffttype, 1		;; We're done if FFTing only
	je	gw_finish_fft

;; Do the inverse FFT

	start_timer 1
	fldz				;; Init SUMOUT
	fstp	SUMOUT
	mov	edx, norm_biglit_array	;; Addr of the big/little flags array
	mov	norm_ptr1, edx		;; Save ptr
	mov	edx, norm_col_mults	;; Addr of the column multipliers
	mov	norm_ptr2, edx		;; Save ptr
	mov	edx, p2cl/128*65536+128/clm*256;; Load loop counter
pfing	mov	ebp, clm*32		;; Set prefetch distance

;; Do inverse FFT levels 8,9
;; On input the 32-byte cache lines hold these data values:
;;	0	256	1	257
;;	512	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	1K	1	1K+1
;;	256	...
;;	...
;;	2K	...
;;	...

;; Do 48 four_complex_unfft macros
;;	distance between fft data elements is 1

c0b:	start_timer 16
	mov	ecx, scratch_area	;; Get address of scratch area
	mov	edi, sincos1		;; Load sin/cos pointer
	mov	eax, 6			;; 6 iterations 8
c3b:	cp_disp four_complex_unfft, 8, blkdst, 2*blkdst, 8, clmblkdst, 2*clmblkdst
	lea	esi, [esi+dist1]	;; Next source pointer
	lea	ecx, [ecx+dist1]	;; Next dest pointer
	add	dl, 256/2/clm		;; Test loop counter
	jnc	c3b			;; Iterate if necessary
	lea	esi, [esi-clm*2*dist1+4*blkdst];; Next source pointer
	lea	ecx, [ecx-clm*2*dist1+4*clmblkdst];; Next dest pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	add	ah, 256/8		;; Test loop counter
	jnc	c3b			;; Iterate if necessary
	lea	ecx, [ecx-8*4*clmblkdst+clmblkdst32];; Next dest pointer
	dec	al			;; Test outer loop counter
	jnz	c3b			;; Iterate if necessary
	lea	ecx, [esi-48*4*blkdst]	;; Create prefetch pointer
	mov	esi, scratch_area	;; Restore source pointer
	end_timer 16

;; Do inverse FFT levels 6,7
;; On input the 32-byte cache lines hold these data values:
;;	0	1K	1	1K+1
;;	256	...
;;	...
;;	2K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	4K	1	4K+1
;;	256	...
;;	...
;;	8K	...
;;	...

;; Do 48 four_complex_unfft macros
;;	distance between fft data elements is 4

	start_timer 17
	mov	edi, sincos2		;; Load sin/cos pointer
	mov	eax, 6			;; 6 iters of 2 iters of 4
c5a:	touch				;; Preload TLB
c5b:	disp four_complex_unfft, 8, 4*clmblkdst, 8*clmblkdst
	loop2	c5b, clmblkdst		;; 2 * 2*clm iterations
	add	edx, 80000000h		;; Test middle loop counter (2 iters)
	jnc	c5a			;; Iterate if necessary
	lea	esi, [esi-4*clmblkdst+16*clmblkdst];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	add	eax, 80000000h		;; Test loop counter
	jnc	c5a			;; Iterate if necessary
	lea	esi, [esi-2*16*clmblkdst+clmblkdst32];; Next source pointer
	dec	al
	jnz	c5a
	lea	esi, [esi-6*clmblkdst32];; Restore source pointer
	end_timer 17

;; Do inverse FFT levels 4,5
;; On input the 32-byte cache lines hold these data values:
;;	0	4K	1	4K+1
;;	256	...
;;	...
;;	8K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	16K	1	16K+1
;;	256	...
;;	...
;;	32K	...
;;	...

;; Do 48 four_complex_unfft macros
;;	distance between fft data elements is 16

	start_timer 18
	mov	edi, sincos3		;; Load sin/cos pointer
	mov	eax, 3			;; 3 iterations of 16
c7a:	touch				;; Preload TLB
c7b:	disp four_complex_unfft, 8, 16*clmblkdst, clmblkdst32
	loop2	c7b, clmblkdst		;; 2 * 2*clm iterations
	add	eax, 80000000h/4	;; Test middle loop counter (8 iters)
	jnc	c7a			;; Iterate if necessary
	lea	esi, [esi-16*clmblkdst+2*clmblkdst32];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al
	jnz	c7a
	lea	esi, [esi-3*2*clmblkdst32];; Restore source pointer
	end_timer 18

;; Do inverse FFT levels 1,2,3
;; On input the 32-byte cache lines hold these data values:
;;	0	16K	1	16K+1
;;	256	...
;;	...
;;	32K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	48K	1	48K+1
;;	256
;;	...
;;	48K-256

;; Do 64 three_complex_last_unfft macros
;;	distance between fft data elements is 64

	start_timer 19
	mov	edi, plus1_premults	;; Address of premultiplier table
c1a:	touch				;; Preload TLB
c1b:	three_complex_last_unfft 8, 2*clmblkdst32, 4*clmblkdst32
	loop2edi c1b, clmblkdst, 48	;; 2 * 2*clm iterations bumping edi
	add	eax, 80000000h/8	;; Test loop counter (16 iters)
	jnc	c1a			;; Iterate if necessary
	lea	esi, [esi-32*clmblkdst+clmblkdst32];; Next source pointer
	add	edx, 80000000h		;; Test loop counter (2 iters)
	jnc	c1a			;; Iterate if necessary
	lea	esi, [esi-2*clmblkdst32];; Restore source pointer
pfing	lea	ecx, [ecx-80*blkdst]	;; Restore dest ptr
	end_timer 19

;; Normalize these values

	start_timer 20
	mov	eax, NORMRTN		;; Addr of normalization routine
	call	eax
	sub	eax, eax
	end_timer 20

;; Have we been given permission to start the FFT on the result?
;; If so, do so now while the data is in the L2 cache.

	cmp	POSTFFT, 0		;; Test flag
	je	short copyback		;; Copy data, skip FFT if flag not set
	copy_3_words clm, 1
	cmp	edx, p2cl/128*65536+(128/clm-4/clm)*256;; Delay fft on 8 sets
	jle	yes_fft			;; No delay, do FFT now

;; Copy data from the scratch area back to the FFT data area

copyback:
	copy_scratch_data_to_fft 6, 32, clmblkdst32
	jmp	no_fft			;; Skip forward FFT code

;; Copy data from the FFT data area back to the scratch area

b0b:	copy_fft_data_to_scratch 6, 32, clmblkdst32

;; Do FFT levels 1,2,3
;;
;; On input the 32-byte cache lines hold these data values:
;;	0	48K	1	48K+1
;;	256
;;	...
;;	48K-256
;; On output the 32-byte cache lines hold these data values:
;;	0	16K	1	16K+1
;;	256	...
;;	...
;;	32K	...
;;	...

;; Do 64 three_complex_fft_premult macros
;;	distance between fft data elements is 64

yes_fft:start_timer 21
	mov	edi, plus1_premults	;; Address of premultiplier table
	sub	eax, eax		;; 64 iterations
pfing	lea	ecx, [ecx+80*blkdst]	;; Create prefetch ptr
b1a:	touch				;; Preload TLB
b1b:	three_complex_fft_premult 2*clmblkdst32, 4*clmblkdst32, 8
	loop1edi b1b, clmblkdst, 48	;; 2*clm iterations bumping edi
	add	ah, 256/32		;; Test loop counter
	jnc	b1a			;; Iterate if necessary
	lea	esi, [esi-32*clmblkdst+clmblkdst32];; Next source pointer
	add	eax, 80000000h		;; Test loop counter (2 iters)
	jnc	b1a			;; Iterate if necessary
	lea	esi, [esi-2*clmblkdst32];; Restore source pointer
	end_timer 21

;; Do FFT levels 4,5
;;
;; On input the 32-byte cache lines hold these data values:
;;	0	16K	1	16K+1
;;	256	...
;;	...
;;	32K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	4K	1	4K+1
;;	256	...
;;	...
;;	8K	...
;;	...

;; Do 48 four_complex_fft macros
;;	distance between fft data elements is 16

	start_timer 22
	mov	edi, sincos3		;; Load sin/cos pointer
	mov	eax, 3			;; 3 iterations of 16
b7a:	touch				;; Preload TLB
b7b:	disp four_complex_fft, 16*clmblkdst, clmblkdst32, 8
	loop2	b7b, clmblkdst		;; 2 * 2*clm iterations
	add	eax, 80000000h/4	;; Test middle loop counter (8 iters)
	jnc	b7a			;; Iterate if necessary
	lea	esi, [esi-16*clmblkdst+2*clmblkdst32];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al
	jnz	b7a
	lea	esi, [esi-3*2*clmblkdst32];; Restore source pointer
	end_timer 22

;; Do FFT levels 6,7
;;
;; On input the 32-byte cache lines hold these data values:
;;	0	4K	1	4K+1
;;	256	...
;;	...
;;	8K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	1K	1	1K+1
;;	256	...
;;	...
;;	2K	...
;;	...

;; Do 48 four_complex_fft macros
;;	distance between fft data elements is 4

	start_timer 23
	mov	edi, sincos2		;; Load sin/cos pointer
	mov	eax, 6			;; 6 iters of 2 iters of 4
b5a:	touch				;; Preload TLB
b5b:	disp four_complex_fft, 4*clmblkdst, 8*clmblkdst, 8
	loop2	b5b, clmblkdst		;; 2 * 2*clm iterations
	add	edx, 80000000h		;; Test middle loop counter (2 iters)
	jnc	b5a			;; Iterate if necessary
	lea	esi, [esi-4*clmblkdst+16*clmblkdst];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	add	eax, 80000000h		;; Test middle loop counter (2 iters)
	jnc	b5a			;; Iterate if necessary
	lea	esi, [esi-2*16*clmblkdst+clmblkdst32];; Next source pointer
	dec	al
	jnz	b5a
	lea	esi, [esi-6*clmblkdst32];; Restore source pointer
	end_timer 23

;; Do FFT levels 8,9
;;
;; On input the 32-byte cache lines hold these data values:
;;	0	1K	1	1K+1
;;	256	...
;;	...
;;	2K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	256	1	257
;;	512	...
;;	...

;; Do 48 four_complex_fft macros
;;	distance between fft data elements is 1

	start_timer 24
	mov	edi, sincos1		;; Load sin/cos pointer
pfing	lea	ecx, [ecx-192*blkdst]	;; Create dest ptr from prefetch ptr
	mov	eax, 6			;; 6 iters of 8 iters
b3b:	cp_disp four_complex_fft, clmblkdst, 2*clmblkdst, 8, blkdst, 2*blkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	lea	ecx, [ecx+dist1]	;; Next dest pointer
	add	dl, 256/2/clm		;; Test inner loop counter
	jnc	b3b			;; Iterate if necessary
	lea	esi, [esi-clm*2*dist1+4*clmblkdst];; Next source pointer
	lea	ecx, [ecx-clm*2*dist1+4*blkdst];; Next dest pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	add	ah, 256/8		;; Test loop counter
	jnc	b3b			;; Iterate if necessary
	lea	esi, [esi-8*4*clmblkdst+clmblkdst32];; Next source pointer
	dec	al
	jnz	b3b
	lea	esi, [ecx-48*4*blkdst]	;; Restore source pointer
	end_timer 24

;; Work on next 4 sets of 32 values

no_fft:	lea	esi, [esi+clm*2*dist1]	;; Next source pointer
	dec	dh			;; Test loop counter
	jnz	short more		;; Do more FFTs
	lea	esi, [esi+64]		;; Skip pad every 128 cache lines (4KB)
pfing	mov	ebp, clm*32		;; Set prefetch distance
	sub	edx, 65536		;; Test loop counter
	jz	short fftdn		;; Jump if done looping
	mov	dh, 128/clm		;; Restore cache line counter
more:	set_prefetch_gap
	cmp	norm_ptr1, 0		;; What was the loop start point?
	jne	c0b			;; Do inverse-FFT/norm/FFT iteration
	jmp	b0b			;; Do another FFT-only iteration
fftdn:	lea	esi, [esi-p2cl*2*dist1-p2cl/128*64];; Restore source pointer
	cmp	esi, DESTARG		;; If esi was not restored, then we 
	jne	short fftdn1		;; just finished skipped postfft data
	sub	ebx, ebx
	mov	DIST_TO_FFTSRCARG, ebx
	cmp	norm_ptr1, 0		;; What was the loop start point?
	je	pass2			;; FFT-only, go do pass 2

;; Split the carries into high and low carries (so they do not exceed
;; the desired number of bits per FFT word.  Then add the carries back to the
;; FFT data.

	start_timer 28
	call	gw_carries
	end_timer 28

;; Have we been given permission to start the FFT on the result?
;; If so, set loop to FFT the first few cache lines now that the carries
;; have been added in.

	cmp	POSTFFT, 0		;; Test flag
	je	short fftdn1		;; Skip FFT code if flag not set
	mov	esi, DESTARG		;; Load FFT data address
	mov	edx, 65536+4/clm*256	;; Do 8 previously skipped cache lines
	mov	norm_ptr1, 0		;; Set flag so only forward FFT is done
	copy_4_words
	jmp	b0b

;; Now add in the FFT'ed carries, cleanup and return

fftdn1:	end_timer 1
	jmp	gw_finish_mult
	ENDM


;; Perform a 112K-element FFT.  This is done in two passes.  Pass 1
;; does 9 levels, pass 2 uses common code to perform the last 8 levels.

pass1levels9pfa7 MACRO pass2_macro, p2cl, clmarg
	LOCAL	b0b, b1a, b1b, b2b, b3b, b4b, b5a, b5b, b6a, b6b, b7a, b7b
	LOCAL	c0b, c1a, c1b, c2b, c3b, c4b, c5a, c5b, c6a, c6b, c7a, c7b
	LOCAL	pass2, copyback, yes_fft, no_fft, more, fftdn, fftdn1

	set_FFT_constants clmarg

;; Call FFT code unless the FFT has already been started

	clear_timers
	cmp	DWORD PTR [esi-28][ebx], 0 ;; Test FFT-started flag
	jne	pass2			;; Jump is FFT already started
	mov	norm_ptr1, 0		;; Set FFT-ing flag
	mov	edx, p2cl/128*65536+128/clm*256;; Load loop counter
pfing	mov	ebp, clm*32		;; Set prefetch distance
	jmp	b0b			;; Do the pass 1 fft

;; Do the last 8 FFT levels
;;	do 1 pass2_8_levels_real and 223 pass2_8_levels_complex

pass2:	start_timer 0
	CALLP	pass2_macro
	end_timer 0
	cmp	ffttype, 1		;; We're done if FFTing only
	je	gw_finish_fft

;; Do the inverse FFT

	start_timer 1
	fldz				;; Init SUMOUT
	fstp	SUMOUT
	mov	edx, norm_biglit_array	;; Addr of the big/little flags array
	mov	norm_ptr1, edx		;; Save ptr
	mov	edx, norm_col_mults	;; Addr of the column multipliers
	mov	norm_ptr2, edx		;; Save ptr
	mov	edx, p2cl/128*65536+128/clm*256;; Load loop counter
pfing	mov	ebp, clm*32		;; Set prefetch distance

;; Do inverse FFT levels 8,9
;; On input the 32-byte cache lines hold these data values:
;;	0	256	1	257
;;	512	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	1K	1	1K+1
;;	256	...
;;	...
;;	2K	...
;;	...

;; Do 1 four_real_four_semireal_unfft macros
;;	distance between fft data elements is 1

c0b:	start_timer 16
	mov	ecx, scratch_area	;; Get address of scratch area
c2b:	cp_disp four_real_four_semireal_unfft, 8, blkdst, 2*blkdst, 8, clmblkdst, 2*clmblkdst
	lea	esi, [esi+dist1]	;; Next source pointer
	lea	ecx, [ecx+dist1]	;; Next dest pointer
	add	dl, 256/2/clm		;; Test inner loop counter
	jnc	c2b			;; Iterate if necessary
	lea	esi, [esi-clm*2*dist1+4*blkdst];; Next source pointer
	lea	ecx, [ecx-clm*2*dist1+4*clmblkdst];; Next dest pointer

;; Do 55 four_complex_unfft macros
;;	distance between fft data elements is 1

	mov	edi, sincos1		;; Load sin/cos pointer
	mov	eax, 256/8*256+7	;; 7 iterations of 7 or 8
c3b:	cp_disp four_complex_unfft, 8, blkdst, 2*blkdst, 8, clmblkdst, 2*clmblkdst
	lea	esi, [esi+dist1]	;; Next source pointer
	lea	ecx, [ecx+dist1]	;; Next dest pointer
	add	dl, 256/2/clm		;; Test loop counter
	jnc	c3b			;; Iterate if necessary
	lea	esi, [esi-clm*2*dist1+4*blkdst];; Next source pointer
	lea	ecx, [ecx-clm*2*dist1+4*clmblkdst];; Next dest pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	add	ah, 256/8		;; Test loop counter
	jnc	c3b			;; Iterate if necessary
	lea	ecx, [ecx-8*4*clmblkdst+clmblkdst32];; Next dest pointer
	dec	al			;; Test outer loop counter
	jnz	c3b			;; Iterate if necessary
	lea	ecx, [esi-56*4*blkdst]	;; Create prefetch pointer
	mov	esi, scratch_area	;; Restore source pointer
	end_timer 16

;; Do inverse FFT levels 6,7
;; On input the 32-byte cache lines hold these data values:
;;	0	1K	1	1K+1
;;	256	...
;;	...
;;	2K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	4K	1	4K+1
;;	256	...
;;	...
;;	8K	...
;;	...

;; Do 4 four_real_four_semireal_unfft macros
;;	distance between fft data elements is 4

	start_timer 17
	touch				;; Preload TLB
c4b:	disp four_real_four_semireal_unfft, 8, 4*clmblkdst, 8*clmblkdst
	loop4	c4b, clmblkdst		;; 4 * 2*clm iterations
	lea	esi, [esi-4*clmblkdst+16*clmblkdst];; Next source pointer

;; Do 52 four_complex_unfft macros
;;	distance between fft data elements is 4

	mov	edi, sincos2		;; Load sin/cos pointer
	mov	eax, 80000000h+7	;; 7 iters of 1 or 2 iters of 4
c5a:	touch				;; Preload TLB
c5b:	disp four_complex_unfft, 8, 4*clmblkdst, 8*clmblkdst
	loop2	c5b, clmblkdst		;; 2 * 2*clm iterations
	add	edx, 80000000h		;; Test middle loop counter (2 iters)
	jnc	c5a			;; Iterate if necessary
	lea	esi, [esi-4*clmblkdst+16*clmblkdst];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	add	eax, 80000000h		;; Test loop counter
	jnc	c5a			;; Iterate if necessary
	lea	esi, [esi-2*16*clmblkdst+clmblkdst32];; Next source pointer
	dec	al
	jnz	c5a
	lea	esi, [esi-7*clmblkdst32];; Restore source pointer
	end_timer 17

;; Do inverse FFT levels 4,5
;; On input the 32-byte cache lines hold these data values:
;;	0	4K	1	4K+1
;;	256	...
;;	...
;;	8K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	16K	1	16K+1
;;	256	...
;;	...
;;	32K	...
;;	...

;; Do 8 eight_reals_unfft_2 macros
;;	distance between fft data elements is 16

	start_timer 18
	mov	al, 4			;; Load loop counter
c6a:	touch				;; Preload TLB
c6b:	disp eight_reals_unfft_2, 8*clmblkdst, 8, 16*clmblkdst
	loop2	c6b, clmblkdst		;; 2 * 2*clm iterations
	dec	al
	jnz	c6a
	lea	esi, [esi-8*clmblkdst+clmblkdst32];; Next source pointer

;; Do 48 four_complex_unfft macros
;;	distance between fft data elements is 16

	mov	edi, sincos3		;; Load sin/cos pointer
	mov	eax, 3			;; 3 iterations of 16
c7a:	touch				;; Preload TLB
c7b:	disp four_complex_unfft, 8, 16*clmblkdst, clmblkdst32
	loop2	c7b, clmblkdst		;; 2 * 2*clm iterations
	add	eax, 80000000h/4	;; Test middle loop counter (8 iters)
	jnc	c7a			;; Iterate if necessary
	lea	esi, [esi-16*clmblkdst+2*clmblkdst32];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al
	jnz	c7a
	lea	esi, [esi-3*2*clmblkdst32-clmblkdst32];; Restore source pointer
	end_timer 18

;; Do inverse FFT levels 1,2,3
;; On input the 32-byte cache lines hold these data values:
;;	0	16K	1	16K+1
;;	256	...
;;	...
;;	32K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	56K	1	56K+1
;;	256
;;	...
;;	56K-256

;; Do 64 seven_reals_last_unfft macros
;;	distance between fft data elements is 64

	start_timer 19
	mov	al, 32			;; Load loop counter
c1a:	touch				;; Preload TLB
c1b:	two_seven_reals_last_unfft 8, clmblkdst32
	loop1	c1b, clmblkdst		;; 2*clm iterations
	dec	al			;; Test loop counter
	jnz	c1a			;; Iterate if necessary
	lea	esi, [esi-32*clmblkdst]	;; Restore source pointer
pfing	lea	ecx, [ecx-87*blkdst]	;; Restore dest ptr
	end_timer 19

;; Normalize these values

	start_timer 20
	mov	eax, NORMRTN		;; Addr of normalization routine
	call	eax
	sub	eax, eax
	end_timer 20

;; Have we been given permission to start the FFT on the result?
;; If so, do so now while the data is in the L2 cache.

	cmp	POSTFFT, 0		;; Test flag
	je	short copyback		;; Copy data, skip FFT if flag not set
	copy_3_words clm, 1
	cmp	edx, p2cl/128*65536+(128/clm-4/clm)*256;; Delay fft on 8 sets
	jle	yes_fft			;; No delay, do FFT now

;; Copy data from the scratch area back to the FFT data area

copyback:
	copy_scratch_data_to_fft 7, 32, clmblkdst32
	jmp	no_fft			;; Skip forward FFT code

;; Copy data from the FFT data area back to the scratch area

b0b:	copy_fft_data_to_scratch 7, 32, clmblkdst32

;; Do FFT levels 1,2,3
;;
;; On input the 32-byte cache lines hold these data values:
;;	0	56K	1	56K+1
;;	256
;;	...
;;	56K-256
;; On output the 32-byte cache lines hold these data values:
;;	0	16K	1	16K+1
;;	256	...
;;	...
;;	32K	...
;;	...

;; Do 64 seven_reals_fft macros
;;	distance between fft data elements is 64

yes_fft:start_timer 21
	mov	eax, 32			;; 32 iterations
pfing	lea	ecx, [ecx+87*blkdst]	;; Create prefetch ptr
b1a:	touch				;; Preload TLB
b1b:	two_seven_reals_fft clmblkdst32, 8
	loop1	b1b, clmblkdst		;; 2*clm iterations
	dec	al			;; Test loop counter
	jnz	b1a			;; Iterate if necessary
	lea	esi, [esi-32*clmblkdst]	;; Restore source pointer
	end_timer 21

;; Do FFT levels 4,5
;;
;; On input the 32-byte cache lines hold these data values:
;;	0	16K	1	16K+1
;;	256	...
;;	...
;;	32K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	4K	1	4K+1
;;	256	...
;;	...
;;	8K	...
;;	...

;; Do 8 eight_reals_fft_2 macros
;;	distance between fft data elements is 16

	start_timer 22
	mov	al, 4			;; Load loop counter
b6a:	touch				;; Preload TLB
b6b:	disp eight_reals_fft_2, 8*clmblkdst, 16*clmblkdst, 8
	loop2	b6b, clmblkdst		;; 2 * 2*clm iterations
	dec	al
	jnz	b6a
	lea	esi, [esi-8*clmblkdst+clmblkdst32];; Next source pointer

;; Do 48 four_complex_fft macros
;;	distance between fft data elements is 16

	mov	edi, sincos3		;; Load sin/cos pointer
	mov	eax, 3			;; 3 iterations of 16
b7a:	touch				;; Preload TLB
b7b:	disp four_complex_fft, 16*clmblkdst, clmblkdst32, 8
	loop1	b7b, clmblkdst		;; 2*clm iterations
	add	ah, 256/16		;; Test middle loop counter
	jnc	b7a			;; Iterate if necessary
	lea	esi, [esi-16*clmblkdst+2*clmblkdst32];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al
	jnz	b7a
	lea	esi, [esi-3*2*clmblkdst32-clmblkdst32];; Restore source pointer
	end_timer 22

;; Do FFT levels 6,7
;;
;; On input the 32-byte cache lines hold these data values:
;;	0	4K	1	4K+1
;;	256	...
;;	...
;;	8K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	1K	1	1K+1
;;	256	...
;;	...
;;	2K	...
;;	...

;; Do 4 four_real_four_semireal_fft macros
;;	distance between fft data elements is 4

	start_timer 23
	touch				;; Preload TLB
b4b:	disp four_real_four_semireal_fft, 4*clmblkdst, 8*clmblkdst, 8
	loop4	b4b, clmblkdst		;; 4 * 2*clm iterations
	lea	esi, [esi-4*clmblkdst+16*clmblkdst];; Next source pointer

;; Do 52 four_complex_fft macros
;;	distance between fft data elements is 4

	mov	edi, sincos2		;; Load sin/cos pointer
	mov	eax, 80000000h+7	;; 7 iters of 1 or 2 iters of 4
b5a:	touch				;; Preload TLB
b5b:	disp four_complex_fft, 4*clmblkdst, 8*clmblkdst, 8
	loop1	b5b, clmblkdst		;; 2*clm iterations
	add	ah, 256/4		;; Test middle loop counter
	jnc	b5a			;; Iterate if necessary
	lea	esi, [esi-4*clmblkdst+16*clmblkdst];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	add	eax, 80000000h		;; Test middle loop counter (2 iters)
	jnc	b5a			;; Iterate if necessary
	lea	esi, [esi-2*16*clmblkdst+clmblkdst32];; Next source pointer
	dec	al
	jnz	b5a
	lea	esi, [esi-7*clmblkdst32];; Restore source pointer
	end_timer 23

;; Do FFT levels 8,9
;;
;; On input the 32-byte cache lines hold these data values:
;;	0	1K	1	1K+1
;;	256	...
;;	...
;;	2K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	256	1	257
;;	512	...
;;	...

;; Do 1 four_real_four_semireal_fft macros
;;	distance between fft data elements is 1

	start_timer 24
pfing	lea	ecx, [ecx-224*blkdst]	;; Create dest ptr from prefetch ptr
b2b:	cp_disp four_real_four_semireal_fft, clmblkdst, 2*clmblkdst, 8, blkdst, 2*blkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	lea	ecx, [ecx+dist1]	;; Next dest pointer
	add	dl, 256/2/clm		;; Test inner loop counter
	jnc	b2b			;; Iterate if necessary
	lea	esi, [esi-clm*2*dist1+4*clmblkdst];; Next source pointer
	lea	ecx, [ecx-clm*2*dist1+4*blkdst];; Next dest pointer

;; Do 55 four_complex_fft macros
;;	distance between fft data elements is 1

	mov	edi, sincos1		;; Load sin/cos pointer
	mov	eax, 256/8*256+7	;; 7 iters of 7 or 8 iters
b3b:	cp_disp four_complex_fft, clmblkdst, 2*clmblkdst, 8, blkdst, 2*blkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	lea	ecx, [ecx+dist1]	;; Next dest pointer
	add	dl, 256/2/clm		;; Test inner loop counter
	jnc	b3b			;; Iterate if necessary
	lea	esi, [esi-clm*2*dist1+4*clmblkdst];; Next source pointer
	lea	ecx, [ecx-clm*2*dist1+4*blkdst];; Next dest pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	add	ah, 256/8		;; Test loop counter
	jnc	b3b			;; Iterate if necessary
	lea	esi, [esi-8*4*clmblkdst+clmblkdst32];; Next source pointer
	dec	al
	jnz	b3b
	lea	esi, [ecx-56*4*blkdst]	;; Restore source pointer
	end_timer 24

;; Work on next 4 sets of 32 values

no_fft:	lea	esi, [esi+clm*2*dist1]	;; Next source pointer
	dec	dh			;; Test loop counter
	jnz	short more		;; Do more FFTs
	lea	esi, [esi+64]		;; Skip pad every 128 cache lines (4KB)
pfing	mov	ebp, clm*32		;; Set prefetch distance
	sub	edx, 65536		;; Test loop counter
	jz	short fftdn		;; Jump if done looping
	mov	dh, 128/clm		;; Restore cache line counter
more:	set_prefetch_gap
	cmp	norm_ptr1, 0		;; What was the loop start point?
	jne	c0b			;; Do inverse-FFT/norm/FFT iteration
	jmp	b0b			;; Do another FFT-only iteration
fftdn:	lea	esi, [esi-p2cl*2*dist1-p2cl/128*64];; Restore source pointer
	cmp	esi, DESTARG		;; If esi was not restored, then we 
	jne	short fftdn1		;; just finished skipped postfft data
	sub	ebx, ebx
	mov	DIST_TO_FFTSRCARG, ebx
	cmp	norm_ptr1, 0		;; What was the loop start point?
	je	pass2			;; FFT-only, go do pass 2

;; Split the carries into high and low carries (so they do not exceed
;; the desired number of bits per FFT word.  Then add the carries back to the
;; FFT data.

	start_timer 28
	call	gw_carries
	end_timer 28

;; Have we been given permission to start the FFT on the result?
;; If so, set loop to FFT the first few cache lines now that the carries
;; have been added in.

	cmp	POSTFFT, 0		;; Test flag
	je	short fftdn1		;; Skip FFT code if flag not set
	mov	esi, DESTARG		;; Load FFT data address
	mov	edx, 65536+4/clm*256	;; Do 8 previously skipped cache lines
	mov	norm_ptr1, 0		;; Set flag so only forward FFT is done
	copy_4_words
	jmp	b0b

;; Now add in the FFT'ed carries, cleanup and return

fftdn1:	end_timer 1
	jmp	gw_finish_mult
	ENDM


;; Perform a 128K-element FFT.  This is done in two passes.  Pass 1
;; does 9 levels, pass 2 uses common code to perform the last 8 levels.

pass1levels9pfa8 MACRO pass2_macro, p2cl, clmarg
	LOCAL	b0b, b1a, b1b, b2b, b3b, b4a, b4b, b5a, b5b, b6a, b6b, b7a, b7b
	LOCAL	c0b, c1a, c1b, c2b, c3b, c4a, c4b, c5a, c5b, c6a, c6b, c7a, c7b
	LOCAL	pass2, copyback, yes_fft, no_fft, more, fftdn, fftdn1

	set_FFT_constants clmarg

;; Call FFT code unless the FFT has already been started

	clear_timers
	cmp	DWORD PTR [esi-28][ebx], 0 ;; Test FFT-started flag
	jne	pass2			;; Jump is FFT already started
	mov	norm_ptr1, 0		;; Set FFT-ing flag
	mov	edx, p2cl/128*65536+128/clm*256;; Load loop counter
pfing	mov	ebp, clm*32		;; Set prefetch distance
	jmp	b0b			;; Do the pass 1 fft

;; Do the last 8 FFT levels
;;	do 1 pass2_8_levels_real and 255 pass2_8_levels_complex

pass2:	start_timer 0
	CALLP	pass2_macro
	end_timer 0
	cmp	ffttype, 1		;; We're done if FFTing only
	je	gw_finish_fft

;; Do the inverse FFT

	start_timer 1
	fldz				;; Init SUMOUT
	fstp	SUMOUT
	mov	edx, norm_biglit_array	;; Addr of the big/little flags array
	mov	norm_ptr1, edx		;; Save ptr
	mov	edx, norm_col_mults	;; Addr of the column multipliers
	mov	norm_ptr2, edx		;; Save ptr
	mov	edx, p2cl/128*65536+128/clm*256;; Load loop counter
pfing	mov	ebp, clm*32		;; Set prefetch distance

;; Do inverse FFT levels 8,9
;; On input the 32-byte cache lines hold these data values:
;;	0	256	1	257
;;	512	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	1K	1	1K+1
;;	256	...
;;	...
;;	2K	...
;;	...

;; Do 1 four_real_four_semireal_unfft macros
;;	distance between fft data elements is 1

c0b:	start_timer 16
	mov	ecx, scratch_area	;; Get address of scratch area
c2b:	cp_disp four_real_four_semireal_unfft, 8, blkdst, 2*blkdst, 8, clmblkdst, 2*clmblkdst
	lea	esi, [esi+dist1]	;; Next source pointer
	lea	ecx, [ecx+dist1]	;; Next dest pointer
	add	dl, 256/2/clm		;; Test inner loop counter
	jnc	c2b			;; Iterate if necessary
	lea	esi, [esi-clm*2*dist1+4*blkdst];; Next source pointer
	lea	ecx, [ecx-clm*2*dist1+4*clmblkdst];; Next dest pointer

;; Do 63 four_complex_unfft macros
;;	distance between fft data elements is 1

	mov	edi, sincos1		;; Load sin/cos pointer
	mov	eax, 256/8*256+8	;; 8 iterations of 7 or 8
c3b:	cp_disp four_complex_unfft, 8, blkdst, 2*blkdst, 8, clmblkdst, 2*clmblkdst
	lea	esi, [esi+dist1]	;; Next source pointer
	lea	ecx, [ecx+dist1]	;; Next dest pointer
	add	dl, 256/2/clm		;; Test loop counter
	jnc	c3b			;; Iterate if necessary
	lea	esi, [esi-clm*2*dist1+4*blkdst];; Next source pointer
	lea	ecx, [ecx-clm*2*dist1+4*clmblkdst];; Next dest pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	add	ah, 256/8		;; Test loop counter
	jnc	c3b			;; Iterate if necessary
	lea	ecx, [ecx-8*4*clmblkdst+clmblkdst32];; Next dest pointer
	dec	al			;; Test outer loop counter
	jnz	c3b			;; Iterate if necessary
	lea	ecx, [esi-64*4*blkdst]	;; Create prefetch pointer
	mov	esi, scratch_area	;; Restore source pointer
	end_timer 16

;; Do inverse FFT levels 6,7
;; On input the 32-byte cache lines hold these data values:
;;	0	1K	1	1K+1
;;	256	...
;;	...
;;	2K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	4K	1	4K+1
;;	256	...
;;	...
;;	8K	...
;;	...

;; Do 4 four_real_four_semireal_unfft macros
;;	distance between fft data elements is 4

	start_timer 17
	mov	al, 2			;; Load loop counter
c4a:	touch				;; Preload TLB
c4b:	disp four_real_four_semireal_unfft, 8, 4*clmblkdst, 8*clmblkdst
	loop2	c4b, clmblkdst		;; 2 * 2*clm iterations
	dec	al
	jnz	c4a
	lea	esi, [esi-4*clmblkdst+16*clmblkdst];; Next source pointer

;; Do 60 four_complex_unfft macros
;;	distance between fft data elements is 4

	mov	edi, sincos2		;; Load sin/cos pointer
	mov	eax, 80000000h+8	;; 8 iters of 1 or 2 iters of 4
c5a:	touch				;; Preload TLB
c5b:	disp four_complex_unfft, 8, 4*clmblkdst, 8*clmblkdst
	loop2	c5b, clmblkdst		;; 2 * 2*clm iterations
	add	edx, 80000000h		;; Test middle loop counter (2 iters)
	jnc	c5a			;; Iterate if necessary
	lea	esi, [esi-4*clmblkdst+16*clmblkdst];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	add	eax, 80000000h		;; Test loop counter
	jnc	c5a			;; Iterate if necessary
	lea	esi, [esi-2*16*clmblkdst+clmblkdst32];; Next source pointer
	dec	al
	jnz	c5a
	lea	esi, [esi-8*clmblkdst32];; Restore source pointer
	end_timer 17

;; Do inverse FFT levels 4,5
;; On input the 32-byte cache lines hold these data values:
;;	0	4K	1	4K+1
;;	256	...
;;	...
;;	8K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	16K	1	16K+1
;;	256	...
;;	...
;;	32K	...
;;	...

;; Do 16 four_real_four_semireal_unfft macros
;;	distance between fft data elements is 16

	start_timer 18
	mov	al, 8			;; Load loop counter
c6a:	touch				;; Preload TLB
c6b:	disp four_real_four_semireal_unfft, 8, 16*clmblkdst, clmblkdst32
	loop2	c6b, clmblkdst		;; 2 * 2*clm iterations
	dec	al
	jnz	c6a
	lea	esi, [esi-16*clmblkdst+2*clmblkdst32];; Next source pointer

;; Do 48 four_complex_unfft macros
;;	distance between fft data elements is 16

	mov	edi, sincos3		;; Load sin/cos pointer
	mov	eax, 3			;; 3 iterations of 16
c7a:	touch				;; Preload TLB
c7b:	disp four_complex_unfft, 8, 16*clmblkdst, clmblkdst32
	loop2	c7b, clmblkdst		;; 2 * 2*clm iterations
	add	eax, 80000000h/4	;; Test middle loop counter (8 iters)
	jnc	c7a			;; Iterate if necessary
	lea	esi, [esi-16*clmblkdst+2*clmblkdst32];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al
	jnz	c7a
	lea	esi, [esi-4*2*clmblkdst32];; Restore source pointer
	end_timer 18

;; Do inverse FFT levels 1,2,3
;; On input the 32-byte cache lines hold these data values:
;;	0	16K	1	16K+1
;;	256	...
;;	...
;;	32K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	64K	1	64K+1
;;	256
;;	...
;;	64K-256

;; Do 64 eight_reals_last_unfft macros
;;	distance between fft data elements is 64

	start_timer 19
c1a:	touch				;; Preload TLB
c1b:	disp eight_reals_last_unfft, 8, 2*clmblkdst32, 4*clmblkdst32
	loop2	c1b, clmblkdst		;; 2 * 2*clm iterations
	add	eax, 80000000h/8	;; Test loop counter (16 iters)
	jnc	c1a			;; Iterate if necessary
	lea	esi, [esi-32*clmblkdst+clmblkdst32];; Next source pointer
	add	edx, 80000000h		;; Test loop counter (2 iters)
	jnc	c1a			;; Iterate if necessary
	lea	esi, [esi-2*clmblkdst32];; Restore source pointer
pfing	lea	ecx, [ecx-96*blkdst]	;; Restore dest ptr
	end_timer 19

;; Normalize these values

	start_timer 20
	mov	eax, NORMRTN		;; Addr of normalization routine
	call	eax
	sub	eax, eax
	end_timer 20

;; Have we been given permission to start the FFT on the result?
;; If so, do so now while the data is in the L2 cache.

	cmp	POSTFFT, 0		;; Test flag
	je	short copyback		;; Copy data, skip FFT if flag not set
	copy_3_words clm, 1
	cmp	edx, p2cl/128*65536+(128/clm-4/clm)*256;; Delay fft on 8 sets
	jle	yes_fft			;; No delay, do FFT now

;; Copy data from the scratch area back to the FFT data area

copyback:
	copy_scratch_data_to_fft 8, 32, clmblkdst32
	jmp	no_fft			;; Skip forward FFT code

;; Copy data from the FFT data area back to the scratch area

b0b:	copy_fft_data_to_scratch 8, 32, clmblkdst32

;; Do FFT levels 1,2,3
;;
;; On input the 32-byte cache lines hold these data values:
;;	0	64K	1	64K+1
;;	256
;;	...
;;	64K-256
;; On output the 32-byte cache lines hold these data values:
;;	0	16K	1	16K+1
;;	256	...
;;	...
;;	32K	...
;;	...

;; Do 64 eight_reals_fft macros
;;	distance between fft data elements is 64

yes_fft:start_timer 21
	sub	eax, eax		;; 64 iterations
pfing	lea	ecx, [ecx+96*blkdst]	;; Create prefetch ptr
b1a:	touch				;; Preload TLB
b1b:	disp eight_reals_fft, 2*clmblkdst32, 4*clmblkdst32, 8
	loop2	b1b, clmblkdst		;; 2 * 2*clm iterations
	add	eax, 80000000h/8	;; Test loop counter (16 iters)
	jnc	b1a			;; Iterate if necessary
	lea	esi, [esi-32*clmblkdst+clmblkdst32];; Next source pointer
	add	edx, 80000000h		;; Test loop counter (2 iters)
	jnc	b1a			;; Iterate if necessary
	lea	esi, [esi-2*clmblkdst32];; Restore source pointer
	end_timer 21

;; Do FFT levels 4,5
;;
;; On input the 32-byte cache lines hold these data values:
;;	0	16K	1	16K+1
;;	256	...
;;	...
;;	32K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	4K	1	4K+1
;;	256	...
;;	...
;;	8K	...
;;	...

;; Do 16 four_real_four_semireal_fft macros
;;	distance between fft data elements is 16

	start_timer 22
	mov	al, 16			;; Load loop counter
b6a:	touch				;; Preload TLB
b6b:	disp four_real_four_semireal_fft, 16*clmblkdst, clmblkdst32, 8
	loop1	b6b, clmblkdst		;; 2*clm iterations
	dec	al
	jnz	b6a
	lea	esi, [esi-16*clmblkdst+2*clmblkdst32];; Next source pointer

;; Do 48 four_complex_fft macros
;;	distance between fft data elements is 16

	mov	edi, sincos3		;; Load sin/cos pointer
	mov	eax, 3			;; 3 iterations of 16
b7a:	touch				;; Preload TLB
b7b:	disp four_complex_fft, 16*clmblkdst, clmblkdst32, 8
	loop1	b7b, clmblkdst		;; 2*clm iterations
	add	ah, 256/16		;; Test middle loop counter
	jnc	b7a			;; Iterate if necessary
	lea	esi, [esi-16*clmblkdst+2*clmblkdst32];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al
	jnz	b7a
	lea	esi, [esi-4*2*clmblkdst32];; Restore source pointer
	end_timer 22

;; Do FFT levels 6,7
;;
;; On input the 32-byte cache lines hold these data values:
;;	0	4K	1	4K+1
;;	256	...
;;	...
;;	8K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	1K	1	1K+1
;;	256	...
;;	...
;;	2K	...
;;	...

;; Do 4 four_real_four_semireal_fft macros
;;	distance between fft data elements is 4

	start_timer 23
	mov	al, 4			;; Load loop counter
b4a:	touch				;; Preload TLB
b4b:	disp four_real_four_semireal_fft, 4*clmblkdst, 8*clmblkdst, 8
	loop1	b4b, clmblkdst		;; 2*clm iterations
	dec	al
	jnz	b4a
	lea	esi, [esi-4*clmblkdst+16*clmblkdst];; Next source pointer

;; Do 60 four_complex_fft macros
;;	distance between fft data elements is 4

	mov	edi, sincos2		;; Load sin/cos pointer
	mov	eax, 80000000h+8	;; 8 iters of 1 or 2 iters of 4
b5a:	touch				;; Preload TLB
b5b:	disp four_complex_fft, 4*clmblkdst, 8*clmblkdst, 8
	loop1	b5b, clmblkdst		;; 2*clm iterations
	add	ah, 256/4		;; Test middle loop counter
	jnc	b5a			;; Iterate if necessary
	lea	esi, [esi-4*clmblkdst+16*clmblkdst];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	add	eax, 80000000h		;; Test middle loop counter (2 iters)
	jnc	b5a			;; Iterate if necessary
	lea	esi, [esi-2*16*clmblkdst+clmblkdst32];; Next source pointer
	dec	al
	jnz	b5a
	lea	esi, [esi-8*clmblkdst32];; Restore source pointer
	end_timer 23

;; Do FFT levels 8,9
;;
;; On input the 32-byte cache lines hold these data values:
;;	0	1K	1	1K+1
;;	256	...
;;	...
;;	2K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	256	1	257
;;	512	...
;;	...

;; Do 1 four_real_four_semireal_fft macros
;;	distance between fft data elements is 1

	start_timer 24
pfing	lea	ecx, [ecx-256*blkdst]	;; Create dest ptr from prefetch ptr
b2b:	cp_disp four_real_four_semireal_fft, clmblkdst, 2*clmblkdst, 8, blkdst, 2*blkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	lea	ecx, [ecx+dist1]	;; Next dest pointer
	add	dl, 256/2/clm		;; Test inner loop counter
	jnc	b2b			;; Iterate if necessary
	lea	esi, [esi-clm*2*dist1+4*clmblkdst];; Next source pointer
	lea	ecx, [ecx-clm*2*dist1+4*blkdst];; Next dest pointer

;; Do 63 four_complex_fft macros
;;	distance between fft data elements is 1

	mov	edi, sincos1		;; Load sin/cos pointer
	mov	eax, 256/8*256+8	;; 8 iters of 7 or 8 iters
b3b:	cp_disp four_complex_fft, clmblkdst, 2*clmblkdst, 8, blkdst, 2*blkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	lea	ecx, [ecx+dist1]	;; Next dest pointer
	add	dl, 256/2/clm		;; Test inner loop counter
	jnc	b3b			;; Iterate if necessary
	lea	esi, [esi-clm*2*dist1+4*clmblkdst];; Next source pointer
	lea	ecx, [ecx-clm*2*dist1+4*blkdst];; Next dest pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	add	ah, 256/8		;; Test loop counter
	jnc	b3b			;; Iterate if necessary
	lea	esi, [esi-8*4*clmblkdst+clmblkdst32];; Next source pointer
	dec	al
	jnz	b3b
	lea	esi, [ecx-64*4*blkdst]	;; Restore source pointer
	end_timer 24

;; Work on next 4 sets of 32 values

no_fft:	lea	esi, [esi+clm*2*dist1]	;; Next source pointer
	dec	dh			;; Test loop counter
	jnz	short more		;; Do more FFTs
	lea	esi, [esi+64]		;; Skip pad every 128 cache lines (4KB)
pfing	mov	ebp, clm*32		;; Set prefetch distance
	sub	edx, 65536		;; Test loop counter
	jz	short fftdn		;; Jump if done looping
	mov	dh, 128/clm		;; Restore cache line counter
more:	set_prefetch_gap
	cmp	norm_ptr1, 0		;; What was the loop start point?
	jne	c0b			;; Do inverse-FFT/norm/FFT iteration
	jmp	b0b			;; Do another FFT-only iteration
fftdn:	lea	esi, [esi-p2cl*2*dist1-p2cl/128*64];; Restore source pointer
	cmp	esi, DESTARG		;; If esi was not restored, then we 
	jne	short fftdn1		;; just finished skipped postfft data
	sub	ebx, ebx
	mov	DIST_TO_FFTSRCARG, ebx
	cmp	norm_ptr1, 0		;; What was the loop start point?
	je	pass2			;; FFT-only, go do pass 2

;; Split the carries into high and low carries (so they do not exceed
;; the desired number of bits per FFT word.  Then add the carries back to the
;; FFT data.

	start_timer 28
	call	gw_carries
	end_timer 28

;; Have we been given permission to start the FFT on the result?
;; If so, set loop to FFT the first few cache lines now that the carries
;; have been added in.

	cmp	POSTFFT, 0		;; Test flag
	je	short fftdn1		;; Skip FFT code if flag not set
	mov	esi, DESTARG		;; Load FFT data address
	mov	edx, 65536+4/clm*256	;; Do 8 previously skipped cache lines
	mov	norm_ptr1, 0		;; Set flag so only forward FFT is done
	copy_4_words
	jmp	b0b

;; Now add in the FFT'ed carries, cleanup and return

fftdn1:	end_timer 1
	jmp	gw_finish_mult
	ENDM


;; Perform a 128K-element negacyclic FFT.  This is done in two passes.  Pass 1
;; does 9 levels, pass 2 uses common code to perform the last 8 levels.

pass1levels9complex4 MACRO pass2_macro, p2cl, clmarg
	LOCAL	b0b, b1a, b1b, b3b, b5a, b5b, b7a, b7b
	LOCAL	c0b, c1a, c1b, c3b, c5a, c5b, c7a, c7b
	LOCAL	pass2, copyback, yes_fft, no_fft, more, fftdn, fftdn1

	set_FFT_constants clmarg

;; Call FFT code unless the FFT has already been started

	clear_timers
	cmp	DWORD PTR [esi-28][ebx], 0 ;; Test FFT-started flag
	jne	pass2			;; Jump is FFT already started
	mov	norm_ptr1, 0		;; Set FFT-ing flag
	mov	edx, p2cl/128*65536+128/clm*256;; Load loop counter
pfing	mov	ebp, clm*32		;; Set prefetch distance
	jmp	b0b			;; Do the pass 1 fft

;; Do the last 8 FFT levels
;;	do 256 pass2_8_levels_complex

pass2:	start_timer 0
	CALLP	pass2_macro
	end_timer 0
	cmp	ffttype, 1		;; We're done if FFTing only
	je	gw_finish_fft

;; Do the inverse FFT

	start_timer 1
	fldz				;; Init SUMOUT
	fstp	SUMOUT
	mov	edx, norm_biglit_array	;; Addr of the big/little flags array
	mov	norm_ptr1, edx		;; Save ptr
	mov	edx, norm_col_mults	;; Addr of the column multipliers
	mov	norm_ptr2, edx		;; Save ptr
	mov	edx, p2cl/128*65536+128/clm*256;; Load loop counter
pfing	mov	ebp, clm*32		;; Set prefetch distance

;; Do inverse FFT levels 8,9
;; On input the 32-byte cache lines hold these data values:
;;	0	256	1	257
;;	512	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	1K	1	1K+1
;;	256	...
;;	...
;;	2K	...
;;	...

;; Do 64 four_complex_unfft macros
;;	distance between fft data elements is 1

c0b:	start_timer 16
	mov	ecx, scratch_area	;; Get address of scratch area
	mov	edi, sincos1		;; Load sin/cos pointer
	mov	eax, 8			;; 8 iterations 8
c3b:	cp_disp four_complex_unfft, 8, blkdst, 2*blkdst, 8, clmblkdst, 2*clmblkdst
	lea	esi, [esi+dist1]	;; Next source pointer
	lea	ecx, [ecx+dist1]	;; Next dest pointer
	add	dl, 256/2/clm		;; Test loop counter
	jnc	c3b			;; Iterate if necessary
	lea	esi, [esi-clm*2*dist1+4*blkdst];; Next source pointer
	lea	ecx, [ecx-clm*2*dist1+4*clmblkdst];; Next dest pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	add	ah, 256/8		;; Test loop counter
	jnc	c3b			;; Iterate if necessary
	lea	ecx, [ecx-8*4*clmblkdst+clmblkdst32];; Next dest pointer
	dec	al			;; Test outer loop counter
	jnz	c3b			;; Iterate if necessary
	lea	ecx, [esi-64*4*blkdst]	;; Create prefetch pointer
	mov	esi, scratch_area	;; Restore source pointer
	end_timer 16

;; Do inverse FFT levels 6,7
;; On input the 32-byte cache lines hold these data values:
;;	0	1K	1	1K+1
;;	256	...
;;	...
;;	2K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	4K	1	4K+1
;;	256	...
;;	...
;;	8K	...
;;	...

;; Do 64 four_complex_unfft macros
;;	distance between fft data elements is 4

	start_timer 17
	mov	edi, sincos2		;; Load sin/cos pointer
	mov	eax, 8			;; 8 iters of 2 iters of 4
c5a:	touch				;; Preload TLB
c5b:	disp four_complex_unfft, 8, 4*clmblkdst, 8*clmblkdst
	loop2	c5b, clmblkdst		;; 2 * 2*clm iterations
	add	edx, 80000000h		;; Test middle loop counter (2 iters)
	jnc	c5a			;; Iterate if necessary
	lea	esi, [esi-4*clmblkdst+16*clmblkdst];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	add	eax, 80000000h		;; Test loop counter
	jnc	c5a			;; Iterate if necessary
	lea	esi, [esi-2*16*clmblkdst+clmblkdst32];; Next source pointer
	dec	al
	jnz	c5a
	lea	esi, [esi-8*clmblkdst32];; Restore source pointer
	end_timer 17

;; Do inverse FFT levels 4,5
;; On input the 32-byte cache lines hold these data values:
;;	0	4K	1	4K+1
;;	256	...
;;	...
;;	8K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	16K	1	16K+1
;;	256	...
;;	...
;;	32K	...
;;	...

;; Do 64 four_complex_unfft macros
;;	distance between fft data elements is 16

	start_timer 18
	mov	edi, sincos3		;; Load sin/cos pointer
	mov	eax, 4			;; 4 iterations of 16
c7a:	touch				;; Preload TLB
c7b:	disp four_complex_unfft, 8, 16*clmblkdst, clmblkdst32
	loop2	c7b, clmblkdst		;; 2 * 2*clm iterations
	add	eax, 80000000h/4	;; Test middle loop counter (8 iters)
	jnc	c7a			;; Iterate if necessary
	lea	esi, [esi-16*clmblkdst+2*clmblkdst32];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al
	jnz	c7a
	lea	esi, [esi-4*2*clmblkdst32];; Restore source pointer
	end_timer 18

;; Do inverse FFT levels 1,2,3
;; On input the 32-byte cache lines hold these data values:
;;	0	16K	1	16K+1
;;	256	...
;;	...
;;	32K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	64K	1	64K+1
;;	256
;;	...
;;	64K-256

;; Do 64 four_complex_last_unfft macros
;;	distance between fft data elements is 64

	start_timer 19
	mov	edi, plus1_premults	;; Address of premultiplier table
c1a:	touch				;; Preload TLB
c1b:	disp four_complex_last_unfft, 8, 2*clmblkdst32, 4*clmblkdst32
	loop1edi c1b, clmblkdst, 64	;; 2*clm iterations bumping edi
	add	ah, 256/32		;; Test loop counter
	jnc	c1a			;; Iterate if necessary
	lea	esi, [esi-32*clmblkdst+clmblkdst32];; Next source pointer
	add	eax, 80000000h		;; Test loop counter (2 iters)
	jnc	c1a			;; Iterate if necessary
	lea	esi, [esi-2*clmblkdst32];; Restore source pointer
pfing	lea	ecx, [ecx-128*blkdst]	;; Restore dest ptr
	end_timer 19

;; Normalize these values

	start_timer 20
	mov	eax, NORMRTN		;; Addr of normalization routine
	call	eax
	sub	eax, eax
	end_timer 20

;; Have we been given permission to start the FFT on the result?
;; If so, do so now while the data is in the L2 cache.

	cmp	POSTFFT, 0		;; Test flag
	je	short copyback		;; Copy data, skip FFT if flag not set
	copy_3_words clm, 1
	cmp	edx, p2cl/128*65536+(128/clm-4/clm)*256;; Delay fft on 8 sets
	jle	yes_fft			;; No delay, do FFT now

;; Copy data from the scratch area back to the FFT data area

copyback:
	copy_scratch_data_to_fft 8, 32, clmblkdst32
	jmp	no_fft			;; Skip forward FFT code

;; Copy data from the FFT data area back to the scratch area

b0b:	copy_fft_data_to_scratch 8, 32, clmblkdst32

;; Do FFT levels 1,2,3
;;
;; On input the 32-byte cache lines hold these data values:
;;	0	64K	1	64K+1
;;	256
;;	...
;;	64K-256
;; On output the 32-byte cache lines hold these data values:
;;	0	16K	1	16K+1
;;	256	...
;;	...
;;	32K	...
;;	...

;; Do 64 four_complex_fft_premult macros
;;	distance between fft data elements is 64

yes_fft:start_timer 21
	mov	edi, plus1_premults	;; Address of premultiplier table
	sub	eax, eax		;; 64 iterations
pfing	lea	ecx, [ecx+128*blkdst]	;; Create prefetch ptr
b1a:	touch				;; Preload TLB
b1b:	disp four_complex_fft_premult, 2*clmblkdst32, 4*clmblkdst32, 8
	loop1edi b1b, clmblkdst, 64	;; 2*clm iterations bumping edi
	add	ah, 256/32		;; Test loop counter
	jnc	b1a			;; Iterate if necessary
	lea	esi, [esi-32*clmblkdst+clmblkdst32];; Next source pointer
	add	eax, 80000000h		;; Test loop counter (2 iters)
	jnc	b1a			;; Iterate if necessary
	lea	esi, [esi-2*clmblkdst32];; Restore source pointer
	end_timer 21

;; Do FFT levels 4,5
;;
;; On input the 32-byte cache lines hold these data values:
;;	0	16K	1	16K+1
;;	256	...
;;	...
;;	32K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	4K	1	4K+1
;;	256	...
;;	...
;;	8K	...
;;	...

;; Do 64 four_complex_fft macros
;;	distance between fft data elements is 16

	start_timer 22
	mov	edi, sincos3		;; Load sin/cos pointer
	mov	eax, 4			;; 4 iterations of 16
b7a:	touch				;; Preload TLB
b7b:	disp four_complex_fft, 16*clmblkdst, clmblkdst32, 8
	loop2	b7b, clmblkdst		;; 2 * 2*clm iterations
	add	eax, 80000000h/4	;; Test middle loop counter (8 iters)
	jnc	b7a			;; Iterate if necessary
	lea	esi, [esi-16*clmblkdst+2*clmblkdst32];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al
	jnz	b7a
	lea	esi, [esi-4*2*clmblkdst32];; Restore source pointer
	end_timer 22

;; Do FFT levels 6,7
;;
;; On input the 32-byte cache lines hold these data values:
;;	0	4K	1	4K+1
;;	256	...
;;	...
;;	8K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	1K	1	1K+1
;;	256	...
;;	...
;;	2K	...
;;	...

;; Do 64 four_complex_fft macros
;;	distance between fft data elements is 4

	start_timer 23
	mov	edi, sincos2		;; Load sin/cos pointer
	mov	eax, 8			;; 8 iters of 2 iters of 4
b5a:	touch				;; Preload TLB
b5b:	disp four_complex_fft, 4*clmblkdst, 8*clmblkdst, 8
	loop2	b5b, clmblkdst		;; 2 * 2*clm iterations
	add	edx, 80000000h		;; Test middle loop counter (2 iters)
	jnc	b5a			;; Iterate if necessary
	lea	esi, [esi-4*clmblkdst+16*clmblkdst];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	add	eax, 80000000h		;; Test middle loop counter (2 iters)
	jnc	b5a			;; Iterate if necessary
	lea	esi, [esi-2*16*clmblkdst+clmblkdst32];; Next source pointer
	dec	al
	jnz	b5a
	lea	esi, [esi-8*clmblkdst32];; Restore source pointer
	end_timer 23

;; Do FFT levels 8,9
;;
;; On input the 32-byte cache lines hold these data values:
;;	0	1K	1	1K+1
;;	256	...
;;	...
;;	2K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	256	1	257
;;	512	...
;;	...

;; Do 64 four_complex_fft macros
;;	distance between fft data elements is 1

	start_timer 24
	mov	edi, sincos1		;; Load sin/cos pointer
pfing	lea	ecx, [ecx-256*blkdst]	;; Create dest ptr from prefetch ptr
	mov	eax, 8			;; 8 iters of 8 iters
b3b:	cp_disp four_complex_fft, clmblkdst, 2*clmblkdst, 8, blkdst, 2*blkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	lea	ecx, [ecx+dist1]	;; Next dest pointer
	add	dl, 256/2/clm		;; Test inner loop counter
	jnc	b3b			;; Iterate if necessary
	lea	esi, [esi-clm*2*dist1+4*clmblkdst];; Next source pointer
	lea	ecx, [ecx-clm*2*dist1+4*blkdst];; Next dest pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	add	ah, 256/8		;; Test loop counter
	jnc	b3b			;; Iterate if necessary
	lea	esi, [esi-8*4*clmblkdst+clmblkdst32];; Next source pointer
	dec	al
	jnz	b3b
	lea	esi, [ecx-64*4*blkdst]	;; Restore source pointer
	end_timer 24

;; Work on next 4 sets of 32 values

no_fft:	lea	esi, [esi+clm*2*dist1]	;; Next source pointer
	dec	dh			;; Test loop counter
	jnz	short more		;; Do more FFTs
	lea	esi, [esi+64]		;; Skip pad every 128 cache lines (4KB)
pfing	mov	ebp, clm*32		;; Set prefetch distance
	sub	edx, 65536		;; Test loop counter
	jz	short fftdn		;; Jump if done looping
	mov	dh, 128/clm		;; Restore cache line counter
more:	set_prefetch_gap
	cmp	norm_ptr1, 0		;; What was the loop start point?
	jne	c0b			;; Do inverse-FFT/norm/FFT iteration
	jmp	b0b			;; Do another FFT-only iteration
fftdn:	lea	esi, [esi-p2cl*2*dist1-p2cl/128*64];; Restore source pointer
	cmp	esi, DESTARG		;; If esi was not restored, then we 
	jne	short fftdn1		;; just finished skipped postfft data
	sub	ebx, ebx
	mov	DIST_TO_FFTSRCARG, ebx
	cmp	norm_ptr1, 0		;; What was the loop start point?
	je	pass2			;; FFT-only, go do pass 2

;; Split the carries into high and low carries (so they do not exceed
;; the desired number of bits per FFT word.  Then add the carries back to the
;; FFT data.

	start_timer 28
	call	gw_carries
	end_timer 28

;; Have we been given permission to start the FFT on the result?
;; If so, set loop to FFT the first few cache lines now that the carries
;; have been added in.

	cmp	POSTFFT, 0		;; Test flag
	je	short fftdn1		;; Skip FFT code if flag not set
	mov	esi, DESTARG		;; Load FFT data address
	mov	edx, 65536+4/clm*256	;; Do 8 previously skipped cache lines
	mov	norm_ptr1, 0		;; Set flag so only forward FFT is done
	copy_4_words
	jmp	b0b

;; Now add in the FFT'ed carries, cleanup and return

fftdn1:	end_timer 1
	jmp	gw_finish_mult
	ENDM


;; Perform a 160K-element FFT.  This is done in two passes.  Pass 1
;; does 10 levels, pass 2 uses common code to perform the last 8 levels.

pass1levels10pfa5 MACRO pass2_macro, p2cl, clmarg
	LOCAL	b0b, b1a, b1b, b2b, b3b, b4a, b4b, b5a, b5b, b6a, b6b
	LOCAL	b7a, b7b, b8a, b8b, b9a, b9b
	LOCAL	c0b, c1a, c1b, c2b, c3b, c4a, c4b, c5a, c5b, c6a, c6b
	LOCAL	c7a, c7b, c8a, c8b, c9a, c9b
	LOCAL	pass2, copyback, yes_fft, no_fft, more, fftdn, fftdn1

	set_FFT_constants clmarg

;; Call FFT code unless the FFT has already been started

	clear_timers
	cmp	DWORD PTR [esi-28][ebx], 0 ;; Test FFT-started flag
	jne	pass2			;; Jump is FFT already started
	mov	norm_ptr1, 0		;; Set FFT-ing flag
	mov	edx, p2cl/128*65536+128/clm*256;; Load loop counter
pfing	mov	ebp, clm*32		;; Set prefetch distance
	jmp	b0b			;; Do the pass 1 fft

;; Do the last 8 FFT levels
;;	do 1 pass2_8_levels_real and 319 pass2_8_levels_complex

pass2:	start_timer 0
	CALLP	pass2_macro
	end_timer 0
	cmp	ffttype, 1		;; We're done if FFTing only
	je	gw_finish_fft

;; Do the inverse FFT

	start_timer 1
	fldz				;; Init SUMOUT
	fstp	SUMOUT
	mov	edx, norm_biglit_array	;; Addr of the big/little flags array
	mov	norm_ptr1, edx		;; Save ptr
	mov	edx, norm_col_mults	;; Addr of the column multipliers
	mov	norm_ptr2, edx		;; Save ptr
	mov	edx, p2cl/128*65536+128/clm*256;; Load loop counter
pfing	mov	ebp, clm*32		;; Set prefetch distance

;; Do inverse FFT levels 9,10
;; On input the 32-byte cache lines hold these data values:
;;	0	256	1	257
;;	512	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	1K	1	1K+1
;;	256	...
;;	...
;;	2K	...
;;	...

;; Do 1 four_real_four_semireal_unfft macros
;;	distance between fft data elements is 1

c0b:	start_timer 15
	mov	ecx, scratch_area	;; Get address of scratch area
c2b:	cp_disp four_real_four_semireal_unfft, 8, blkdst, 2*blkdst, 8, clmblkdst, 2*clmblkdst
	lea	esi, [esi+dist1]	;; Next source pointer
	lea	ecx, [ecx+dist1]	;; Next dest pointer
	add	dl, 256/2/clm		;; Test inner loop counter
	jnc	c2b			;; Iterate if necessary
	lea	esi, [esi-clm*2*dist1+4*blkdst];; Next source pointer
	lea	ecx, [ecx-clm*2*dist1+4*clmblkdst];; Next dest pointer

;; Do 79 four_complex_unfft macros
;;	distance between fft data elements is 1

	mov	edi, sincos1		;; Load sin/cos pointer
	mov	eax, 256/8*256+10	;; 10 iterations of 7 or 8
c3b:	cp_disp four_complex_unfft, 8, blkdst, 2*blkdst, 8, clmblkdst, 2*clmblkdst
	lea	esi, [esi+dist1]	;; Next source pointer
	lea	ecx, [ecx+dist1]	;; Next dest pointer
	add	dl, 256/2/clm		;; Test loop counter
	jnc	c3b			;; Iterate if necessary
	lea	esi, [esi-clm*2*dist1+4*blkdst];; Next source pointer
	lea	ecx, [ecx-clm*2*dist1+4*clmblkdst];; Next dest pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	add	ah, 256/8		;; Test loop counter
	jnc	c3b			;; Iterate if necessary
	lea	ecx, [ecx-8*4*clmblkdst+clmblkdst32];; Next dest pointer
	dec	al			;; Test outer loop counter
	jnz	c3b			;; Iterate if necessary
	lea	ecx, [esi-80*4*blkdst]	;; Create prefetch pointer
	mov	esi, scratch_area	;; Restore source pointer
	end_timer 15

;; Do inverse FFT levels 7,8
;; On input the 32-byte cache lines hold these data values:
;;	0	1K	1	1K+1
;;	256	...
;;	...
;;	2K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	4K	1	4K+1
;;	256	...
;;	...
;;	8K	...
;;	...

;; Do 4 four_real_four_semireal_unfft macros
;;	distance between fft data elements is 4

	start_timer 16
	mov	al, 2			;; Load loop counter
c4a:	touch				;; Preload TLB
c4b:	disp four_real_four_semireal_unfft, 8, 4*clmblkdst, 8*clmblkdst
	loop2	c4b, clmblkdst		;; 2 * 2*clm iterations
	dec	al
	jnz	c4a
	lea	esi, [esi-4*clmblkdst+16*clmblkdst];; Next source pointer

;; Do 76 four_complex_unfft macros
;;	distance between fft data elements is 4

	mov	edi, sincos2		;; Load sin/cos pointer
	mov	eax, 80000000h+10	;; 10 iters of 1 or 2 iters of 4
c5a:	touch				;; Preload TLB
c5b:	disp four_complex_unfft, 8, 4*clmblkdst, 8*clmblkdst
	loop2	c5b, clmblkdst		;; 2 * 2*clm iterations
	add	edx, 80000000h		;; Test middle loop counter (2 iters)
	jnc	c5a			;; Iterate if necessary
	lea	esi, [esi-4*clmblkdst+16*clmblkdst];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	add	eax, 80000000h		;; Test loop counter
	jnc	c5a			;; Iterate if necessary
	lea	esi, [esi-2*16*clmblkdst+clmblkdst32];; Next source pointer
	dec	al
	jnz	c5a
	lea	esi, [esi-10*clmblkdst32];; Restore source pointer
	end_timer 16

;; Do inverse FFT levels 5,6
;; On input the 32-byte cache lines hold these data values:
;;	0	4K	1	4K+1
;;	256	...
;;	...
;;	8K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	16K	1	16K+1
;;	256	...
;;	...
;;	32K	...
;;	...

;; Do 16 four_real_four_semireal_unfft macros
;;	distance between fft data elements is 16

	start_timer 17
	mov	al, 4			;; Load loop counter
c6a:	touch				;; Preload TLB
c6b:	disp four_real_four_semireal_unfft, 8, 16*clmblkdst, clmblkdst32
	loop4	c6b, clmblkdst		;; 4 * 2*clm iterations
	dec	al
	jnz	c6a
	lea	esi, [esi-16*clmblkdst+2*clmblkdst32];; Next source pointer

;; Do 64 four_complex_unfft macros
;;	distance between fft data elements is 16

	mov	edi, sincos3		;; Load sin/cos pointer
	mov	eax, 4			;; 4 iterations of 16
c7a:	touch				;; Preload TLB
c7b:	disp four_complex_unfft, 8, 16*clmblkdst, clmblkdst32
	loop2	c7b, clmblkdst		;; 2 * 2*clm iterations
	add	eax, 80000000h/4	;; Test middle loop counter (8 iters)
	jnc	c7a			;; Iterate if necessary
	lea	esi, [esi-16*clmblkdst+2*clmblkdst32];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al
	jnz	c7a
	lea	esi, [esi-5*2*clmblkdst32];; Restore source pointer
	end_timer 17

;; Do inverse FFT levels 4
;; On input the 32-byte cache lines hold these data values:
;;	0	16K	1	16K+1
;;	256	...
;;	...
;;	32K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	32K	1	32K+1
;;	256	...
;;	...
;;	64K	...
;;	...

;; Do 16 eight_reals_unfft_1 macros
;;	distance between fft data elements is 64

	start_timer 18
	mov	al, 4			;; Load loop counter
c8a:	touch				;; Preload TLB
c8b:	disp eight_reals_unfft_1, 16*clmblkdst, clmblkdst32, 8
	loop4	c8b, clmblkdst		;; 4 * 2*clm iterations
	dec	al
	jnz	c8a
	lea	esi, [esi-16*clmblkdst+2*clmblkdst32];; Next source pointer

;; Do 64 four_complex_unfft_1 macros
;;	distance between fft data elements is 64

	mov	edi, sincos4		;; Load sin/cos pointer
	mov	eax, 2			;; 2 iterations of 32
c9a:	touch				;; Preload TLB
c9b:	disp four_complex_unfft_1, clmblkdst32, 8, 2*clmblkdst32
	loop4	c9b, clmblkdst		;; 4 * 2*clm iterations
	add	eax, 80000000h/4	;; Test middle loop counter (8 iters)
	jnc	c9a			;; Iterate if necessary
	lea	esi, [esi-32*clmblkdst+4*clmblkdst32];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al
	jnz	c9a
	lea	esi, [esi-2*4*clmblkdst32-2*clmblkdst32];; Restore source ptr
	end_timer 18

;; Do inverse FFT levels 1,2,3
;; On input the 32-byte cache lines hold these data values:
;;	0	32K	1	32K+1
;;	256	...
;;	...
;;	64K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	80K	1	80K+1
;;	256
;;	...
;;	80K-256

;; Do 128 five_reals_last_unfft macros
;;	distance between fft data elements is 128

	start_timer 19
c1a:	touch				;; Preload TLB
c1b:	two_five_reals_last_unfft 8, 2*clmblkdst32
	loop1	c1b, clmblkdst		;; 2*clm iterations
	add	ah, 256/32		;; Test loop counter (32 iters)
	jnc	c1a			;; Iterate if necessary
	lea	esi, [esi-32*clmblkdst+clmblkdst32];; Restore source pointer
	add	eax, 80000000h		;; Test loop counter (2 iters)
	jnc	c1a			;; Iterate if necessary
	lea	esi, [esi-2*clmblkdst32];; Restore source pointer
pfing	lea	ecx, [ecx-160*blkdst]	;; Restore dest ptr
	end_timer 19

;; Normalize these values

	start_timer 20
	mov	eax, NORMRTN		;; Addr of normalization routine
	call	eax
	sub	eax, eax
	end_timer 20

;; Have we been given permission to start the FFT on the result?
;; If so, do so now while the data is in the L2 cache.

	cmp	POSTFFT, 0		;; Test flag
	je	short copyback		;; Copy data, skip FFT if flag not set
	copy_3_words clm, 1
	cmp	edx, p2cl/128*65536+(128/clm-4/clm)*256;; Delay fft on 8 sets
	jle	yes_fft			;; No delay, do FFT now

;; Copy data from the scratch area back to the FFT data area

copyback:
	copy_scratch_data_to_fft 10, 32, clmblkdst32
	jmp	no_fft			;; Skip forward FFT code

;; Copy data from the FFT data area back to the scratch area

b0b:	copy_fft_data_to_scratch 10, 32, clmblkdst32

;; Do FFT levels 1,2,3
;;
;; On input the 32-byte cache lines hold these data values:
;;	0	80K	1	80K+1
;;	256
;;	...
;;	80K-256
;; On output the 32-byte cache lines hold these data values:
;;	0	32K	1	32K+1
;;	256	...
;;	...
;;	64K	...
;;	...

;; Do 128 five_reals_fft macros
;;	distance between fft data elements is 128

yes_fft:start_timer 21
	sub	eax, eax		;; 64 iterations
pfing	lea	ecx, [ecx+160*blkdst]	;; Create prefetch ptr
b1a:	touch				;; Preload TLB
b1b:	two_five_reals_fft 2*clmblkdst32, 8
	loop1	b1b, clmblkdst		;; 2*clm iterations
	add	ah, 256/32		;; Test loop counter (32 iters)
	jnc	b1a			;; Iterate if necessary
	lea	esi, [esi-32*clmblkdst+clmblkdst32];; Next source pointer
	add	eax, 80000000h		;; Test loop counter (2 iters)
	jnc	b1a			;; Iterate if necessary
	lea	esi, [esi-2*clmblkdst32];; Restore source pointer
	end_timer 21

;; Do FFT levels 4
;;
;; On input the 32-byte cache lines hold these data values:
;;	0	32K	1	32K+1
;;	256	...
;;	...
;;	64K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	16K	1	16K+1
;;	256	...
;;	...
;;	32K	...
;;	...

;; Do 16 eight_reals_fft_1 macros
;;	distance between fft data elements is 64

	start_timer 22
	mov	al, 4			;; Load loop counter
b8a:	touch				;; Preload TLB
b8b:	disp eight_reals_fft_1, 16*clmblkdst, clmblkdst32, 8
	loop4	b8b, clmblkdst		;; 4 * 2*clm iterations
	dec	al
	jnz	b8a
	lea	esi, [esi-16*clmblkdst+2*clmblkdst32];; Next source pointer

;; Do 64 four_complex_fft_1 macros
;;	distance between fft data elements is 64

	mov	edi, sincos4		;; Load sin/cos pointer
	mov	eax, 2			;; 2 iterations of 32
b9a:	touch				;; Preload TLB
b9b:	disp four_complex_fft_1, clmblkdst32, 2*clmblkdst32, 8
	loop4	b9b, clmblkdst		;; 4 * 2*clm iterations
	add	eax, 80000000h/4	;; Test middle loop counter (8 iters)
	jnc	b9a			;; Iterate if necessary
	lea	esi, [esi-32*clmblkdst+4*clmblkdst32];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al
	jnz	b9a
	lea	esi, [esi-2*4*clmblkdst32-2*clmblkdst32];; Restore source ptr
	end_timer 22

;; Do FFT levels 5,6
;;
;; On input the 32-byte cache lines hold these data values:
;;	0	16K	1	16K+1
;;	256	...
;;	...
;;	32K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	4K	1	4K+1
;;	256	...
;;	...
;;	8K	...
;;	...

;; Do 16 four_real_four_semireal_fft macros
;;	distance between fft data elements is 16

	start_timer 23
	mov	al, 4			;; Load loop counter
b6a:	touch				;; Preload TLB
b6b:	disp four_real_four_semireal_fft, 16*clmblkdst, clmblkdst32, 8
	loop4	b6b, clmblkdst		;; 4 * 2*clm iterations
	dec	al
	jnz	b6a
	lea	esi, [esi-16*clmblkdst+2*clmblkdst32];; Next source pointer

;; Do 64 four_complex_fft macros
;;	distance between fft data elements is 16

	mov	edi, sincos3		;; Load sin/cos pointer
	mov	eax, 4			;; 4 iterations of 16
b7a:	touch				;; Preload TLB
b7b:	disp four_complex_fft, 16*clmblkdst, clmblkdst32, 8
	loop2	b7b, clmblkdst		;; 2 * 2*clm iterations
	add	eax, 80000000h/4	;; Test middle loop counter (8 iters)
	jnc	b7a			;; Iterate if necessary
	lea	esi, [esi-16*clmblkdst+2*clmblkdst32];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al
	jnz	b7a
	lea	esi, [esi-5*2*clmblkdst32];; Restore source pointer
	end_timer 23

;; Do FFT levels 7,8
;;
;; On input the 32-byte cache lines hold these data values:
;;	0	4K	1	4K+1
;;	256	...
;;	...
;;	8K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	1K	1	1K+1
;;	256	...
;;	...
;;	2K	...
;;	...

;; Do 4 four_real_four_semireal_fft macros
;;	distance between fft data elements is 4

	start_timer 24
	mov	al, 2			;; Load loop counter
b4a:	touch				;; Preload TLB
b4b:	disp four_real_four_semireal_fft, 4*clmblkdst, 8*clmblkdst, 8
	loop2	b4b, clmblkdst		;; 2 * 2*clm iterations
	dec	al
	jnz	b4a
	lea	esi, [esi-4*clmblkdst+16*clmblkdst];; Next source pointer

;; Do 76 four_complex_fft macros
;;	distance between fft data elements is 4

	mov	edi, sincos2		;; Load sin/cos pointer
	mov	eax, 80000000h+10	;; 10 iters of 1 or 2 iters of 4
b5a:	touch				;; Preload TLB
b5b:	disp four_complex_fft, 4*clmblkdst, 8*clmblkdst, 8
	loop2	b5b, clmblkdst		;; 2*clm iterations
	add	edx, 80000000h		;; Test middle loop counter (2 iters)
	jnc	b5a			;; Iterate if necessary
	lea	esi, [esi-4*clmblkdst+16*clmblkdst];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	add	eax, 80000000h		;; Test middle loop counter (2 iters)
	jnc	b5a			;; Iterate if necessary
	lea	esi, [esi-2*16*clmblkdst+clmblkdst32];; Next source pointer
	dec	al
	jnz	b5a
	lea	esi, [esi-10*clmblkdst32]	;; Restore source pointer
	end_timer 24

;; Do FFT levels 9,10
;;
;; On input the 32-byte cache lines hold these data values:
;;	0	1K	1	1K+1
;;	256	...
;;	...
;;	2K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	256	1	257
;;	512	...
;;	...

;; Do 1 four_real_four_semireal_fft macros
;;	distance between fft data elements is 1

	start_timer 25
pfing	lea	ecx, [ecx-320*blkdst]	;; Create dest ptr from prefetch ptr
b2b:	cp_disp four_real_four_semireal_fft, clmblkdst, 2*clmblkdst, 8, blkdst, 2*blkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	lea	ecx, [ecx+dist1]	;; Next dest pointer
	add	dl, 256/2/clm		;; Test inner loop counter
	jnc	b2b			;; Iterate if necessary
	lea	esi, [esi-clm*2*dist1+4*clmblkdst];; Next source pointer
	lea	ecx, [ecx-clm*2*dist1+4*blkdst];; Next dest pointer

;; Do 79 four_complex_fft macros
;;	distance between fft data elements is 1

	mov	edi, sincos1		;; Load sin/cos pointer
	mov	eax, 256/8*256+10	;; 10 iters of 7 or 8 iters
b3b:	cp_disp four_complex_fft, clmblkdst, 2*clmblkdst, 8, blkdst, 2*blkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	lea	ecx, [ecx+dist1]	;; Next dest pointer
	add	dl, 256/2/clm		;; Test inner loop counter
	jnc	b3b			;; Iterate if necessary
	lea	esi, [esi-clm*2*dist1+4*clmblkdst];; Next source pointer
	lea	ecx, [ecx-clm*2*dist1+4*blkdst];; Next dest pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	add	ah, 256/8		;; Test loop counter
	jnc	b3b			;; Iterate if necessary
	lea	esi, [esi-8*4*clmblkdst+clmblkdst32];; Next source pointer
	dec	al
	jnz	b3b
	lea	esi, [ecx-80*4*blkdst]	;; Restore source pointer
	end_timer 25

;; Work on next 4 sets of 32 values

no_fft:	lea	esi, [esi+clm*2*dist1]	;; Next source pointer
	dec	dh			;; Test loop counter
	jnz	short more		;; Do more FFTs
	lea	esi, [esi+64]		;; Skip pad every 128 cache lines (4KB)
pfing	mov	ebp, clm*32		;; Set prefetch distance
	sub	edx, 65536		;; Test loop counter
	jz	short fftdn		;; Jump if done looping
	mov	dh, 128/clm		;; Restore cache line counter
more:	set_prefetch_gap
	cmp	norm_ptr1, 0		;; What was the loop start point?
	jne	c0b			;; Do inverse-FFT/norm/FFT iteration
	jmp	b0b			;; Do another FFT-only iteration
fftdn:	lea	esi, [esi-p2cl*2*dist1-p2cl/128*64];; Restore source pointer
	cmp	esi, DESTARG		;; If esi was not restored, then we 
	jne	short fftdn1		;; just finished skipped postfft data
	sub	ebx, ebx
	mov	DIST_TO_FFTSRCARG, ebx
	cmp	norm_ptr1, 0		;; What was the loop start point?
	je	pass2			;; FFT-only, go do pass 2

;; Split the carries into high and low carries (so they do not exceed
;; the desired number of bits per FFT word.  Then add the carries back to the
;; FFT data.

	start_timer 28
	call	gw_carries
	end_timer 28

;; Have we been given permission to start the FFT on the result?
;; If so, set loop to FFT the first few cache lines now that the carries
;; have been added in.

	cmp	POSTFFT, 0		;; Test flag
	je	short fftdn1		;; Skip FFT code if flag not set
	mov	esi, DESTARG		;; Load FFT data address
	mov	edx, 65536+4/clm*256	;; Do 8 previously skipped cache lines
	mov	norm_ptr1, 0		;; Set flag so only forward FFT is done
	copy_4_words
	jmp	b0b

;; Now add in the FFT'ed carries, cleanup and return

fftdn1:	end_timer 1
	jmp	gw_finish_mult
	ENDM


;; Perform a 192K-element FFT.  This is done in two passes.  Pass 1
;; does 10 levels, pass 2 uses common code to perform the last 8 levels.

pass1levels10pfa6 MACRO pass2_macro, p2cl, clmarg
	LOCAL	b0b, b1a, b1b, b2b, b3b, b4a, b4b, b5a, b5b, b6a, b6b
	LOCAL	b7a, b7b, b8a, b8b, b9a, b9b
	LOCAL	c0b, c1a, c1b, c2b, c3b, c4a, c4b, c5a, c5b, c6a, c6b
	LOCAL	c7a, c7b, c8a, c8b, c9a, c9b
	LOCAL	pass2, copyback, yes_fft, no_fft, more, fftdn, fftdn1

	set_FFT_constants clmarg

;; Call FFT code unless the FFT has already been started

	clear_timers
	cmp	DWORD PTR [esi-28][ebx], 0 ;; Test FFT-started flag
	jne	pass2			;; Jump is FFT already started
	mov	norm_ptr1, 0		;; Set FFT-ing flag
	mov	edx, p2cl/128*65536+128/clm*256;; Load loop counter
pfing	mov	ebp, clm*32		;; Set prefetch distance
	jmp	b0b			;; Do the pass 1 fft

;; Do the last 8 FFT levels
;;	do 1 pass2_8_levels_real and 383 pass2_8_levels_complex

pass2:	start_timer 0
	CALLP	pass2_macro
	end_timer 0
	cmp	ffttype, 1		;; We're done if FFTing only
	je	gw_finish_fft

;; Do the inverse FFT

	start_timer 1
	fldz				;; Init SUMOUT
	fstp	SUMOUT
	mov	edx, norm_biglit_array	;; Addr of the big/little flags array
	mov	norm_ptr1, edx		;; Save ptr
	mov	edx, norm_col_mults	;; Addr of the column multipliers
	mov	norm_ptr2, edx		;; Save ptr
	mov	edx, p2cl/128*65536+128/clm*256;; Load loop counter
pfing	mov	ebp, clm*32		;; Set prefetch distance

;; Do inverse FFT levels 9,10
;; On input the 32-byte cache lines hold these data values:
;;	0	256	1	257
;;	512	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	1K	1	1K+1
;;	256	...
;;	...
;;	2K	...
;;	...

;; Do 1 four_real_four_semireal_unfft macros
;;	distance between fft data elements is 1

c0b:	start_timer 15
	mov	ecx, scratch_area	;; Get address of scratch area
c2b:	cp_disp four_real_four_semireal_unfft, 8, blkdst, 2*blkdst, 8, clmblkdst, 2*clmblkdst
	lea	esi, [esi+dist1]	;; Next source pointer
	lea	ecx, [ecx+dist1]	;; Next dest pointer
	add	dl, 256/2/clm		;; Test inner loop counter
	jnc	c2b			;; Iterate if necessary
	lea	esi, [esi-clm*2*dist1+4*blkdst];; Next source pointer
	lea	ecx, [ecx-clm*2*dist1+4*clmblkdst];; Next dest pointer

;; Do 95 four_complex_unfft macros
;;	distance between fft data elements is 1

	mov	edi, sincos1		;; Load sin/cos pointer
	mov	eax, 256/8*256+12	;; 12 iterations of 7 or 8
c3b:	cp_disp four_complex_unfft, 8, blkdst, 2*blkdst, 8, clmblkdst, 2*clmblkdst
	lea	esi, [esi+dist1]	;; Next source pointer
	lea	ecx, [ecx+dist1]	;; Next dest pointer
	add	dl, 256/2/clm		;; Test loop counter
	jnc	c3b			;; Iterate if necessary
	lea	esi, [esi-clm*2*dist1+4*blkdst];; Next source pointer
	lea	ecx, [ecx-clm*2*dist1+4*clmblkdst];; Next dest pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	add	ah, 256/8		;; Test loop counter
	jnc	c3b			;; Iterate if necessary
	lea	ecx, [ecx-8*4*clmblkdst+clmblkdst32];; Next dest pointer
	dec	al			;; Test outer loop counter
	jnz	c3b			;; Iterate if necessary
	lea	ecx, [esi-96*4*blkdst]	;; Create prefetch pointer
	mov	esi, scratch_area	;; Restore source pointer
	end_timer 15

;; Do inverse FFT levels 7,8
;; On input the 32-byte cache lines hold these data values:
;;	0	1K	1	1K+1
;;	256	...
;;	...
;;	2K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	4K	1	4K+1
;;	256	...
;;	...
;;	8K	...
;;	...

;; Do 4 four_real_four_semireal_unfft macros
;;	distance between fft data elements is 4

	start_timer 16
	mov	eax, 2			;; Load loop counter
c4a:	touch				;; Preload TLB
c4b:	disp four_real_four_semireal_unfft, 8, 4*clmblkdst, 8*clmblkdst
	loop2	c4b, clmblkdst		;; 2 * 2*clm iterations
	dec	al
	jnz	c4a
	lea	esi, [esi-4*clmblkdst+16*clmblkdst];; Next source pointer

;; Do 92 four_complex_unfft macros
;;	distance between fft data elements is 4

	mov	edi, sincos2		;; Load sin/cos pointer
	mov	eax, 80000000h+12	;; 12 iters of 1 or 2 iters of 4
c5a:	touch				;; Preload TLB
c5b:	disp four_complex_unfft, 8, 4*clmblkdst, 8*clmblkdst
	loop2	c5b, clmblkdst		;; 2 * 2*clm iterations
	add	edx, 80000000h		;; Test middle loop counter (2 iters)
	jnc	c5a			;; Iterate if necessary
	lea	esi, [esi-4*clmblkdst+16*clmblkdst];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	add	eax, 80000000h		;; Test loop counter
	jnc	c5a			;; Iterate if necessary
	lea	esi, [esi-2*16*clmblkdst+clmblkdst32];; Next source pointer
	dec	al
	jnz	c5a
	lea	esi, [esi-12*clmblkdst32];; Restore source pointer
	end_timer 16

;; Do inverse FFT levels 5,6
;; On input the 32-byte cache lines hold these data values:
;;	0	4K	1	4K+1
;;	256	...
;;	...
;;	8K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	16K	1	16K+1
;;	256	...
;;	...
;;	32K	...
;;	...

;; Do 16 four_real_four_semireal_unfft macros
;;	distance between fft data elements is 16

	start_timer 17
	mov	al, 8			;; Load loop counter
c6a:	touch				;; Preload TLB
c6b:	disp four_real_four_semireal_unfft, 8, 16*clmblkdst, clmblkdst32
	loop2	c6b, clmblkdst		;; 2 * 2*clm iterations
	dec	al
	jnz	c6a
	lea	esi, [esi-16*clmblkdst+2*clmblkdst32];; Next source pointer

;; Do 80 four_complex_unfft macros
;;	distance between fft data elements is 16

	mov	edi, sincos3		;; Load sin/cos pointer
	mov	eax, 5			;; 5 iterations of 16
c7a:	touch				;; Preload TLB
c7b:	disp four_complex_unfft, 8, 16*clmblkdst, clmblkdst32
	loop2	c7b, clmblkdst		;; 2 * 2*clm iterations
	add	eax, 80000000h/4	;; Test middle loop counter (8 iters)
	jnc	c7a			;; Iterate if necessary
	lea	esi, [esi-16*clmblkdst+2*clmblkdst32];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al
	jnz	c7a
	lea	esi, [esi-6*2*clmblkdst32];; Restore source pointer
	end_timer 17

;; Do inverse FFT levels 4
;; On input the 32-byte cache lines hold these data values:
;;	0	16K	1	16K+1
;;	256	...
;;	...
;;	32K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	32K	1	32K+1
;;	256	...
;;	...
;;	64K	...
;;	...

;; Do 32 four_real_four_semireal_unfft_1 macros
;;	distance between fft data elements is 64

	start_timer 18
	mov	al, 8			;; Load loop counter
c8a:	touch				;; Preload TLB
c8b:	disp four_real_four_semireal_unfft_1, clmblkdst32, 8, 2*clmblkdst32
	loop4	c8b, clmblkdst		;; 4 * 2*clm iterations
	dec	al
	jnz	c8a
	lea	esi, [esi-32*clmblkdst+4*clmblkdst32];; Next source pointer

;; Do 64 four_complex_unfft_1 macros
;;	distance between fft data elements is 64

	mov	edi, sincos4		;; Load sin/cos pointer
	mov	eax, 2			;; 2 iterations of 32
c9a:	touch				;; Preload TLB
c9b:	disp four_complex_unfft_1, clmblkdst32, 8, 2*clmblkdst32
	loop4	c9b, clmblkdst		;; 4 * 2*clm iterations
	add	eax, 80000000h/4	;; Test middle loop counter (8 iters)
	jnc	c9a			;; Iterate if necessary
	lea	esi, [esi-32*clmblkdst+4*clmblkdst32];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al
	jnz	c9a
	lea	esi, [esi-3*4*clmblkdst32];; Restore source pointer
	end_timer 18

;; Do inverse FFT levels 1,2,3
;; On input the 32-byte cache lines hold these data values:
;;	0	32K	1	32K+1
;;	256	...
;;	...
;;	64K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	96K	1	96K+1
;;	256
;;	...
;;	96K-256

;; Do 128 six_reals_last_unfft macros
;;	distance between fft data elements is 128

	start_timer 19
c1a:	touch				;; Preload TLB
c1b:	disp six_reals_last_unfft, 8, 4*clmblkdst32, 8*clmblkdst32
	loop2	c1b, clmblkdst		;; 2 * 2*clm iterations
	add	eax, 80000000h/8	;; Test loop counter (16 iters)
	jnc	c1a			;; Iterate if necessary
	lea	esi, [esi-32*clmblkdst+clmblkdst32];; Next source pointer
	add	edx, 80000000h/2	;; Test loop counter (4 iters)
	jnc	c1a			;; Iterate if necessary
	lea	esi, [esi-4*clmblkdst32];; Restore source pointer
pfing	lea	ecx, [ecx-184*blkdst]	;; Restore dest ptr
	end_timer 19

;; Normalize these values

	start_timer 20
	mov	eax, NORMRTN		;; Addr of normalization routine
	call	eax
	sub	eax, eax
	end_timer 20

;; Have we been given permission to start the FFT on the result?
;; If so, do so now while the data is in the L2 cache.

	cmp	POSTFFT, 0		;; Test flag
	je	short copyback		;; Copy data, skip FFT if flag not set
	copy_3_words clm, 1
	cmp	edx, p2cl/128*65536+(128/clm-4/clm)*256;; Delay fft on 8 sets
	jle	yes_fft			;; No delay, do FFT now

;; Copy data from the scratch area back to the FFT data area

copyback:
	copy_scratch_data_to_fft 12, 32, clmblkdst32
	jmp	no_fft			;; Skip forward FFT code

;; Copy data from the FFT data area back to the scratch area

b0b:	copy_fft_data_to_scratch 12, 32, clmblkdst32

;; Do FFT levels 1,2,3
;;
;; On input the 32-byte cache lines hold these data values:
;;	0	96K	1	96K+1
;;	256
;;	...
;;	96K-256
;; On output the 32-byte cache lines hold these data values:
;;	0	32K	1	32K+1
;;	256	...
;;	...
;;	64K	...
;;	...

;; Do 128 six_reals_fft macros
;;	distance between fft data elements is 128

yes_fft:start_timer 21
	sub	eax, eax		;; 128 iterations
pfing	lea	ecx, [ecx+184*blkdst]	;; Create prefetch ptr
b1a:	touch				;; Preload TLB
b1b:	disp six_reals_fft, 4*clmblkdst32, 8*clmblkdst32, 8
	loop2	b1b, clmblkdst		;; 2 * 2*clm iterations
	add	eax, 80000000h/8	;; Test loop counter (16 iters)
	jnc	b1a			;; Iterate if necessary
	lea	esi, [esi-32*clmblkdst+clmblkdst32];; Next source pointer
	add	edx, 80000000h/2	;; Test loop counter (4 iters)
	jnc	b1a			;; Iterate if necessary
	lea	esi, [esi-4*clmblkdst32];; Restore source pointer
	end_timer 21

;; Do FFT levels 4
;;
;; On input the 32-byte cache lines hold these data values:
;;	0	32K	1	32K+1
;;	256	...
;;	...
;;	64K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	16K	1	16K+1
;;	256	...
;;	...
;;	32K	...
;;	...

;; Do 32 four_real_four_semireal_fft_1 macros
;;	distance between fft data elements is 64

	start_timer 22
	mov	al, 8			;; Load loop counter
b8a:	touch				;; Preload TLB
b8b:	disp four_real_four_semireal_fft_1, clmblkdst32, 2*clmblkdst32, 8
	loop4	b8b, clmblkdst		;; 4 * 2*clm iterations
	dec	al
	jnz	b8a
	lea	esi, [esi-32*clmblkdst+4*clmblkdst32];; Next source pointer

;; Do 64 four_complex_fft_1 macros
;;	distance between fft data elements is 64

	mov	edi, sincos4		;; Load sin/cos pointer
	mov	eax, 2			;; 2 iterations of 32
b9a:	touch				;; Preload TLB
b9b:	disp four_complex_fft_1, clmblkdst32, 2*clmblkdst32, 8
	loop2	b9b, clmblkdst		;; 2 * 2*clm iterations
	add	eax, 80000000h/8	;; Test middle loop counter (16 iters)
	jnc	b9a			;; Iterate if necessary
	lea	esi, [esi-32*clmblkdst+4*clmblkdst32];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al
	jnz	b9a
	lea	esi, [esi-3*4*clmblkdst32];; Restore source pointer
	end_timer 22

;; Do FFT levels 5,6
;;
;; On input the 32-byte cache lines hold these data values:
;;	0	16K	1	16K+1
;;	256	...
;;	...
;;	32K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	4K	1	4K+1
;;	256	...
;;	...
;;	8K	...
;;	...

;; Do 16 four_real_four_semireal_fft macros
;;	distance between fft data elements is 16

	start_timer 23
	mov	al, 8			;; Load loop counter
b6a:	touch				;; Preload TLB
b6b:	disp four_real_four_semireal_fft, 16*clmblkdst, clmblkdst32, 8
	loop2	b6b, clmblkdst		;; 2 * 2*clm iterations
	dec	al
	jnz	b6a
	lea	esi, [esi-16*clmblkdst+2*clmblkdst32];; Next source pointer

;; Do 80 four_complex_fft macros
;;	distance between fft data elements is 16

	mov	edi, sincos3		;; Load sin/cos pointer
	mov	eax, 5			;; 5 iterations of 16
b7a:	touch				;; Preload TLB
b7b:	disp four_complex_fft, 16*clmblkdst, clmblkdst32, 8
	loop2	b7b, clmblkdst		;; 2 * 2*clm iterations
	add	eax, 80000000h/4	;; Test middle loop counter (8 iters)
	jnc	b7a			;; Iterate if necessary
	lea	esi, [esi-16*clmblkdst+2*clmblkdst32];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al
	jnz	b7a
	lea	esi, [esi-6*2*clmblkdst32];; Restore source pointer
	end_timer 23

;; Do FFT levels 7,8
;;
;; On input the 32-byte cache lines hold these data values:
;;	0	4K	1	4K+1
;;	256	...
;;	...
;;	8K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	1K	1	1K+1
;;	256	...
;;	...
;;	2K	...
;;	...

;; Do 4 four_real_four_semireal_fft macros
;;	distance between fft data elements is 4

	start_timer 24
	mov	al, 2			;; Load loop counter
b4a:	touch				;; Preload TLB
b4b:	disp four_real_four_semireal_fft, 4*clmblkdst, 8*clmblkdst, 8
	loop2	b4b, clmblkdst		;; 2 * 2*clm iterations
	dec	al
	jnz	b4a
	lea	esi, [esi-4*clmblkdst+16*clmblkdst];; Next source pointer

;; Do 92 four_complex_fft macros
;;	distance between fft data elements is 4

	mov	edi, sincos2		;; Load sin/cos pointer
	mov	eax, 80000000h+12	;; 12 iters of 1 or 2 iters of 4
b5a:	touch				;; Preload TLB
b5b:	disp four_complex_fft, 4*clmblkdst, 8*clmblkdst, 8
	loop2	b5b, clmblkdst		;; 2*clm iterations
	add	edx, 80000000h		;; Test middle loop counter (2 iters)
	jnc	b5a			;; Iterate if necessary
	lea	esi, [esi-4*clmblkdst+16*clmblkdst];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	add	eax, 80000000h		;; Test middle loop counter (2 iters)
	jnc	b5a			;; Iterate if necessary
	lea	esi, [esi-2*16*clmblkdst+clmblkdst32];; Next source pointer
	dec	al
	jnz	b5a
	lea	esi, [esi-12*clmblkdst32];; Restore source pointer
	end_timer 24

;; Do FFT levels 9,10
;;
;; On input the 32-byte cache lines hold these data values:
;;	0	1K	1	1K+1
;;	256	...
;;	...
;;	2K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	256	1	257
;;	512	...
;;	...

;; Do 1 four_real_four_semireal_fft macros
;;	distance between fft data elements is 1

	start_timer 25
pfing	lea	ecx, [ecx-384*blkdst]	;; Create dest ptr from prefetch ptr
b2b:	cp_disp four_real_four_semireal_fft, clmblkdst, 2*clmblkdst, 8, blkdst, 2*blkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	lea	ecx, [ecx+dist1]	;; Next dest pointer
	add	dl, 256/2/clm		;; Test inner loop counter
	jnc	b2b			;; Iterate if necessary
	lea	esi, [esi-clm*2*dist1+4*clmblkdst];; Next source pointer
	lea	ecx, [ecx-clm*2*dist1+4*blkdst];; Next dest pointer

;; Do 95 four_complex_fft macros
;;	distance between fft data elements is 1

	mov	edi, sincos1		;; Load sin/cos pointer
	mov	eax, 256/8*256+12	;; 12 iters of 7 or 8 iters
b3b:	cp_disp four_complex_fft, clmblkdst, 2*clmblkdst, 8, blkdst, 2*blkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	lea	ecx, [ecx+dist1]	;; Next dest pointer
	add	dl, 256/2/clm		;; Test inner loop counter
	jnc	b3b			;; Iterate if necessary
	lea	esi, [esi-clm*2*dist1+4*clmblkdst];; Next source pointer
	lea	ecx, [ecx-clm*2*dist1+4*blkdst];; Next dest pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	add	ah, 256/8		;; Test loop counter
	jnc	b3b			;; Iterate if necessary
	lea	esi, [esi-8*4*clmblkdst+clmblkdst32];; Next source pointer
	dec	al
	jnz	b3b
	lea	esi, [ecx-96*4*blkdst]	;; Restore source pointer
	end_timer 25

;; Work on next 4 sets of 32 values

no_fft:	lea	esi, [esi+clm*2*dist1]	;; Next source pointer
	dec	dh			;; Test loop counter
	jnz	short more		;; Do more FFTs
	lea	esi, [esi+64]		;; Skip pad every 128 cache lines (4KB)
pfing	mov	ebp, clm*32		;; Set prefetch distance
	sub	edx, 65536		;; Test loop counter
	jz	short fftdn		;; Jump if done looping
	mov	dh, 128/clm		;; Restore cache line counter
more:	set_prefetch_gap
	cmp	norm_ptr1, 0		;; What was the loop start point?
	jne	c0b			;; Do inverse-FFT/norm/FFT iteration
	jmp	b0b			;; Do another FFT-only iteration
fftdn:	lea	esi, [esi-p2cl*2*dist1-p2cl/128*64];; Restore source pointer
	cmp	esi, DESTARG		;; If esi was not restored, then we 
	jne	short fftdn1		;; just finished skipped postfft data
	sub	ebx, ebx
	mov	DIST_TO_FFTSRCARG, ebx
	cmp	norm_ptr1, 0		;; What was the loop start point?
	je	pass2			;; FFT-only, go do pass 2

;; Split the carries into high and low carries (so they do not exceed
;; the desired number of bits per FFT word.  Then add the carries back to the
;; FFT data.

	start_timer 28
	call	gw_carries
	end_timer 28

;; Have we been given permission to start the FFT on the result?
;; If so, set loop to FFT the first few cache lines now that the carries
;; have been added in.

	cmp	POSTFFT, 0		;; Test flag
	je	short fftdn1		;; Skip FFT code if flag not set
	mov	esi, DESTARG		;; Load FFT data address
	mov	edx, 65536+4/clm*256	;; Do 8 previously skipped cache lines
	mov	norm_ptr1, 0		;; Set flag so only forward FFT is done
	copy_4_words
	jmp	b0b

;; Now add in the FFT'ed carries, cleanup and return

fftdn1:	end_timer 1
	jmp	gw_finish_mult
	ENDM


;; Perform a 192K-element negacyclic FFT.  This is done in two passes.  Pass 1
;; does 10 levels, pass 2 uses common code to perform the last 8 levels.

pass1levels10complex3 MACRO pass2_macro, p2cl, clmarg
	LOCAL	b0b, b1a, b1b, b3b, b5a, b5b, b7a, b7b, b9a, b9b
	LOCAL	c0b, c1a, c1b, c3b, c5a, c5b, c7a, c7b, c9a, c9b
	LOCAL	pass2, copyback, yes_fft, no_fft, more, fftdn, fftdn1

	set_FFT_constants clmarg

;; Call FFT code unless the FFT has already been started

	clear_timers
	cmp	DWORD PTR [esi-28][ebx], 0 ;; Test FFT-started flag
	jne	pass2			;; Jump is FFT already started
	mov	norm_ptr1, 0		;; Set FFT-ing flag
	mov	edx, p2cl/128*65536+128/clm*256;; Load loop counter
pfing	mov	ebp, clm*32		;; Set prefetch distance
	jmp	b0b			;; Do the pass 1 fft

;; Do the last 8 FFT levels
;;	do 384 pass2_8_levels_complex

pass2:	start_timer 0
	CALLP	pass2_macro
	end_timer 0
	cmp	ffttype, 1		;; We're done if FFTing only
	je	gw_finish_fft

;; Do the inverse FFT

	start_timer 1
	fldz				;; Init SUMOUT
	fstp	SUMOUT
	mov	edx, norm_biglit_array	;; Addr of the big/little flags array
	mov	norm_ptr1, edx		;; Save ptr
	mov	edx, norm_col_mults	;; Addr of the column multipliers
	mov	norm_ptr2, edx		;; Save ptr
	mov	edx, p2cl/128*65536+128/clm*256;; Load loop counter
pfing	mov	ebp, clm*32		;; Set prefetch distance

;; Do inverse FFT levels 9,10
;; On input the 32-byte cache lines hold these data values:
;;	0	256	1	257
;;	512	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	1K	1	1K+1
;;	256	...
;;	...
;;	2K	...
;;	...

;; Do 96 four_complex_unfft macros
;;	distance between fft data elements is 1

c0b:	start_timer 15
	mov	ecx, scratch_area	;; Get address of scratch area
	mov	edi, sincos1		;; Load sin/cos pointer
	mov	eax, 12			;; 12 iterations 8
c3b:	cp_disp four_complex_unfft, 8, blkdst, 2*blkdst, 8, clmblkdst, 2*clmblkdst
	lea	esi, [esi+dist1]	;; Next source pointer
	lea	ecx, [ecx+dist1]	;; Next dest pointer
	add	dl, 256/2/clm		;; Test loop counter
	jnc	c3b			;; Iterate if necessary
	lea	esi, [esi-clm*2*dist1+4*blkdst];; Next source pointer
	lea	ecx, [ecx-clm*2*dist1+4*clmblkdst];; Next dest pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	add	ah, 256/8		;; Test loop counter
	jnc	c3b			;; Iterate if necessary
	lea	ecx, [ecx-8*4*clmblkdst+clmblkdst32];; Next dest pointer
	dec	al			;; Test outer loop counter
	jnz	c3b			;; Iterate if necessary
	lea	ecx, [esi-96*4*blkdst]	;; Create prefetch pointer
	mov	esi, scratch_area	;; Restore source pointer
	end_timer 15

;; Do inverse FFT levels 7,8
;; On input the 32-byte cache lines hold these data values:
;;	0	1K	1	1K+1
;;	256	...
;;	...
;;	2K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	4K	1	4K+1
;;	256	...
;;	...
;;	8K	...
;;	...

;; Do 96 four_complex_unfft macros
;;	distance between fft data elements is 4

	start_timer 16
	mov	edi, sincos2		;; Load sin/cos pointer
	mov	eax, 12			;; 12 iters of 2 iters of 4
c5a:	touch				;; Preload TLB
c5b:	disp four_complex_unfft, 8, 4*clmblkdst, 8*clmblkdst
	loop2	c5b, clmblkdst		;; 2 * 2*clm iterations
	add	edx, 80000000h		;; Test middle loop counter (2 iters)
	jnc	c5a			;; Iterate if necessary
	lea	esi, [esi-4*clmblkdst+16*clmblkdst];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	add	eax, 80000000h		;; Test loop counter
	jnc	c5a			;; Iterate if necessary
	lea	esi, [esi-2*16*clmblkdst+clmblkdst32];; Next source pointer
	dec	al
	jnz	c5a
	lea	esi, [esi-12*clmblkdst32];; Restore source pointer
	end_timer 16

;; Do inverse FFT levels 5,6
;; On input the 32-byte cache lines hold these data values:
;;	0	4K	1	4K+1
;;	256	...
;;	...
;;	8K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	16K	1	16K+1
;;	256	...
;;	...
;;	32K	...
;;	...

;; Do 96 four_complex_unfft macros
;;	distance between fft data elements is 16

	start_timer 17
	mov	edi, sincos3		;; Load sin/cos pointer
	mov	eax, 6			;; 6 iterations of 16
c7a:	touch				;; Preload TLB
c7b:	disp four_complex_unfft, 8, 16*clmblkdst, clmblkdst32
	loop2	c7b, clmblkdst		;; 2 * 2*clm iterations
	add	eax, 80000000h/4	;; Test middle loop counter (8 iters)
	jnc	c7a			;; Iterate if necessary
	lea	esi, [esi-16*clmblkdst+2*clmblkdst32];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al
	jnz	c7a
	lea	esi, [esi-6*2*clmblkdst32];; Restore source pointer
	end_timer 17

;; Do inverse FFT levels 4
;; On input the 32-byte cache lines hold these data values:
;;	0	16K	1	16K+1
;;	256	...
;;	...
;;	32K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	32K	1	32K+1
;;	256	...
;;	...
;;	64K	...
;;	...

;; Do 96 four_complex_unfft_1 macros
;;	distance between fft data elements is 64

	start_timer 18
	mov	edi, sincos4		;; Load sin/cos pointer
	mov	eax, 3			;; 3 iterations of 32
c9a:	touch				;; Preload TLB
c9b:	disp four_complex_unfft_1, clmblkdst32, 8, 2*clmblkdst32
	loop4	c9b, clmblkdst		;; 4 * 2*clm iterations
	add	eax, 80000000h/4	;; Test middle loop counter (8 iters)
	jnc	c9a			;; Iterate if necessary
	lea	esi, [esi-32*clmblkdst+4*clmblkdst32];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al
	jnz	c9a
	lea	esi, [esi-3*4*clmblkdst32];; Restore source pointer
	end_timer 18

;; Do inverse FFT levels 1,2,3
;; On input the 32-byte cache lines hold these data values:
;;	0	32K	1	32K+1
;;	256	...
;;	...
;;	64K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	128K	1	128K+1
;;	256
;;	...
;;	128K-256

;; Do 128 three_complex_last_unfft macros
;;	distance between fft data elements is 128

	start_timer 19
	mov	edi, plus1_premults	;; Address of premultiplier table
c1a:	touch				;; Preload TLB
c1b:	three_complex_last_unfft 8, 4*clmblkdst32, 8*clmblkdst32
	loop2edi c1b, clmblkdst, 48	;; 2 * 2*clm iterations bumping edi
	add	eax, 80000000h/8	;; Test loop counter (16 iters)
	jnc	c1a			;; Iterate if necessary
	lea	esi, [esi-32*clmblkdst+clmblkdst32];; Next source pointer
	add	edx, 80000000h/2	;; Test loop counter (4 iters)
	jnc	c1a			;; Iterate if necessary
	lea	esi, [esi-4*clmblkdst32];; Restore source pointer
pfing	lea	ecx, [ecx-184*blkdst]	;; Restore dest ptr
	end_timer 19

;; Normalize these values

	start_timer 20
	mov	eax, NORMRTN		;; Addr of normalization routine
	call	eax
	sub	eax, eax
	end_timer 20

;; Have we been given permission to start the FFT on the result?
;; If so, do so now while the data is in the L2 cache.

	cmp	POSTFFT, 0		;; Test flag
	je	short copyback		;; Copy data, skip FFT if flag not set
	copy_3_words clm, 1
	cmp	edx, p2cl/128*65536+(128/clm-4/clm)*256;; Delay fft on 8 sets
	jle	yes_fft			;; No delay, do FFT now

;; Copy data from the scratch area back to the FFT data area

copyback:
	copy_scratch_data_to_fft 12, 32, clmblkdst32
	jmp	no_fft			;; Skip forward FFT code

;; Copy data from the FFT data area back to the scratch area

b0b:	copy_fft_data_to_scratch 12, 32, clmblkdst32

;; Do FFT levels 1,2,3
;;
;; On input the 32-byte cache lines hold these data values:
;;	0	128K	1	128K+1
;;	256
;;	...
;;	128K-256
;; On output the 32-byte cache lines hold these data values:
;;	0	32K	1	32K+1
;;	256	...
;;	...
;;	64K	...
;;	...

;; Do 128 three_complex_fft_premult macros
;;	distance between fft data elements is 128

yes_fft:start_timer 21
	mov	edi, plus1_premults	;; Address of premultiplier table
	sub	eax, eax		;; 128 iterations
pfing	lea	ecx, [ecx+176*blkdst]	;; Create prefetch ptr
b1a:	touch				;; Preload TLB
b1b:	three_complex_fft_premult 4*clmblkdst32, 8*clmblkdst32, 8
	loop2edi b1b, clmblkdst, 48	;; 2 * 2*clm iterations bumping edi
	add	eax, 80000000h/8	;; Test loop counter (16 iters)
	jnc	b1a			;; Iterate if necessary
	lea	esi, [esi-32*clmblkdst+clmblkdst32];; Next source pointer
	add	edx, 80000000h/2	;; Test loop counter (4 iters)
	jnc	b1a			;; Iterate if necessary
	lea	esi, [esi-4*clmblkdst32];; Restore source pointer
	end_timer 21

;; Do FFT levels 4
;;
;; On input the 32-byte cache lines hold these data values:
;;	0	32K	1	32K+1
;;	256	...
;;	...
;;	64K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	16K	1	16K+1
;;	256	...
;;	...
;;	32K	...
;;	...

;; Do 96 four_complex_fft_1 macros
;;	distance between fft data elements is 64

	start_timer 22
	mov	edi, sincos4		;; Load sin/cos pointer
	mov	eax, 3			;; 3 iterations of 32
b9a:	touch				;; Preload TLB
b9b:	disp four_complex_fft_1, clmblkdst32, 2*clmblkdst32, 8
	loop2	b9b, clmblkdst		;; 2 * 2*clm iterations
	add	eax, 80000000h/8	;; Test middle loop counter (16 iters)
	jnc	b9a			;; Iterate if necessary
	lea	esi, [esi-32*clmblkdst+4*clmblkdst32];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al
	jnz	b9a
	lea	esi, [esi-3*4*clmblkdst32];; Restore source pointer
	end_timer 22

;; Do FFT levels 5,6
;;
;; On input the 32-byte cache lines hold these data values:
;;	0	16K	1	16K+1
;;	256	...
;;	...
;;	32K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	4K	1	4K+1
;;	256	...
;;	...
;;	8K	...
;;	...

;; Do 96 four_complex_fft macros
;;	distance between fft data elements is 16

	start_timer 23
	mov	edi, sincos3		;; Load sin/cos pointer
	mov	eax, 6			;; 6 iterations of 16
b7a:	touch				;; Preload TLB
b7b:	disp four_complex_fft, 16*clmblkdst, clmblkdst32, 8
	loop2	b7b, clmblkdst		;; 2 * 2*clm iterations
	add	eax, 80000000h/4	;; Test middle loop counter (8 iters)
	jnc	b7a			;; Iterate if necessary
	lea	esi, [esi-16*clmblkdst+2*clmblkdst32];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al
	jnz	b7a
	lea	esi, [esi-6*2*clmblkdst32];; Restore source pointer
	end_timer 23

;; Do FFT levels 7,8
;;
;; On input the 32-byte cache lines hold these data values:
;;	0	4K	1	4K+1
;;	256	...
;;	...
;;	8K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	1K	1	1K+1
;;	256	...
;;	...
;;	2K	...
;;	...

;; Do 96 four_complex_fft macros
;;	distance between fft data elements is 4

	start_timer 24
	mov	edi, sincos2		;; Load sin/cos pointer
	mov	eax, 12			;; 12 iters of 2 iters of 4
b5a:	touch				;; Preload TLB
b5b:	disp four_complex_fft, 4*clmblkdst, 8*clmblkdst, 8
	loop2	b5b, clmblkdst		;; 2 * 2*clm iterations
	add	edx, 80000000h		;; Test middle loop counter (2 iters)
	jnc	b5a			;; Iterate if necessary
	lea	esi, [esi-4*clmblkdst+16*clmblkdst];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	add	eax, 80000000h		;; Test middle loop counter (2 iters)
	jnc	b5a			;; Iterate if necessary
	lea	esi, [esi-2*16*clmblkdst+clmblkdst32];; Next source pointer
	dec	al
	jnz	b5a
	lea	esi, [esi-12*clmblkdst32];; Restore source pointer
	end_timer 24

;; Do FFT levels 9,10
;;
;; On input the 32-byte cache lines hold these data values:
;;	0	1K	1	1K+1
;;	256	...
;;	...
;;	2K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	256	1	257
;;	512	...
;;	...

;; Do 96 four_complex_fft macros
;;	distance between fft data elements is 1

	start_timer 25
	mov	edi, sincos1		;; Load sin/cos pointer
pfing	lea	ecx, [ecx-384*blkdst]	;; Create dest ptr from prefetch ptr
	mov	eax, 12			;; 12 iters of 8 iters
b3b:	cp_disp four_complex_fft, clmblkdst, 2*clmblkdst, 8, blkdst, 2*blkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	lea	ecx, [ecx+dist1]	;; Next dest pointer
	add	dl, 256/2/clm		;; Test inner loop counter
	jnc	b3b			;; Iterate if necessary
	lea	esi, [esi-clm*2*dist1+4*clmblkdst];; Next source pointer
	lea	ecx, [ecx-clm*2*dist1+4*blkdst];; Next dest pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	add	ah, 256/8		;; Test loop counter
	jnc	b3b			;; Iterate if necessary
	lea	esi, [esi-8*4*clmblkdst+clmblkdst32];; Next source pointer
	dec	al
	jnz	b3b
	lea	esi, [ecx-96*4*blkdst]	;; Restore source pointer
	end_timer 25

;; Work on next 4 sets of 32 values

no_fft:	lea	esi, [esi+clm*2*dist1]	;; Next source pointer
	dec	dh			;; Test loop counter
	jnz	short more		;; Do more FFTs
	lea	esi, [esi+64]		;; Skip pad every 128 cache lines (4KB)
pfing	mov	ebp, clm*32		;; Set prefetch distance
	sub	edx, 65536		;; Test loop counter
	jz	short fftdn		;; Jump if done looping
	mov	dh, 128/clm		;; Restore cache line counter
more:	set_prefetch_gap
	cmp	norm_ptr1, 0		;; What was the loop start point?
	jne	c0b			;; Do inverse-FFT/norm/FFT iteration
	jmp	b0b			;; Do another FFT-only iteration
fftdn:	lea	esi, [esi-p2cl*2*dist1-p2cl/128*64];; Restore source pointer
	cmp	esi, DESTARG		;; If esi was not restored, then we 
	jne	short fftdn1		;; just finished skipped postfft data
	sub	ebx, ebx
	mov	DIST_TO_FFTSRCARG, ebx
	cmp	norm_ptr1, 0		;; What was the loop start point?
	je	pass2			;; FFT-only, go do pass 2

;; Split the carries into high and low carries (so they do not exceed
;; the desired number of bits per FFT word.  Then add the carries back to the
;; FFT data.

	start_timer 28
	call	gw_carries
	end_timer 28

;; Have we been given permission to start the FFT on the result?
;; If so, set loop to FFT the first few cache lines now that the carries
;; have been added in.

	cmp	POSTFFT, 0		;; Test flag
	je	short fftdn1		;; Skip FFT code if flag not set
	mov	esi, DESTARG		;; Load FFT data address
	mov	edx, 65536+4/clm*256	;; Do 8 previously skipped cache lines
	mov	norm_ptr1, 0		;; Set flag so only forward FFT is done
	copy_4_words
	jmp	b0b

;; Now add in the FFT'ed carries, cleanup and return

fftdn1:	end_timer 1
	jmp	gw_finish_mult
	ENDM


;; Perform a 224K-element FFT.  This is done in two passes.  Pass 1
;; does 10 levels, pass 2 uses common code to perform the last 8 levels.

pass1levels10pfa7 MACRO pass2_macro, p2cl, clmarg
	LOCAL	b0b, b1a, b1b, b2b, b3b, b4a, b4b, b5a, b5b, b6a, b6b
	LOCAL	b7a, b7b, b8a, b8b, b9a, b9b
	LOCAL	c0b, c1a, c1b, c2b, c3b, c4a, c4b, c5a, c5b, c6a, c6b
	LOCAL	c7a, c7b, c8a, c8b, c9a, c9b
	LOCAL	pass2, copyback, yes_fft, no_fft, more, fftdn, fftdn1

	set_FFT_constants clmarg

;; Call FFT code unless the FFT has already been started

	clear_timers
	cmp	DWORD PTR [esi-28][ebx], 0 ;; Test FFT-started flag
	jne	pass2			;; Jump is FFT already started
	mov	norm_ptr1, 0		;; Set FFT-ing flag
	mov	edx, p2cl/128*65536+128/clm*256;; Load loop counter
pfing	mov	ebp, clm*32		;; Set prefetch distance
	jmp	b0b			;; Do the pass 1 fft

;; Do the last 8 FFT levels
;;	do 1 pass2_8_levels_real and 447 pass2_8_levels_complex

pass2:	start_timer 0
	CALLP	pass2_macro
	end_timer 0
	cmp	ffttype, 1		;; We're done if FFTing only
	je	gw_finish_fft

;; Do the inverse FFT

	start_timer 1
	fldz				;; Init SUMOUT
	fstp	SUMOUT
	mov	edx, norm_biglit_array	;; Addr of the big/little flags array
	mov	norm_ptr1, edx		;; Save ptr
	mov	edx, norm_col_mults	;; Addr of the column multipliers
	mov	norm_ptr2, edx		;; Save ptr
	mov	edx, p2cl/128*65536+128/clm*256;; Load loop counter
pfing	mov	ebp, clm*32		;; Set prefetch distance

;; Do inverse FFT levels 9,10
;; On input the 32-byte cache lines hold these data values:
;;	0	256	1	257
;;	512	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	1K	1	1K+1
;;	256	...
;;	...
;;	2K	...
;;	...

;; Do 1 four_real_four_semireal_unfft macros
;;	distance between fft data elements is 1

c0b:	start_timer 15
	mov	ecx, scratch_area	;; Get address of scratch area
c2b:	cp_disp four_real_four_semireal_unfft, 8, blkdst, 2*blkdst, 8, clmblkdst, 2*clmblkdst
	lea	esi, [esi+dist1]	;; Next source pointer
	lea	ecx, [ecx+dist1]	;; Next dest pointer
	add	dl, 256/2/clm		;; Test inner loop counter
	jnc	c2b			;; Iterate if necessary
	lea	esi, [esi-clm*2*dist1+4*blkdst];; Next source pointer
	lea	ecx, [ecx-clm*2*dist1+4*clmblkdst];; Next dest pointer

;; Do 111 four_complex_unfft macros
;;	distance between fft data elements is 1

	mov	edi, sincos1		;; Load sin/cos pointer
	mov	eax, 256/8*256+14	;; 14 iterations of 7 or 8
c3b:	cp_disp four_complex_unfft, 8, blkdst, 2*blkdst, 8, clmblkdst, 2*clmblkdst
	lea	esi, [esi+dist1]	;; Next source pointer
	lea	ecx, [ecx+dist1]	;; Next dest pointer
	add	dl, 256/2/clm		;; Test loop counter
	jnc	c3b			;; Iterate if necessary
	lea	esi, [esi-clm*2*dist1+4*blkdst];; Next source pointer
	lea	ecx, [ecx-clm*2*dist1+4*clmblkdst];; Next dest pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	add	ah, 256/8		;; Test loop counter
	jnc	c3b			;; Iterate if necessary
	lea	ecx, [ecx-8*4*clmblkdst+clmblkdst32];; Next dest pointer
	dec	al			;; Test outer loop counter
	jnz	c3b			;; Iterate if necessary
	lea	ecx, [esi-112*4*blkdst]	;; Create prefetch pointer
	mov	esi, scratch_area	;; Restore source pointer
	end_timer 15

;; Do inverse FFT levels 7,8
;; On input the 32-byte cache lines hold these data values:
;;	0	1K	1	1K+1
;;	256	...
;;	...
;;	2K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	4K	1	4K+1
;;	256	...
;;	...
;;	8K	...
;;	...

;; Do 4 four_real_four_semireal_unfft macros
;;	distance between fft data elements is 4

	start_timer 16
	mov	al, 2			;; Load loop counter
c4a:	touch				;; Preload TLB
c4b:	disp four_real_four_semireal_unfft, 8, 4*clmblkdst, 8*clmblkdst
	loop2	c4b, clmblkdst		;; 2 * 2*clm iterations
	dec	al
	jnz	c4a
	lea	esi, [esi-4*clmblkdst+16*clmblkdst];; Next source pointer

;; Do 108 four_complex_unfft macros
;;	distance between fft data elements is 4

	mov	edi, sincos2		;; Load sin/cos pointer
	mov	eax, 80000000h+14	;; 14 iters of 1 or 2 iters of 4
c5a:	touch				;; Preload TLB
c5b:	disp four_complex_unfft, 8, 4*clmblkdst, 8*clmblkdst
	loop2	c5b, clmblkdst		;; 2 * 2*clm iterations
	add	edx, 80000000h		;; Test middle loop counter (2 iters)
	jnc	c5a			;; Iterate if necessary
	lea	esi, [esi-4*clmblkdst+16*clmblkdst];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	add	eax, 80000000h		;; Test loop counter
	jnc	c5a			;; Iterate if necessary
	lea	esi, [esi-2*16*clmblkdst+clmblkdst32];; Next source pointer
	dec	al
	jnz	c5a
	lea	esi, [esi-14*clmblkdst32];; Restore source pointer
	end_timer 16

;; Do inverse FFT levels 5,6
;; On input the 32-byte cache lines hold these data values:
;;	0	4K	1	4K+1
;;	256	...
;;	...
;;	8K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	16K	1	16K+1
;;	256	...
;;	...
;;	32K	...
;;	...

;; Do 16 four_real_four_semireal_unfft macros
;;	distance between fft data elements is 16

	start_timer 17
	mov	al, 4			;; Load loop counter
c6a:	touch				;; Preload TLB
c6b:	disp four_real_four_semireal_unfft, 8, 16*clmblkdst, clmblkdst32
	loop4	c6b, clmblkdst		;; 4 * 2*clm iterations
	dec	al
	jnz	c6a
	lea	esi, [esi-16*clmblkdst+2*clmblkdst32];; Next source pointer

;; Do 96 four_complex_unfft macros
;;	distance between fft data elements is 16

	mov	edi, sincos3		;; Load sin/cos pointer
	mov	eax, 6			;; 6 iterations of 16
c7a:	touch				;; Preload TLB
c7b:	disp four_complex_unfft, 8, 16*clmblkdst, clmblkdst32
	loop2	c7b, clmblkdst		;; 2 * 2*clm iterations
	add	eax, 80000000h/4	;; Test middle loop counter (8 iters)
	jnc	c7a			;; Iterate if necessary
	lea	esi, [esi-16*clmblkdst+2*clmblkdst32];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al
	jnz	c7a
	lea	esi, [esi-7*2*clmblkdst32];; Restore source pointer
	end_timer 17

;; Do inverse FFT levels 4
;; On input the 32-byte cache lines hold these data values:
;;	0	16K	1	16K+1
;;	256	...
;;	...
;;	32K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	32K	1	32K+1
;;	256	...
;;	...
;;	64K	...
;;	...

;; Do 16 eight_reals_unfft_1 macros
;;	distance between fft data elements is 64

	start_timer 18
	mov	al, 4			;; Load loop counter
c8a:	touch				;; Preload TLB
c8b:	disp eight_reals_unfft_1, 16*clmblkdst, clmblkdst32, 8
	loop4	c8b, clmblkdst		;; 4 * 2*clm iterations
	dec	al
	jnz	c8a
	lea	esi, [esi-16*clmblkdst+2*clmblkdst32];; Next source pointer

;; Do 96 four_complex_unfft_1 macros
;;	distance between fft data elements is 64

	mov	edi, sincos4		;; Load sin/cos pointer
	mov	eax, 3			;; 3 iterations of 32
c9a:	touch				;; Preload TLB
c9b:	disp four_complex_unfft_1, clmblkdst32, 8, 2*clmblkdst32
	loop2	c9b, clmblkdst		;; 4 * 2*clm iterations
	add	eax, 80000000h/8	;; Test middle loop counter (16 iters)
	jnc	c9a			;; Iterate if necessary
	lea	esi, [esi-32*clmblkdst+4*clmblkdst32];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al
	jnz	c9a
	lea	esi, [esi-3*4*clmblkdst32-2*clmblkdst32];; Restore source ptr
	end_timer 18

;; Do inverse FFT levels 1,2,3
;; On input the 32-byte cache lines hold these data values:
;;	0	32K	1	32K+1
;;	256	...
;;	...
;;	64K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	112K	1	112K+1
;;	256
;;	...
;;	112K-256

;; Do 128 seven_reals_last_unfft macros
;;	distance between fft data elements is 128

	start_timer 19
c1a:	touch				;; Preload TLB
c1b:	two_seven_reals_last_unfft 8, 2*clmblkdst32
	loop1	c1b, clmblkdst		;; 2*clm iterations
	add	ah, 256/32		;; Test loop counter
	jnc	c1a			;; Iterate if necessary
	lea	esi, [esi-32*clmblkdst+clmblkdst32];; Next source pointer
	add	eax, 80000000h		;; Test loop counter (2 iters)
	jnc	c1a			;; Iterate if necessary
	lea	esi, [esi-2*clmblkdst32];; Restore source pointer
pfing	lea	ecx, [ecx-224*blkdst]	;; Restore dest ptr
	end_timer 19

;; Normalize these values

	start_timer 20
	mov	eax, NORMRTN		;; Addr of normalization routine
	call	eax
	sub	eax, eax
	end_timer 20

;; Have we been given permission to start the FFT on the result?
;; If so, do so now while the data is in the L2 cache.

	cmp	POSTFFT, 0		;; Test flag
	je	short copyback		;; Copy data, skip FFT if flag not set
	copy_3_words clm, 1
	cmp	edx, p2cl/128*65536+(128/clm-4/clm)*256;; Delay fft on 8 sets
	jle	yes_fft			;; No delay, do FFT now

;; Copy data from the scratch area back to the FFT data area

copyback:
	copy_scratch_data_to_fft 14, 32, clmblkdst32
	jmp	no_fft			;; Skip forward FFT code

;; Copy data from the FFT data area back to the scratch area

b0b:	copy_fft_data_to_scratch 14, 32, clmblkdst32

;; Do FFT levels 1,2,3
;;
;; On input the 32-byte cache lines hold these data values:
;;	0	112K	1	112K+1
;;	256
;;	...
;;	112K-256
;; On output the 32-byte cache lines hold these data values:
;;	0	32K	1	32K+1
;;	256	...
;;	...
;;	64K	...
;;	...

;; Do 128 seven_reals_fft macros
;;	distance between fft data elements is 128

yes_fft:start_timer 21
	sub	eax, eax		;; 64 iterations
pfing	lea	ecx, [ecx+224*blkdst]	;; Create prefetch ptr
b1a:	touch				;; Preload TLB
b1b:	two_seven_reals_fft 2*clmblkdst32, 8
	loop1	b1b, clmblkdst		;; 2*clm iterations
	add	ah, 256/32		;; Test loop counter (32 iters)
	jnc	b1a			;; Iterate if necessary
	lea	esi, [esi-32*clmblkdst+clmblkdst32];; Next source pointer
	add	eax, 80000000h		;; Test loop counter (2 iters)
	jnc	b1a			;; Iterate if necessary
	lea	esi, [esi-2*clmblkdst32];; Restore source pointer
	end_timer 21

;; Do FFT levels 4
;;
;; On input the 32-byte cache lines hold these data values:
;;	0	32K	1	32K+1
;;	256	...
;;	...
;;	64K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	16K	1	16K+1
;;	256	...
;;	...
;;	32K	...
;;	...

;; Do 16 eight_reals_fft_1 macros
;;	distance between fft data elements is 64

	start_timer 22
	mov	al, 4			;; Load loop counter
b8a:	touch				;; Preload TLB
b8b:	disp eight_reals_fft_1, 16*clmblkdst, clmblkdst32, 8
	loop4	b8b, clmblkdst		;; 4 * 2*clm iterations
	dec	al
	jnz	b8a
	lea	esi, [esi-16*clmblkdst+2*clmblkdst32];; Next source pointer

;; Do 96 four_complex_fft_1 macros
;;	distance between fft data elements is 64

	mov	edi, sincos4		;; Load sin/cos pointer
	mov	eax, 3			;; 3 iterations of 32
b9a:	touch				;; Preload TLB
b9b:	disp four_complex_fft_1, clmblkdst32, 2*clmblkdst32, 8
	loop2	b9b, clmblkdst		;; 2 * 2*clm iterations
	add	eax, 80000000h/8	;; Test middle loop counter (16 iters)
	jnc	b9a			;; Iterate if necessary
	lea	esi, [esi-32*clmblkdst+4*clmblkdst32];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al
	jnz	b9a
	lea	esi, [esi-3*4*clmblkdst32-2*clmblkdst32];; Restore source ptr
	end_timer 22

;; Do FFT levels 5,6
;;
;; On input the 32-byte cache lines hold these data values:
;;	0	16K	1	16K+1
;;	256	...
;;	...
;;	32K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	4K	1	4K+1
;;	256	...
;;	...
;;	8K	...
;;	...

;; Do 16 four_real_four_semireal_fft macros
;;	distance between fft data elements is 16

	start_timer 23
	mov	al, 4			;; Load loop counter
b6a:	touch				;; Preload TLB
b6b:	disp four_real_four_semireal_fft, 16*clmblkdst, clmblkdst32, 8
	loop4	b6b, clmblkdst		;; 4 * 2*clm iterations
	dec	al
	jnz	b6a
	lea	esi, [esi-16*clmblkdst+2*clmblkdst32];; Next source pointer

;; Do 96 four_complex_fft macros
;;	distance between fft data elements is 16

	mov	edi, sincos3		;; Load sin/cos pointer
	mov	eax, 6			;; 6 iterations of 16
b7a:	touch				;; Preload TLB
b7b:	disp four_complex_fft, 16*clmblkdst, clmblkdst32, 8
	loop2	b7b, clmblkdst		;; 2 * 2*clm iterations
	add	eax, 80000000h/4	;; Test middle loop counter (8 iters)
	jnc	b7a			;; Iterate if necessary
	lea	esi, [esi-16*clmblkdst+2*clmblkdst32];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al
	jnz	b7a
	lea	esi, [esi-7*2*clmblkdst32];; Restore source pointer
	end_timer 23

;; Do FFT levels 7,8
;;
;; On input the 32-byte cache lines hold these data values:
;;	0	4K	1	4K+1
;;	256	...
;;	...
;;	8K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	1K	1	1K+1
;;	256	...
;;	...
;;	2K	...
;;	...

;; Do 4 four_real_four_semireal_fft macros
;;	distance between fft data elements is 4

	start_timer 24
	mov	al, 2			;; Load loop counter
b4a:	touch				;; Preload TLB
b4b:	disp four_real_four_semireal_fft, 4*clmblkdst, 8*clmblkdst, 8
	loop2	b4b, clmblkdst		;; 2 * 2*clm iterations
	dec	al
	jnz	b4a
	lea	esi, [esi-4*clmblkdst+16*clmblkdst];; Next source pointer

;; Do 108 four_complex_fft macros
;;	distance between fft data elements is 4

	mov	edi, sincos2		;; Load sin/cos pointer
	mov	eax, 80000000h+14	;; 14 iters of 1 or 2 iters of 4
b5a:	touch				;; Preload TLB
b5b:	disp four_complex_fft, 4*clmblkdst, 8*clmblkdst, 8
	loop2	b5b, clmblkdst		;; 2*clm iterations
	add	edx, 80000000h		;; Test middle loop counter (2 iters)
	jnc	b5a			;; Iterate if necessary
	lea	esi, [esi-4*clmblkdst+16*clmblkdst];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	add	eax, 80000000h		;; Test middle loop counter (2 iters)
	jnc	b5a			;; Iterate if necessary
	lea	esi, [esi-2*16*clmblkdst+clmblkdst32];; Next source pointer
	dec	al
	jnz	b5a
	lea	esi, [esi-14*clmblkdst32];; Restore source pointer
	end_timer 24

;; Do FFT levels 9,10
;;
;; On input the 32-byte cache lines hold these data values:
;;	0	1K	1	1K+1
;;	256	...
;;	...
;;	2K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	256	1	257
;;	512	...
;;	...

;; Do 1 four_real_four_semireal_fft macros
;;	distance between fft data elements is 1

	start_timer 25
pfing	lea	ecx, [ecx-448*blkdst]	;; Create dest ptr from prefetch ptr
b2b:	cp_disp four_real_four_semireal_fft, clmblkdst, 2*clmblkdst, 8, blkdst, 2*blkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	lea	ecx, [ecx+dist1]	;; Next dest pointer
	add	dl, 256/2/clm		;; Test inner loop counter
	jnc	b2b			;; Iterate if necessary
	lea	esi, [esi-clm*2*dist1+4*clmblkdst];; Next source pointer
	lea	ecx, [ecx-clm*2*dist1+4*blkdst];; Next dest pointer

;; Do 111 four_complex_fft macros
;;	distance between fft data elements is 1

	mov	edi, sincos1		;; Load sin/cos pointer
	mov	eax, 256/8*256+14	;; 14 iters of 7 or 8 iters
b3b:	cp_disp four_complex_fft, clmblkdst, 2*clmblkdst, 8, blkdst, 2*blkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	lea	ecx, [ecx+dist1]	;; Next dest pointer
	add	dl, 256/2/clm		;; Test inner loop counter
	jnc	b3b			;; Iterate if necessary
	lea	esi, [esi-clm*2*dist1+4*clmblkdst];; Next source pointer
	lea	ecx, [ecx-clm*2*dist1+4*blkdst];; Next dest pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	add	ah, 256/8		;; Test loop counter
	jnc	b3b			;; Iterate if necessary
	lea	esi, [esi-8*4*clmblkdst+clmblkdst32];; Next source pointer
	dec	al
	jnz	b3b
	lea	esi, [ecx-112*4*blkdst]	;; Restore source pointer
	end_timer 25

;; Work on next 4 sets of 32 values

no_fft:	lea	esi, [esi+clm*2*dist1]	;; Next source pointer
	dec	dh			;; Test loop counter
	jnz	short more		;; Do more FFTs
	lea	esi, [esi+64]		;; Skip pad every 128 cache lines (4KB)
pfing	mov	ebp, clm*32		;; Set prefetch distance
	sub	edx, 65536		;; Test loop counter
	jz	short fftdn		;; Jump if done looping
	mov	dh, 128/clm		;; Restore cache line counter
more:	set_prefetch_gap
	cmp	norm_ptr1, 0		;; What was the loop start point?
	jne	c0b			;; Do inverse-FFT/norm/FFT iteration
	jmp	b0b			;; Do another FFT-only iteration
fftdn:	lea	esi, [esi-p2cl*2*dist1-p2cl/128*64];; Restore source pointer
	cmp	esi, DESTARG		;; If esi was not restored, then we 
	jne	short fftdn1		;; just finished skipped postfft data
	sub	ebx, ebx
	mov	DIST_TO_FFTSRCARG, ebx
	cmp	norm_ptr1, 0		;; What was the loop start point?
	je	pass2			;; FFT-only, go do pass 2

;; Split the carries into high and low carries (so they do not exceed
;; the desired number of bits per FFT word.  Then add the carries back to the
;; FFT data.

	start_timer 28
	call	gw_carries
	end_timer 28

;; Have we been given permission to start the FFT on the result?
;; If so, set loop to FFT the first few cache lines now that the carries
;; have been added in.

	cmp	POSTFFT, 0		;; Test flag
	je	short fftdn1		;; Skip FFT code if flag not set
	mov	esi, DESTARG		;; Load FFT data address
	mov	edx, 65536+4/clm*256	;; Do 8 previously skipped cache lines
	mov	norm_ptr1, 0		;; Set flag so only forward FFT is done
	copy_4_words
	jmp	b0b

;; Now add in the FFT'ed carries, cleanup and return

fftdn1:	end_timer 1
	jmp	gw_finish_mult
	ENDM


;; Perform a 256K-element FFT.  This is done in two passes.  Pass 1
;; does 10 levels, pass 2 uses common code to perform the last 8 levels.

pass1levels10pfa8 MACRO pass2_macro, p2cl, clmarg
	LOCAL	b0b, b1a, b1b, b2b, b3b, b4a, b4b, b5a, b5b, b6a, b6b
	LOCAL	b7a, b7b, b8a, b8b, b9a, b9b
	LOCAL	c0b, c1a, c1b, c2b, c3b, c4a, c4b, c5a, c5b, c6a, c6b
	LOCAL	c7a, c7b, c8a, c8b, c9a, c9b
	LOCAL	pass2, copyback, yes_fft, no_fft, more, fftdn, fftdn1

	set_FFT_constants clmarg

;; Call FFT code unless the FFT has already been started

	clear_timers
	cmp	DWORD PTR [esi-28][ebx], 0 ;; Test FFT-started flag
	jne	pass2			;; Jump is FFT already started
	mov	norm_ptr1, 0		;; Set FFT-ing flag
	mov	edx, p2cl/128*65536+128/clm*256;; Load loop counter
pfing	mov	ebp, clm*32		;; Set prefetch distance
	jmp	b0b			;; Do the pass 1 fft

;; Do the last 8 FFT levels
;;	do 1 pass2_8_levels_real and 511 pass2_8_levels_complex

pass2:	start_timer 0
	CALLP	pass2_macro
	end_timer 0
	cmp	ffttype, 1		;; We're done if FFTing only
	je	gw_finish_fft

;; Do the inverse FFT

	start_timer 1
	fldz				;; Init SUMOUT
	fstp	SUMOUT
	mov	edx, norm_biglit_array	;; Addr of the big/little flags array
	mov	norm_ptr1, edx		;; Save ptr
	mov	edx, norm_col_mults	;; Addr of the column multipliers
	mov	norm_ptr2, edx		;; Save ptr
	mov	edx, p2cl/128*65536+128/clm*256;; Load loop counter
pfing	mov	ebp, clm*32		;; Set prefetch distance

;; Do inverse FFT levels 9,10
;; On input the 32-byte cache lines hold these data values:
;;	0	256	1	257
;;	512	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	1K	1	1K+1
;;	256	...
;;	...
;;	2K	...
;;	...

;; Do 1 four_real_four_semireal_unfft macros
;;	distance between fft data elements is 1

c0b:	start_timer 15
	mov	ecx, scratch_area	;; Get address of scratch area
c2b:	cp_disp four_real_four_semireal_unfft, 8, blkdst, 2*blkdst, 8, clmblkdst, 2*clmblkdst
	lea	esi, [esi+dist1]	;; Next source pointer
	lea	ecx, [ecx+dist1]	;; Next dest pointer
	add	dl, 256/2/clm		;; Test inner loop counter
	jnc	c2b			;; Iterate if necessary
	lea	esi, [esi-clm*2*dist1+4*blkdst];; Next source pointer
	lea	ecx, [ecx-clm*2*dist1+4*clmblkdst];; Next dest pointer

;; Do 127 four_complex_unfft macros
;;	distance between fft data elements is 1

	mov	edi, sincos1		;; Load sin/cos pointer
	mov	eax, 256/8*256+16	;; 16 iterations of 7 or 8
c3b:	cp_disp four_complex_unfft, 8, blkdst, 2*blkdst, 8, clmblkdst, 2*clmblkdst
	lea	esi, [esi+dist1]	;; Next source pointer
	lea	ecx, [ecx+dist1]	;; Next dest pointer
	add	dl, 256/2/clm		;; Test loop counter
	jnc	c3b			;; Iterate if necessary
	lea	esi, [esi-clm*2*dist1+4*blkdst];; Next source pointer
	lea	ecx, [ecx-clm*2*dist1+4*clmblkdst];; Next dest pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	add	ah, 256/8		;; Test loop counter
	jnc	c3b			;; Iterate if necessary
	lea	ecx, [ecx-8*4*clmblkdst+clmblkdst32];; Next dest pointer
	dec	al			;; Test outer loop counter
	jnz	c3b			;; Iterate if necessary
	lea	ecx, [esi-128*4*blkdst]	;; Create prefetch pointer
	mov	esi, scratch_area	;; Restore source pointer
	end_timer 15

;; Do inverse FFT levels 7,8
;; On input the 32-byte cache lines hold these data values:
;;	0	1K	1	1K+1
;;	256	...
;;	...
;;	2K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	4K	1	4K+1
;;	256	...
;;	...
;;	8K	...
;;	...

;; Do 4 four_real_four_semireal_unfft macros
;;	distance between fft data elements is 4

	start_timer 16
	mov	al, 2			;; Load loop counter
c4a:	touch				;; Preload TLB
c4b:	disp four_real_four_semireal_unfft, 8, 4*clmblkdst, 8*clmblkdst
	loop2	c4b, clmblkdst		;; 2 * 2*clm iterations
	dec	al
	jnz	c4a
	lea	esi, [esi-4*clmblkdst+16*clmblkdst];; Next source pointer

;; Do 124 four_complex_unfft macros
;;	distance between fft data elements is 4

	mov	edi, sincos2		;; Load sin/cos pointer
	mov	eax, 80000000h+16	;; 16 iters of 1 or 2 iters of 4
c5a:	touch				;; Preload TLB
c5b:	disp four_complex_unfft, 8, 4*clmblkdst, 8*clmblkdst
	loop2	c5b, clmblkdst		;; 2 * 2*clm iterations
	add	edx, 80000000h		;; Test middle loop counter (2 iters)
	jnc	c5a			;; Iterate if necessary
	lea	esi, [esi-4*clmblkdst+16*clmblkdst];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	add	eax, 80000000h		;; Test loop counter
	jnc	c5a			;; Iterate if necessary
	lea	esi, [esi-2*16*clmblkdst+clmblkdst32];; Next source pointer
	dec	al
	jnz	c5a
	lea	esi, [esi-16*clmblkdst32];; Restore source pointer
	end_timer 16

;; Do inverse FFT levels 5,6
;; On input the 32-byte cache lines hold these data values:
;;	0	4K	1	4K+1
;;	256	...
;;	...
;;	8K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	16K	1	16K+1
;;	256	...
;;	...
;;	32K	...
;;	...

;; Do 16 four_real_four_semireal_unfft macros
;;	distance between fft data elements is 16

	start_timer 17
	mov	al, 8			;; Load loop counter
c6a:	touch				;; Preload TLB
c6b:	disp four_real_four_semireal_unfft, 8, 16*clmblkdst, clmblkdst32
	loop2	c6b, clmblkdst		;; 2 * 2*clm iterations
	dec	al
	jnz	c6a
	lea	esi, [esi-16*clmblkdst+2*clmblkdst32];; Next source pointer

;; Do 112 four_complex_unfft macros
;;	distance between fft data elements is 16

	mov	edi, sincos3		;; Load sin/cos pointer
	mov	eax, 7			;; 7 iterations of 16
c7a:	touch				;; Preload TLB
c7b:	disp four_complex_unfft, 8, 16*clmblkdst, clmblkdst32
	loop2	c7b, clmblkdst		;; 2 * 2*clm iterations
	add	eax, 80000000h/4	;; Test middle loop counter (8 iters)
	jnc	c7a			;; Iterate if necessary
	lea	esi, [esi-16*clmblkdst+2*clmblkdst32];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al
	jnz	c7a
	lea	esi, [esi-8*2*clmblkdst32];; Restore source pointer
	end_timer 17

;; Do inverse FFT levels 4
;; On input the 32-byte cache lines hold these data values:
;;	0	16K	1	16K+1
;;	256	...
;;	...
;;	32K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	32K	1	32K+1
;;	256	...
;;	...
;;	64K	...
;;	...

;; Do 32 four_real_four_semireal_unfft_1 macros
;;	distance between fft data elements is 64

	start_timer 18
	mov	al, 16			;; Load loop counter
c8a:	touch				;; Preload TLB
c8b:	disp four_real_four_semireal_unfft_1, clmblkdst32, 8, 2*clmblkdst32
	loop2	c8b, clmblkdst		;; 2 * 2*clm iterations
	dec	al
	jnz	c8a
	lea	esi, [esi-32*clmblkdst+4*clmblkdst32];; Next source pointer

;; Do 96 four_complex_unfft_1 macros
;;	distance between fft data elements is 64

	mov	edi, sincos4		;; Load sin/cos pointer
	mov	eax, 3			;; 3 iterations of 32
c9a:	touch				;; Preload TLB
c9b:	disp four_complex_unfft_1, clmblkdst32, 8, 2*clmblkdst32
	loop2	c9b, clmblkdst		;; 2 * 2*clm iterations
	add	eax, 80000000h/8	;; Test middle loop counter (16 iters)
	jnc	c9a			;; Iterate if necessary
	lea	esi, [esi-32*clmblkdst+4*clmblkdst32];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al
	jnz	c9a
	lea	esi, [esi-4*4*clmblkdst32];; Restore source pointer
	end_timer 18

;; Do inverse FFT levels 1,2,3
;; On input the 32-byte cache lines hold these data values:
;;	0	32K	1	32K+1
;;	256	...
;;	...
;;	64K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	128K	1	128K+1
;;	256
;;	...
;;	128K-256

;; Do 128 eight_reals_last_unfft macros
;;	distance between fft data elements is 128

	start_timer 19
c1a:	touch				;; Preload TLB
c1b:	disp eight_reals_last_unfft, 8, 4*clmblkdst32, 8*clmblkdst32
	loop2	c1b, clmblkdst		;; 2 * 2*clm iterations
	add	eax, 80000000h/8	;; Test loop counter (16 iters)
	jnc	c1a			;; Iterate if necessary
	lea	esi, [esi-32*clmblkdst+clmblkdst32];; Next source pointer
	add	edx, 80000000h/2	;; Test loop counter (4 iters)
	jnc	c1a			;; Iterate if necessary
	lea	esi, [esi-4*clmblkdst32];; Restore source pointer
pfing	lea	ecx, [ecx-256*blkdst]	;; Restore dest ptr
	end_timer 19

;; Normalize these values

	start_timer 20
	mov	eax, NORMRTN		;; Addr of normalization routine
	call	eax
	sub	eax, eax
	end_timer 20

;; Have we been given permission to start the FFT on the result?
;; If so, do so now while the data is in the L2 cache.

	cmp	POSTFFT, 0		;; Test flag
	je	short copyback		;; Copy data, skip FFT if flag not set
	copy_3_words clm, 1
	cmp	edx, p2cl/128*65536+(128/clm-4/clm)*256;; Delay fft on 8 sets
	jle	yes_fft			;; No delay, do FFT now

;; Copy data from the scratch area back to the FFT data area

copyback:
	copy_scratch_data_to_fft 16, 32, clmblkdst32
	jmp	no_fft			;; Skip forward FFT code

;; Copy data from the FFT data area back to the scratch area

b0b:	copy_fft_data_to_scratch 16, 32, clmblkdst32

;; Do FFT levels 1,2,3
;;
;; On input the 32-byte cache lines hold these data values:
;;	0	128K	1	128K+1
;;	256
;;	...
;;	128K-256
;; On output the 32-byte cache lines hold these data values:
;;	0	32K	1	32K+1
;;	256	...
;;	...
;;	64K	...
;;	...

;; Do 128 eight_reals_fft macros
;;	distance between fft data elements is 128

yes_fft:start_timer 21
	sub	eax, eax		;; 128 iterations
pfing	lea	ecx, [ecx+256*blkdst]	;; Create prefetch ptr
b1a:	touch				;; Preload TLB
b1b:	disp eight_reals_fft, 4*clmblkdst32, 8*clmblkdst32, 8
	loop2	b1b, clmblkdst		;; 2 * 2*clm iterations
	add	eax, 80000000h/8	;; Test loop counter (16 iters)
	jnc	b1a			;; Iterate if necessary
	lea	esi, [esi-32*clmblkdst+clmblkdst32];; Next source pointer
	add	edx, 80000000h/2	;; Test loop counter (4 iters)
	jnc	b1a			;; Iterate if necessary
	lea	esi, [esi-4*clmblkdst32];; Restore source pointer
	end_timer 21

;; Do FFT levels 4
;;
;; On input the 32-byte cache lines hold these data values:
;;	0	32K	1	32K+1
;;	256	...
;;	...
;;	64K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	16K	1	16K+1
;;	256	...
;;	...
;;	32K	...
;;	...

;; Do 32 four_real_four_semireal_fft_1 macros
;;	distance between fft data elements is 64

	start_timer 22
	mov	al, 16			;; Load loop counter
b8a:	touch				;; Preload TLB
b8b:	disp four_real_four_semireal_fft_1, clmblkdst32, 2*clmblkdst32, 8
	loop2	b8b, clmblkdst		;; 2 * 2*clm iterations
	dec	al
	jnz	b8a
	lea	esi, [esi-32*clmblkdst+4*clmblkdst32];; Next source pointer

;; Do 96 four_complex_fft_1 macros
;;	distance between fft data elements is 64

	mov	edi, sincos4		;; Load sin/cos pointer
	mov	eax, 3			;; 3 iterations of 32
b9a:	touch				;; Preload TLB
b9b:	disp four_complex_fft_1, clmblkdst32, 2*clmblkdst32, 8
	loop2	b9b, clmblkdst		;; 2 * 2*clm iterations
	add	eax, 80000000h/8	;; Test middle loop counter (16 iters)
	jnc	b9a			;; Iterate if necessary
	lea	esi, [esi-32*clmblkdst+4*clmblkdst32];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al
	jnz	b9a
	lea	esi, [esi-4*4*clmblkdst32];; Restore source pointer
	end_timer 22

;; Do FFT levels 5,6
;;
;; On input the 32-byte cache lines hold these data values:
;;	0	16K	1	16K+1
;;	256	...
;;	...
;;	32K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	4K	1	4K+1
;;	256	...
;;	...
;;	8K	...
;;	...

;; Do 16 four_real_four_semireal_fft macros
;;	distance between fft data elements is 16

	start_timer 23
	mov	al, 8			;; Load loop counter
b6a:	touch				;; Preload TLB
b6b:	disp four_real_four_semireal_fft, 16*clmblkdst, clmblkdst32, 8
	loop2	b6b, clmblkdst		;; 2 * 2*clm iterations
	dec	al
	jnz	b6a
	lea	esi, [esi-16*clmblkdst+2*clmblkdst32];; Next source pointer

;; Do 112 four_complex_fft macros
;;	distance between fft data elements is 16

	mov	edi, sincos3		;; Load sin/cos pointer
	mov	eax, 7			;; 7 iterations of 16
b7a:	touch				;; Preload TLB
b7b:	disp four_complex_fft, 16*clmblkdst, clmblkdst32, 8
	loop2	b7b, clmblkdst		;; 2 * 2*clm iterations
	add	eax, 80000000h/4	;; Test middle loop counter (8 iters)
	jnc	b7a			;; Iterate if necessary
	lea	esi, [esi-16*clmblkdst+2*clmblkdst32];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al
	jnz	b7a
	lea	esi, [esi-8*2*clmblkdst32];; Restore source pointer
	end_timer 23

;; Do FFT levels 7,8
;;
;; On input the 32-byte cache lines hold these data values:
;;	0	4K	1	4K+1
;;	256	...
;;	...
;;	8K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	1K	1	1K+1
;;	256	...
;;	...
;;	2K	...
;;	...

;; Do 4 four_real_four_semireal_fft macros
;;	distance between fft data elements is 4

	start_timer 24
	mov	al, 2			;; Load loop counter
b4a:	touch				;; Preload TLB
b4b:	disp four_real_four_semireal_fft, 4*clmblkdst, 8*clmblkdst, 8
	loop2	b4b, clmblkdst		;; 2 * 2*clm iterations
	dec	al
	jnz	b4a
	lea	esi, [esi-4*clmblkdst+16*clmblkdst];; Next source pointer

;; Do 124 four_complex_fft macros
;;	distance between fft data elements is 4

	mov	edi, sincos2		;; Load sin/cos pointer
	mov	eax, 80000000h+16	;; 16 iters of 1 or 2 iters of 4
b5a:	touch				;; Preload TLB
b5b:	disp four_complex_fft, 4*clmblkdst, 8*clmblkdst, 8
	loop2	b5b, clmblkdst		;; 2*clm iterations
	add	edx, 80000000h		;; Test middle loop counter (2 iters)
	jnc	b5a			;; Iterate if necessary
	lea	esi, [esi-4*clmblkdst+16*clmblkdst];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	add	eax, 80000000h		;; Test middle loop counter (2 iters)
	jnc	b5a			;; Iterate if necessary
	lea	esi, [esi-2*16*clmblkdst+clmblkdst32];; Next source pointer
	dec	al
	jnz	b5a
	lea	esi, [esi-16*clmblkdst32];; Restore source pointer
	end_timer 24

;; Do FFT levels 9,10
;;
;; On input the 32-byte cache lines hold these data values:
;;	0	1K	1	1K+1
;;	256	...
;;	...
;;	2K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	256	1	257
;;	512	...
;;	...

;; Do 1 four_real_four_semireal_fft macros
;;	distance between fft data elements is 1

	start_timer 25
pfing	lea	ecx, [ecx-512*blkdst]	;; Create dest ptr from prefetch ptr
b2b:	cp_disp four_real_four_semireal_fft, clmblkdst, 2*clmblkdst, 8, blkdst, 2*blkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	lea	ecx, [ecx+dist1]	;; Next dest pointer
	add	dl, 256/2/clm		;; Test inner loop counter
	jnc	b2b			;; Iterate if necessary
	lea	esi, [esi-clm*2*dist1+4*clmblkdst];; Next source pointer
	lea	ecx, [ecx-clm*2*dist1+4*blkdst];; Next dest pointer

;; Do 127 four_complex_fft macros
;;	distance between fft data elements is 1

	mov	edi, sincos1		;; Load sin/cos pointer
	mov	eax, 256/8*256+16	;; 16 iters of 7 or 8 iters
b3b:	cp_disp four_complex_fft, clmblkdst, 2*clmblkdst, 8, blkdst, 2*blkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	lea	ecx, [ecx+dist1]	;; Next dest pointer
	add	dl, 256/2/clm		;; Test inner loop counter
	jnc	b3b			;; Iterate if necessary
	lea	esi, [esi-clm*2*dist1+4*clmblkdst];; Next source pointer
	lea	ecx, [ecx-clm*2*dist1+4*blkdst];; Next dest pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	add	ah, 256/8		;; Test loop counter
	jnc	b3b			;; Iterate if necessary
	lea	esi, [esi-8*4*clmblkdst+clmblkdst32];; Next source pointer
	dec	al
	jnz	b3b
	lea	esi, [ecx-128*4*blkdst]	;; Restore source pointer
	end_timer 25

;; Work on next 4 sets of 32 values

no_fft:	lea	esi, [esi+clm*2*dist1]	;; Next source pointer
	dec	dh			;; Test loop counter
	jnz	short more		;; Do more FFTs
	lea	esi, [esi+64]		;; Skip pad every 128 cache lines (4KB)
pfing	mov	ebp, clm*32		;; Set prefetch distance
	sub	edx, 65536		;; Test loop counter
	jz	short fftdn		;; Jump if done looping
	mov	dh, 128/clm		;; Restore cache line counter
more:	set_prefetch_gap
	cmp	norm_ptr1, 0		;; What was the loop start point?
	jne	c0b			;; Do inverse-FFT/norm/FFT iteration
	jmp	b0b			;; Do another FFT-only iteration
fftdn:	lea	esi, [esi-p2cl*2*dist1-p2cl/128*64];; Restore source pointer
	cmp	esi, DESTARG		;; If esi was not restored, then we 
	jne	short fftdn1		;; just finished skipped postfft data
	sub	ebx, ebx
	mov	DIST_TO_FFTSRCARG, ebx
	cmp	norm_ptr1, 0		;; What was the loop start point?
	je	pass2			;; FFT-only, go do pass 2

;; Split the carries into high and low carries (so they do not exceed
;; the desired number of bits per FFT word.  Then add the carries back to the
;; FFT data.

	start_timer 28
	call	gw_carries
	end_timer 28

;; Have we been given permission to start the FFT on the result?
;; If so, set loop to FFT the first few cache lines now that the carries
;; have been added in.

	cmp	POSTFFT, 0		;; Test flag
	je	short fftdn1		;; Skip FFT code if flag not set
	mov	esi, DESTARG		;; Load FFT data address
	mov	edx, 65536+4/clm*256	;; Do 8 previously skipped cache lines
	mov	norm_ptr1, 0		;; Set flag so only forward FFT is done
	copy_4_words
	jmp	b0b

;; Now add in the FFT'ed carries, cleanup and return

fftdn1:	end_timer 1
	jmp	gw_finish_mult
	ENDM


;; Perform a 256K-element negacyclic FFT.  This is done in two passes.  Pass 1
;; does 10 levels, pass 2 uses common code to perform the last 8 levels.

pass1levels10complex4 MACRO pass2_macro, p2cl, clmarg
	LOCAL	b0b, b1a, b1b, b3b, b5a, b5b, b7a, b7b, b9a, b9b
	LOCAL	c0b, c1a, c1b, c3b, c5a, c5b, c7a, c7b, c9a, c9b
	LOCAL	pass2, copyback, yes_fft, no_fft, more, fftdn, fftdn1

	set_FFT_constants clmarg

;; Call FFT code unless the FFT has already been started

	clear_timers
	cmp	DWORD PTR [esi-28][ebx], 0 ;; Test FFT-started flag
	jne	pass2			;; Jump is FFT already started
	mov	norm_ptr1, 0		;; Set FFT-ing flag
	mov	edx, p2cl/128*65536+128/clm*256;; Load loop counter
pfing	mov	ebp, clm*32		;; Set prefetch distance
	jmp	b0b			;; Do the pass 1 fft

;; Do the last 8 FFT levels
;;	do 512 pass2_8_levels_complex

pass2:	start_timer 0
	CALLP	pass2_macro
	end_timer 0
	cmp	ffttype, 1		;; We're done if FFTing only
	je	gw_finish_fft

;; Do the inverse FFT

	start_timer 1
	fldz				;; Init SUMOUT
	fstp	SUMOUT
	mov	edx, norm_biglit_array	;; Addr of the big/little flags array
	mov	norm_ptr1, edx		;; Save ptr
	mov	edx, norm_col_mults	;; Addr of the column multipliers
	mov	norm_ptr2, edx		;; Save ptr
	mov	edx, p2cl/128*65536+128/clm*256;; Load loop counter
pfing	mov	ebp, clm*32		;; Set prefetch distance

;; Do inverse FFT levels 9,10
;; On input the 32-byte cache lines hold these data values:
;;	0	256	1	257
;;	512	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	1K	1	1K+1
;;	256	...
;;	...
;;	2K	...
;;	...

;; Do 128 four_complex_unfft macros
;;	distance between fft data elements is 1

c0b:	start_timer 15
	mov	ecx, scratch_area	;; Get address of scratch area
	mov	edi, sincos1		;; Load sin/cos pointer
	mov	eax, 16			;; 16 iterations 8
c3b:	cp_disp four_complex_unfft, 8, blkdst, 2*blkdst, 8, clmblkdst, 2*clmblkdst
	lea	esi, [esi+dist1]	;; Next source pointer
	lea	ecx, [ecx+dist1]	;; Next dest pointer
	add	dl, 256/2/clm		;; Test loop counter
	jnc	c3b			;; Iterate if necessary
	lea	esi, [esi-clm*2*dist1+4*blkdst];; Next source pointer
	lea	ecx, [ecx-clm*2*dist1+4*clmblkdst];; Next dest pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	add	ah, 256/8		;; Test loop counter
	jnc	c3b			;; Iterate if necessary
	lea	ecx, [ecx-8*4*clmblkdst+clmblkdst32];; Next dest pointer
	dec	al			;; Test outer loop counter
	jnz	c3b			;; Iterate if necessary
	lea	ecx, [esi-128*4*blkdst]	;; Create prefetch pointer
	mov	esi, scratch_area	;; Restore source pointer
	end_timer 15

;; Do inverse FFT levels 7,8
;; On input the 32-byte cache lines hold these data values:
;;	0	1K	1	1K+1
;;	256	...
;;	...
;;	2K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	4K	1	4K+1
;;	256	...
;;	...
;;	8K	...
;;	...

;; Do 128 four_complex_unfft macros
;;	distance between fft data elements is 4

	start_timer 16
	mov	edi, sincos2		;; Load sin/cos pointer
	mov	eax, 16			;; 16 iters of 2 iters of 4
c5a:	touch				;; Preload TLB
c5b:	disp four_complex_unfft, 8, 4*clmblkdst, 8*clmblkdst
	loop2	c5b, clmblkdst		;; 2 * 2*clm iterations
	add	edx, 80000000h		;; Test middle loop counter (2 iters)
	jnc	c5a			;; Iterate if necessary
	lea	esi, [esi-4*clmblkdst+16*clmblkdst];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	add	eax, 80000000h		;; Test loop counter
	jnc	c5a			;; Iterate if necessary
	lea	esi, [esi-2*16*clmblkdst+clmblkdst32];; Next source pointer
	dec	al
	jnz	c5a
	lea	esi, [esi-16*clmblkdst32];; Restore source pointer
	end_timer 16

;; Do inverse FFT levels 5,6
;; On input the 32-byte cache lines hold these data values:
;;	0	4K	1	4K+1
;;	256	...
;;	...
;;	8K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	16K	1	16K+1
;;	256	...
;;	...
;;	32K	...
;;	...

;; Do 128 four_complex_unfft macros
;;	distance between fft data elements is 16

	start_timer 17
	mov	edi, sincos3		;; Load sin/cos pointer
	mov	eax, 8			;; 8 iterations of 16
c7a:	touch				;; Preload TLB
c7b:	disp four_complex_unfft, 8, 16*clmblkdst, clmblkdst32
	loop2	c7b, clmblkdst		;; 2 * 2*clm iterations
	add	eax, 80000000h/4	;; Test middle loop counter (8 iters)
	jnc	c7a			;; Iterate if necessary
	lea	esi, [esi-16*clmblkdst+2*clmblkdst32];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al
	jnz	c7a
	lea	esi, [esi-8*2*clmblkdst32];; Restore source pointer
	end_timer 17

;; Do inverse FFT levels 4
;; On input the 32-byte cache lines hold these data values:
;;	0	16K	1	16K+1
;;	256	...
;;	...
;;	32K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	32K	1	32K+1
;;	256	...
;;	...
;;	64K	...
;;	...

;; Do 128 four_complex_unfft_1 macros
;;	distance between fft data elements is 64

	start_timer 18
	mov	edi, sincos4		;; Load sin/cos pointer
	mov	eax, 4			;; 4 iterations of 32
c9a:	touch				;; Preload TLB
c9b:	disp four_complex_unfft_1, clmblkdst32, 8, 2*clmblkdst32
	loop2	c9b, clmblkdst		;; 2 * 2*clm iterations
	add	eax, 80000000h/8	;; Test middle loop counter (16 iters)
	jnc	c9a			;; Iterate if necessary
	lea	esi, [esi-32*clmblkdst+4*clmblkdst32];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al
	jnz	c9a
	lea	esi, [esi-4*4*clmblkdst32];; Restore source pointer
	end_timer 18

;; Do inverse FFT levels 1,2,3
;; On input the 32-byte cache lines hold these data values:
;;	0	32K	1	32K+1
;;	256	...
;;	...
;;	64K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	128K	1	128K+1
;;	256
;;	...
;;	128K-256

;; Do 128 four_complex_last_unfft macros
;;	distance between fft data elements is 128

	start_timer 19
	mov	edi, plus1_premults	;; Address of premultiplier table
c1a:	touch				;; Preload TLB
c1b:	disp four_complex_last_unfft, 8, 4*clmblkdst32, 8*clmblkdst32
	loop2edi c1b, clmblkdst, 64	;; 2 * 2*clm iterations bumping edi
	add	eax, 80000000h/8	;; Test loop counter (16 iters)
	jnc	c1a			;; Iterate if necessary
	lea	esi, [esi-32*clmblkdst+clmblkdst32];; Next source pointer
	add	edx, 80000000h/2	;; Test loop counter (4 iters)
	jnc	c1a			;; Iterate if necessary
	lea	esi, [esi-4*clmblkdst32];; Restore source pointer
pfing	lea	ecx, [ecx-256*blkdst]	;; Restore dest ptr
	end_timer 19

;; Normalize these values

	start_timer 20
	mov	eax, NORMRTN		;; Addr of normalization routine
	call	eax
	sub	eax, eax
	end_timer 20

;; Have we been given permission to start the FFT on the result?
;; If so, do so now while the data is in the L2 cache.

	cmp	POSTFFT, 0		;; Test flag
	je	short copyback		;; Copy data, skip FFT if flag not set
	copy_3_words clm, 1
	cmp	edx, p2cl/128*65536+(128/clm-4/clm)*256;; Delay fft on 8 sets
	jle	yes_fft			;; No delay, do FFT now

;; Copy data from the scratch area back to the FFT data area

copyback:
	copy_scratch_data_to_fft 16, 32, clmblkdst32
	jmp	no_fft			;; Skip forward FFT code

;; Copy data from the FFT data area back to the scratch area

b0b:	copy_fft_data_to_scratch 16, 32, clmblkdst32

;; Do FFT levels 1,2,3
;;
;; On input the 32-byte cache lines hold these data values:
;;	0	128K	1	128K+1
;;	256
;;	...
;;	128K-256
;; On output the 32-byte cache lines hold these data values:
;;	0	32K	1	32K+1
;;	256	...
;;	...
;;	64K	...
;;	...

;; Do 128 four_complex_fft_premult macros
;;	distance between fft data elements is 128

yes_fft:start_timer 21
	mov	edi, plus1_premults	;; Address of premultiplier table
	sub	eax, eax		;; 128 iterations
pfing	lea	ecx, [ecx+256*blkdst]	;; Create prefetch ptr
b1a:	touch				;; Preload TLB
b1b:	disp four_complex_fft_premult, 4*clmblkdst32, 8*clmblkdst32, 8
	loop2edi b1b, clmblkdst, 64	;; 2 * 2*clm iterations bumping edi
	add	eax, 80000000h/8	;; Test loop counter (16 iters)
	jnc	b1a			;; Iterate if necessary
	lea	esi, [esi-32*clmblkdst+clmblkdst32];; Next source pointer
	add	edx, 80000000h/2	;; Test loop counter (4 iters)
	jnc	b1a			;; Iterate if necessary
	lea	esi, [esi-4*clmblkdst32];; Restore source pointer
	end_timer 21

;; Do FFT levels 4
;;
;; On input the 32-byte cache lines hold these data values:
;;	0	32K	1	32K+1
;;	256	...
;;	...
;;	64K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	16K	1	16K+1
;;	256	...
;;	...
;;	32K	...
;;	...

;; Do 128 four_complex_fft_1 macros
;;	distance between fft data elements is 64

	start_timer 22
	mov	edi, sincos4		;; Load sin/cos pointer
	mov	eax, 4			;; 4 iterations of 32
b9a:	touch				;; Preload TLB
b9b:	disp four_complex_fft_1, clmblkdst32, 2*clmblkdst32, 8
	loop2	b9b, clmblkdst		;; 2 * 2*clm iterations
	add	eax, 80000000h/8	;; Test middle loop counter (16 iters)
	jnc	b9a			;; Iterate if necessary
	lea	esi, [esi-32*clmblkdst+4*clmblkdst32];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al
	jnz	b9a
	lea	esi, [esi-4*4*clmblkdst32];; Restore source pointer
	end_timer 22

;; Do FFT levels 5,6
;;
;; On input the 32-byte cache lines hold these data values:
;;	0	16K	1	16K+1
;;	256	...
;;	...
;;	32K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	4K	1	4K+1
;;	256	...
;;	...
;;	8K	...
;;	...

;; Do 128 four_complex_fft macros
;;	distance between fft data elements is 16

	start_timer 23
	mov	edi, sincos3		;; Load sin/cos pointer
	mov	eax, 8			;; 8 iterations of 16
b7a:	touch				;; Preload TLB
b7b:	disp four_complex_fft, 16*clmblkdst, clmblkdst32, 8
	loop2	b7b, clmblkdst		;; 2 * 2*clm iterations
	add	eax, 80000000h/4	;; Test middle loop counter (8 iters)
	jnc	b7a			;; Iterate if necessary
	lea	esi, [esi-16*clmblkdst+2*clmblkdst32];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al
	jnz	b7a
	lea	esi, [esi-8*2*clmblkdst32];; Restore source pointer
	end_timer 23

;; Do FFT levels 7,8
;;
;; On input the 32-byte cache lines hold these data values:
;;	0	4K	1	4K+1
;;	256	...
;;	...
;;	8K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	1K	1	1K+1
;;	256	...
;;	...
;;	2K	...
;;	...

;; Do 128 four_complex_fft macros
;;	distance between fft data elements is 4

	start_timer 24
	mov	edi, sincos2		;; Load sin/cos pointer
	mov	eax, 16			;; 16 iters of 2 iters of 4
b5a:	touch				;; Preload TLB
b5b:	disp four_complex_fft, 4*clmblkdst, 8*clmblkdst, 8
	loop2	b5b, clmblkdst		;; 2 * 2*clm iterations
	add	edx, 80000000h		;; Test middle loop counter (2 iters)
	jnc	b5a			;; Iterate if necessary
	lea	esi, [esi-4*clmblkdst+16*clmblkdst];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	add	eax, 80000000h		;; Test middle loop counter (2 iters)
	jnc	b5a			;; Iterate if necessary
	lea	esi, [esi-2*16*clmblkdst+clmblkdst32];; Next source pointer
	dec	al
	jnz	b5a
	lea	esi, [esi-16*clmblkdst32];; Restore source pointer
	end_timer 24

;; Do FFT levels 9,10
;;
;; On input the 32-byte cache lines hold these data values:
;;	0	1K	1	1K+1
;;	256	...
;;	...
;;	2K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	256	1	257
;;	512	...
;;	...

;; Do 128 four_complex_fft macros
;;	distance between fft data elements is 1

	start_timer 25
	mov	edi, sincos1		;; Load sin/cos pointer
pfing	lea	ecx, [ecx-512*blkdst]	;; Create dest ptr from prefetch ptr
	mov	eax, 16			;; 16 iters of 8 iters
b3b:	cp_disp four_complex_fft, clmblkdst, 2*clmblkdst, 8, blkdst, 2*blkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	lea	ecx, [ecx+dist1]	;; Next dest pointer
	add	dl, 256/2/clm		;; Test inner loop counter
	jnc	b3b			;; Iterate if necessary
	lea	esi, [esi-clm*2*dist1+4*clmblkdst];; Next source pointer
	lea	ecx, [ecx-clm*2*dist1+4*blkdst];; Next dest pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	add	ah, 256/8		;; Test loop counter
	jnc	b3b			;; Iterate if necessary
	lea	esi, [esi-8*4*clmblkdst+clmblkdst32];; Next source pointer
	dec	al
	jnz	b3b
	lea	esi, [ecx-128*4*blkdst]	;; Restore source pointer
	end_timer 25

;; Work on next 4 sets of 32 values

no_fft:	lea	esi, [esi+clm*2*dist1]	;; Next source pointer
	dec	dh			;; Test loop counter
	jnz	short more		;; Do more FFTs
	lea	esi, [esi+64]		;; Skip pad every 128 cache lines (4KB)
pfing	mov	ebp, clm*32		;; Set prefetch distance
	sub	edx, 65536		;; Test loop counter
	jz	short fftdn		;; Jump if done looping
	mov	dh, 128/clm		;; Restore cache line counter
more:	set_prefetch_gap
	cmp	norm_ptr1, 0		;; What was the loop start point?
	jne	c0b			;; Do inverse-FFT/norm/FFT iteration
	jmp	b0b			;; Do another FFT-only iteration
fftdn:	lea	esi, [esi-p2cl*2*dist1-p2cl/128*64];; Restore source pointer
	cmp	esi, DESTARG		;; If esi was not restored, then we 
	jne	short fftdn1		;; just finished skipped postfft data
	sub	ebx, ebx
	mov	DIST_TO_FFTSRCARG, ebx
	cmp	norm_ptr1, 0		;; What was the loop start point?
	je	pass2			;; FFT-only, go do pass 2

;; Split the carries into high and low carries (so they do not exceed
;; the desired number of bits per FFT word.  Then add the carries back to the
;; FFT data.

	start_timer 28
	call	gw_carries
	end_timer 28

;; Have we been given permission to start the FFT on the result?
;; If so, set loop to FFT the first few cache lines now that the carries
;; have been added in.

	cmp	POSTFFT, 0		;; Test flag
	je	short fftdn1		;; Skip FFT code if flag not set
	mov	esi, DESTARG		;; Load FFT data address
	mov	edx, 65536+4/clm*256	;; Do 8 previously skipped cache lines
	mov	norm_ptr1, 0		;; Set flag so only forward FFT is done
	copy_4_words
	jmp	b0b

;; Now add in the FFT'ed carries, cleanup and return

fftdn1:	end_timer 1
	jmp	gw_finish_mult
	ENDM

;;
;; Utility macros
;;

;; Set various constants used in the FFT.  Note that a clmarg of 0 really
;; means used a clm of one but don't do any prefetching.

set_FFT_constants MACRO clmarg
	IF clmarg EQ 0
	PREFETCHING = 0
	clm = 1
	ELSE
	clm = clmarg
	IFNDEF PFETCH
	PREFETCHING = 0
	ELSE
	PREFETCHING = 1
	ENDIF
	ENDIF
	clmblkdst = (32*clm)
	clmblkdst32 = (32*clmblkdst+64)
	ENDM

;; Copy data from scratch area to FFT data area
;; esi = scratch area pointer
;; ecx = FFT data area pointer
;; Returns with esi = FFT data area pointer

copy_scratch_data_to_fft MACRO numsets, numlines, incr
	LOCAL	a0b
	pusher	ecx			;; Save FFT data ptr
	mov	esi, scratch_area	;; Get address of scratch area
	mov	eax, numsets		;; Load counter
a0b:	fld	QWORD PTR [esi]		;; Copy a cache line
	fstp	QWORD PTR [ecx]
	fld	QWORD PTR [esi+8]
	fstp	QWORD PTR [ecx+8]
	fld	QWORD PTR [esi+16]
	fstp	QWORD PTR [ecx+16]
	fld	QWORD PTR [esi+24]
	fstp	QWORD PTR [ecx+24]
	lea	esi, [esi+32]		;; Next source pointer
	lea	ecx, [ecx+32]		;; Next dest pointer
	loopclm	dl, clm, a0b		;; Iterate if necessary
	lea	ecx, [ecx-clm*32+blkdst];; Next dest pointer
	IFB <numlines>
	dec	eax			;; Test loop counter
	jnz	short a0b		;; Iterate if necessary
	ELSE
	add	ah, 256/numlines	;; Test loop counter
	jnc	short a0b		;; Iterate if necessary
	lea	esi, [esi-numlines*clm*32+incr];; Next source pointer
	dec	al			;; Test loop counter
	jnz	short a0b		;; Iterate if necessary
	ENDIF
	popper	esi			;; Restore FFT data ptr
	ENDM

;; Copy data from FFT data area to scratch area
;; esi = FFT data area pointer
;; Returns with esi = scratch area pointer
;; and ecx = FFT data area pointer

copy_fft_data_to_scratch MACRO numsets, numlines, incr
	LOCAL	a0b
	pusher	esi			;; Save FFT data ptr
	add	esi, DIST_TO_FFTSRCARG
	mov	ecx, scratch_area	;; Get address of scratch area
	mov	eax, numsets		;; Load counter
a0b:	fld	QWORD PTR [esi]		;; Copy a cache line
	fstp	QWORD PTR [ecx]
	fld	QWORD PTR [esi+8]
	fstp	QWORD PTR [ecx+8]
	fld	QWORD PTR [esi+16]
	fstp	QWORD PTR [ecx+16]
	fld	QWORD PTR [esi+24]
	fstp	QWORD PTR [ecx+24]
	lea	esi, [esi+32]		;; Next source pointer
	lea	ecx, [ecx+32]		;; Next dest pointer
	loopclm	dl, clm, a0b		;; Iterate if necessary
	lea	esi, [esi-clm*32+blkdst];; Next source pointer
	IFB <numlines>
	dec	eax			;; Test loop counter
	jnz	short a0b		;; Iterate if necessary
	ELSE
	add	ah, 256/numlines	;; Test loop counter
	jnc	short a0b		;; Iterate if necessary
	lea	ecx, [ecx-numlines*clm*32+incr];; Next source pointer
	dec	al			;; Test loop counter
	jnz	short a0b		;; Iterate if necessary
	ENDIF
	popper	ecx			;; Restore FFT data ptr as prefetch ptr
	mov	esi, scratch_area	;; Set source ptr to scratch area
	ENDM

;; Set ebp to make sure we prefetch the correct data.  This macro checks
;; for the 64 byte gap that occurs every 4KB and the extra 64 byte gap
;; that occurs on the very last iteration.

set_prefetch_gap MACRO
	LOCAL	nogap	
	IF PREFETCHING NE 0
	cmp	dh, 1			;; Check for upcoming 64 byte gap
	jne	short nogap		;; Skip if there is no 64 byte gap
	lea	ebp, [ebp+64]		;; Skip pad bytes each 4KB
	cmp	edx, 1*65536+1*256	;; Check last iteration
	jne	short nogap		;; Skip if not last
	lea	ebp, [ebp+64]		;; Skip pad bytes after block
nogap:
	ENDIF
	ENDM

;; Preload a TLB so that later calls to prefetcht1 do their work

touch	MACRO
	IF PREFETCHING NE 0
	cmp	esp, DWORD PTR [ecx+(clm-1)*32]
	ENDIF
	ENDM

;; Loop 2*clm times.  Prefetch a cache line every 2 macro calls.
;; Uses top bit of al, top clm+1 bits of dl.

loop1	MACRO label, incr
	loop1edi label, incr, 0
	ENDM

loop1edi MACRO label, incr, incr_edi

	IF	PREFETCHING EQ 0 OR clm EQ 1
	lea	esi, [esi+dist1]	;; Next source pointer
	add	dl, 256/2/clm		;; Test inner loop counter
	jnc	label			;; Iterate if necessary
	bump	esi, -clm*2*dist1+incr	;; Next source pointer
	bump	edi, incr_edi		;; Next premultiplier
pfing	prefetcht1 [ecx][ebp]		;; Prefetch a cache line
pfing	lea	ecx, [ecx+blkdst]	;; Next prefetch pointer

	ELSE
	IF	clm GE 2
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test inner loop counter
	jnc	label			;; Iterate if necessary
	prefetcht1 [ecx][ebp]		;; Prefetch a cache line
	lea	ecx, [ecx+32]
	add	dl, 256/clm		;; Test loop counter
	jnc	label			;; Iterate if necessary
	bump	esi, -clm*2*dist1+incr	;; Next source pointer
	bump	edi, incr_edi		;; Next premultiplier
	lea	ecx, [ecx-clm*32+blkdst];; Next prefetch pointer
	ENDIF
	ENDIF

	ENDM

;; Loop 2 * 2*clm times.  Prefetch a cache line every 4 macro calls.
;; Uses top 2 bits of al, top clm+2 bits of dl, top bit of ah.

loop2	MACRO label, incr
	loop2edi label, incr, 0
	ENDM

loop2edi MACRO label, incr, incr_edi

	IF	PREFETCHING EQ 0 OR clm EQ 1
	lea	esi, [esi+dist1]	;; Next source pointer
	IF incr NE clm*2*dist1 OR incr_edi NE 0
	add	dl, 256/2/clm		;; Test inner loop counter
	jnc	label			;; Iterate if necessary
	bump	esi, -clm*2*dist1+incr	;; Next source pointer
	bump	edi, incr_edi		;; Next premultiplier
	add	al, 256/2		;; Test loop counter
	jnc	label			;; Iterate if necessary
	ELSE
	add	dl, 256/2/clm/2		;; Test inner loop counter
	jnc	label			;; Iterate if necessary
	ENDIF
pfing	prefetcht1 [ecx][ebp]		;; Prefetch a cache line
pfing	lea	ecx, [ecx+blkdst]	;; Next prefetch pointer

	ELSE
	IF	clm EQ 2
	lea	esi, [esi+dist1]	;; Next source pointer
	add	dl, 256/2/clm		;; Test inner loop counter
	jnc	label			;; Iterate if necessary
	bump	esi, -clm*2*dist1+incr	;; Next source pointer
	bump	edi, incr_edi		;; Next premultiplier
	prefetcht1 [ecx][ebp]		;; Prefetch a cache line
	lea	ecx, [ecx+32]
	add	al, 256/2		;; Test loop counter
	jnc	label			;; Iterate if necessary
	lea	ecx, [ecx-clm*32+blkdst];; Next prefetch pointer

	ELSE
	IF	clm GE 4
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2/2		;; Test inner loop counter
	jnc	label			;; Iterate if necessary
	prefetcht1 [ecx][ebp]		;; Prefetch a cache line
	lea	ecx, [ecx+32]
	IF incr NE clm*2*dist1 OR incr_edi NE 0
	add	dl, 256/(clm/2)		;; Test loop counter
	jnc	label			;; Iterate if necessary
	bump	esi, -clm*2*dist1+incr	;; Next source pointer
	bump	edi, incr_edi		;; Next premultiplier
	add	ah, 256/2		;; Test loop counter
	jnc	label			;; Iterate if necessary
	ELSE
	add	dl, 256/(clm/2)/2	;; Test loop counter
	jnc	label			;; Iterate if necessary
	ENDIF
	lea	ecx, [ecx-clm*32+blkdst];; Next prefetch pointer
	ENDIF
	ENDIF
	ENDIF

	ENDM

;; Loop 4 * 2*clm times.  Prefetch a cache line every 8 macro calls.
;; Uses top 2 bits of al, top clm+3 bits of dl, top bit of ah.

loop4	MACRO label, incr

	IF	PREFETCHING EQ 0 OR clm EQ 1
	lea	esi, [esi+dist1]	;; Next source pointer
	IF incr NE clm*2*dist1
	add	al, 256/2/clm		;; Test inner loop counter
	jnc	label			;; Iterate if necessary
	lea	esi, [esi-clm*2*dist1+incr];; Next source pointer
	add	dl, 256/4		;; Test loop counter
	jnc	label			;; Iterate if necessary
	ELSE
	add	dl, 256/2/clm/4		;; Test inner loop counter
	jnc	label			;; Iterate if necessary
	ENDIF
pfing	prefetcht1 [ecx][ebp]		;; Prefetch a cache line
pfing	lea	ecx, [ecx+blkdst]	;; Next prefetch pointer

	ELSE
	IF	clm EQ 2
	lea	esi, [esi+dist1]	;; Next source pointer
	IF incr NE clm*2*dist1
	add	dl, 256/2/clm		;; Test inner loop counter
	jnc	label			;; Iterate if necessary
	lea	esi, [esi-clm*2*dist1+incr];; Next source pointer
	add	al, 256/2		;; Test loop counter
	jnc	label			;; Iterate if necessary
	ELSE
	add	dl, 256/2/clm/2		;; Test inner loop counter
	jnc	label			;; Iterate if necessary
	ENDIF
	prefetcht1 [ecx][ebp]		;; Prefetch a cache line
	lea	ecx, [ecx+32]
	add	ah, 256/2		;; Test loop counter
	jnc	label			;; Iterate if necessary
	lea	ecx, [ecx-clm*32+blkdst];; Next prefetch pointer

	ELSE
	IF	clm EQ 4
	lea	esi, [esi+dist1]	;; Next source pointer
	add	dl, 256/2/clm		;; Test inner loop counter
	jnc	label			;; Iterate if necessary
	bump	esi, -clm*2*dist1+incr	;; Next source pointer
	prefetcht1 [ecx][ebp]		;; Prefetch a cache line
	lea	ecx, [ecx+32]
	add	al, 256/4		;; Test loop counter
	jnc	label			;; Iterate if necessary
	lea	ecx, [ecx-clm*32+blkdst];; Next prefetch pointer
	ENDIF
	ENDIF
	ENDIF

	ENDM
