; Copyright 1998-2024 - Mersenne Research, Inc.  All rights reserved
; Author:  George Woltman
; Email: woltman@alum.mit.edu
;
; These macros do the first pass of two-pass FFTs.
;

; ********************************************************
; ********************************************************
; *******************  PASS 1 MACROS  ********************
; ********************************************************
; ********************************************************


;;***********************************************************************
;; Pass 1 macros
;;
;; p2cl = number of cache lines in a pass 2 block
;; clm = cache line multiplier (num cache lines processed each iteration)
;;***********************************************************************


;; Perform a 5K-element FFT.  This is done in two passes.  Pass 1
;; does 5 levels, pass 2 uses common code to perform the last 8 levels.

pass1levels5pfa5 MACRO pass2_macro, p2cl, clm
	LOCAL	b0b, b1b, b2b, b3b, c0b, c1b, c2b, c3b
	LOCAL	pass2, no_fft, more, fftdn, fftdn1

;; Call FFT code unless the FFT has already been started

	clear_timers
	cmp	DWORD PTR [esi-28][ebx], 0 ;; Test FFT-started flag
	jne	pass2			;; Jump is FFT already started
	mov	norm_ptr1, 0		;; Set FFT-ing flag
	mov	edx, p2cl/128*65536+128/clm*256;; Load loop counter
	jmp	b0b			;; Do the pass 1 fft

;; Do the last 8 FFT levels
;;	do 1 pass2_8_levels_real and 9 pass2_8_levels_complex

pass2:	start_timer 0
	CALLP	pass2_macro
	end_timer 0
	cmp	ffttype, 1		;; We're done if FFTing only
	je	gw_finish_fft

;; Do the inverse FFT

	start_timer 1
	fldz				;; Init SUMOUT
	fstp	SUMOUT
	mov	edx, norm_biglit_array	;; Addr of the big/little flags array
	mov	norm_ptr1, edx		;; Save ptr
	mov	edx, norm_col_mults	;; Addr of the column multipliers
	mov	norm_ptr2, edx		;; Save ptr
	mov	edx, p2cl/128*65536+128/clm*256;; Load loop counter

;; Do inverse FFT levels 4,5
;; On input the 32-byte cache lines hold these data values:
;;	0	256	1	257
;;	512	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	512	1	513
;;	256	...
;;	1K	...
;;	...

;; Do 1 four_real_unfft macros
;;	distance between fft data elements is 1
;;	For better pipelining we call eight_reals_unfft_2 which works
;;	on two sets of four_reals at the same time.

c0b:	start_timer 18
c2b:	disp eight_reals_unfft_2, dist1, 8, blkdst
	lea	esi, [esi+2*dist1]	;; Next source pointer
	loopclm	dl, clm, c2b
	lea	esi, [esi-clm*2*dist1+2*blkdst];; Next source pointer

;; Do 2 four_complex_unfft macros
;;	distance between fft data elements is 1

	mov	edi, sincos1		;; Load sin/cos pointer
	mov	al, 2			;; 2 iterations
c3b:	disp four_complex_unfft, 8, blkdst, 2*blkdst
	lea	esi, [esi+dist1]	;; Next source pointer
	add	dl, 256/2/clm		;; Test loop counter
	jnc	c3b			;; Iterate if necessary
	lea	esi, [esi-clm*2*dist1+4*blkdst];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al			;; Test outer loop counter
	jnz	c3b			;; Iterate if necessary
	lea	esi, [esi-2*4*blkdst-2*blkdst];; Restore source pointer
	end_timer 18

;; Do inverse FFT levels 1,2,3
;; On input the 32-byte cache lines hold these data values:
;;	0	512	1	513
;;	256	...
;;	1K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	2.5K	1	3K+1
;;	256
;;	...
;;	3K-256

;; Do 4 five_reals_last_unfft macros
;;	distance between fft data elements is 4

	start_timer 19
	mov	al, 2			;; Load loop counter
c1b:	two_five_reals_last_unfft 8, 2*blkdst
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test loop counter
	jnc	c1b			;; Iterate if necessary
	IFDEF PFETCH
	prefetcht1 [esi-32+clm*32]
;	prefetcht1 [esi-32+clm*32+2*blkdst]
	prefetcht1 [esi-32+clm*32+4*blkdst]
;	prefetcht1 [esi-32+clm*32+6*blkdst]
	prefetcht1 [esi-32+clm*32+8*blkdst]
	ENDIF
	loopclm	dl, clm, c1b
	lea	esi, [esi-clm*2*dist1+blkdst];; Next source pointer
	dec	al			;; Test loop counter
	jnz	c1b			;; Iterate if necessary
	lea	esi, [esi-2*blkdst]	;; Restore source pointer
	end_timer 19

;; Normalize these values

	start_timer 20
	mov	eax, NORMRTN		;; Addr of normalization routine
	call	eax
	sub	eax, eax
	end_timer 20

;; Have we been given permission to start the FFT on the result?
;; If so, do so now while the data is in the L2 cache.

	cmp	POSTFFT, 0		;; Test flag
	je	no_fft			;; Skip FFT code if flag not set
	cmp	edx, p2cl/128*65536+(128/clm-4/clm)*256;; Delay fft on 8 sets
	jg	no_fft			;; Jump if delaying FFT
	copy_3_words clm, 0

;; Do FFT levels 1,2,3
;;
;; On input the 32-byte cache lines hold these data values:
;;	0	2.5K	1	2.5K+1
;;	256
;;	...
;;	2.5K-256
;; On output the 32-byte cache lines hold these data values:
;;	0	512	1	513
;;	256	...
;;	1K	...
;;	...

;; Do 4 five_reals_first_fft macros
;;	distance between fft data elements is 4

b0b:	start_timer 21
	mov	ebx, DIST_TO_FFTSRCARG
	mov	al, 2			;; 4 iterations
b1b:	two_five_reals_first_fft 2*blkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	add	dl, 256/2/clm		;; Test loop counter
	jnc	b1b			;; Iterate if necessary
	lea	esi, [esi-clm*2*dist1+blkdst];; Next source pointer
	dec	al			;; Test loop counter
	jnz	b1b			;; Iterate if necessary
	lea	esi, [esi-2*blkdst]	;; Restore source pointer
	end_timer 21

;; Do FFT levels 4,5
;;
;; On input the 32-byte cache lines hold these data values:
;;	0	512	1	513
;;	256	...
;;	1K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	256	1	257
;;	512	...
;;	...

;; Do 1 four_real_fft macros
;;	distance between fft data elements is 1
;;	For better pipelining we call eight_reals_fft_2 which works
;;	on two sets of four_reals at the same time.

	start_timer 22
b2b:	disp eight_reals_fft_2, dist1, blkdst, 8
	lea	esi, [esi+2*dist1]	;; Next source pointer
	IFDEF PFETCH
;	prefetcht1 [esi-32+clm*32]
	prefetcht1 [esi-32+clm*32+blkdst]
	ENDIF
	loopclm	dl, clm, b2b
	lea	esi, [esi-clm*2*dist1+2*blkdst];; Next source pointer

;; Do 2 four_complex_fft macros
;;	distance between fft data elements is 1

	mov	edi, sincos1		;; Load sin/cos pointer
	mov	al, 2			;; 2 iterations
b3b:	disp four_complex_fft, blkdst, 2*blkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test inner loop counter
	jnc	b3b			;; Iterate if necessary
	IFDEF PFETCH
;	prefetcht1 [esi-32+clm*32]
	prefetcht1 [esi-32+clm*32+blkdst]
;	prefetcht1 [esi-32+clm*32+2*blkdst]
	prefetcht1 [esi-32+clm*32+3*blkdst]
	ENDIF
	loopclm	dl, clm, b3b
	lea	esi, [esi-clm*2*dist1+4*blkdst];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al
	jnz	b3b
	lea	esi, [esi-2*4*blkdst-2*blkdst];; Restore source pointer
	end_timer 22

;; Work on next 4 sets of 32 values

no_fft:	lea	esi, [esi+clm*2*dist1]	;; Next source pointer
	dec	dh			;; Test loop counter
	jnz	short more		;; Do more FFTs
	lea	esi, [esi+64]		;; Skip pad every 128 cache lines (4KB)
	sub	edx, 65536		;; Test loop counter
	jz	short fftdn		;; Jump if done looping
	mov	dh, 128/clm		;; Restore cache line counter
more:	cmp	norm_ptr1, 0		;; What was the loop start point?
	jne	c0b			;; Do inverse-FFT/norm/FFT iteration
	jmp	b0b			;; Do another FFT-only iteration
fftdn:	lea	esi, [esi-p2cl*2*dist1-p2cl/128*64];; Restore source pointer
	cmp	esi, DESTARG		;; If esi was not restored, then we 
	jne	short fftdn1		;; just finished skipped postfft data
	sub	ebx, ebx
	mov	DIST_TO_FFTSRCARG, ebx
	cmp	norm_ptr1, 0		;; What was the loop start point?
	je	pass2			;; FFT-only, go do pass 2

;; Split the carries into high and low carries (so they do not exceed
;; the desired number of bits per FFT word.  Then add the carries back to the
;; FFT data.

	start_timer 28
	call	gw_carries
	end_timer 28

;; Have we been given permission to start the FFT on the result?
;; If so, set loop to FFT the first few cache lines now that the carries
;; have been added in.

	cmp	POSTFFT, 0		;; Test flag
	je	short fftdn1		;; Skip FFT code if flag not set
	mov	esi, DESTARG		;; Load FFT data address
	mov	edx, 65536+4/clm*256	;; Do 8 previously skipped cache lines
	mov	norm_ptr1, 0		;; Set flag so only forward FFT is done
	copy_4_words
	jmp	b0b

;; Now add in the FFT'ed carries, cleanup and return

fftdn1:	end_timer 1
	jmp	gw_finish_mult
	ENDM


;; Perform a 6K-element FFT.  This is done in two passes.  Pass 1
;; does 5 levels, pass 2 uses common code to perform the last 8 levels.

pass1levels5pfa6 MACRO pass2_macro, p2cl, clm
	LOCAL	b0b, b1b, b2b, b3b, c0b, c1b, c2b, c3b
	LOCAL	pass2, no_fft, more, fftdn, fftdn1

;; Call FFT code unless the FFT has already been started

	clear_timers
	cmp	DWORD PTR [esi-28][ebx], 0 ;; Test FFT-started flag
	jne	pass2			;; Jump is FFT already started
	mov	norm_ptr1, 0		;; Set FFT-ing flag
	mov	edx, p2cl/128*65536+128/clm*256;; Load loop counter
	jmp	b0b			;; Do the pass 1 fft

;; Do the last 8 FFT levels
;;	do 1 pass2_8_levels_real and 11 pass2_8_levels_complex

pass2:	start_timer 0
	CALLP	pass2_macro
	end_timer 0
	cmp	ffttype, 1		;; We're done if FFTing only
	je	gw_finish_fft

;; Do the inverse FFT

	start_timer 1
	fldz				;; Init SUMOUT
	fstp	SUMOUT
	mov	edx, norm_biglit_array	;; Addr of the big/little flags array
	mov	norm_ptr1, edx		;; Save ptr
	mov	edx, norm_col_mults	;; Addr of the column multipliers
	mov	norm_ptr2, edx		;; Save ptr
	mov	edx, p2cl/128*65536+128/clm*256;; Load loop counter

;; Do inverse FFT levels 4,5
;; On input the 32-byte cache lines hold these data values:
;;	0	256	1	257
;;	512	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	1K	1	1K+1
;;	256	...
;;	...
;;	2K	...
;;	...

;; Do 1 four_real_four_semireal_unfft macros
;;	distance between fft data elements is 1

c0b:	start_timer 18
c2b:	disp four_real_four_semireal_unfft, 8, blkdst, 2*blkdst
	lea	esi, [esi+dist1]	;; Next source pointer
	add	dl, 256/2/clm		;; Test inner loop counter
	jnc	c2b			;; Iterate if necessary
	lea	esi, [esi-clm*2*dist1+4*blkdst];; Next source pointer

;; Do 2 four_complex_unfft macros
;;	distance between fft data elements is 1

	mov	edi, sincos1		;; Load sin/cos pointer
	mov	al, 2			;; 2 iterations
c3b:	disp four_complex_unfft, 8, blkdst, 2*blkdst
	lea	esi, [esi+dist1]	;; Next source pointer
	add	dl, 256/2/clm		;; Test loop counter
	jnc	c3b			;; Iterate if necessary
	lea	esi, [esi-clm*2*dist1+4*blkdst];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al			;; Test outer loop counter
	jnz	c3b			;; Iterate if necessary
	lea	esi, [esi-3*4*blkdst]	;; Restore source pointer
	end_timer 18

;; Do inverse FFT levels 1,2,3
;; On input the 32-byte cache lines hold these data values:
;;	0	1K	1	1K+1
;;	256	...
;;	...
;;	2K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	3K	1	3K+1
;;	256
;;	...
;;	3K-256

;; Do 4 six_reals_last_unfft macros
;;	distance between fft data elements is 4

	start_timer 19
	mov	al, 4			;; Load loop counter
c1b:	disp six_reals_last_unfft, 8, 4*blkdst, 8*blkdst
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test loop counter
	jnc	c1b			;; Iterate if necessary
	IFDEF PFETCH
	prefetcht1 [esi-32+clm*32]
;	prefetcht1 [esi-32+clm*32+4*blkdst]
	prefetcht1 [esi-32+clm*32+8*blkdst]
	ENDIF
	loopclm	dl, clm, c1b
	lea	esi, [esi-clm*2*dist1+blkdst];; Next source pointer
	dec	al			;; Test loop counter
	jnz	c1b			;; Iterate if necessary
	lea	esi, [esi-4*blkdst]	;; Restore source pointer
	end_timer 19

;; Normalize these values

	start_timer 20
	mov	eax, NORMRTN		;; Addr of normalization routine
	call	eax
	sub	eax, eax
	end_timer 20

;; Have we been given permission to start the FFT on the result?
;; If so, do so now while the data is in the L2 cache.

	cmp	POSTFFT, 0		;; Test flag
	je	no_fft			;; Skip FFT code if flag not set
	cmp	edx, p2cl/128*65536+(128/clm-4/clm)*256;; Delay fft on 8 sets
	jg	no_fft			;; Jump if delaying FFT
	copy_3_words clm, 0

;; Do FFT levels 1,2,3
;;
;; On input the 32-byte cache lines hold these data values:
;;	0	3K	1	3K+1
;;	256
;;	...
;;	3K-256
;; On output the 32-byte cache lines hold these data values:
;;	0	1K	1	1K+1
;;	256	...
;;	...
;;	2K	...
;;	...

;; Do 4 six_reals_first_fft macros
;;	distance between fft data elements is 4

b0b:	start_timer 21
	mov	ebx, DIST_TO_FFTSRCARG
	mov	al, 4			;; 4 iterations
b1b:	disp six_reals_first_fft, 4*blkdst, 8*blkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	add	dl, 256/2/clm		;; Test loop counter
	jnc	b1b			;; Iterate if necessary
	lea	esi, [esi-clm*2*dist1+blkdst];; Next source pointer
	dec	al			;; Test loop counter
	jnz	b1b			;; Iterate if necessary
	lea	esi, [esi-4*blkdst]	;; Restore source pointer
	end_timer 21

;; Do FFT levels 4,5
;;
;; On input the 32-byte cache lines hold these data values:
;;	0	1K	1	1K+1
;;	256	...
;;	...
;;	2K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	256	1	257
;;	512	...
;;	...

;; Do 1 four_real_four_semireal_fft macros
;;	distance between fft data elements is 1

	start_timer 22
b2b:	disp four_real_four_semireal_fft, blkdst, 2*blkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test inner loop counter
	jnc	b2b			;; Iterate if necessary
	IFDEF PFETCH
;	prefetcht1 [esi-32+clm*32]
	prefetcht1 [esi-32+clm*32+blkdst]
;	prefetcht1 [esi-32+clm*32+2*blkdst]
	prefetcht1 [esi-32+clm*32+3*blkdst]
	ENDIF
	loopclm	dl, clm, b2b
	lea	esi, [esi-clm*2*dist1+4*blkdst];; Next source pointer

;; Do 2 four_complex_fft macros
;;	distance between fft data elements is 1

	mov	edi, sincos1		;; Load sin/cos pointer
	mov	al, 2			;; 2 iterations
b3b:	disp four_complex_fft, blkdst, 2*blkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test inner loop counter
	jnc	b3b			;; Iterate if necessary
	IFDEF PFETCH
;	prefetcht1 [esi-32+clm*32]
	prefetcht1 [esi-32+clm*32+blkdst]
;	prefetcht1 [esi-32+clm*32+2*blkdst]
	prefetcht1 [esi-32+clm*32+3*blkdst]
	ENDIF
	loopclm	dl, clm, b3b
	lea	esi, [esi-clm*2*dist1+4*blkdst];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al
	jnz	b3b
	lea	esi, [esi-3*4*blkdst]	;; Restore source pointer
	end_timer 22

;; Work on next 4 sets of 32 values

no_fft:	lea	esi, [esi+clm*2*dist1]	;; Next source pointer
	dec	dh			;; Test loop counter
	jnz	short more		;; Do more FFTs
	lea	esi, [esi+64]		;; Skip pad every 128 cache lines (4KB)
	sub	edx, 65536		;; Test loop counter
	jz	short fftdn		;; Jump if done looping
	mov	dh, 128/clm		;; Restore cache line counter
more:	cmp	norm_ptr1, 0		;; What was the loop start point?
	jne	c0b			;; Do inverse-FFT/norm/FFT iteration
	jmp	b0b			;; Do another FFT-only iteration
fftdn:	lea	esi, [esi-p2cl*2*dist1-p2cl/128*64];; Restore source pointer
	cmp	esi, DESTARG		;; If esi was not restored, then we 
	jne	short fftdn1		;; just finished skipped postfft data
	sub	ebx, ebx
	mov	DIST_TO_FFTSRCARG, ebx
	cmp	norm_ptr1, 0		;; What was the loop start point?
	je	pass2			;; FFT-only, go do pass 2

;; Split the carries into high and low carries (so they do not exceed
;; the desired number of bits per FFT word.  Then add the carries back to the
;; FFT data.

	start_timer 28
	call	gw_carries
	end_timer 28

;; Have we been given permission to start the FFT on the result?
;; If so, set loop to FFT the first few cache lines now that the carries
;; have been added in.

	cmp	POSTFFT, 0		;; Test flag
	je	short fftdn1		;; Skip FFT code if flag not set
	mov	esi, DESTARG		;; Load FFT data address
	mov	edx, 65536+4/clm*256	;; Do 8 previously skipped cache lines
	mov	norm_ptr1, 0		;; Set flag so only forward FFT is done
	copy_4_words
	jmp	b0b

;; Now add in the FFT'ed carries, cleanup and return

fftdn1:	end_timer 1
	jmp	gw_finish_mult
	ENDM


;; Perform a 6K-element negacyclic FFT.  This is done in two passes.  Pass 1
;; does 5 levels, pass 2 uses common code to perform the last 8 levels.

pass1levels5complex3 MACRO pass2_macro, p2cl, clm
	LOCAL	b0b, b1b, b3b, c0b, c1b, c3b
	LOCAL	pass2, no_fft, more, fftdn, fftdn1

;; Call FFT code unless the FFT has already been started

	clear_timers
	cmp	DWORD PTR [esi-28][ebx], 0 ;; Test FFT-started flag
	jne	pass2			;; Jump is FFT already started
	mov	norm_ptr1, 0		;; Set FFT-ing flag
	mov	edx, p2cl/128*65536+128/clm*256;; Load loop counter
	jmp	b0b			;; Do the pass 1 fft

;; Do the last 8 FFT levels
;;	do 12 pass2_8_levels_complex

pass2:	start_timer 0
	CALLP	pass2_macro
	end_timer 0
	cmp	ffttype, 1		;; We're done if FFTing only
	je	gw_finish_fft

;; Do the inverse FFT

	start_timer 1
	fldz				;; Init SUMOUT
	fstp	SUMOUT
	mov	edx, norm_biglit_array	;; Addr of the big/little flags array
	mov	norm_ptr1, edx		;; Save ptr
	mov	edx, norm_col_mults	;; Addr of the column multipliers
	mov	norm_ptr2, edx		;; Save ptr
	mov	edx, p2cl/128*65536+128/clm*256;; Load loop counter

;; Do inverse FFT levels 4,5
;; On input the 32-byte cache lines hold these data values:
;;	0	256	1	257
;;	512	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	1K	1	1K+1
;;	256	...
;;	...
;;	2K	...
;;	...

;; Do 3 four_complex_unfft macros
;;	distance between fft data elements is 1

c0b:	start_timer 18
	mov	edi, sincos1		;; Load sin/cos pointer
	mov	al, 3			;; 3 iterations
c3b:	disp four_complex_unfft, 8, blkdst, 2*blkdst
	lea	esi, [esi+dist1]	;; Next source pointer
	add	dl, 256/2/clm		;; Test loop counter
	jnc	c3b			;; Iterate if necessary
	lea	esi, [esi-clm*2*dist1+4*blkdst];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al			;; Test outer loop counter
	jnz	c3b			;; Iterate if necessary
	lea	esi, [esi-3*4*blkdst]	;; Restore source pointer
	end_timer 18

;; Do inverse FFT levels 1,2,3
;; On input the 32-byte cache lines hold these data values:
;;	0	1K	1	1K+1
;;	256	...
;;	...
;;	2K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	3K	1	3K+1
;;	256
;;	...
;;	3K-256

;; Do 4 three_complex_last_unfft macros
;;	distance between fft data elements is 4

	start_timer 19
	mov	edi, plus1_premults	;; Address of premultiplier table
	mov	al, 4			;; Load loop counter
c1b:	three_complex_last_unfft 8, 4*blkdst, 8*blkdst
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test loop counter
	jnc	c1b			;; Iterate if necessary
	IFDEF PFETCH
	prefetcht1 [esi-32+clm*32]
;	prefetcht1 [esi-32+clm*32+4*blkdst]
	prefetcht1 [esi-32+clm*32+8*blkdst]
	ENDIF
	loopclm	dl, clm, c1b
	lea	esi, [esi-clm*2*dist1+blkdst];; Next source pointer
	lea	edi, [edi+48]		;; Next premultiplier pointer
	dec	al			;; Test loop counter
	jnz	c1b			;; Iterate if necessary
	lea	esi, [esi-4*blkdst]	;; Restore source pointer
	end_timer 19

;; Normalize these values

	start_timer 20
	mov	eax, NORMRTN		;; Addr of normalization routine
	call	eax
	sub	eax, eax
	end_timer 20

;; Have we been given permission to start the FFT on the result?
;; If so, do so now while the data is in the L2 cache.

	cmp	POSTFFT, 0		;; Test flag
	je	no_fft			;; Skip FFT code if flag not set
	cmp	edx, p2cl/128*65536+(128/clm-4/clm)*256;; Delay fft on 8 sets
	jg	no_fft			;; Jump if delaying FFT
	copy_3_words clm, 0

;; Do FFT levels 1,2,3
;;
;; On input the 32-byte cache lines hold these data values:
;;	0	3K	1	3K+1
;;	256
;;	...
;;	3K-256
;; On output the 32-byte cache lines hold these data values:
;;	0	1K	1	1K+1
;;	256	...
;;	...
;;	2K	...
;;	...

;; Do 4 three_complex_first_fft macros
;;	distance between fft data elements is 4

b0b:	start_timer 21
	mov	ebx, DIST_TO_FFTSRCARG
	mov	edi, plus1_premults	;; Address of premultiplier table
	mov	al, 4			;; 4 iterations
b1b:	three_complex_first_fft 4*blkdst, 8*blkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	add	dl, 256/2/clm		;; Test loop counter
	jnc	b1b			;; Iterate if necessary
	lea	esi, [esi-clm*2*dist1+blkdst];; Next source pointer
	lea	edi, [edi+48]		;; Next premultiplier pointer
	dec	al			;; Test loop counter
	jnz	b1b			;; Iterate if necessary
	lea	esi, [esi-4*blkdst]	;; Restore source pointer
	end_timer 21

;; Do FFT levels 4,5
;;
;; On input the 32-byte cache lines hold these data values:
;;	0	1K	1	1K+1
;;	256	...
;;	...
;;	2K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	256	1	257
;;	512	...
;;	...

;; Do 3 four_complex_fft macros
;;	distance between fft data elements is 1

	start_timer 22
	mov	edi, sincos1		;; Load sin/cos pointer
	mov	al, 3			;; 3 iterations
b3b:	disp four_complex_fft, blkdst, 2*blkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test inner loop counter
	jnc	b3b			;; Iterate if necessary
	IFDEF PFETCH
;	prefetcht1 [esi-32+clm*32]
	prefetcht1 [esi-32+clm*32+blkdst]
;	prefetcht1 [esi-32+clm*32+2*blkdst]
	prefetcht1 [esi-32+clm*32+3*blkdst]
	ENDIF
	loopclm	dl, clm, b3b
	lea	esi, [esi-clm*2*dist1+4*blkdst];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al
	jnz	b3b
	lea	esi, [esi-3*4*blkdst]	;; Restore source pointer
	end_timer 22

;; Work on next 4 sets of 32 values

no_fft:	lea	esi, [esi+clm*2*dist1]	;; Next source pointer
	dec	dh			;; Test loop counter
	jnz	short more		;; Do more FFTs
	lea	esi, [esi+64]		;; Skip pad every 128 cache lines (4KB)
	sub	edx, 65536		;; Test loop counter
	jz	short fftdn		;; Jump if done looping
	mov	dh, 128/clm		;; Restore cache line counter
more:	cmp	norm_ptr1, 0		;; What was the loop start point?
	jne	c0b			;; Do inverse-FFT/norm/FFT iteration
	jmp	b0b			;; Do another FFT-only iteration
fftdn:	lea	esi, [esi-p2cl*2*dist1-p2cl/128*64];; Restore source pointer
	cmp	esi, DESTARG		;; If esi was not restored, then we 
	jne	short fftdn1		;; just finished skipped postfft data
	sub	ebx, ebx
	mov	DIST_TO_FFTSRCARG, ebx
	cmp	norm_ptr1, 0		;; What was the loop start point?
	je	pass2			;; FFT-only, go do pass 2

;; Split the carries into high and low carries (so they do not exceed
;; the desired number of bits per FFT word.  Then add the carries back to the
;; FFT data.

	start_timer 28
	call	gw_carries
	end_timer 28

;; Have we been given permission to start the FFT on the result?
;; If so, set loop to FFT the first few cache lines now that the carries
;; have been added in.

	cmp	POSTFFT, 0		;; Test flag
	je	short fftdn1		;; Skip FFT code if flag not set
	mov	esi, DESTARG		;; Load FFT data address
	mov	edx, 65536+4/clm*256	;; Do 8 previously skipped cache lines
	mov	norm_ptr1, 0		;; Set flag so only forward FFT is done
	copy_4_words
	jmp	b0b

;; Now add in the FFT'ed carries, cleanup and return

fftdn1:	end_timer 1
	jmp	gw_finish_mult
	ENDM


;; Perform a 7K-element FFT.  This is done in two passes.  Pass 1
;; does 5 levels, pass 2 uses common code to perform the last 8 levels.

pass1levels5pfa7 MACRO pass2_macro, p2cl, clm
	LOCAL	b0b, b1b, b2b, b3b, c0b, c1b, c2b, c3b
	LOCAL	pass2, no_fft, more, fftdn, fftdn1

;; Call FFT code unless the FFT has already been started

	clear_timers
	cmp	DWORD PTR [esi-28][ebx], 0 ;; Test FFT-started flag
	jne	pass2			;; Jump is FFT already started
	mov	norm_ptr1, 0		;; Set FFT-ing flag
	mov	edx, p2cl/128*65536+128/clm*256;; Load loop counter
	jmp	b0b			;; Do the pass 1 fft

;; Do the last 8 FFT levels
;;	do 1 pass2_8_levels_real and 13 pass2_8_levels_complex

pass2:	start_timer 0
	CALLP	pass2_macro
	end_timer 0
	cmp	ffttype, 1		;; We're done if FFTing only
	je	gw_finish_fft

;; Do the inverse FFT

	start_timer 1
	fldz				;; Init SUMOUT
	fstp	SUMOUT
	mov	edx, norm_biglit_array	;; Addr of the big/little flags array
	mov	norm_ptr1, edx		;; Save ptr
	mov	edx, norm_col_mults	;; Addr of the column multipliers
	mov	norm_ptr2, edx		;; Save ptr
	mov	edx, p2cl/128*65536+128/clm*256;; Load loop counter

;; Do inverse FFT levels 4,5
;; On input the 32-byte cache lines hold these data values:
;;	0	256	1	257
;;	512	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	512	1	513
;;	256	...
;;	1K	...
;;	...

;; Do 1 four_real_unfft macros
;;	distance between fft data elements is 1
;;	For better pipelining we call eight_reals_unfft_2 which works
;;	on two sets of four_reals at the same time.

c0b:	start_timer 18
c2b:	disp eight_reals_unfft_2, dist1, 8, blkdst
	lea	esi, [esi+2*dist1]	;; Next source pointer
	loopclm	dl, clm, c2b
	lea	esi, [esi-clm*2*dist1+2*blkdst];; Next source pointer

;; Do 3 four_complex_unfft macros
;;	distance between fft data elements is 1

	mov	edi, sincos1		;; Load sin/cos pointer
	mov	al, 3			;; 2 iterations
c3b:	disp four_complex_unfft, 8, blkdst, 2*blkdst
	lea	esi, [esi+dist1]	;; Next source pointer
	add	dl, 256/2/clm		;; Test loop counter
	jnc	c3b			;; Iterate if necessary
	lea	esi, [esi-clm*2*dist1+4*blkdst];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al			;; Test outer loop counter
	jnz	c3b			;; Iterate if necessary
	lea	esi, [esi-3*4*blkdst-2*blkdst];; Restore source pointer
	end_timer 18

;; Do inverse FFT levels 1,2,3
;; On input the 32-byte cache lines hold these data values:
;;	0	512	1	513
;;	256	...
;;	1K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	3.5K	1	3.5K+1
;;	256
;;	...
;;	3.5K-256

;; Do 4 seven_reals_last_unfft macros
;;	distance between fft data elements is 4

	start_timer 19
	mov	al, 2			;; Load loop counter
c1b:	two_seven_reals_last_unfft 8, 2*blkdst
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test loop counter
	jnc	c1b			;; Iterate if necessary
	IFDEF PFETCH
	prefetcht1 [esi-32+clm*32]
;	prefetcht1 [esi-32+clm*32+2*blkdst]
	prefetcht1 [esi-32+clm*32+4*blkdst]
;	prefetcht1 [esi-32+clm*32+6*blkdst]
	prefetcht1 [esi-32+clm*32+8*blkdst]
;	prefetcht1 [esi-32+clm*32+10*blkdst]
	prefetcht1 [esi-32+clm*32+12*blkdst]
	ENDIF
	loopclm	dl, clm, c1b
	lea	esi, [esi-clm*2*dist1+blkdst];; Next source pointer
	dec	al			;; Test loop counter
	jnz	c1b			;; Iterate if necessary
	lea	esi, [esi-2*blkdst]	;; Restore source pointer
	end_timer 19

;; Normalize these values

	start_timer 20
	mov	eax, NORMRTN		;; Addr of normalization routine
	call	eax
	sub	eax, eax
	end_timer 20

;; Have we been given permission to start the FFT on the result?
;; If so, do so now while the data is in the L2 cache.

	cmp	POSTFFT, 0		;; Test flag
	je	no_fft			;; Skip FFT code if flag not set
	cmp	edx, p2cl/128*65536+(128/clm-4/clm)*256;; Delay fft on 8 sets
	jg	no_fft			;; Jump if delaying FFT
	copy_3_words clm, 0

;; Do FFT levels 1,2,3
;;
;; On input the 32-byte cache lines hold these data values:
;;	0	3.5K	1	3.5K+1
;;	256
;;	...
;;	3.5K-256
;; On output the 32-byte cache lines hold these data values:
;;	0	512	1	513
;;	256	...
;;	1K	...
;;	...

;; Do 4 seven_reals_first_fft macros
;;	distance between fft data elements is 4

b0b:	start_timer 21
	mov	ebx, DIST_TO_FFTSRCARG
	mov	al, 2			;; 4 iterations
b1b:	two_seven_reals_first_fft 2*blkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	add	dl, 256/2/clm		;; Test loop counter
	jnc	b1b			;; Iterate if necessary
	lea	esi, [esi-clm*2*dist1+blkdst];; Next source pointer
	dec	al			;; Test loop counter
	jnz	b1b			;; Iterate if necessary
	lea	esi, [esi-2*blkdst]	;; Restore source pointer
	end_timer 21

;; Do FFT levels 4,5
;;
;; On input the 32-byte cache lines hold these data values:
;;	0	512	1	513
;;	256	...
;;	1K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	256	1	257
;;	512	...
;;	...

;; Do 1 four_real_fft macros
;;	distance between fft data elements is 1
;;	For better pipelining we call eight_reals_fft_2 which works
;;	on two sets of four_reals at the same time.

	start_timer 22
b2b:	disp eight_reals_fft_2, dist1, blkdst, 8
	lea	esi, [esi+2*dist1]	;; Next source pointer
	IFDEF PFETCH
;	prefetcht1 [esi-32+clm*32]
	prefetcht1 [esi-32+clm*32+blkdst]
	ENDIF
	loopclm	dl, clm, b2b
	lea	esi, [esi-clm*2*dist1+2*blkdst];; Next source pointer

;; Do 3 four_complex_fft macros
;;	distance between fft data elements is 1

	mov	edi, sincos1		;; Load sin/cos pointer
	mov	al, 3			;; 2 iterations
b3b:	disp four_complex_fft, blkdst, 2*blkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test inner loop counter
	jnc	b3b			;; Iterate if necessary
	IFDEF PFETCH
;	prefetcht1 [esi-32+clm*32]
	prefetcht1 [esi-32+clm*32+blkdst]
;	prefetcht1 [esi-32+clm*32+2*blkdst]
	prefetcht1 [esi-32+clm*32+3*blkdst]
	ENDIF
	loopclm	dl, clm, b3b
	lea	esi, [esi-clm*2*dist1+4*blkdst];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al
	jnz	b3b
	lea	esi, [esi-3*4*blkdst-2*blkdst];; Restore source pointer
	end_timer 22

;; Work on next 4 sets of 32 values

no_fft:	lea	esi, [esi+clm*2*dist1]	;; Next source pointer
	dec	dh			;; Test loop counter
	jnz	short more		;; Do more FFTs
	lea	esi, [esi+64]		;; Skip pad every 128 cache lines (4KB)
	sub	edx, 65536		;; Test loop counter
	jz	short fftdn		;; Jump if done looping
	mov	dh, 128/clm		;; Restore cache line counter
more:	cmp	norm_ptr1, 0		;; What was the loop start point?
	jne	c0b			;; Do inverse-FFT/norm/FFT iteration
	jmp	b0b			;; Do another FFT-only iteration
fftdn:	lea	esi, [esi-p2cl*2*dist1-p2cl/128*64];; Restore source pointer
	cmp	esi, DESTARG		;; If esi was not restored, then we 
	jne	short fftdn1		;; just finished skipped postfft data
	sub	ebx, ebx
	mov	DIST_TO_FFTSRCARG, ebx
	cmp	norm_ptr1, 0		;; What was the loop start point?
	je	pass2			;; FFT-only, go do pass 2

;; Split the carries into high and low carries (so they do not exceed
;; the desired number of bits per FFT word.  Then add the carries back to the
;; FFT data.

	start_timer 28
	call	gw_carries
	end_timer 28

;; Have we been given permission to start the FFT on the result?
;; If so, set loop to FFT the first few cache lines now that the carries
;; have been added in.

	cmp	POSTFFT, 0		;; Test flag
	je	short fftdn1		;; Skip FFT code if flag not set
	mov	esi, DESTARG		;; Load FFT data address
	mov	edx, 65536+4/clm*256	;; Do 8 previously skipped cache lines
	mov	norm_ptr1, 0		;; Set flag so only forward FFT is done
	copy_4_words
	jmp	b0b

;; Now add in the FFT'ed carries, cleanup and return

fftdn1:	end_timer 1
	jmp	gw_finish_mult
	ENDM


;; Perform a 8K-element FFT.  This is done in two passes.  Pass 1
;; does 5 levels, pass 2 uses common code to perform the last 8 levels.

pass1levels5pfa8 MACRO pass2_macro, p2cl, clm
	LOCAL	b0b, b1b, b2b, b3b, c0b, c1b, c2b, c3b
	LOCAL	pass2, no_fft, more, fftdn, fftdn1

;; Call FFT code unless the FFT has already been started

	clear_timers
	cmp	DWORD PTR [esi-28][ebx], 0 ;; Test FFT-started flag
	jne	pass2			;; Jump is FFT already started
	mov	norm_ptr1, 0		;; Set FFT-ing flag
	mov	edx, p2cl/128*65536+128/clm*256;; Load loop counter
	jmp	b0b			;; Do the pass 1 fft

;; Do the last 8 FFT levels
;;	do 1 pass2_8_levels_real and 15 pass2_8_levels_complex

pass2:	start_timer 0
	CALLP	pass2_macro
	end_timer 0
	cmp	ffttype, 1		;; We're done if FFTing only
	je	gw_finish_fft

;; Do the inverse FFT

	start_timer 1
	fldz				;; Init SUMOUT
	fstp	SUMOUT
	mov	edx, norm_biglit_array	;; Addr of the big/little flags array
	mov	norm_ptr1, edx		;; Save ptr
	mov	edx, norm_col_mults	;; Addr of the column multipliers
	mov	norm_ptr2, edx		;; Save ptr
	mov	edx, p2cl/128*65536+128/clm*256;; Load loop counter

;; Do inverse FFT levels 4,5
;; On input the 32-byte cache lines hold these data values:
;;	0	256	1	257
;;	512	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	1K	1	1K+1
;;	256	...
;;	...
;;	2K	...
;;	...

;; Do 1 four_real_four_semireal_unfft macros
;;	distance between fft data elements is 1

c0b:	start_timer 18
c2b:	disp four_real_four_semireal_unfft, 8, blkdst, 2*blkdst
	lea	esi, [esi+dist1]	;; Next source pointer
	add	dl, 256/2/clm		;; Test inner loop counter
	jnc	c2b			;; Iterate if necessary
	lea	esi, [esi-clm*2*dist1+4*blkdst];; Next source pointer

;; Do 3 four_complex_unfft macros
;;	distance between fft data elements is 1

	mov	edi, sincos1		;; Load sin/cos pointer
	mov	al, 3			;; 3 iterations
c3b:	disp four_complex_unfft, 8, blkdst, 2*blkdst
	lea	esi, [esi+dist1]	;; Next source pointer
	add	dl, 256/2/clm		;; Test loop counter
	jnc	c3b			;; Iterate if necessary
	lea	esi, [esi-clm*2*dist1+4*blkdst];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al			;; Test outer loop counter
	jnz	c3b			;; Iterate if necessary
	lea	esi, [esi-4*4*blkdst]	;; Restore source pointer
	end_timer 18

;; Do inverse FFT levels 1,2,3
;; On input the 32-byte cache lines hold these data values:
;;	0	1K	1	1K+1
;;	256	...
;;	...
;;	2K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	4K	1	4K+1
;;	256
;;	...
;;	4K-256

;; Do 4 eight_reals_last_unfft macros
;;	distance between fft data elements is 4

	start_timer 19
	mov	al, 4			;; Load loop counter
c1b:	disp eight_reals_last_unfft, 8, 4*blkdst, 8*blkdst
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test loop counter
	jnc	c1b			;; Iterate if necessary
	IFDEF PFETCH
	prefetcht1 [esi-32+clm*32]
;	prefetcht1 [esi-32+clm*32+4*blkdst]
	prefetcht1 [esi-32+clm*32+8*blkdst]
;	prefetcht1 [esi-32+clm*32+12*blkdst]
	ENDIF
	loopclm	dl, clm, c1b
	lea	esi, [esi-clm*2*dist1+blkdst];; Next source pointer
	dec	al			;; Test loop counter
	jnz	c1b			;; Iterate if necessary
	lea	esi, [esi-4*blkdst]	;; Restore source pointer
	end_timer 19

;; Normalize these values

	start_timer 20
	mov	eax, NORMRTN		;; Addr of normalization routine
	call	eax
	sub	eax, eax
	end_timer 20

;; Have we been given permission to start the FFT on the result?
;; If so, do so now while the data is in the L2 cache.

	cmp	POSTFFT, 0		;; Test flag
	je	no_fft			;; Skip FFT code if flag not set
	cmp	edx, p2cl/128*65536+(128/clm-4/clm)*256;; Delay fft on 8 sets
	jg	no_fft			;; Jump if delaying FFT
	copy_3_words clm, 0

;; Do FFT levels 1,2,3
;;
;; On input the 32-byte cache lines hold these data values:
;;	0	4K	1	4K+1
;;	256
;;	...
;;	4K-256
;; On output the 32-byte cache lines hold these data values:
;;	0	1K	1	1K+1
;;	256	...
;;	...
;;	2K	...
;;	...

;; Do 4 eight_reals_first_fft macros
;;	distance between fft data elements is 4

b0b:	start_timer 21
	mov	ebx, DIST_TO_FFTSRCARG
	mov	al, 4			;; 4 iterations
b1b:	disp eight_reals_first_fft, 4*blkdst, 8*blkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	add	dl, 256/2/clm		;; Test loop counter
	jnc	b1b			;; Iterate if necessary
	lea	esi, [esi-clm*2*dist1+blkdst];; Next source pointer
	dec	al			;; Test loop counter
	jnz	b1b			;; Iterate if necessary
	lea	esi, [esi-4*blkdst]	;; Restore source pointer
	end_timer 21

;; Do FFT levels 4,5
;;
;; On input the 32-byte cache lines hold these data values:
;;	0	1K	1	1K+1
;;	256	...
;;	...
;;	2K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	256	1	257
;;	512	...
;;	...

;; Do 1 four_real_four_semireal_fft macros
;;	distance between fft data elements is 1

	start_timer 22
b2b:	disp four_real_four_semireal_fft, blkdst, 2*blkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test inner loop counter
	jnc	b2b			;; Iterate if necessary
	IFDEF PFETCH
;	prefetcht1 [esi-32+clm*32]
	prefetcht1 [esi-32+clm*32+blkdst]
;	prefetcht1 [esi-32+clm*32+2*blkdst]
	prefetcht1 [esi-32+clm*32+3*blkdst]
	ENDIF
	loopclm	dl, clm, b2b
	lea	esi, [esi-clm*2*dist1+4*blkdst];; Next source pointer

;; Do 3 four_complex_fft macros
;;	distance between fft data elements is 1

	mov	edi, sincos1		;; Load sin/cos pointer
	mov	al, 3			;; 3 iterations
b3b:	disp four_complex_fft, blkdst, 2*blkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test inner loop counter
	jnc	b3b			;; Iterate if necessary
	IFDEF PFETCH
;	prefetcht1 [esi-32+clm*32]
	prefetcht1 [esi-32+clm*32+blkdst]
;	prefetcht1 [esi-32+clm*32+2*blkdst]
	prefetcht1 [esi-32+clm*32+3*blkdst]
	ENDIF
	loopclm	dl, clm, b3b
	lea	esi, [esi-clm*2*dist1+4*blkdst];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al
	jnz	b3b
	lea	esi, [esi-4*4*blkdst]	;; Restore source pointer
	end_timer 22

;; Work on next 4 sets of 32 values

no_fft:	lea	esi, [esi+clm*2*dist1]	;; Next source pointer
	dec	dh			;; Test loop counter
	jnz	short more		;; Do more FFTs
	lea	esi, [esi+64]		;; Skip pad every 128 cache lines (4KB)
	sub	edx, 65536		;; Test loop counter
	jz	short fftdn		;; Jump if done looping
	mov	dh, 128/clm		;; Restore cache line counter
more:	cmp	norm_ptr1, 0		;; What was the loop start point?
	jne	c0b			;; Do inverse-FFT/norm/FFT iteration
	jmp	b0b			;; Do another FFT-only iteration
fftdn:	lea	esi, [esi-p2cl*2*dist1-p2cl/128*64];; Restore source pointer
	cmp	esi, DESTARG		;; If esi was not restored, then we 
	jne	short fftdn1		;; just finished skipped postfft data
	sub	ebx, ebx
	mov	DIST_TO_FFTSRCARG, ebx
	cmp	norm_ptr1, 0		;; What was the loop start point?
	je	pass2			;; FFT-only, go do pass 2

;; Split the carries into high and low carries (so they do not exceed
;; the desired number of bits per FFT word.  Then add the carries back to the
;; FFT data.

	start_timer 28
	call	gw_carries
	end_timer 28

;; Have we been given permission to start the FFT on the result?
;; If so, set loop to FFT the first few cache lines now that the carries
;; have been added in.

	cmp	POSTFFT, 0		;; Test flag
	je	short fftdn1		;; Skip FFT code if flag not set
	mov	esi, DESTARG		;; Load FFT data address
	mov	edx, 65536+4/clm*256	;; Do 8 previously skipped cache lines
	mov	norm_ptr1, 0		;; Set flag so only forward FFT is done
	copy_4_words
	jmp	b0b

;; Now add in the FFT'ed carries, cleanup and return

fftdn1:	end_timer 1
	jmp	gw_finish_mult
	ENDM


;; Perform a 8K-element negacyclic FFT.  This is done in two passes.  Pass 1
;; does 5 levels, pass 2 uses common code to perform the last 8 levels.

pass1levels5complex4 MACRO pass2_macro, p2cl, clm
	LOCAL	b0b, b1b, b3b, c0b, c1b, c3b
	LOCAL	pass2, no_fft, more, fftdn, fftdn1

;; Call FFT code unless the FFT has already been started

	clear_timers
	cmp	DWORD PTR [esi-28][ebx], 0 ;; Test FFT-started flag
	jne	pass2			;; Jump is FFT already started
	mov	norm_ptr1, 0		;; Set FFT-ing flag
	mov	edx, p2cl/128*65536+128/clm*256;; Load loop counter
	jmp	b0b			;; Do the pass 1 fft

;; Do the last 8 FFT levels
;;	do 16 pass2_8_levels_complex

pass2:	start_timer 0
	CALLP	pass2_macro
	end_timer 0
	cmp	ffttype, 1		;; We're done if FFTing only
	je	gw_finish_fft

;; Do the inverse FFT

	start_timer 1
	fldz				;; Init SUMOUT
	fstp	SUMOUT
	mov	edx, norm_biglit_array	;; Addr of the big/little flags array
	mov	norm_ptr1, edx		;; Save ptr
	mov	edx, norm_col_mults	;; Addr of the column multipliers
	mov	norm_ptr2, edx		;; Save ptr
	mov	edx, p2cl/128*65536+128/clm*256;; Load loop counter

;; Do inverse FFT levels 4,5
;; On input the 32-byte cache lines hold these data values:
;;	0	256	1	257
;;	512	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	1K	1	1K+1
;;	256	...
;;	...
;;	2K	...
;;	...

;; Do 4 four_complex_unfft macros
;;	distance between fft data elements is 1

c0b:	start_timer 18
	mov	edi, sincos1		;; Load sin/cos pointer
	mov	al, 4			;; 4 iterations
c3b:	disp four_complex_unfft, 8, blkdst, 2*blkdst
	lea	esi, [esi+dist1]	;; Next source pointer
	add	dl, 256/2/clm		;; Test loop counter
	jnc	c3b			;; Iterate if necessary
	lea	esi, [esi-clm*2*dist1+4*blkdst];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al			;; Test outer loop counter
	jnz	c3b			;; Iterate if necessary
	lea	esi, [esi-4*4*blkdst]	;; Restore source pointer
	end_timer 18

;; Do inverse FFT levels 1,2,3
;; On input the 32-byte cache lines hold these data values:
;;	0	1K	1	1K+1
;;	256	...
;;	...
;;	2K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	4K	1	4K+1
;;	256
;;	...
;;	4K-256

;; Do 4 four_complex_last_unfft macros
;;	distance between fft data elements is 4

	start_timer 19
	mov	edi, plus1_premults	;; Address of premultiplier table
	mov	al, 4			;; Load loop counter
c1b:	disp four_complex_last_unfft, 8, 4*blkdst, 8*blkdst
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test loop counter
	jnc	c1b			;; Iterate if necessary
	IFDEF PFETCH
	prefetcht1 [esi-32+clm*32]
;	prefetcht1 [esi-32+clm*32+4*blkdst]
	prefetcht1 [esi-32+clm*32+8*blkdst]
;	prefetcht1 [esi-32+clm*32+12*blkdst]
	ENDIF
	loopclm	dl, clm, c1b
	lea	esi, [esi-clm*2*dist1+blkdst];; Next source pointer
	lea	edi, [edi+64]		;; Next premultiplier pointer
	dec	al			;; Test loop counter
	jnz	c1b			;; Iterate if necessary
	lea	esi, [esi-4*blkdst]	;; Restore source pointer
	end_timer 19

;; Normalize these values

	start_timer 20
	mov	eax, NORMRTN		;; Addr of normalization routine
	call	eax
	sub	eax, eax
	end_timer 20

;; Have we been given permission to start the FFT on the result?
;; If so, do so now while the data is in the L2 cache.

	cmp	POSTFFT, 0		;; Test flag
	je	no_fft			;; Skip FFT code if flag not set
	cmp	edx, p2cl/128*65536+(128/clm-4/clm)*256;; Delay fft on 8 sets
	jg	no_fft			;; Jump if delaying FFT
	copy_3_words clm, 0

;; Do FFT levels 1,2,3
;;
;; On input the 32-byte cache lines hold these data values:
;;	0	4K	1	4K+1
;;	256
;;	...
;;	4K-256
;; On output the 32-byte cache lines hold these data values:
;;	0	1K	1	1K+1
;;	256	...
;;	...
;;	2K	...
;;	...

;; Do 4 four_complex_first_fft macros
;;	distance between fft data elements is 4

b0b:	start_timer 21
	mov	ebx, DIST_TO_FFTSRCARG
	mov	edi, plus1_premults	;; Address of premultiplier table
	mov	al, 4			;; 4 iterations
b1b:	disp four_complex_first_fft, 4*blkdst, 8*blkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	add	dl, 256/2/clm		;; Test loop counter
	jnc	b1b			;; Iterate if necessary
	lea	esi, [esi-clm*2*dist1+blkdst];; Next source pointer
	lea	edi, [edi+64]		;; Next premultiplier pointer
	dec	al			;; Test loop counter
	jnz	b1b			;; Iterate if necessary
	lea	esi, [esi-4*blkdst]	;; Restore source pointer
	end_timer 21

;; Do FFT levels 4,5
;;
;; On input the 32-byte cache lines hold these data values:
;;	0	1K	1	1K+1
;;	256	...
;;	...
;;	2K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	256	1	257
;;	512	...
;;	...

;; Do 4 four_complex_fft macros
;;	distance between fft data elements is 1

	start_timer 22
	mov	edi, sincos1		;; Load sin/cos pointer
	mov	al, 4			;; 4 iterations
b3b:	disp four_complex_fft, blkdst, 2*blkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test inner loop counter
	jnc	b3b			;; Iterate if necessary
	IFDEF PFETCH
;	prefetcht1 [esi-32+clm*32]
	prefetcht1 [esi-32+clm*32+blkdst]
;	prefetcht1 [esi-32+clm*32+2*blkdst]
	prefetcht1 [esi-32+clm*32+3*blkdst]
	ENDIF
	loopclm	dl, clm, b3b
	lea	esi, [esi-clm*2*dist1+4*blkdst];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al
	jnz	b3b
	lea	esi, [esi-4*4*blkdst]	;; Restore source pointer
	end_timer 22

;; Work on next 4 sets of 32 values

no_fft:	lea	esi, [esi+clm*2*dist1]	;; Next source pointer
	dec	dh			;; Test loop counter
	jnz	short more		;; Do more FFTs
	lea	esi, [esi+64]		;; Skip pad every 128 cache lines (4KB)
	sub	edx, 65536		;; Test loop counter
	jz	short fftdn		;; Jump if done looping
	mov	dh, 128/clm		;; Restore cache line counter
more:	cmp	norm_ptr1, 0		;; What was the loop start point?
	jne	c0b			;; Do inverse-FFT/norm/FFT iteration
	jmp	b0b			;; Do another FFT-only iteration
fftdn:	lea	esi, [esi-p2cl*2*dist1-p2cl/128*64];; Restore source pointer
	cmp	esi, DESTARG		;; If esi was not restored, then we 
	jne	short fftdn1		;; just finished skipped postfft data
	sub	ebx, ebx
	mov	DIST_TO_FFTSRCARG, ebx
	cmp	norm_ptr1, 0		;; What was the loop start point?
	je	pass2			;; FFT-only, go do pass 2

;; Split the carries into high and low carries (so they do not exceed
;; the desired number of bits per FFT word.  Then add the carries back to the
;; FFT data.

	start_timer 28
	call	gw_carries
	end_timer 28

;; Have we been given permission to start the FFT on the result?
;; If so, set loop to FFT the first few cache lines now that the carries
;; have been added in.

	cmp	POSTFFT, 0		;; Test flag
	je	short fftdn1		;; Skip FFT code if flag not set
	mov	esi, DESTARG		;; Load FFT data address
	mov	edx, 65536+4/clm*256	;; Do 8 previously skipped cache lines
	mov	norm_ptr1, 0		;; Set flag so only forward FFT is done
	copy_4_words
	jmp	b0b

;; Now add in the FFT'ed carries, cleanup and return

fftdn1:	end_timer 1
	jmp	gw_finish_mult
	ENDM


;; Perform a 10K-element FFT.  This is done in two passes.  Pass 1
;; does 6 levels, pass 2 uses common code to perform the last 8 levels.

pass1levels6pfa5 MACRO pass2_macro, p2cl, clm
	LOCAL	b0b, b1b, b2b, b3b, b4b, b5b, c0b, c1b, c2b, c3b, c4b, c5b
	LOCAL	pass2, no_fft, more, fftdn, fftdn1

;; Call FFT code unless the FFT has already been started

	clear_timers
	cmp	DWORD PTR [esi-28][ebx], 0 ;; Test FFT-started flag
	jne	pass2			;; Jump is FFT already started
	mov	norm_ptr1, 0		;; Set FFT-ing flag
	mov	edx, p2cl/128*65536+128/clm*256;; Load loop counter
	jmp	b0b			;; Do the pass 1 fft

;; Do the last 8 FFT levels
;;	do 1 pass2_8_levels_real and 19 pass2_8_levels_complex

pass2:	start_timer 0
	CALLP	pass2_macro
	end_timer 0
	cmp	ffttype, 1		;; We're done if FFTing only
	je	gw_finish_fft

;; Do the inverse FFT

	start_timer 1
	fldz				;; Init SUMOUT
	fstp	SUMOUT
	mov	edx, norm_biglit_array	;; Addr of the big/little flags array
	mov	norm_ptr1, edx		;; Save ptr
	mov	edx, norm_col_mults	;; Addr of the column multipliers
	mov	norm_ptr2, edx		;; Save ptr
	mov	edx, p2cl/128*65536+128/clm*256;; Load loop counter

;; Do inverse FFT levels 5,6
;; On input the 32-byte cache lines hold these data values:
;;	0	256	1	257
;;	512	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	1K	1	1K+1
;;	256	...
;;	...
;;	2K	...
;;	...

;; Do 1 four_real_four_semireal_unfft macros
;;	distance between fft data elements is 1

c0b:	start_timer 17
c2b:	disp four_real_four_semireal_unfft, 8, blkdst, 2*blkdst
	lea	esi, [esi+dist1]	;; Next source pointer
	add	dl, 256/2/clm		;; Test inner loop counter
	jnc	c2b			;; Iterate if necessary
	lea	esi, [esi-clm*2*dist1+4*blkdst];; Next source pointer

;; Do 4 four_complex_unfft macros
;;	distance between fft data elements is 1

	mov	edi, sincos1		;; Load sin/cos pointer
	mov	al, 4			;; 4 iterations
c3b:	disp four_complex_unfft, 8, blkdst, 2*blkdst
	lea	esi, [esi+dist1]	;; Next source pointer
	add	dl, 256/2/clm		;; Test loop counter
	jnc	c3b			;; Iterate if necessary
	lea	esi, [esi-clm*2*dist1+4*blkdst];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al			;; Test outer loop counter
	jnz	c3b			;; Iterate if necessary
	lea	esi, [esi-5*4*blkdst]	;; Restore source pointer
	end_timer 17

;; Do inverse FFT levels 4
;; On input the 32-byte cache lines hold these data values:
;;	0	1K	1	1K+1
;;	256	...
;;	...
;;	2K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	2K	1	2K+1
;;	256	...
;;	...
;;	4K	...
;;	...

;; Do 1 eight_reals_unfft_1 macros
;;	distance between fft data elements is 4

	start_timer 18
c4b:	disp eight_reals_unfft_1, blkdst, 2*blkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test inner loop counter
	jnc	c4b			;; Iterate if necessary
	IFDEF PFETCH
	prefetcht1 [esi-32+clm*32]
;	prefetcht1 [esi-32+clm*32+blkdst]
	prefetcht1 [esi-32+clm*32+2*blkdst]
;	prefetcht1 [esi-32+clm*32+3*blkdst]
	ENDIF
	loopclm	dl, clm, c4b
	lea	esi, [esi-clm*2*dist1+4*blkdst];; Next source pointer

;; Do 4 four_complex_unfft_1 macros
;;	distance between fft data elements is 4

	mov	edi, sincos2		;; Load sin/cos pointer
	mov	eax, 2			;; 2 iterations of 2
c5b:	disp four_complex_unfft_1, 2*blkdst, 8, 4*blkdst
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test inner loop counter
	jnc	c5b			;; Iterate if necessary
	IFDEF PFETCH
	prefetcht1 [esi-32+clm*32]
;	prefetcht1 [esi-32+clm*32+2*blkdst]
	prefetcht1 [esi-32+clm*32+4*blkdst]
;	prefetcht1 [esi-32+clm*32+6*blkdst]
	ENDIF
	loopclm	dl, clm, c5b
	lea	esi, [esi-clm*2*dist1+blkdst];; Next source pointer
	add	ah, 256/2		;; Test middle loop counter
	jnc	c5b			;; Iterate if necessary
	lea	esi, [esi-2*blkdst+8*blkdst];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al
	jnz	c5b
	lea	esi, [esi-2*8*blkdst-4*blkdst];; Restore source pointer
	end_timer 18

;; Do inverse FFT levels 1,2,3
;; On input the 32-byte cache lines hold these data values:
;;	0	2K	1	2K+1
;;	256	...
;;	...
;;	4K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	5K	1	5K+1
;;	256
;;	...
;;	5K-256

;; Do 8 five_reals_last_unfft macros
;;	distance between fft data elements is 8

	start_timer 19
	mov	al, 4			;; Load loop counter
c1b:	two_five_reals_last_unfft 8, 4*blkdst
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test loop counter
	jnc	c1b			;; Iterate if necessary
	IFDEF PFETCH
	prefetcht1 [esi-32+clm*32]
;	prefetcht1 [esi-32+clm*32+4*blkdst]
	prefetcht1 [esi-32+clm*32+8*blkdst]
;	prefetcht1 [esi-32+clm*32+12*blkdst]
	prefetcht1 [esi-32+clm*32+16*blkdst]
	ENDIF
	loopclm	dl, clm, c1b
	lea	esi, [esi-clm*2*dist1+blkdst];; Next source pointer
	dec	al			;; Test loop counter
	jnz	c1b			;; Iterate if necessary
	lea	esi, [esi-4*blkdst]	;; Restore source pointer
	end_timer 19

;; Normalize these values

	start_timer 20
	mov	eax, NORMRTN		;; Addr of normalization routine
	call	eax
	sub	eax, eax
	end_timer 20

;; Have we been given permission to start the FFT on the result?
;; If so, do so now while the data is in the L2 cache.

	cmp	POSTFFT, 0		;; Test flag
	je	no_fft			;; Skip FFT code if flag not set
	cmp	edx, p2cl/128*65536+(128/clm-4/clm)*256;; Delay fft on 8 sets
	jg	no_fft			;; Jump if delaying FFT
	copy_3_words clm, 0

;; Do FFT levels 1,2,3
;;
;; On input the 32-byte cache lines hold these data values:
;;	0	5K	1	5K+1
;;	256
;;	...
;;	5K-256
;; On output the 32-byte cache lines hold these data values:
;;	0	2K	1	2K+1
;;	256	...
;;	...
;;	4K	...
;;	...

;; Do 8 five_reals_first_fft macros
;;	distance between fft data elements is 8

b0b:	start_timer 21
	mov	ebx, DIST_TO_FFTSRCARG
	mov	al, 4			;; 4 iterations
b1b:	two_five_reals_first_fft 4*blkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	add	dl, 256/2/clm		;; Test loop counter
	jnc	b1b			;; Iterate if necessary
	lea	esi, [esi-clm*2*dist1+blkdst];; Next source pointer
	dec	al			;; Test loop counter
	jnz	b1b			;; Iterate if necessary
	lea	esi, [esi-4*blkdst]	;; Restore source pointer
	end_timer 21

;; Do FFT levels 4
;;
;; On input the 32-byte cache lines hold these data values:
;;	0	2K	1	2K+1
;;	256	...
;;	...
;;	4K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	1K	1	1K+1
;;	256	...
;;	...
;;	2K	...
;;	...

;; Do 1 eight_reals_fft_1 macros
;;	distance between fft data elements is 4

	start_timer 22
b4b:	disp eight_reals_fft_1, blkdst, 2*blkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test inner loop counter
	jnc	b4b			;; Iterate if necessary
	IFDEF PFETCH
;	prefetcht1 [esi-32+clm*32]
	prefetcht1 [esi-32+clm*32+blkdst]
;	prefetcht1 [esi-32+clm*32+2*blkdst]
	prefetcht1 [esi-32+clm*32+3*blkdst]
	ENDIF
	loopclm	dl, clm, b4b
	lea	esi, [esi-clm*2*dist1+4*blkdst];; Next source pointer

;; Do 4 four_complex_fft_1 macros
;;	distance between fft data elements is 4

	mov	edi, sincos2		;; Load sin/cos pointer
	mov	eax, 2			;; 2 iterations of 2
b5b:	disp four_complex_fft_1, 2*blkdst, 4*blkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test inner loop counter
	jnc	b5b			;; Iterate if necessary
	IFDEF PFETCH
;	prefetcht1 [esi-32+clm*32]
	prefetcht1 [esi-32+clm*32+2*blkdst]
;	prefetcht1 [esi-32+clm*32+4*blkdst]
	prefetcht1 [esi-32+clm*32+6*blkdst]
	ENDIF
	loopclm	dl, clm, b5b
	lea	esi, [esi-clm*2*dist1+blkdst];; Next source pointer
	add	ah, 256/2		;; Test middle loop counter
	jnc	b5b			;; Iterate if necessary
	lea	esi, [esi-2*blkdst+8*blkdst];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al
	jnz	b5b
	lea	esi, [esi-2*8*blkdst-4*blkdst];; Restore source pointer
	end_timer 22

;; Do FFT levels 5,6
;;
;; On input the 32-byte cache lines hold these data values:
;;	0	1K	1	1K+1
;;	256	...
;;	...
;;	2K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	256	1	257
;;	512	...
;;	...

;; Do 1 four_real_four_semireal_fft macros
;;	distance between fft data elements is 1

	start_timer 23
b2b:	disp four_real_four_semireal_fft, blkdst, 2*blkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test inner loop counter
	jnc	b2b			;; Iterate if necessary
	IFDEF PFETCH
;	prefetcht1 [esi-32+clm*32]
	prefetcht1 [esi-32+clm*32+blkdst]
;	prefetcht1 [esi-32+clm*32+2*blkdst]
	prefetcht1 [esi-32+clm*32+3*blkdst]
	ENDIF
	loopclm	dl, clm, b2b
	lea	esi, [esi-clm*2*dist1+4*blkdst];; Next source pointer

;; Do 4 four_complex_fft macros
;;	distance between fft data elements is 1

	mov	edi, sincos1		;; Load sin/cos pointer
	mov	al, 4			;; 4 iterations
b3b:	disp four_complex_fft, blkdst, 2*blkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test inner loop counter
	jnc	b3b			;; Iterate if necessary
	IFDEF PFETCH
;	prefetcht1 [esi-32+clm*32]
	prefetcht1 [esi-32+clm*32+blkdst]
;	prefetcht1 [esi-32+clm*32+2*blkdst]
	prefetcht1 [esi-32+clm*32+3*blkdst]
	ENDIF
	loopclm	dl, clm, b3b
	lea	esi, [esi-clm*2*dist1+4*blkdst];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al
	jnz	b3b
	lea	esi, [esi-5*4*blkdst]	;; Restore source pointer
	end_timer 23

;; Work on next 4 sets of 32 values

no_fft:	lea	esi, [esi+clm*2*dist1]	;; Next source pointer
	dec	dh			;; Test loop counter
	jnz	short more		;; Do more FFTs
	lea	esi, [esi+64]		;; Skip pad every 128 cache lines (4KB)
	sub	edx, 65536		;; Test loop counter
	jz	short fftdn		;; Jump if done looping
	mov	dh, 128/clm		;; Restore cache line counter
more:	cmp	norm_ptr1, 0		;; What was the loop start point?
	jne	c0b			;; Do inverse-FFT/norm/FFT iteration
	jmp	b0b			;; Do another FFT-only iteration
fftdn:	lea	esi, [esi-p2cl*2*dist1-p2cl/128*64];; Restore source pointer
	cmp	esi, DESTARG		;; If esi was not restored, then we 
	jne	short fftdn1		;; just finished skipped postfft data
	sub	ebx, ebx
	mov	DIST_TO_FFTSRCARG, ebx
	cmp	norm_ptr1, 0		;; What was the loop start point?
	je	pass2			;; FFT-only, go do pass 2

;; Split the carries into high and low carries (so they do not exceed
;; the desired number of bits per FFT word.  Then add the carries back to the
;; FFT data.

	start_timer 28
	call	gw_carries
	end_timer 28

;; Have we been given permission to start the FFT on the result?
;; If so, set loop to FFT the first few cache lines now that the carries
;; have been added in.

	cmp	POSTFFT, 0		;; Test flag
	je	short fftdn1		;; Skip FFT code if flag not set
	mov	esi, DESTARG		;; Load FFT data address
	mov	edx, 65536+4/clm*256	;; Do 8 previously skipped cache lines
	mov	norm_ptr1, 0		;; Set flag so only forward FFT is done
	copy_4_words
	jmp	b0b

;; Now add in the FFT'ed carries, cleanup and return

fftdn1:	end_timer 1
	jmp	gw_finish_mult
	ENDM


;; Perform a 12K-element FFT.  This is done in two passes.  Pass 1
;; does 6 levels, pass 2 uses common code to perform the last 8 levels.

pass1levels6pfa6 MACRO pass2_macro, p2cl, clm
	LOCAL	b0b, b1b, b2b, b3b, b4b, b5b, c0b, c1b, c2b, c3b, c4b, c5b
	LOCAL	pass2, no_fft, more, fftdn, fftdn1

;; Call FFT code unless the FFT has already been started

	clear_timers
	cmp	DWORD PTR [esi-28][ebx], 0 ;; Test FFT-started flag
	jne	pass2			;; Jump is FFT already started
	mov	norm_ptr1, 0		;; Set FFT-ing flag
	mov	edx, p2cl/128*65536+128/clm*256;; Load loop counter
	jmp	b0b			;; Do the pass 1 fft

;; Do the last 8 FFT levels
;;	do 1 pass2_8_levels_real and 23 pass2_8_levels_complex

pass2:	start_timer 0
	CALLP	pass2_macro
	end_timer 0
	cmp	ffttype, 1		;; We're done if FFTing only
	je	gw_finish_fft

;; Do the inverse FFT

	start_timer 1
	fldz				;; Init SUMOUT
	fstp	SUMOUT
	mov	edx, norm_biglit_array	;; Addr of the big/little flags array
	mov	norm_ptr1, edx		;; Save ptr
	mov	edx, norm_col_mults	;; Addr of the column multipliers
	mov	norm_ptr2, edx		;; Save ptr
	mov	edx, p2cl/128*65536+128/clm*256;; Load loop counter

;; Do inverse FFT levels 5,6
;; On input the 32-byte cache lines hold these data values:
;;	0	256	1	257
;;	512	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	1K	1	1K+1
;;	256	...
;;	...
;;	2K	...
;;	...

;; Do 1 four_real_four_semireal_unfft macros
;;	distance between fft data elements is 1

c0b:	start_timer 17
c2b:	disp four_real_four_semireal_unfft, 8, blkdst, 2*blkdst
	lea	esi, [esi+dist1]	;; Next source pointer
	add	dl, 256/2/clm		;; Test inner loop counter
	jnc	c2b			;; Iterate if necessary
	lea	esi, [esi-clm*2*dist1+4*blkdst];; Next source pointer

;; Do 5 four_complex_unfft macros
;;	distance between fft data elements is 1

	mov	edi, sincos1		;; Load sin/cos pointer
	mov	al, 5			;; 5 iterations
c3b:	disp four_complex_unfft, 8, blkdst, 2*blkdst
	lea	esi, [esi+dist1]	;; Next source pointer
	add	dl, 256/2/clm		;; Test loop counter
	jnc	c3b			;; Iterate if necessary
	lea	esi, [esi-clm*2*dist1+4*blkdst];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al			;; Test outer loop counter
	jnz	c3b			;; Iterate if necessary
	lea	esi, [esi-6*4*blkdst]	;; Restore source pointer
	end_timer 17

;; Do inverse FFT levels 4
;; On input the 32-byte cache lines hold these data values:
;;	0	1K	1	1K+1
;;	256	...
;;	...
;;	2K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	2K	1	2K+1
;;	256	...
;;	...
;;	4K	...
;;	...

;; Do 2 four_real_four_semireal_unfft_1 macros
;;	distance between fft data elements is 4

	start_timer 18
	mov	al, 2			;; Load loop counter
c4b:	disp four_real_four_semireal_unfft_1, 2*blkdst, 8, 4*blkdst
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test inner loop counter
	jnc	c4b			;; Iterate if necessary
	IFDEF PFETCH
	prefetcht1 [esi-32+clm*32]
;	prefetcht1 [esi-32+clm*32+2*blkdst]
	prefetcht1 [esi-32+clm*32+4*blkdst]
;	prefetcht1 [esi-32+clm*32+6*blkdst]
	ENDIF
	loopclm	dl, clm, c4b
	lea	esi, [esi-clm*2*dist1+blkdst];; Next source pointer
	dec	al
	jnz	c4b
	lea	esi, [esi-2*blkdst+8*blkdst];; Next source pointer

;; Do 4 four_complex_unfft_1 macros
;;	distance between fft data elements is 4

	mov	edi, sincos2		;; Load sin/cos pointer
	mov	eax, 2			;; 2 iterations of 2
c5b:	disp four_complex_unfft_1, 2*blkdst, 8, 4*blkdst
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test inner loop counter
	jnc	c5b			;; Iterate if necessary
	IFDEF PFETCH
	prefetcht1 [esi-32+clm*32]
;	prefetcht1 [esi-32+clm*32+2*blkdst]
	prefetcht1 [esi-32+clm*32+4*blkdst]
;	prefetcht1 [esi-32+clm*32+6*blkdst]
	ENDIF
	loopclm	dl, clm, c5b
	lea	esi, [esi-clm*2*dist1+blkdst];; Next source pointer
	add	ah, 256/2		;; Test middle loop counter
	jnc	c5b			;; Iterate if necessary
	lea	esi, [esi-2*blkdst+8*blkdst];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al
	jnz	c5b
	lea	esi, [esi-3*8*blkdst]	;; Restore source pointer
	end_timer 18

;; Do inverse FFT levels 1,2,3
;; On input the 32-byte cache lines hold these data values:
;;	0	2K	1	2K+1
;;	256	...
;;	...
;;	4K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	6K	1	6K+1
;;	256
;;	...
;;	6K-256

;; Do 8 six_reals_last_unfft macros
;;	distance between fft data elements is 8

	start_timer 19
	mov	al, 8			;; Load loop counter
c1b:	disp six_reals_last_unfft, 8, 8*blkdst, 16*blkdst
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test loop counter
	jnc	c1b			;; Iterate if necessary
	IFDEF PFETCH
	prefetcht1 [esi-32+clm*32]
;	prefetcht1 [esi-32+clm*32+8*blkdst]
	prefetcht1 [esi-32+clm*32+16*blkdst]
;	prefetcht1 [esi-32+clm*32+24*blkdst]
	ENDIF
	loopclm	dl, clm, c1b
	lea	esi, [esi-clm*2*dist1+blkdst];; Next source pointer
	dec	al			;; Test loop counter
	jnz	c1b			;; Iterate if necessary
	lea	esi, [esi-8*blkdst]	;; Restore source pointer
	end_timer 19

;; Normalize these values

	start_timer 20
	mov	eax, NORMRTN		;; Addr of normalization routine
	call	eax
	sub	eax, eax
	end_timer 20

;; Have we been given permission to start the FFT on the result?
;; If so, do so now while the data is in the L2 cache.

	cmp	POSTFFT, 0		;; Test flag
	je	no_fft			;; Skip FFT code if flag not set
	cmp	edx, p2cl/128*65536+(128/clm-4/clm)*256;; Delay fft on 8 sets
	jg	no_fft			;; Jump if delaying FFT
	copy_3_words clm, 0

;; Do FFT levels 1,2,3
;;
;; On input the 32-byte cache lines hold these data values:
;;	0	6K	1	6K+1
;;	256
;;	...
;;	6K-256
;; On output the 32-byte cache lines hold these data values:
;;	0	2K	1	2K+1
;;	256	...
;;	...
;;	4K	...
;;	...

;; Do 8 six_reals_first_fft macros
;;	distance between fft data elements is 8

b0b:	start_timer 21
	mov	ebx, DIST_TO_FFTSRCARG
	mov	al, 8			;; 8 iterations
b1b:	disp six_reals_first_fft, 8*blkdst, 16*blkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	add	dl, 256/2/clm		;; Test loop counter
	jnc	b1b			;; Iterate if necessary
	lea	esi, [esi-clm*2*dist1+blkdst];; Next source pointer
	dec	al			;; Test loop counter
	jnz	b1b			;; Iterate if necessary
	lea	esi, [esi-8*blkdst]	;; Restore source pointer
	end_timer 21

;; Do FFT levels 4
;;
;; On input the 32-byte cache lines hold these data values:
;;	0	2K	1	2K+1
;;	256	...
;;	...
;;	4K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	1K	1	1K+1
;;	256	...
;;	...
;;	2K	...
;;	...

;; Do 2 four_real_four_semireal_fft_1 macros
;;	distance between fft data elements is 4

	start_timer 22
	mov	al, 2			;; Load loop counter
b4b:	disp four_real_four_semireal_fft_1, 2*blkdst, 4*blkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test inner loop counter
	jnc	b4b			;; Iterate if necessary
	IFDEF PFETCH
;	prefetcht1 [esi-32+clm*32]
	prefetcht1 [esi-32+clm*32+2*blkdst]
;	prefetcht1 [esi-32+clm*32+4*blkdst]
	prefetcht1 [esi-32+clm*32+6*blkdst]
	ENDIF
	loopclm	dl, clm, b4b
	lea	esi, [esi-clm*2*dist1+blkdst];; Next source pointer
	dec	al
	jnz	b4b
	lea	esi, [esi-2*blkdst+8*blkdst];; Next source pointer

;; Do 4 four_complex_fft_1 macros
;;	distance between fft data elements is 4

	mov	edi, sincos2		;; Load sin/cos pointer
	mov	eax, 2			;; 2 iterations of 2
b5b:	disp four_complex_fft_1, 2*blkdst, 4*blkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test inner loop counter
	jnc	b5b			;; Iterate if necessary
	IFDEF PFETCH
;	prefetcht1 [esi-32+clm*32]
	prefetcht1 [esi-32+clm*32+2*blkdst]
;	prefetcht1 [esi-32+clm*32+4*blkdst]
	prefetcht1 [esi-32+clm*32+6*blkdst]
	ENDIF
	loopclm	dl, clm, b5b
	lea	esi, [esi-clm*2*dist1+blkdst];; Next source pointer
	add	ah, 256/2		;; Test middle loop counter
	jnc	b5b			;; Iterate if necessary
	lea	esi, [esi-2*blkdst+8*blkdst];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al
	jnz	b5b
	lea	esi, [esi-3*8*blkdst]	;; Restore source pointer
	end_timer 22

;; Do FFT levels 5,6
;;
;; On input the 32-byte cache lines hold these data values:
;;	0	1K	1	1K+1
;;	256	...
;;	...
;;	2K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	256	1	257
;;	512	...
;;	...

;; Do 1 four_real_four_semireal_fft macros
;;	distance between fft data elements is 1

	start_timer 23
b2b:	disp four_real_four_semireal_fft, blkdst, 2*blkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test inner loop counter
	jnc	b2b			;; Iterate if necessary
	IFDEF PFETCH
;	prefetcht1 [esi-32+clm*32]
	prefetcht1 [esi-32+clm*32+blkdst]
;	prefetcht1 [esi-32+clm*32+2*blkdst]
	prefetcht1 [esi-32+clm*32+3*blkdst]
	ENDIF
	loopclm	dl, clm, b2b
	lea	esi, [esi-clm*2*dist1+4*blkdst];; Next source pointer

;; Do 5 four_complex_fft macros
;;	distance between fft data elements is 1

	mov	edi, sincos1		;; Load sin/cos pointer
	mov	al, 5			;; 5 iterations
b3b:	disp four_complex_fft, blkdst, 2*blkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test inner loop counter
	jnc	b3b			;; Iterate if necessary
	IFDEF PFETCH
;	prefetcht1 [esi-32+clm*32]
	prefetcht1 [esi-32+clm*32+blkdst]
;	prefetcht1 [esi-32+clm*32+2*blkdst]
	prefetcht1 [esi-32+clm*32+3*blkdst]
	ENDIF
	loopclm	dl, clm, b3b
	lea	esi, [esi-clm*2*dist1+4*blkdst];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al
	jnz	b3b
	lea	esi, [esi-6*4*blkdst]	;; Restore source pointer
	end_timer 23

;; Work on next 4 sets of 32 values

no_fft:	lea	esi, [esi+clm*2*dist1]	;; Next source pointer
	dec	dh			;; Test loop counter
	jnz	short more		;; Do more FFTs
	lea	esi, [esi+64]		;; Skip pad every 128 cache lines (4KB)
	sub	edx, 65536		;; Test loop counter
	jz	short fftdn		;; Jump if done looping
	mov	dh, 128/clm		;; Restore cache line counter
more:	cmp	norm_ptr1, 0		;; What was the loop start point?
	jne	c0b			;; Do inverse-FFT/norm/FFT iteration
	jmp	b0b			;; Do another FFT-only iteration
fftdn:	lea	esi, [esi-p2cl*2*dist1-p2cl/128*64];; Restore source pointer
	cmp	esi, DESTARG		;; If esi was not restored, then we 
	jne	short fftdn1		;; just finished skipped postfft data
	sub	ebx, ebx
	mov	DIST_TO_FFTSRCARG, ebx
	cmp	norm_ptr1, 0		;; What was the loop start point?
	je	pass2			;; FFT-only, go do pass 2

;; Split the carries into high and low carries (so they do not exceed
;; the desired number of bits per FFT word.  Then add the carries back to the
;; FFT data.

	start_timer 28
	call	gw_carries
	end_timer 28

;; Have we been given permission to start the FFT on the result?
;; If so, set loop to FFT the first few cache lines now that the carries
;; have been added in.

	cmp	POSTFFT, 0		;; Test flag
	je	short fftdn1		;; Skip FFT code if flag not set
	mov	esi, DESTARG		;; Load FFT data address
	mov	edx, 65536+4/clm*256	;; Do 8 previously skipped cache lines
	mov	norm_ptr1, 0		;; Set flag so only forward FFT is done
	copy_4_words
	jmp	b0b

;; Now add in the FFT'ed carries, cleanup and return

fftdn1:	end_timer 1
	jmp	gw_finish_mult
	ENDM


;; Perform a 12K-element negacyclic FFT.  This is done in two passes.  Pass 1
;; does 6 levels, pass 2 uses common code to perform the last 8 levels.

pass1levels6complex3 MACRO pass2_macro, p2cl, clm
	LOCAL	b0b, b1b, b3b, b5b, c0b, c1b, c3b, c5b
	LOCAL	pass2, no_fft, more, fftdn, fftdn1

;; Call FFT code unless the FFT has already been started

	clear_timers
	cmp	DWORD PTR [esi-28][ebx], 0 ;; Test FFT-started flag
	jne	pass2			;; Jump is FFT already started
	mov	norm_ptr1, 0		;; Set FFT-ing flag
	mov	edx, p2cl/128*65536+128/clm*256;; Load loop counter
	jmp	b0b			;; Do the pass 1 fft

;; Do the last 8 FFT levels
;;	do 24 pass2_8_levels_complex

pass2:	start_timer 0
	CALLP	pass2_macro
	end_timer 0
	cmp	ffttype, 1		;; We're done if FFTing only
	je	gw_finish_fft

;; Do the inverse FFT

	start_timer 1
	fldz				;; Init SUMOUT
	fstp	SUMOUT
	mov	edx, norm_biglit_array	;; Addr of the big/little flags array
	mov	norm_ptr1, edx		;; Save ptr
	mov	edx, norm_col_mults	;; Addr of the column multipliers
	mov	norm_ptr2, edx		;; Save ptr
	mov	edx, p2cl/128*65536+128/clm*256;; Load loop counter

;; Do inverse FFT levels 5,6
;; On input the 32-byte cache lines hold these data values:
;;	0	256	1	257
;;	512	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	1K	1	1K+1
;;	256	...
;;	...
;;	2K	...
;;	...

;; Do 8 four_complex_unfft macros
;;	distance between fft data elements is 1

c0b:	start_timer 17
	mov	edi, sincos1		;; Load sin/cos pointer
	mov	al, 6			;; 6 iterations
c3b:	disp four_complex_unfft, 8, blkdst, 2*blkdst
	lea	esi, [esi+dist1]	;; Next source pointer
	add	dl, 256/2/clm		;; Test loop counter
	jnc	c3b			;; Iterate if necessary
	lea	esi, [esi-clm*2*dist1+4*blkdst];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al			;; Test outer loop counter
	jnz	c3b			;; Iterate if necessary
	lea	esi, [esi-6*4*blkdst]	;; Restore source pointer
	end_timer 17

;; Do inverse FFT levels 4
;; On input the 32-byte cache lines hold these data values:
;;	0	1K	1	1K+1
;;	256	...
;;	...
;;	2K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	2K	1	2K+1
;;	256	...
;;	...
;;	4K	...
;;	...

;; Do 6 four_complex_unfft_1 macros
;;	distance between fft data elements is 4

	start_timer 18
	mov	edi, sincos2		;; Load sin/cos pointer
	mov	eax, 3			;; 3 iterations of 2
c5b:	disp four_complex_unfft_1, 2*blkdst, 8, 4*blkdst
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test inner loop counter
	jnc	c5b			;; Iterate if necessary
	IFDEF PFETCH
	prefetcht1 [esi-32+clm*32]
;	prefetcht1 [esi-32+clm*32+2*blkdst]
	prefetcht1 [esi-32+clm*32+4*blkdst]
;	prefetcht1 [esi-32+clm*32+6*blkdst]
	ENDIF
	loopclm	dl, clm, c5b
	lea	esi, [esi-clm*2*dist1+blkdst];; Next source pointer
	add	ah, 256/2		;; Test middle loop counter
	jnc	c5b			;; Iterate if necessary
	lea	esi, [esi-2*blkdst+8*blkdst];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al
	jnz	c5b
	lea	esi, [esi-3*8*blkdst]	;; Restore source pointer
	end_timer 18

;; Do inverse FFT levels 1,2,3
;; On input the 32-byte cache lines hold these data values:
;;	0	2K	1	2K+1
;;	256	...
;;	...
;;	4K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	6K	1	6K+1
;;	256
;;	...
;;	6K-256

;; Do 8 three_complex_last_unfft macros
;;	distance between fft data elements is 8

	start_timer 19
	mov	edi, plus1_premults	;; Address of premultiplier table
	mov	al, 8			;; Load loop counter
c1b:	three_complex_last_unfft 8, 8*blkdst, 16*blkdst
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test loop counter
	jnc	c1b			;; Iterate if necessary
	IFDEF PFETCH
	prefetcht1 [esi-32+clm*32]
;	prefetcht1 [esi-32+clm*32+8*blkdst]
	prefetcht1 [esi-32+clm*32+16*blkdst]
	ENDIF
	loopclm	dl, clm, c1b
	lea	esi, [esi-clm*2*dist1+blkdst];; Next source pointer
	lea	edi, [edi+48]		;; Next premultiplier pointer
	dec	al			;; Test loop counter
	jnz	c1b			;; Iterate if necessary
	lea	esi, [esi-8*blkdst]	;; Restore source pointer
	end_timer 19

;; Normalize these values

	start_timer 20
	mov	eax, NORMRTN		;; Addr of normalization routine
	call	eax
	sub	eax, eax
	end_timer 20

;; Have we been given permission to start the FFT on the result?
;; If so, do so now while the data is in the L2 cache.

	cmp	POSTFFT, 0		;; Test flag
	je	no_fft			;; Skip FFT code if flag not set
	cmp	edx, p2cl/128*65536+(128/clm-4/clm)*256;; Delay fft on 8 sets
	jg	no_fft			;; Jump if delaying FFT
	copy_3_words clm, 0

;; Do FFT levels 1,2,3
;;
;; On input the 32-byte cache lines hold these data values:
;;	0	6K	1	6K+1
;;	256
;;	...
;;	6K-256
;; On output the 32-byte cache lines hold these data values:
;;	0	2K	1	2K+1
;;	256	...
;;	...
;;	4K	...
;;	...

;; Do 8 three_complex_first_fft macros
;;	distance between fft data elements is 8

b0b:	start_timer 21
	mov	ebx, DIST_TO_FFTSRCARG
	mov	edi, plus1_premults	;; Address of premultiplier table
	mov	al, 8			;; 8 iterations
b1b:	three_complex_first_fft 8*blkdst, 16*blkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	add	dl, 256/2/clm		;; Test loop counter
	jnc	b1b			;; Iterate if necessary
	lea	esi, [esi-clm*2*dist1+blkdst];; Next source pointer
	lea	edi, [edi+48]		;; Next premultiplier pointer
	dec	al			;; Test loop counter
	jnz	b1b			;; Iterate if necessary
	lea	esi, [esi-8*blkdst]	;; Restore source pointer
	end_timer 21

;; Do FFT levels 4
;;
;; On input the 32-byte cache lines hold these data values:
;;	0	2K	1	2K+1
;;	256	...
;;	...
;;	4K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	1K	1	1K+1
;;	256	...
;;	...
;;	2K	...
;;	...

;; Do 6 four_complex_fft_1 macros
;;	distance between fft data elements is 4

	start_timer 22
	mov	edi, sincos2		;; Load sin/cos pointer
	mov	eax, 3			;; 3 iterations of 2
b5b:	disp four_complex_fft_1, 2*blkdst, 4*blkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test inner loop counter
	jnc	b5b			;; Iterate if necessary
	IFDEF PFETCH
;	prefetcht1 [esi-32+clm*32]
	prefetcht1 [esi-32+clm*32+2*blkdst]
;	prefetcht1 [esi-32+clm*32+4*blkdst]
	prefetcht1 [esi-32+clm*32+6*blkdst]
	ENDIF
	loopclm	dl, clm, b5b
	lea	esi, [esi-clm*2*dist1+blkdst];; Next source pointer
	add	ah, 256/2		;; Test middle loop counter
	jnc	b5b			;; Iterate if necessary
	lea	esi, [esi-2*blkdst+8*blkdst];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al
	jnz	b5b
	lea	esi, [esi-3*8*blkdst]	;; Restore source pointer
	end_timer 22

;; Do FFT levels 5,6
;;
;; On input the 32-byte cache lines hold these data values:
;;	0	1K	1	1K+1
;;	256	...
;;	...
;;	2K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	256	1	257
;;	512	...
;;	...

;; Do 6 four_complex_fft macros
;;	distance between fft data elements is 1

	start_timer 23
	mov	edi, sincos1		;; Load sin/cos pointer
	mov	al, 6			;; 6 iterations
b3b:	disp four_complex_fft, blkdst, 2*blkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test inner loop counter
	jnc	b3b			;; Iterate if necessary
	IFDEF PFETCH
;	prefetcht1 [esi-32+clm*32]
	prefetcht1 [esi-32+clm*32+blkdst]
;	prefetcht1 [esi-32+clm*32+2*blkdst]
	prefetcht1 [esi-32+clm*32+3*blkdst]
	ENDIF
	loopclm	dl, clm, b3b
	lea	esi, [esi-clm*2*dist1+4*blkdst];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al
	jnz	b3b
	lea	esi, [esi-6*4*blkdst]	;; Restore source pointer
	end_timer 23

;; Work on next 4 sets of 32 values

no_fft:	lea	esi, [esi+clm*2*dist1]	;; Next source pointer
	dec	dh			;; Test loop counter
	jnz	short more		;; Do more FFTs
	lea	esi, [esi+64]		;; Skip pad every 128 cache lines (4KB)
	sub	edx, 65536		;; Test loop counter
	jz	short fftdn		;; Jump if done looping
	mov	dh, 128/clm		;; Restore cache line counter
more:	cmp	norm_ptr1, 0		;; What was the loop start point?
	jne	c0b			;; Do inverse-FFT/norm/FFT iteration
	jmp	b0b			;; Do another FFT-only iteration
fftdn:	lea	esi, [esi-p2cl*2*dist1-p2cl/128*64];; Restore source pointer
	cmp	esi, DESTARG		;; If esi was not restored, then we 
	jne	short fftdn1		;; just finished skipped postfft data
	sub	ebx, ebx
	mov	DIST_TO_FFTSRCARG, ebx
	cmp	norm_ptr1, 0		;; What was the loop start point?
	je	pass2			;; FFT-only, go do pass 2

;; Split the carries into high and low carries (so they do not exceed
;; the desired number of bits per FFT word.  Then add the carries back to the
;; FFT data.

	start_timer 28
	call	gw_carries
	end_timer 28

;; Have we been given permission to start the FFT on the result?
;; If so, set loop to FFT the first few cache lines now that the carries
;; have been added in.

	cmp	POSTFFT, 0		;; Test flag
	je	short fftdn1		;; Skip FFT code if flag not set
	mov	esi, DESTARG		;; Load FFT data address
	mov	edx, 65536+4/clm*256	;; Do 8 previously skipped cache lines
	mov	norm_ptr1, 0		;; Set flag so only forward FFT is done
	copy_4_words
	jmp	b0b

;; Now add in the FFT'ed carries, cleanup and return

fftdn1:	end_timer 1
	jmp	gw_finish_mult
	ENDM


;; Perform a 14K-element FFT.  This is done in two passes.  Pass 1
;; does 6 levels, pass 2 uses common code to perform the last 8 levels.

pass1levels6pfa7 MACRO pass2_macro, p2cl, clm
	LOCAL	b0b, b1b, b2b, b3b, b4b, b5b, c0b, c1b, c2b, c3b, c4b, c5b
	LOCAL	pass2, no_fft, more, fftdn, fftdn1

;; Call FFT code unless the FFT has already been started

	clear_timers
	cmp	DWORD PTR [esi-28][ebx], 0 ;; Test FFT-started flag
	jne	pass2			;; Jump is FFT already started
	mov	norm_ptr1, 0		;; Set FFT-ing flag
	mov	edx, p2cl/128*65536+128/clm*256;; Load loop counter
	jmp	b0b			;; Do the pass 1 fft

;; Do the last 8 FFT levels
;;	do 1 pass2_8_levels_real and 27 pass2_8_levels_complex

pass2:	start_timer 0
	CALLP	pass2_macro
	end_timer 0
	cmp	ffttype, 1		;; We're done if FFTing only
	je	gw_finish_fft

;; Do the inverse FFT

	start_timer 1
	fldz				;; Init SUMOUT
	fstp	SUMOUT
	mov	edx, norm_biglit_array	;; Addr of the big/little flags array
	mov	norm_ptr1, edx		;; Save ptr
	mov	edx, norm_col_mults	;; Addr of the column multipliers
	mov	norm_ptr2, edx		;; Save ptr
	mov	edx, p2cl/128*65536+128/clm*256;; Load loop counter

;; Do inverse FFT levels 5,6
;; On input the 32-byte cache lines hold these data values:
;;	0	256	1	257
;;	512	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	1K	1	1K+1
;;	256	...
;;	...
;;	2K	...
;;	...

;; Do 1 four_real_four_semireal_unfft macros
;;	distance between fft data elements is 1

c0b:	start_timer 17
c2b:	disp four_real_four_semireal_unfft, 8, blkdst, 2*blkdst
	lea	esi, [esi+dist1]	;; Next source pointer
	add	dl, 256/2/clm		;; Test inner loop counter
	jnc	c2b			;; Iterate if necessary
	lea	esi, [esi-clm*2*dist1+4*blkdst];; Next source pointer

;; Do 6 four_complex_unfft macros
;;	distance between fft data elements is 1

	mov	edi, sincos1		;; Load sin/cos pointer
	mov	al, 6			;; 6 iterations
c3b:	disp four_complex_unfft, 8, blkdst, 2*blkdst
	lea	esi, [esi+dist1]	;; Next source pointer
	add	dl, 256/2/clm		;; Test loop counter
	jnc	c3b			;; Iterate if necessary
	lea	esi, [esi-clm*2*dist1+4*blkdst];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al			;; Test outer loop counter
	jnz	c3b			;; Iterate if necessary
	lea	esi, [esi-7*4*blkdst]	;; Restore source pointer
	end_timer 17

;; Do inverse FFT levels 4
;; On input the 32-byte cache lines hold these data values:
;;	0	1K	1	1K+1
;;	256	...
;;	...
;;	2K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	2K	1	2K+1
;;	256	...
;;	...
;;	4K	...
;;	...

;; Do 1 eight_reals_unfft_1 macros
;;	distance between fft data elements is 4

	start_timer 18
c4b:	disp eight_reals_unfft_1, blkdst, 2*blkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test inner loop counter
	jnc	c4b			;; Iterate if necessary
	IFDEF PFETCH
	prefetcht1 [esi-32+clm*32]
;	prefetcht1 [esi-32+clm*32+blkdst]
	prefetcht1 [esi-32+clm*32+2*blkdst]
;	prefetcht1 [esi-32+clm*32+3*blkdst]
	ENDIF
	loopclm	dl, clm, c4b
	lea	esi, [esi-clm*2*dist1+4*blkdst];; Next source pointer

;; Do 6 four_complex_unfft_1 macros
;;	distance between fft data elements is 4

	mov	edi, sincos2		;; Load sin/cos pointer
	mov	eax, 3			;; 3 iterations of 2
c5b:	disp four_complex_unfft_1, 2*blkdst, 8, 4*blkdst
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test inner loop counter
	jnc	c5b			;; Iterate if necessary
	IFDEF PFETCH
	prefetcht1 [esi-32+clm*32]
;	prefetcht1 [esi-32+clm*32+2*blkdst]
	prefetcht1 [esi-32+clm*32+4*blkdst]
;	prefetcht1 [esi-32+clm*32+6*blkdst]
	ENDIF
	loopclm	dl, clm, c5b
	lea	esi, [esi-clm*2*dist1+blkdst];; Next source pointer
	add	ah, 256/2		;; Test middle loop counter
	jnc	c5b			;; Iterate if necessary
	lea	esi, [esi-2*blkdst+8*blkdst];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al
	jnz	c5b
	lea	esi, [esi-3*8*blkdst-4*blkdst];; Restore source pointer
	end_timer 18

;; Do inverse FFT levels 1,2,3
;; On input the 32-byte cache lines hold these data values:
;;	0	2K	1	2K+1
;;	256	...
;;	...
;;	4K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	7K	1	7K+1
;;	256
;;	...
;;	7K-256

;; Do 8 seven_reals_last_unfft macros
;;	distance between fft data elements is 8

	start_timer 19
	mov	al, 4			;; Load loop counter
c1b:	two_seven_reals_last_unfft 8, 4*blkdst
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test loop counter
	jnc	c1b			;; Iterate if necessary
	IFDEF PFETCH
	prefetcht1 [esi-32+clm*32]
;	prefetcht1 [esi-32+clm*32+4*blkdst]
	prefetcht1 [esi-32+clm*32+8*blkdst]
;	prefetcht1 [esi-32+clm*32+12*blkdst]
	prefetcht1 [esi-32+clm*32+16*blkdst]
;	prefetcht1 [esi-32+clm*32+20*blkdst]
	prefetcht1 [esi-32+clm*32+24*blkdst]
	ENDIF
	loopclm	dl, clm, c1b
	lea	esi, [esi-clm*2*dist1+blkdst];; Next source pointer
	dec	al			;; Test loop counter
	jnz	c1b			;; Iterate if necessary
	lea	esi, [esi-4*blkdst]	;; Restore source pointer
	end_timer 19

;; Normalize these values

	start_timer 20
	mov	eax, NORMRTN		;; Addr of normalization routine
	call	eax
	sub	eax, eax
	end_timer 20

;; Have we been given permission to start the FFT on the result?
;; If so, do so now while the data is in the L2 cache.

	cmp	POSTFFT, 0		;; Test flag
	je	no_fft			;; Skip FFT code if flag not set
	cmp	edx, p2cl/128*65536+(128/clm-4/clm)*256;; Delay fft on 8 sets
	jg	no_fft			;; Jump if delaying FFT
	copy_3_words clm, 0

;; Do FFT levels 1,2,3
;;
;; On input the 32-byte cache lines hold these data values:
;;	0	7K	1	7K+1
;;	256
;;	...
;;	7K-256
;; On output the 32-byte cache lines hold these data values:
;;	0	2K	1	2K+1
;;	256	...
;;	...
;;	4K	...
;;	...

;; Do 8 seven_reals_first_fft macros
;;	distance between fft data elements is 8

b0b:	start_timer 21
	mov	ebx, DIST_TO_FFTSRCARG
	mov	al, 4			;; 4 iterations
b1b:	two_seven_reals_first_fft 4*blkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	add	dl, 256/2/clm		;; Test loop counter
	jnc	b1b			;; Iterate if necessary
	lea	esi, [esi-clm*2*dist1+blkdst];; Next source pointer
	dec	al			;; Test loop counter
	jnz	b1b			;; Iterate if necessary
	lea	esi, [esi-4*blkdst]	;; Restore source pointer
	end_timer 21

;; Do FFT levels 4
;;
;; On input the 32-byte cache lines hold these data values:
;;	0	2K	1	2K+1
;;	256	...
;;	...
;;	4K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	1K	1	1K+1
;;	256	...
;;	...
;;	2K	...
;;	...

;; Do 1 eight_reals_fft_1 macros
;;	distance between fft data elements is 4

	start_timer 22
b4b:	disp eight_reals_fft_1, blkdst, 2*blkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test inner loop counter
	jnc	b4b			;; Iterate if necessary
	IFDEF PFETCH
;	prefetcht1 [esi-32+clm*32]
	prefetcht1 [esi-32+clm*32+blkdst]
;	prefetcht1 [esi-32+clm*32+2*blkdst]
	prefetcht1 [esi-32+clm*32+3*blkdst]
	ENDIF
	loopclm	dl, clm, b4b
	lea	esi, [esi-clm*2*dist1+4*blkdst];; Next source pointer

;; Do 6 four_complex_fft_1 macros
;;	distance between fft data elements is 4

	mov	edi, sincos2		;; Load sin/cos pointer
	mov	eax, 3			;; 3 iterations of 2
b5b:	disp four_complex_fft_1, 2*blkdst, 4*blkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test inner loop counter
	jnc	b5b			;; Iterate if necessary
	IFDEF PFETCH
;	prefetcht1 [esi-32+clm*32]
	prefetcht1 [esi-32+clm*32+2*blkdst]
;	prefetcht1 [esi-32+clm*32+4*blkdst]
	prefetcht1 [esi-32+clm*32+6*blkdst]
	ENDIF
	loopclm	dl, clm, b5b
	lea	esi, [esi-clm*2*dist1+blkdst];; Next source pointer
	add	ah, 256/2		;; Test middle loop counter
	jnc	b5b			;; Iterate if necessary
	lea	esi, [esi-2*blkdst+8*blkdst];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al
	jnz	b5b
	lea	esi, [esi-3*8*blkdst-4*blkdst];; Restore source pointer
	end_timer 22

;; Do FFT levels 5,6
;;
;; On input the 32-byte cache lines hold these data values:
;;	0	1K	1	1K+1
;;	256	...
;;	...
;;	2K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	256	1	257
;;	512	...
;;	...

;; Do 1 four_real_four_semireal_fft macros
;;	distance between fft data elements is 1

	start_timer 23
b2b:	disp four_real_four_semireal_fft, blkdst, 2*blkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test inner loop counter
	jnc	b2b			;; Iterate if necessary
	IFDEF PFETCH
;	prefetcht1 [esi-32+clm*32]
	prefetcht1 [esi-32+clm*32+blkdst]
;	prefetcht1 [esi-32+clm*32+2*blkdst]
	prefetcht1 [esi-32+clm*32+3*blkdst]
	ENDIF
	loopclm	dl, clm, b2b
	lea	esi, [esi-clm*2*dist1+4*blkdst];; Next source pointer

;; Do 6 four_complex_fft macros
;;	distance between fft data elements is 1

	mov	edi, sincos1		;; Load sin/cos pointer
	mov	al, 6			;; 6 iterations
b3b:	disp four_complex_fft, blkdst, 2*blkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test inner loop counter
	jnc	b3b			;; Iterate if necessary
	IFDEF PFETCH
;	prefetcht1 [esi-32+clm*32]
	prefetcht1 [esi-32+clm*32+blkdst]
;	prefetcht1 [esi-32+clm*32+2*blkdst]
	prefetcht1 [esi-32+clm*32+3*blkdst]
	ENDIF
	loopclm	dl, clm, b3b
	lea	esi, [esi-clm*2*dist1+4*blkdst];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al
	jnz	b3b
	lea	esi, [esi-7*4*blkdst]	;; Restore source pointer
	end_timer 23

;; Work on next 4 sets of 32 values

no_fft:	lea	esi, [esi+clm*2*dist1]	;; Next source pointer
	dec	dh			;; Test loop counter
	jnz	short more		;; Do more FFTs
	lea	esi, [esi+64]		;; Skip pad every 128 cache lines (4KB)
	sub	edx, 65536		;; Test loop counter
	jz	short fftdn		;; Jump if done looping
	mov	dh, 128/clm		;; Restore cache line counter
more:	cmp	norm_ptr1, 0		;; What was the loop start point?
	jne	c0b			;; Do inverse-FFT/norm/FFT iteration
	jmp	b0b			;; Do another FFT-only iteration
fftdn:	lea	esi, [esi-p2cl*2*dist1-p2cl/128*64];; Restore source pointer
	cmp	esi, DESTARG		;; If esi was not restored, then we 
	jne	short fftdn1		;; just finished skipped postfft data
	sub	ebx, ebx
	mov	DIST_TO_FFTSRCARG, ebx
	cmp	norm_ptr1, 0		;; What was the loop start point?
	je	pass2			;; FFT-only, go do pass 2

;; Split the carries into high and low carries (so they do not exceed
;; the desired number of bits per FFT word.  Then add the carries back to the
;; FFT data.

	start_timer 28
	call	gw_carries
	end_timer 28

;; Have we been given permission to start the FFT on the result?
;; If so, set loop to FFT the first few cache lines now that the carries
;; have been added in.

	cmp	POSTFFT, 0		;; Test flag
	je	short fftdn1		;; Skip FFT code if flag not set
	mov	esi, DESTARG		;; Load FFT data address
	mov	edx, 65536+4/clm*256	;; Do 8 previously skipped cache lines
	mov	norm_ptr1, 0		;; Set flag so only forward FFT is done
	copy_4_words
	jmp	b0b

;; Now add in the FFT'ed carries, cleanup and return

fftdn1:	end_timer 1
	jmp	gw_finish_mult
	ENDM


;; Perform a 16K-element FFT.  This is done in two passes.  Pass 1
;; does 6 levels, pass 2 uses common code to perform the last 8 levels.

pass1levels6pfa8 MACRO pass2_macro, p2cl, clm
	LOCAL	b0b, b1b, b2b, b3b, b4b, b5b, c0b, c1b, c2b, c3b, c4b, c5b
	LOCAL	pass2, no_fft, more, fftdn, fftdn1

;; Call FFT code unless the FFT has already been started

	clear_timers
	cmp	DWORD PTR [esi-28][ebx], 0 ;; Test FFT-started flag
	jne	pass2			;; Jump is FFT already started
	mov	norm_ptr1, 0		;; Set FFT-ing flag
	mov	edx, p2cl/128*65536+128/clm*256;; Load loop counter
	jmp	b0b			;; Do the pass 1 fft

;; Do the last 8 FFT levels
;;	do 1 pass2_8_levels_real and 31 pass2_8_levels_complex

pass2:	start_timer 0
	CALLP	pass2_macro
	end_timer 0
	cmp	ffttype, 1		;; We're done if FFTing only
	je	gw_finish_fft

;; Do the inverse FFT

	start_timer 1
	fldz				;; Init SUMOUT
	fstp	SUMOUT
	mov	edx, norm_biglit_array	;; Addr of the big/little flags array
	mov	norm_ptr1, edx		;; Save ptr
	mov	edx, norm_col_mults	;; Addr of the column multipliers
	mov	norm_ptr2, edx		;; Save ptr
	mov	edx, p2cl/128*65536+128/clm*256;; Load loop counter

;; Do inverse FFT levels 5,6
;; On input the 32-byte cache lines hold these data values:
;;	0	256	1	257
;;	512	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	1K	1	1K+1
;;	256	...
;;	...
;;	2K	...
;;	...

;; Do 1 four_real_four_semireal_unfft macros
;;	distance between fft data elements is 1

c0b:	start_timer 17
c2b:	disp four_real_four_semireal_unfft, 8, blkdst, 2*blkdst
	lea	esi, [esi+dist1]	;; Next source pointer
	add	dl, 256/2/clm		;; Test inner loop counter
	jnc	c2b			;; Iterate if necessary
	lea	esi, [esi-clm*2*dist1+4*blkdst];; Next source pointer

;; Do 7 four_complex_unfft macros
;;	distance between fft data elements is 1

	mov	edi, sincos1		;; Load sin/cos pointer
	mov	al, 7			;; 7 iterations
c3b:	disp four_complex_unfft, 8, blkdst, 2*blkdst
	lea	esi, [esi+dist1]	;; Next source pointer
	add	dl, 256/2/clm		;; Test loop counter
	jnc	c3b			;; Iterate if necessary
	lea	esi, [esi-clm*2*dist1+4*blkdst];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al			;; Test outer loop counter
	jnz	c3b			;; Iterate if necessary
	lea	esi, [esi-8*4*blkdst]	;; Restore source pointer
	end_timer 17

;; Do inverse FFT levels 4
;; On input the 32-byte cache lines hold these data values:
;;	0	1K	1	1K+1
;;	256	...
;;	...
;;	2K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	2K	1	2K+1
;;	256	...
;;	...
;;	4K	...
;;	...

;; Do 2 four_real_four_semireal_unfft_1 macros
;;	distance between fft data elements is 4

	start_timer 18
	mov	al, 2			;; Load loop counter
c4b:	disp four_real_four_semireal_unfft_1, 2*blkdst, 8, 4*blkdst
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test inner loop counter
	jnc	c4b			;; Iterate if necessary
	IFDEF PFETCH
	prefetcht1 [esi-32+clm*32]
;	prefetcht1 [esi-32+clm*32+2*blkdst]
	prefetcht1 [esi-32+clm*32+4*blkdst]
;	prefetcht1 [esi-32+clm*32+6*blkdst]
	ENDIF
	loopclm	dl, clm, c4b
	lea	esi, [esi-clm*2*dist1+blkdst];; Next source pointer
	dec	al
	jnz	c4b
	lea	esi, [esi-2*blkdst+8*blkdst];; Next source pointer

;; Do 6 four_complex_unfft_1 macros
;;	distance between fft data elements is 4

	mov	edi, sincos2		;; Load sin/cos pointer
	mov	eax, 3			;; 3 iterations of 2
c5b:	disp four_complex_unfft_1, 2*blkdst, 8, 4*blkdst
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test inner loop counter
	jnc	c5b			;; Iterate if necessary
	IFDEF PFETCH
	prefetcht1 [esi-32+clm*32]
;	prefetcht1 [esi-32+clm*32+2*blkdst]
	prefetcht1 [esi-32+clm*32+4*blkdst]
;	prefetcht1 [esi-32+clm*32+6*blkdst]
	ENDIF
	loopclm	dl, clm, c5b
	lea	esi, [esi-clm*2*dist1+blkdst];; Next source pointer
	add	ah, 256/2		;; Test middle loop counter
	jnc	c5b			;; Iterate if necessary
	lea	esi, [esi-2*blkdst+8*blkdst];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al
	jnz	c5b
	lea	esi, [esi-4*8*blkdst]	;; Restore source pointer
	end_timer 18

;; Do inverse FFT levels 1,2,3
;; On input the 32-byte cache lines hold these data values:
;;	0	2K	1	2K+1
;;	256	...
;;	...
;;	4K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	8K	1	8K+1
;;	256
;;	...
;;	8K-256

;; Do 8 eight_reals_last_unfft macros
;;	distance between fft data elements is 8

	start_timer 19
	mov	al, 8			;; Load loop counter
c1b:	disp eight_reals_last_unfft, 8, 8*blkdst, 16*blkdst
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test loop counter
	jnc	c1b			;; Iterate if necessary
	IFDEF PFETCH
	prefetcht1 [esi-32+clm*32]
;	prefetcht1 [esi-32+clm*32+8*blkdst]
	prefetcht1 [esi-32+clm*32+16*blkdst]
;	prefetcht1 [esi-32+clm*32+24*blkdst]
	ENDIF
	loopclm	dl, clm, c1b
	lea	esi, [esi-clm*2*dist1+blkdst];; Next source pointer
	dec	al			;; Test loop counter
	jnz	c1b			;; Iterate if necessary
	lea	esi, [esi-8*blkdst]	;; Restore source pointer
	end_timer 19

;; Normalize these values

	start_timer 20
	mov	eax, NORMRTN		;; Addr of normalization routine
	call	eax
	sub	eax, eax
	end_timer 20

;; Have we been given permission to start the FFT on the result?
;; If so, do so now while the data is in the L2 cache.

	cmp	POSTFFT, 0		;; Test flag
	je	no_fft			;; Skip FFT code if flag not set
	cmp	edx, p2cl/128*65536+(128/clm-4/clm)*256;; Delay fft on 8 sets
	jg	no_fft			;; Jump if delaying FFT
	copy_3_words clm, 0

;; Do FFT levels 1,2,3
;;
;; On input the 32-byte cache lines hold these data values:
;;	0	8K	1	8K+1
;;	256
;;	...
;;	8K-256
;; On output the 32-byte cache lines hold these data values:
;;	0	2K	1	2K+1
;;	256	...
;;	...
;;	4K	...
;;	...

;; Do 8 eight_reals_first_fft macros
;;	distance between fft data elements is 8

b0b:	start_timer 21
	mov	ebx, DIST_TO_FFTSRCARG
	mov	al, 8			;; 8 iterations
b1b:	disp eight_reals_first_fft, 8*blkdst, 16*blkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	add	dl, 256/2/clm		;; Test loop counter
	jnc	b1b			;; Iterate if necessary
	lea	esi, [esi-clm*2*dist1+blkdst];; Next source pointer
	dec	al			;; Test loop counter
	jnz	b1b			;; Iterate if necessary
	lea	esi, [esi-8*blkdst]	;; Restore source pointer
	end_timer 21

;; Do FFT levels 4
;;
;; On input the 32-byte cache lines hold these data values:
;;	0	2K	1	2K+1
;;	256	...
;;	...
;;	4K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	1K	1	1K+1
;;	256	...
;;	...
;;	2K	...
;;	...

;; Do 2 four_real_four_semireal_fft_1 macros
;;	distance between fft data elements is 4

	start_timer 22
	mov	al, 2			;; Load loop counter
b4b:	disp four_real_four_semireal_fft_1, 2*blkdst, 4*blkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test inner loop counter
	jnc	b4b			;; Iterate if necessary
	IFDEF PFETCH
;	prefetcht1 [esi-32+clm*32]
	prefetcht1 [esi-32+clm*32+2*blkdst]
;	prefetcht1 [esi-32+clm*32+4*blkdst]
	prefetcht1 [esi-32+clm*32+6*blkdst]
	ENDIF
	loopclm	dl, clm, b4b
	lea	esi, [esi-clm*2*dist1+blkdst];; Next source pointer
	dec	al
	jnz	b4b
	lea	esi, [esi-2*blkdst+8*blkdst];; Next source pointer

;; Do 6 four_complex_fft_1 macros
;;	distance between fft data elements is 4

	mov	edi, sincos2		;; Load sin/cos pointer
	mov	eax, 3			;; 3 iterations of 2
b5b:	disp four_complex_fft_1, 2*blkdst, 4*blkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test inner loop counter
	jnc	b5b			;; Iterate if necessary
	IFDEF PFETCH
;	prefetcht1 [esi-32+clm*32]
	prefetcht1 [esi-32+clm*32+2*blkdst]
;	prefetcht1 [esi-32+clm*32+4*blkdst]
	prefetcht1 [esi-32+clm*32+6*blkdst]
	ENDIF
	loopclm	dl, clm, b5b
	lea	esi, [esi-clm*2*dist1+blkdst];; Next source pointer
	add	ah, 256/2		;; Test middle loop counter
	jnc	b5b			;; Iterate if necessary
	lea	esi, [esi-2*blkdst+8*blkdst];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al
	jnz	b5b
	lea	esi, [esi-4*8*blkdst]	;; Restore source pointer
	end_timer 22

;; Do FFT levels 5,6
;;
;; On input the 32-byte cache lines hold these data values:
;;	0	1K	1	1K+1
;;	256	...
;;	...
;;	2K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	256	1	257
;;	512	...
;;	...

;; Do 1 four_real_four_semireal_fft macros
;;	distance between fft data elements is 1

	start_timer 23
b2b:	disp four_real_four_semireal_fft, blkdst, 2*blkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test inner loop counter
	jnc	b2b			;; Iterate if necessary
	IFDEF PFETCH
;	prefetcht1 [esi-32+clm*32]
	prefetcht1 [esi-32+clm*32+blkdst]
;	prefetcht1 [esi-32+clm*32+2*blkdst]
	prefetcht1 [esi-32+clm*32+3*blkdst]
	ENDIF
	loopclm	dl, clm, b2b
	lea	esi, [esi-clm*2*dist1+4*blkdst];; Next source pointer

;; Do 7 four_complex_fft macros
;;	distance between fft data elements is 1

	mov	edi, sincos1		;; Load sin/cos pointer
	mov	al, 7			;; 7 iterations
b3b:	disp four_complex_fft, blkdst, 2*blkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test inner loop counter
	jnc	b3b			;; Iterate if necessary
	IFDEF PFETCH
;	prefetcht1 [esi-32+clm*32]
	prefetcht1 [esi-32+clm*32+blkdst]
;	prefetcht1 [esi-32+clm*32+2*blkdst]
	prefetcht1 [esi-32+clm*32+3*blkdst]
	ENDIF
	loopclm	dl, clm, b3b
	lea	esi, [esi-clm*2*dist1+4*blkdst];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al
	jnz	b3b
	lea	esi, [esi-8*4*blkdst]	;; Restore source pointer
	end_timer 23

;; Work on next 4 sets of 32 values

no_fft:	lea	esi, [esi+clm*2*dist1]	;; Next source pointer
	dec	dh			;; Test loop counter
	jnz	short more		;; Do more FFTs
	lea	esi, [esi+64]		;; Skip pad every 128 cache lines (4KB)
	sub	edx, 65536		;; Test loop counter
	jz	short fftdn		;; Jump if done looping
	mov	dh, 128/clm		;; Restore cache line counter
more:	cmp	norm_ptr1, 0		;; What was the loop start point?
	jne	c0b			;; Do inverse-FFT/norm/FFT iteration
	jmp	b0b			;; Do another FFT-only iteration
fftdn:	lea	esi, [esi-p2cl*2*dist1-p2cl/128*64];; Restore source pointer
	cmp	esi, DESTARG		;; If esi was not restored, then we 
	jne	short fftdn1		;; just finished skipped postfft data
	sub	ebx, ebx
	mov	DIST_TO_FFTSRCARG, ebx
	cmp	norm_ptr1, 0		;; What was the loop start point?
	je	pass2			;; FFT-only, go do pass 2

;; Split the carries into high and low carries (so they do not exceed
;; the desired number of bits per FFT word.  Then add the carries back to the
;; FFT data.

	start_timer 28
	call	gw_carries
	end_timer 28

;; Have we been given permission to start the FFT on the result?
;; If so, set loop to FFT the first few cache lines now that the carries
;; have been added in.

	cmp	POSTFFT, 0		;; Test flag
	je	short fftdn1		;; Skip FFT code if flag not set
	mov	esi, DESTARG		;; Load FFT data address
	mov	edx, 65536+4/clm*256	;; Do 8 previously skipped cache lines
	mov	norm_ptr1, 0		;; Set flag so only forward FFT is done
	copy_4_words
	jmp	b0b

;; Now add in the FFT'ed carries, cleanup and return

fftdn1:	end_timer 1
	jmp	gw_finish_mult
	ENDM


;; Perform a 16K-element negacyclic FFT.  This is done in two passes.  Pass 1
;; does 6 levels, pass 2 uses common code to perform the last 8 levels.

pass1levels6complex4 MACRO pass2_macro, p2cl, clm
	LOCAL	b0b, b1b, b3b, b5b, c0b, c1b, c3b, c5b
	LOCAL	pass2, no_fft, more, fftdn, fftdn1

;; Call FFT code unless the FFT has already been started

	clear_timers
	cmp	DWORD PTR [esi-28][ebx], 0 ;; Test FFT-started flag
	jne	pass2			;; Jump is FFT already started
	mov	norm_ptr1, 0		;; Set FFT-ing flag
	mov	edx, p2cl/128*65536+128/clm*256;; Load loop counter
	jmp	b0b			;; Do the pass 1 fft

;; Do the last 8 FFT levels
;;	do 32 pass2_8_levels_complex

pass2:	start_timer 0
	CALLP	pass2_macro
	end_timer 0
	cmp	ffttype, 1		;; We're done if FFTing only
	je	gw_finish_fft

;; Do the inverse FFT

	start_timer 1
	fldz				;; Init SUMOUT
	fstp	SUMOUT
	mov	edx, norm_biglit_array	;; Addr of the big/little flags array
	mov	norm_ptr1, edx		;; Save ptr
	mov	edx, norm_col_mults	;; Addr of the column multipliers
	mov	norm_ptr2, edx		;; Save ptr
	mov	edx, p2cl/128*65536+128/clm*256;; Load loop counter

;; Do inverse FFT levels 5,6
;; On input the 32-byte cache lines hold these data values:
;;	0	256	1	257
;;	512	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	1K	1	1K+1
;;	256	...
;;	...
;;	2K	...
;;	...

;; Do 8 four_complex_unfft macros
;;	distance between fft data elements is 1

c0b:	start_timer 17
	mov	edi, sincos1		;; Load sin/cos pointer
	mov	al, 8			;; 8 iterations
c3b:	disp four_complex_unfft, 8, blkdst, 2*blkdst
	lea	esi, [esi+dist1]	;; Next source pointer
	add	dl, 256/2/clm		;; Test loop counter
	jnc	c3b			;; Iterate if necessary
	lea	esi, [esi-clm*2*dist1+4*blkdst];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al			;; Test outer loop counter
	jnz	c3b			;; Iterate if necessary
	lea	esi, [esi-8*4*blkdst]	;; Restore source pointer
	end_timer 17

;; Do inverse FFT levels 4
;; On input the 32-byte cache lines hold these data values:
;;	0	1K	1	1K+1
;;	256	...
;;	...
;;	2K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	2K	1	2K+1
;;	256	...
;;	...
;;	4K	...
;;	...

;; Do 8 four_complex_unfft_1 macros
;;	distance between fft data elements is 4

	start_timer 18
	mov	edi, sincos2		;; Load sin/cos pointer
	mov	eax, 4			;; 4 iterations of 2
c5b:	disp four_complex_unfft_1, 2*blkdst, 8, 4*blkdst
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test inner loop counter
	jnc	c5b			;; Iterate if necessary
	IFDEF PFETCH
	prefetcht1 [esi-32+clm*32]
;	prefetcht1 [esi-32+clm*32+2*blkdst]
	prefetcht1 [esi-32+clm*32+4*blkdst]
;	prefetcht1 [esi-32+clm*32+6*blkdst]
	ENDIF
	loopclm	dl, clm, c5b
	lea	esi, [esi-clm*2*dist1+blkdst];; Next source pointer
	add	ah, 256/2		;; Test middle loop counter
	jnc	c5b			;; Iterate if necessary
	lea	esi, [esi-2*blkdst+8*blkdst];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al
	jnz	c5b
	lea	esi, [esi-4*8*blkdst]	;; Restore source pointer
	end_timer 18

;; Do inverse FFT levels 1,2,3
;; On input the 32-byte cache lines hold these data values:
;;	0	2K	1	2K+1
;;	256	...
;;	...
;;	4K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	8K	1	8K+1
;;	256
;;	...
;;	8K-256

;; Do 8 four_complex_last_unfft macros
;;	distance between fft data elements is 8

	start_timer 19
	mov	edi, plus1_premults	;; Address of premultiplier table
	mov	al, 8			;; Load loop counter
c1b:	disp four_complex_last_unfft, 8, 8*blkdst, 16*blkdst
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test loop counter
	jnc	c1b			;; Iterate if necessary
	IFDEF PFETCH
	prefetcht1 [esi-32+clm*32]
;	prefetcht1 [esi-32+clm*32+8*blkdst]
	prefetcht1 [esi-32+clm*32+16*blkdst]
;	prefetcht1 [esi-32+clm*32+24*blkdst]
	ENDIF
	loopclm	dl, clm, c1b
	lea	esi, [esi-clm*2*dist1+blkdst];; Next source pointer
	lea	edi, [edi+64]		;; Next premultiplier pointer
	dec	al			;; Test loop counter
	jnz	c1b			;; Iterate if necessary
	lea	esi, [esi-8*blkdst]	;; Restore source pointer
	end_timer 19

;; Normalize these values

	start_timer 20
	mov	eax, NORMRTN		;; Addr of normalization routine
	call	eax
	sub	eax, eax
	end_timer 20

;; Have we been given permission to start the FFT on the result?
;; If so, do so now while the data is in the L2 cache.

	cmp	POSTFFT, 0		;; Test flag
	je	no_fft			;; Skip FFT code if flag not set
	cmp	edx, p2cl/128*65536+(128/clm-4/clm)*256;; Delay fft on 8 sets
	jg	no_fft			;; Jump if delaying FFT
	copy_3_words clm, 0

;; Do FFT levels 1,2,3
;;
;; On input the 32-byte cache lines hold these data values:
;;	0	8K	1	8K+1
;;	256
;;	...
;;	8K-256
;; On output the 32-byte cache lines hold these data values:
;;	0	2K	1	2K+1
;;	256	...
;;	...
;;	4K	...
;;	...

;; Do 8 four_complex_first_fft macros
;;	distance between fft data elements is 8

b0b:	start_timer 21
	mov	ebx, DIST_TO_FFTSRCARG
	mov	edi, plus1_premults	;; Address of premultiplier table
	mov	al, 8			;; 8 iterations
b1b:	disp four_complex_first_fft, 8*blkdst, 16*blkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	add	dl, 256/2/clm		;; Test loop counter
	jnc	b1b			;; Iterate if necessary
	lea	esi, [esi-clm*2*dist1+blkdst];; Next source pointer
	lea	edi, [edi+64]		;; Next premultiplier pointer
	dec	al			;; Test loop counter
	jnz	b1b			;; Iterate if necessary
	lea	esi, [esi-8*blkdst]	;; Restore source pointer
	end_timer 21

;; Do FFT levels 4
;;
;; On input the 32-byte cache lines hold these data values:
;;	0	2K	1	2K+1
;;	256	...
;;	...
;;	4K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	1K	1	1K+1
;;	256	...
;;	...
;;	2K	...
;;	...

;; Do 8 four_complex_fft_1 macros
;;	distance between fft data elements is 4

	start_timer 22
	mov	edi, sincos2		;; Load sin/cos pointer
	mov	eax, 4			;; 4 iterations of 2
b5b:	disp four_complex_fft_1, 2*blkdst, 4*blkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test inner loop counter
	jnc	b5b			;; Iterate if necessary
	IFDEF PFETCH
;	prefetcht1 [esi-32+clm*32]
	prefetcht1 [esi-32+clm*32+2*blkdst]
;	prefetcht1 [esi-32+clm*32+4*blkdst]
	prefetcht1 [esi-32+clm*32+6*blkdst]
	ENDIF
	loopclm	dl, clm, b5b
	lea	esi, [esi-clm*2*dist1+blkdst];; Next source pointer
	add	ah, 256/2		;; Test middle loop counter
	jnc	b5b			;; Iterate if necessary
	lea	esi, [esi-2*blkdst+8*blkdst];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al
	jnz	b5b
	lea	esi, [esi-4*8*blkdst]	;; Restore source pointer
	end_timer 22

;; Do FFT levels 5,6
;;
;; On input the 32-byte cache lines hold these data values:
;;	0	1K	1	1K+1
;;	256	...
;;	...
;;	2K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	256	1	257
;;	512	...
;;	...

;; Do 8 four_complex_fft macros
;;	distance between fft data elements is 1

	start_timer 23
	mov	edi, sincos1		;; Load sin/cos pointer
	mov	al, 8			;; 8 iterations
b3b:	disp four_complex_fft, blkdst, 2*blkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test inner loop counter
	jnc	b3b			;; Iterate if necessary
	IFDEF PFETCH
;	prefetcht1 [esi-32+clm*32]
	prefetcht1 [esi-32+clm*32+blkdst]
;	prefetcht1 [esi-32+clm*32+2*blkdst]
	prefetcht1 [esi-32+clm*32+3*blkdst]
	ENDIF
	loopclm	dl, clm, b3b
	lea	esi, [esi-clm*2*dist1+4*blkdst];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al
	jnz	b3b
	lea	esi, [esi-8*4*blkdst]	;; Restore source pointer
	end_timer 23

;; Work on next 4 sets of 32 values

no_fft:	lea	esi, [esi+clm*2*dist1]	;; Next source pointer
	dec	dh			;; Test loop counter
	jnz	short more		;; Do more FFTs
	lea	esi, [esi+64]		;; Skip pad every 128 cache lines (4KB)
	sub	edx, 65536		;; Test loop counter
	jz	short fftdn		;; Jump if done looping
	mov	dh, 128/clm		;; Restore cache line counter
more:	cmp	norm_ptr1, 0		;; What was the loop start point?
	jne	c0b			;; Do inverse-FFT/norm/FFT iteration
	jmp	b0b			;; Do another FFT-only iteration
fftdn:	lea	esi, [esi-p2cl*2*dist1-p2cl/128*64];; Restore source pointer
	cmp	esi, DESTARG		;; If esi was not restored, then we 
	jne	short fftdn1		;; just finished skipped postfft data
	sub	ebx, ebx
	mov	DIST_TO_FFTSRCARG, ebx
	cmp	norm_ptr1, 0		;; What was the loop start point?
	je	pass2			;; FFT-only, go do pass 2

;; Split the carries into high and low carries (so they do not exceed
;; the desired number of bits per FFT word.  Then add the carries back to the
;; FFT data.

	start_timer 28
	call	gw_carries
	end_timer 28

;; Have we been given permission to start the FFT on the result?
;; If so, set loop to FFT the first few cache lines now that the carries
;; have been added in.

	cmp	POSTFFT, 0		;; Test flag
	je	short fftdn1		;; Skip FFT code if flag not set
	mov	esi, DESTARG		;; Load FFT data address
	mov	edx, 65536+4/clm*256	;; Do 8 previously skipped cache lines
	mov	norm_ptr1, 0		;; Set flag so only forward FFT is done
	copy_4_words
	jmp	b0b

;; Now add in the FFT'ed carries, cleanup and return

fftdn1:	end_timer 1
	jmp	gw_finish_mult
	ENDM


;; Perform a 20K-element FFT.  This is done in two passes.  Pass 1
;; does 7 levels, pass 2 uses common code to perform the last 8 levels.

pass1levels7pfa5 MACRO pass2_macro, p2cl, clm
	LOCAL	b0b, b1b, b2b, b3b, b4b, b5b, c0b, c1b, c2b, c3b, c4b, c5b
	LOCAL	pass2, no_fft, more, fftdn, fftdn1

;; Call FFT code unless the FFT has already been started

	clear_timers
	cmp	DWORD PTR [esi-28][ebx], 0 ;; Test FFT-started flag
	jne	pass2			;; Jump is FFT already started
	mov	norm_ptr1, 0		;; Set FFT-ing flag
	mov	edx, p2cl/128*65536+128/clm*256;; Load loop counter
	jmp	b0b			;; Do the pass 1 fft

;; Do the last 8 FFT levels
;;	do 1 pass2_8_levels_real and 39 pass2_8_levels_complex

pass2:	start_timer 0
	CALLP	pass2_macro
	end_timer 0
	cmp	ffttype, 1		;; We're done if FFTing only
	je	gw_finish_fft

;; Do the inverse FFT

	start_timer 1
	fldz				;; Init SUMOUT
	fstp	SUMOUT
	mov	edx, norm_biglit_array	;; Addr of the big/little flags array
	mov	norm_ptr1, edx		;; Save ptr
	mov	edx, norm_col_mults	;; Addr of the column multipliers
	mov	norm_ptr2, edx		;; Save ptr
	mov	edx, p2cl/128*65536+128/clm*256;; Load loop counter

;; Do inverse FFT levels 6,7
;; On input the 32-byte cache lines hold these data values:
;;	0	256	1	257
;;	512	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	1K	1	1K+1
;;	256	...
;;	...
;;	2K	...
;;	...

;; Do 1 four_real_four_semireal_unfft macros
;;	distance between fft data elements is 1

c0b:	start_timer 17
c2b:	disp four_real_four_semireal_unfft, 8, blkdst, 2*blkdst
	lea	esi, [esi+dist1]	;; Next source pointer
	add	dl, 256/2/clm		;; Test inner loop counter
	jnc	c2b			;; Iterate if necessary
	lea	esi, [esi-clm*2*dist1+4*blkdst];; Next source pointer

;; Do 9 four_complex_unfft macros
;;	distance between fft data elements is 1

	mov	edi, sincos1		;; Load sin/cos pointer
	mov	al, 9			;; 9 iterations
c3b:	disp four_complex_unfft, 8, blkdst, 2*blkdst
	lea	esi, [esi+dist1]	;; Next source pointer
	add	dl, 256/2/clm		;; Test loop counter
	jnc	c3b			;; Iterate if necessary
	lea	esi, [esi-clm*2*dist1+4*blkdst];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al			;; Test outer loop counter
	jnz	c3b			;; Iterate if necessary
	lea	esi, [esi-10*4*blkdst]	;; Restore source pointer
	end_timer 17

;; Do inverse FFT levels 4,5
;; On input the 32-byte cache lines hold these data values:
;;	0	1K	1	1K+1
;;	256	...
;;	...
;;	2K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	4K	1	4K+1
;;	256	...
;;	...
;;	8K	...
;;	...

;; Do 2 eight_reals_unfft_2 macros
;;	distance between fft data elements is 4

	start_timer 18
	mov	al, 2			;; Load loop counter
c4b:	disp eight_reals_unfft_2, 2*blkdst, 8, 4*blkdst
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test inner loop counter
	jnc	c4b			;; Iterate if necessary
	IFDEF PFETCH
	prefetcht1 [esi-32+clm*32]
;	prefetcht1 [esi-32+clm*32+2*blkdst]
	prefetcht1 [esi-32+clm*32+4*blkdst]
;	prefetcht1 [esi-32+clm*32+6*blkdst]
	ENDIF
	loopclm	dl, clm, c4b
	lea	esi, [esi-clm*2*dist1+blkdst];; Next source pointer
	dec	al
	jnz	c4b
	lea	esi, [esi-2*blkdst+8*blkdst];; Next source pointer

;; Do 8 four_complex_unfft macros
;;	distance between fft data elements is 4

	mov	edi, sincos2		;; Load sin/cos pointer
	mov	eax, 2			;; 2 iterations of 4
c5b:	disp four_complex_unfft, 8, 4*blkdst, 8*blkdst
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test inner loop counter
	jnc	c5b			;; Iterate if necessary
	IFDEF PFETCH
	prefetcht1 [esi-32+clm*32]
;	prefetcht1 [esi-32+clm*32+4*blkdst]
	prefetcht1 [esi-32+clm*32+8*blkdst]
;	prefetcht1 [esi-32+clm*32+12*blkdst]
	ENDIF
	loopclm	dl, clm, c5b
	lea	esi, [esi-clm*2*dist1+blkdst];; Next source pointer
	add	ah, 256/4		;; Test middle loop counter
	jnc	c5b			;; Iterate if necessary
	lea	esi, [esi-4*blkdst+16*blkdst];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al
	jnz	c5b
	lea	esi, [esi-2*16*blkdst-8*blkdst];; Restore source pointer
	end_timer 18

;; Do inverse FFT levels 1,2,3
;; On input the 32-byte cache lines hold these data values:
;;	0	4K	1	4K+1
;;	256	...
;;	...
;;	8K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	10K	1	10K+1
;;	256
;;	...
;;	10K-256

;; Do 16 five_reals_last_unfft macros
;;	distance between fft data elements is 16

	start_timer 19
	mov	al, 8			;; Load loop counter
c1b:	two_five_reals_last_unfft 8, 8*blkdst
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test loop counter
	jnc	c1b			;; Iterate if necessary
	IFDEF PFETCH
	prefetcht1 [esi-32+clm*32]
;	prefetcht1 [esi-32+clm*32+8*blkdst]
	prefetcht1 [esi-32+clm*32+16*blkdst]
;	prefetcht1 [esi-32+clm*32+24*blkdst]
	prefetcht1 [esi-32+clm*32+32*blkdst]
	ENDIF
	loopclm	dl, clm, c1b
	lea	esi, [esi-clm*2*dist1+blkdst];; Next source pointer
	dec	al			;; Test loop counter
	jnz	c1b			;; Iterate if necessary
	lea	esi, [esi-8*blkdst]	;; Restore source pointer
	end_timer 19

;; Normalize these values

	start_timer 20
	mov	eax, NORMRTN		;; Addr of normalization routine
	call	eax
	sub	eax, eax
	end_timer 20

;; Have we been given permission to start the FFT on the result?
;; If so, do so now while the data is in the L2 cache.

	cmp	POSTFFT, 0		;; Test flag
	je	no_fft			;; Skip FFT code if flag not set
	cmp	edx, p2cl/128*65536+(128/clm-4/clm)*256;; Delay fft on 8 sets
	jg	no_fft			;; Jump if delaying FFT
	copy_3_words clm, 0

;; Do FFT levels 1,2,3
;;
;; On input the 32-byte cache lines hold these data values:
;;	0	10K	1	10K+1
;;	256
;;	...
;;	10K-256
;; On output the 32-byte cache lines hold these data values:
;;	0	4K	1	4K+1
;;	256	...
;;	...
;;	8K	...
;;	...

;; Do 16 five_reals_first_fft macros
;;	distance between fft data elements is 16

b0b:	start_timer 21
	mov	ebx, DIST_TO_FFTSRCARG
	mov	al, 8			;; 8 iterations
b1b:	two_five_reals_first_fft 8*blkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	add	dl, 256/2/clm		;; Test loop counter
	jnc	b1b			;; Iterate if necessary
	lea	esi, [esi-clm*2*dist1+blkdst];; Next source pointer
	dec	al			;; Test loop counter
	jnz	b1b			;; Iterate if necessary
	lea	esi, [esi-8*blkdst]	;; Restore source pointer
	end_timer 21

;; Do FFT levels 4,5
;;
;; On input the 32-byte cache lines hold these data values:
;;	0	4K	1	4K+1
;;	256	...
;;	...
;;	8K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	1K	1	1K+1
;;	256	...
;;	...
;;	2K	...
;;	...

;; Do 2 eight_reals_fft_2 macros
;;	distance between fft data elements is 4

	start_timer 22
	mov	al, 2			;; Load loop counter
b4b:	disp eight_reals_fft_2, 2*blkdst, 4*blkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test inner loop counter
	jnc	b4b			;; Iterate if necessary
	IFDEF PFETCH
;	prefetcht1 [esi-32+clm*32]
	prefetcht1 [esi-32+clm*32+2*blkdst]
;	prefetcht1 [esi-32+clm*32+4*blkdst]
	prefetcht1 [esi-32+clm*32+6*blkdst]
	ENDIF
	loopclm	dl, clm, b4b
	lea	esi, [esi-clm*2*dist1+blkdst];; Next source pointer
	dec	al
	jnz	b4b
	lea	esi, [esi-2*blkdst+8*blkdst];; Next source pointer

;; Do 8 four_complex_fft macros
;;	distance between fft data elements is 4

	mov	edi, sincos2		;; Load sin/cos pointer
	mov	eax, 2			;; 2 iterations of 4
b5b:	disp four_complex_fft, 4*blkdst, 8*blkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test inner loop counter
	jnc	b5b			;; Iterate if necessary
	IFDEF PFETCH
;	prefetcht1 [esi-32+clm*32]
	prefetcht1 [esi-32+clm*32+4*blkdst]
;	prefetcht1 [esi-32+clm*32+8*blkdst]
	prefetcht1 [esi-32+clm*32+12*blkdst]
	ENDIF
	loopclm	dl, clm, b5b
	lea	esi, [esi-clm*2*dist1+blkdst];; Next source pointer
	add	ah, 256/4		;; Test middle loop counter
	jnc	b5b			;; Iterate if necessary
	lea	esi, [esi-4*blkdst+16*blkdst];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al
	jnz	b5b
	lea	esi, [esi-2*16*blkdst-8*blkdst];; Restore source pointer
	end_timer 22

;; Do FFT levels 6,7
;;
;; On input the 32-byte cache lines hold these data values:
;;	0	1K	1	1K+1
;;	256	...
;;	...
;;	2K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	256	1	257
;;	512	...
;;	...

;; Do 1 four_real_four_semireal_fft macros
;;	distance between fft data elements is 1

	start_timer 23
b2b:	disp four_real_four_semireal_fft, blkdst, 2*blkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test inner loop counter
	jnc	b2b			;; Iterate if necessary
	IFDEF PFETCH
;	prefetcht1 [esi-32+clm*32]
	prefetcht1 [esi-32+clm*32+blkdst]
;	prefetcht1 [esi-32+clm*32+2*blkdst]
	prefetcht1 [esi-32+clm*32+3*blkdst]
	ENDIF
	loopclm	dl, clm, b2b
	lea	esi, [esi-clm*2*dist1+4*blkdst];; Next source pointer

;; Do 9 four_complex_fft macros
;;	distance between fft data elements is 1

	mov	edi, sincos1		;; Load sin/cos pointer
	mov	al, 9			;; 9 iterations
b3b:	disp four_complex_fft, blkdst, 2*blkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test inner loop counter
	jnc	b3b			;; Iterate if necessary
	IFDEF PFETCH
;	prefetcht1 [esi-32+clm*32]
	prefetcht1 [esi-32+clm*32+blkdst]
;	prefetcht1 [esi-32+clm*32+2*blkdst]
	prefetcht1 [esi-32+clm*32+3*blkdst]
	ENDIF
	loopclm	dl, clm, b3b
	lea	esi, [esi-clm*2*dist1+4*blkdst];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al
	jnz	b3b
	lea	esi, [esi-10*4*blkdst]	;; Restore source pointer
	end_timer 23

;; Work on next 4 sets of 32 values

no_fft:	lea	esi, [esi+clm*2*dist1]	;; Next source pointer
	dec	dh			;; Test loop counter
	jnz	short more		;; Do more FFTs
	lea	esi, [esi+64]		;; Skip pad every 128 cache lines (4KB)
	sub	edx, 65536		;; Test loop counter
	jz	short fftdn		;; Jump if done looping
	mov	dh, 128/clm		;; Restore cache line counter
more:	cmp	norm_ptr1, 0		;; What was the loop start point?
	jne	c0b			;; Do inverse-FFT/norm/FFT iteration
	jmp	b0b			;; Do another FFT-only iteration
fftdn:	lea	esi, [esi-p2cl*2*dist1-p2cl/128*64];; Restore source pointer
	cmp	esi, DESTARG		;; If esi was not restored, then we 
	jne	short fftdn1		;; just finished skipped postfft data
	sub	ebx, ebx
	mov	DIST_TO_FFTSRCARG, ebx
	cmp	norm_ptr1, 0		;; What was the loop start point?
	je	pass2			;; FFT-only, go do pass 2

;; Split the carries into high and low carries (so they do not exceed
;; the desired number of bits per FFT word.  Then add the carries back to the
;; FFT data.

	start_timer 28
	call	gw_carries
	end_timer 28

;; Have we been given permission to start the FFT on the result?
;; If so, set loop to FFT the first few cache lines now that the carries
;; have been added in.

	cmp	POSTFFT, 0		;; Test flag
	je	short fftdn1		;; Skip FFT code if flag not set
	mov	esi, DESTARG		;; Load FFT data address
	mov	edx, 65536+4/clm*256	;; Do 8 previously skipped cache lines
	mov	norm_ptr1, 0		;; Set flag so only forward FFT is done
	copy_4_words
	jmp	b0b

;; Now add in the FFT'ed carries, cleanup and return

fftdn1:	end_timer 1
	jmp	gw_finish_mult
	ENDM


;; Perform a 24K-element FFT.  This is done in two passes.  Pass 1
;; does 7 levels, pass 2 uses common code to perform the last 8 levels.

pass1levels7pfa6 MACRO pass2_macro, p2cl, clm
	LOCAL	b0b, b1b, b2b, b3b, b4b, b5b, c0b, c1b, c2b, c3b, c4b, c5b
	LOCAL	pass2, no_fft, more, fftdn, fftdn1

;; Call FFT code unless the FFT has already been started

	clear_timers
	cmp	DWORD PTR [esi-28][ebx], 0 ;; Test FFT-started flag
	jne	pass2			;; Jump is FFT already started
	mov	norm_ptr1, 0		;; Set FFT-ing flag
	mov	edx, p2cl/128*65536+128/clm*256;; Load loop counter
	jmp	b0b			;; Do the pass 1 fft

;; Do the last 8 FFT levels
;;	do 1 pass2_8_levels_real and 63 pass2_8_levels_complex

pass2:	start_timer 0
	CALLP	pass2_macro
	end_timer 0
	cmp	ffttype, 1		;; We're done if FFTing only
	je	gw_finish_fft

;; Do the inverse FFT

	start_timer 1
	fldz				;; Init SUMOUT
	fstp	SUMOUT
	mov	edx, norm_biglit_array	;; Addr of the big/little flags array
	mov	norm_ptr1, edx		;; Save ptr
	mov	edx, norm_col_mults	;; Addr of the column multipliers
	mov	norm_ptr2, edx		;; Save ptr
	mov	edx, p2cl/128*65536+128/clm*256;; Load loop counter

;; Do inverse FFT levels 6,7
;; On input the 32-byte cache lines hold these data values:
;;	0	256	1	257
;;	512	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	1K	1	1K+1
;;	256	...
;;	...
;;	2K	...
;;	...

;; Do 1 four_real_four_semireal_unfft macros
;;	distance between fft data elements is 1

c0b:	start_timer 17
c2b:	disp four_real_four_semireal_unfft, 8, blkdst, 2*blkdst
	lea	esi, [esi+dist1]	;; Next source pointer
	add	dl, 256/2/clm		;; Test inner loop counter
	jnc	c2b			;; Iterate if necessary
	lea	esi, [esi-clm*2*dist1+4*blkdst];; Next source pointer

;; Do 11 four_complex_unfft macros
;;	distance between fft data elements is 1

	mov	edi, sincos1		;; Load sin/cos pointer
	mov	al, 11			;; 11 iterations
c3b:	disp four_complex_unfft, 8, blkdst, 2*blkdst
	lea	esi, [esi+dist1]	;; Next source pointer
	add	dl, 256/2/clm		;; Test loop counter
	jnc	c3b			;; Iterate if necessary
	lea	esi, [esi-clm*2*dist1+4*blkdst];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al			;; Test outer loop counter
	jnz	c3b			;; Iterate if necessary
	lea	esi, [esi-12*4*blkdst]	;; Restore source pointer
	end_timer 17

;; Do inverse FFT levels 4,5
;; On input the 32-byte cache lines hold these data values:
;;	0	1K	1	1K+1
;;	256	...
;;	...
;;	2K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	4K	1	4K+1
;;	256	...
;;	...
;;	8K	...
;;	...

;; Do 4 four_real_four_semireal_unfft macros
;;	distance between fft data elements is 4

	start_timer 18
	mov	al, 4			;; Load loop counter
c4b:	disp four_real_four_semireal_unfft, 8, 4*blkdst, 8*blkdst
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test inner loop counter
	jnc	c4b			;; Iterate if necessary
	IFDEF PFETCH
	prefetcht1 [esi-32+clm*32]
;	prefetcht1 [esi-32+clm*32+4*blkdst]
	prefetcht1 [esi-32+clm*32+8*blkdst]
;	prefetcht1 [esi-32+clm*32+12*blkdst]
	ENDIF
	loopclm	dl, clm, c4b
	lea	esi, [esi-clm*2*dist1+blkdst];; Next source pointer
	dec	al
	jnz	c4b
	lea	esi, [esi-4*blkdst+16*blkdst];; Next source pointer

;; Do 8 four_complex_unfft macros
;;	distance between fft data elements is 4

	mov	edi, sincos2		;; Load sin/cos pointer
	mov	eax, 2			;; 2 iterations of 4
c5b:	disp four_complex_unfft, 8, 4*blkdst, 8*blkdst
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test inner loop counter
	jnc	c5b			;; Iterate if necessary
	IFDEF PFETCH
	prefetcht1 [esi-32+clm*32]
;	prefetcht1 [esi-32+clm*32+4*blkdst]
	prefetcht1 [esi-32+clm*32+8*blkdst]
;	prefetcht1 [esi-32+clm*32+12*blkdst]
	ENDIF
	loopclm	dl, clm, c5b
	lea	esi, [esi-clm*2*dist1+blkdst];; Next source pointer
	add	ah, 256/4		;; Test middle loop counter
	jnc	c5b			;; Iterate if necessary
	lea	esi, [esi-4*blkdst+16*blkdst];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al
	jnz	c5b
	lea	esi, [esi-3*16*blkdst]	;; Restore source pointer
	end_timer 18

;; Do inverse FFT levels 1,2,3
;; On input the 32-byte cache lines hold these data values:
;;	0	4K	1	4K+1
;;	256	...
;;	...
;;	8K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	12K	1	12K+1
;;	256
;;	...
;;	12K-256

;; Do 16 six_reals_last_unfft macros
;;	distance between fft data elements is 16

	start_timer 19
	mov	al, 16			;; Load loop counter
c1b:	disp six_reals_last_unfft, 8, 16*blkdst, 32*blkdst
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test loop counter
	jnc	c1b			;; Iterate if necessary
	IFDEF PFETCH
	prefetcht1 [esi-32+clm*32]
;	prefetcht1 [esi-32+clm*32+16*blkdst]
	prefetcht1 [esi-32+clm*32+32*blkdst]
;	prefetcht1 [esi-32+clm*32+48*blkdst]
	ENDIF
	loopclm	dl, clm, c1b
	lea	esi, [esi-clm*2*dist1+blkdst];; Next source pointer
	dec	al			;; Test loop counter
	jnz	c1b			;; Iterate if necessary
	lea	esi, [esi-16*blkdst]	;; Restore source pointer
	end_timer 19

;; Normalize these values

	start_timer 20
	mov	eax, NORMRTN		;; Addr of normalization routine
	call	eax
	sub	eax, eax
	end_timer 20

;; Have we been given permission to start the FFT on the result?
;; If so, do so now while the data is in the L2 cache.

	cmp	POSTFFT, 0		;; Test flag
	je	no_fft			;; Skip FFT code if flag not set
	cmp	edx, p2cl/128*65536+(128/clm-4/clm)*256;; Delay fft on 8 sets
	jg	no_fft			;; Jump if delaying FFT
	copy_3_words clm, 0

;; Do FFT levels 1,2,3
;;
;; On input the 32-byte cache lines hold these data values:
;;	0	12K	1	12K+1
;;	256
;;	...
;;	12K-256
;; On output the 32-byte cache lines hold these data values:
;;	0	4K	1	4K+1
;;	256	...
;;	...
;;	8K	...
;;	...

;; Do 16 six_reals_first_fft macros
;;	distance between fft data elements is 16

b0b:	start_timer 21
	mov	ebx, DIST_TO_FFTSRCARG
	mov	al, 16			;; 16 iterations
b1b:	disp six_reals_first_fft, 16*blkdst, 32*blkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	add	dl, 256/2/clm		;; Test loop counter
	jnc	b1b			;; Iterate if necessary
	lea	esi, [esi-clm*2*dist1+blkdst];; Next source pointer
	dec	al			;; Test loop counter
	jnz	b1b			;; Iterate if necessary
	lea	esi, [esi-16*blkdst]	;; Restore source pointer
	end_timer 21

;; Do FFT levels 4,5
;;
;; On input the 32-byte cache lines hold these data values:
;;	0	4K	1	4K+1
;;	256	...
;;	...
;;	8K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	1K	1	1K+1
;;	256	...
;;	...
;;	2K	...
;;	...

;; Do 4 four_real_four_semireal_fft macros
;;	distance between fft data elements is 4

	start_timer 22
	mov	al, 4			;; Load loop counter
b4b:	disp four_real_four_semireal_fft, 4*blkdst, 8*blkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test inner loop counter
	jnc	b4b			;; Iterate if necessary
	IFDEF PFETCH
;	prefetcht1 [esi-32+clm*32]
	prefetcht1 [esi-32+clm*32+4*blkdst]
;	prefetcht1 [esi-32+clm*32+8*blkdst]
	prefetcht1 [esi-32+clm*32+12*blkdst]
	ENDIF
	loopclm	dl, clm, b4b
	lea	esi, [esi-clm*2*dist1+blkdst];; Next source pointer
	dec	al
	jnz	b4b
	lea	esi, [esi-4*blkdst+16*blkdst];; Next source pointer

;; Do 8 four_complex_fft macros
;;	distance between fft data elements is 4

	mov	edi, sincos2		;; Load sin/cos pointer
	mov	eax, 2			;; 2 iterations of 4
b5b:	disp four_complex_fft, 4*blkdst, 8*blkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test inner loop counter
	jnc	b5b			;; Iterate if necessary
	IFDEF PFETCH
;	prefetcht1 [esi-32+clm*32]
	prefetcht1 [esi-32+clm*32+4*blkdst]
;	prefetcht1 [esi-32+clm*32+8*blkdst]
	prefetcht1 [esi-32+clm*32+12*blkdst]
	ENDIF
	loopclm	dl, clm, b5b
	lea	esi, [esi-clm*2*dist1+blkdst];; Next source pointer
	add	ah, 256/4		;; Test middle loop counter
	jnc	b5b			;; Iterate if necessary
	lea	esi, [esi-4*blkdst+16*blkdst];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al
	jnz	b5b
	lea	esi, [esi-3*16*blkdst]	;; Restore source pointer
	end_timer 22

;; Do FFT levels 6,7
;;
;; On input the 32-byte cache lines hold these data values:
;;	0	1K	1	1K+1
;;	256	...
;;	...
;;	2K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	256	1	257
;;	512	...
;;	...

;; Do 1 four_real_four_semireal_fft macros
;;	distance between fft data elements is 1

	start_timer 23
b2b:	disp four_real_four_semireal_fft, blkdst, 2*blkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test inner loop counter
	jnc	b2b			;; Iterate if necessary
	IFDEF PFETCH
;	prefetcht1 [esi-32+clm*32]
	prefetcht1 [esi-32+clm*32+blkdst]
;	prefetcht1 [esi-32+clm*32+2*blkdst]
	prefetcht1 [esi-32+clm*32+3*blkdst]
	ENDIF
	loopclm	dl, clm, b2b
	lea	esi, [esi-clm*2*dist1+4*blkdst];; Next source pointer

;; Do 11 four_complex_fft macros
;;	distance between fft data elements is 1

	mov	edi, sincos1		;; Load sin/cos pointer
	mov	al, 11			;; 11 iterations
b3b:	disp four_complex_fft, blkdst, 2*blkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test inner loop counter
	jnc	b3b			;; Iterate if necessary
	IFDEF PFETCH
;	prefetcht1 [esi-32+clm*32]
	prefetcht1 [esi-32+clm*32+blkdst]
;	prefetcht1 [esi-32+clm*32+2*blkdst]
	prefetcht1 [esi-32+clm*32+3*blkdst]
	ENDIF
	loopclm	dl, clm, b3b
	lea	esi, [esi-clm*2*dist1+4*blkdst];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al
	jnz	b3b
	lea	esi, [esi-12*4*blkdst]	;; Restore source pointer
	end_timer 23

;; Work on next 4 sets of 32 values

no_fft:	lea	esi, [esi+clm*2*dist1]	;; Next source pointer
	dec	dh			;; Test loop counter
	jnz	short more		;; Do more FFTs
	lea	esi, [esi+64]		;; Skip pad every 128 cache lines (4KB)
	sub	edx, 65536		;; Test loop counter
	jz	short fftdn		;; Jump if done looping
	mov	dh, 128/clm		;; Restore cache line counter
more:	cmp	norm_ptr1, 0		;; What was the loop start point?
	jne	c0b			;; Do inverse-FFT/norm/FFT iteration
	jmp	b0b			;; Do another FFT-only iteration
fftdn:	lea	esi, [esi-p2cl*2*dist1-p2cl/128*64];; Restore source pointer
	cmp	esi, DESTARG		;; If esi was not restored, then we 
	jne	short fftdn1		;; just finished skipped postfft data
	sub	ebx, ebx
	mov	DIST_TO_FFTSRCARG, ebx
	cmp	norm_ptr1, 0		;; What was the loop start point?
	je	pass2			;; FFT-only, go do pass 2

;; Split the carries into high and low carries (so they do not exceed
;; the desired number of bits per FFT word.  Then add the carries back to the
;; FFT data.

	start_timer 28
	call	gw_carries
	end_timer 28

;; Have we been given permission to start the FFT on the result?
;; If so, set loop to FFT the first few cache lines now that the carries
;; have been added in.

	cmp	POSTFFT, 0		;; Test flag
	je	short fftdn1		;; Skip FFT code if flag not set
	mov	esi, DESTARG		;; Load FFT data address
	mov	edx, 65536+4/clm*256	;; Do 8 previously skipped cache lines
	mov	norm_ptr1, 0		;; Set flag so only forward FFT is done
	copy_4_words
	jmp	b0b

;; Now add in the FFT'ed carries, cleanup and return

fftdn1:	end_timer 1
	jmp	gw_finish_mult
	ENDM


;; Perform a 24K-element negacyclic FFT.  This is done in two passes.  Pass 1
;; does 7 levels, pass 2 uses common code to perform the last 8 levels.

pass1levels7complex3 MACRO pass2_macro, p2cl, clm
	LOCAL	b0b, b1b, b3b, b5b, c0b, c1b, c3b, c5b
	LOCAL	pass2, no_fft, more, fftdn, fftdn1

;; Call FFT code unless the FFT has already been started

	clear_timers
	cmp	DWORD PTR [esi-28][ebx], 0 ;; Test FFT-started flag
	jne	pass2			;; Jump is FFT already started
	mov	norm_ptr1, 0		;; Set FFT-ing flag
	mov	edx, p2cl/128*65536+128/clm*256;; Load loop counter
	jmp	b0b			;; Do the pass 1 fft

;; Do the last 8 FFT levels
;;	do 48 pass2_8_levels_complex

pass2:	start_timer 0
	CALLP	pass2_macro
	end_timer 0
	cmp	ffttype, 1		;; We're done if FFTing only
	je	gw_finish_fft

;; Do the inverse FFT

	start_timer 1
	fldz				;; Init SUMOUT
	fstp	SUMOUT
	mov	edx, norm_biglit_array	;; Addr of the big/little flags array
	mov	norm_ptr1, edx		;; Save ptr
	mov	edx, norm_col_mults	;; Addr of the column multipliers
	mov	norm_ptr2, edx		;; Save ptr
	mov	edx, p2cl/128*65536+128/clm*256;; Load loop counter

;; Do inverse FFT levels 6,7
;; On input the 32-byte cache lines hold these data values:
;;	0	256	1	257
;;	512	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	1K	1	1K+1
;;	256	...
;;	...
;;	2K	...
;;	...

;; Do 12 four_complex_unfft macros
;;	distance between fft data elements is 1

c0b:	start_timer 17
	mov	edi, sincos1		;; Load sin/cos pointer
	mov	al, 12			;; 12 iterations
c3b:	disp four_complex_unfft, 8, blkdst, 2*blkdst
	lea	esi, [esi+dist1]	;; Next source pointer
	add	dl, 256/2/clm		;; Test loop counter
	jnc	c3b			;; Iterate if necessary
	lea	esi, [esi-clm*2*dist1+4*blkdst];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al			;; Test outer loop counter
	jnz	c3b			;; Iterate if necessary
	lea	esi, [esi-12*4*blkdst]	;; Restore source pointer
	end_timer 17

;; Do inverse FFT levels 4,5
;; On input the 32-byte cache lines hold these data values:
;;	0	1K	1	1K+1
;;	256	...
;;	...
;;	2K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	4K	1	4K+1
;;	256	...
;;	...
;;	8K	...
;;	...

;; Do 12 four_complex_unfft macros
;;	distance between fft data elements is 4

	start_timer 18
	mov	edi, sincos2		;; Load sin/cos pointer
	mov	eax, 3			;; 3 iterations of 4
c5b:	disp four_complex_unfft, 8, 4*blkdst, 8*blkdst
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test inner loop counter
	jnc	c5b			;; Iterate if necessary
	IFDEF PFETCH
	prefetcht1 [esi-32+clm*32]
;	prefetcht1 [esi-32+clm*32+4*blkdst]
	prefetcht1 [esi-32+clm*32+8*blkdst]
;	prefetcht1 [esi-32+clm*32+12*blkdst]
	ENDIF
	loopclm	dl, clm, c5b
	lea	esi, [esi-clm*2*dist1+blkdst];; Next source pointer
	add	ah, 256/4		;; Test middle loop counter
	jnc	c5b			;; Iterate if necessary
	lea	esi, [esi-4*blkdst+16*blkdst];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al
	jnz	c5b
	lea	esi, [esi-3*16*blkdst]	;; Restore source pointer
	end_timer 18

;; Do inverse FFT levels 1,2,3
;; On input the 32-byte cache lines hold these data values:
;;	0	4K	1	4K+1
;;	256	...
;;	...
;;	8K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	12K	1	12K+1
;;	256
;;	...
;;	12K-256

;; Do 16 three_complex_last_unfft macros
;;	distance between fft data elements is 16

	start_timer 19
	mov	edi, plus1_premults	;; Address of premultiplier table
	mov	al, 16			;; Load loop counter
c1b:	three_complex_last_unfft 8, 16*blkdst, 32*blkdst
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test loop counter
	jnc	c1b			;; Iterate if necessary
	IFDEF PFETCH
	prefetcht1 [esi-32+clm*32]
;	prefetcht1 [esi-32+clm*32+16*blkdst]
	prefetcht1 [esi-32+clm*32+32*blkdst]
	ENDIF
	loopclm	dl, clm, c1b
	lea	esi, [esi-clm*2*dist1+blkdst];; Next source pointer
	lea	edi, [edi+48]		;; Next premultiplier pointer
	dec	al			;; Test loop counter
	jnz	c1b			;; Iterate if necessary
	lea	esi, [esi-16*blkdst]	;; Restore source pointer
	end_timer 19

;; Normalize these values

	start_timer 20
	mov	eax, NORMRTN		;; Addr of normalization routine
	call	eax
	sub	eax, eax
	end_timer 20

;; Have we been given permission to start the FFT on the result?
;; If so, do so now while the data is in the L2 cache.

	cmp	POSTFFT, 0		;; Test flag
	je	no_fft			;; Skip FFT code if flag not set
	cmp	edx, p2cl/128*65536+(128/clm-4/clm)*256;; Delay fft on 8 sets
	jg	no_fft			;; Jump if delaying FFT
	copy_3_words clm, 0

;; Do FFT levels 1,2,3
;;
;; On input the 32-byte cache lines hold these data values:
;;	0	12K	1	12K+1
;;	256
;;	...
;;	12K-256
;; On output the 32-byte cache lines hold these data values:
;;	0	4K	1	4K+1
;;	256	...
;;	...
;;	8K	...
;;	...

;; Do 16 three_complex_first_fft macros
;;	distance between fft data elements is 16

b0b:	start_timer 21
	mov	ebx, DIST_TO_FFTSRCARG
	mov	edi, plus1_premults	;; Address of premultiplier table
	mov	al, 16			;; 16 iterations
b1b:	three_complex_first_fft 16*blkdst, 32*blkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	add	dl, 256/2/clm		;; Test loop counter
	jnc	b1b			;; Iterate if necessary
	lea	esi, [esi-clm*2*dist1+blkdst];; Next source pointer
	lea	edi, [edi+48]		;; Next premultiplier pointer
	dec	al			;; Test loop counter
	jnz	b1b			;; Iterate if necessary
	lea	esi, [esi-16*blkdst]	;; Restore source pointer
	end_timer 21

;; Do FFT levels 4,5
;;
;; On input the 32-byte cache lines hold these data values:
;;	0	4K	1	4K+1
;;	256	...
;;	...
;;	8K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	1K	1	1K+1
;;	256	...
;;	...
;;	2K	...
;;	...

;; Do 12 four_complex_fft macros
;;	distance between fft data elements is 4

	start_timer 22
	mov	edi, sincos2		;; Load sin/cos pointer
	mov	eax, 3			;; 3 iterations of 4
b5b:	disp four_complex_fft, 4*blkdst, 8*blkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test inner loop counter
	jnc	b5b			;; Iterate if necessary
	IFDEF PFETCH
;	prefetcht1 [esi-32+clm*32]
	prefetcht1 [esi-32+clm*32+4*blkdst]
;	prefetcht1 [esi-32+clm*32+8*blkdst]
	prefetcht1 [esi-32+clm*32+12*blkdst]
	ENDIF
	loopclm	dl, clm, b5b
	lea	esi, [esi-clm*2*dist1+blkdst];; Next source pointer
	add	ah, 256/4		;; Test middle loop counter
	jnc	b5b			;; Iterate if necessary
	lea	esi, [esi-4*blkdst+16*blkdst];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al
	jnz	b5b
	lea	esi, [esi-3*16*blkdst]	;; Restore source pointer
	end_timer 22

;; Do FFT levels 6,7
;;
;; On input the 32-byte cache lines hold these data values:
;;	0	1K	1	1K+1
;;	256	...
;;	...
;;	2K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	256	1	257
;;	512	...
;;	...

;; Do 12 four_complex_fft macros
;;	distance between fft data elements is 1

	start_timer 23
	mov	edi, sincos1		;; Load sin/cos pointer
	mov	al, 12			;; 12 iterations
b3b:	disp four_complex_fft, blkdst, 2*blkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test inner loop counter
	jnc	b3b			;; Iterate if necessary
	IFDEF PFETCH
;	prefetcht1 [esi-32+clm*32]
	prefetcht1 [esi-32+clm*32+blkdst]
;	prefetcht1 [esi-32+clm*32+2*blkdst]
	prefetcht1 [esi-32+clm*32+3*blkdst]
	ENDIF
	loopclm	dl, clm, b3b
	lea	esi, [esi-clm*2*dist1+4*blkdst];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al
	jnz	b3b
	lea	esi, [esi-12*4*blkdst]	;; Restore source pointer
	end_timer 23

;; Work on next 4 sets of 32 values

no_fft:	lea	esi, [esi+clm*2*dist1]	;; Next source pointer
	dec	dh			;; Test loop counter
	jnz	short more		;; Do more FFTs
	lea	esi, [esi+64]		;; Skip pad every 128 cache lines (4KB)
	sub	edx, 65536		;; Test loop counter
	jz	short fftdn		;; Jump if done looping
	mov	dh, 128/clm		;; Restore cache line counter
more:	cmp	norm_ptr1, 0		;; What was the loop start point?
	jne	c0b			;; Do inverse-FFT/norm/FFT iteration
	jmp	b0b			;; Do another FFT-only iteration
fftdn:	lea	esi, [esi-p2cl*2*dist1-p2cl/128*64];; Restore source pointer
	cmp	esi, DESTARG		;; If esi was not restored, then we 
	jne	short fftdn1		;; just finished skipped postfft data
	sub	ebx, ebx
	mov	DIST_TO_FFTSRCARG, ebx
	cmp	norm_ptr1, 0		;; What was the loop start point?
	je	pass2			;; FFT-only, go do pass 2

;; Split the carries into high and low carries (so they do not exceed
;; the desired number of bits per FFT word.  Then add the carries back to the
;; FFT data.

	start_timer 28
	call	gw_carries
	end_timer 28

;; Have we been given permission to start the FFT on the result?
;; If so, set loop to FFT the first few cache lines now that the carries
;; have been added in.

	cmp	POSTFFT, 0		;; Test flag
	je	short fftdn1		;; Skip FFT code if flag not set
	mov	esi, DESTARG		;; Load FFT data address
	mov	edx, 65536+4/clm*256	;; Do 8 previously skipped cache lines
	mov	norm_ptr1, 0		;; Set flag so only forward FFT is done
	copy_4_words
	jmp	b0b

;; Now add in the FFT'ed carries, cleanup and return

fftdn1:	end_timer 1
	jmp	gw_finish_mult
	ENDM


;; Perform a 28K-element FFT.  This is done in two passes.  Pass 1
;; does 7 levels, pass 2 uses common code to perform the last 8 levels.

pass1levels7pfa7 MACRO pass2_macro, p2cl, clm
	LOCAL	b0b, b1b, b2b, b3b, b4b, b5b, c0b, c1b, c2b, c3b, c4b, c5b
	LOCAL	pass2, no_fft, more, fftdn, fftdn1

;; Call FFT code unless the FFT has already been started

	clear_timers
	cmp	DWORD PTR [esi-28][ebx], 0 ;; Test FFT-started flag
	jne	pass2			;; Jump is FFT already started
	mov	norm_ptr1, 0		;; Set FFT-ing flag
	mov	edx, p2cl/128*65536+128/clm*256;; Load loop counter
	jmp	b0b			;; Do the pass 1 fft

;; Do the last 8 FFT levels
;;	do 1 pass2_8_levels_real and 55 pass2_8_levels_complex

pass2:	start_timer 0
	CALLP	pass2_macro
	end_timer 0
	cmp	ffttype, 1		;; We're done if FFTing only
	je	gw_finish_fft

;; Do the inverse FFT

	start_timer 1
	fldz				;; Init SUMOUT
	fstp	SUMOUT
	mov	edx, norm_biglit_array	;; Addr of the big/little flags array
	mov	norm_ptr1, edx		;; Save ptr
	mov	edx, norm_col_mults	;; Addr of the column multipliers
	mov	norm_ptr2, edx		;; Save ptr
	mov	edx, p2cl/128*65536+128/clm*256;; Load loop counter

;; Do inverse FFT levels 6,7
;; On input the 32-byte cache lines hold these data values:
;;	0	256	1	257
;;	512	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	1K	1	1K+1
;;	256	...
;;	...
;;	2K	...
;;	...

;; Do 1 four_real_four_semireal_unfft macros
;;	distance between fft data elements is 1

c0b:	start_timer 17
c2b:	disp four_real_four_semireal_unfft, 8, blkdst, 2*blkdst
	lea	esi, [esi+dist1]	;; Next source pointer
	add	dl, 256/2/clm		;; Test inner loop counter
	jnc	c2b			;; Iterate if necessary
	lea	esi, [esi-clm*2*dist1+4*blkdst];; Next source pointer

;; Do 13 four_complex_unfft macros
;;	distance between fft data elements is 1

	mov	edi, sincos1		;; Load sin/cos pointer
	mov	al, 13			;; 13 iterations
c3b:	disp four_complex_unfft, 8, blkdst, 2*blkdst
	lea	esi, [esi+dist1]	;; Next source pointer
	add	dl, 256/2/clm		;; Test loop counter
	jnc	c3b			;; Iterate if necessary
	lea	esi, [esi-clm*2*dist1+4*blkdst];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al			;; Test outer loop counter
	jnz	c3b			;; Iterate if necessary
	lea	esi, [esi-14*4*blkdst]	;; Restore source pointer
	end_timer 17

;; Do inverse FFT levels 4,5
;; On input the 32-byte cache lines hold these data values:
;;	0	1K	1	1K+1
;;	256	...
;;	...
;;	2K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	4K	1	4K+1
;;	256	...
;;	...
;;	8K	...
;;	...

;; Do 2 eight_reals_unfft_2 macros
;;	distance between fft data elements is 4

	start_timer 18
	mov	al, 2			;; Load loop counter
c4b:	disp eight_reals_unfft_2, 2*blkdst, 8, 4*blkdst
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test inner loop counter
	jnc	c4b			;; Iterate if necessary
	IFDEF PFETCH
	prefetcht1 [esi-32+clm*32]
;	prefetcht1 [esi-32+clm*32+2*blkdst]
	prefetcht1 [esi-32+clm*32+4*blkdst]
;	prefetcht1 [esi-32+clm*32+6*blkdst]
	ENDIF
	loopclm	dl, clm, c4b
	lea	esi, [esi-clm*2*dist1+blkdst];; Next source pointer
	dec	al
	jnz	c4b
	lea	esi, [esi-2*blkdst+8*blkdst];; Next source pointer

;; Do 12 four_complex_unfft macros
;;	distance between fft data elements is 4

	mov	edi, sincos2		;; Load sin/cos pointer
	mov	eax, 3			;; 3 iterations of 4
c5b:	disp four_complex_unfft, 8, 4*blkdst, 8*blkdst
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test inner loop counter
	jnc	c5b			;; Iterate if necessary
	IFDEF PFETCH
	prefetcht1 [esi-32+clm*32]
;	prefetcht1 [esi-32+clm*32+4*blkdst]
	prefetcht1 [esi-32+clm*32+8*blkdst]
;	prefetcht1 [esi-32+clm*32+12*blkdst]
	ENDIF
	loopclm	dl, clm, c5b
	lea	esi, [esi-clm*2*dist1+blkdst];; Next source pointer
	add	ah, 256/4		;; Test middle loop counter
	jnc	c5b			;; Iterate if necessary
	lea	esi, [esi-4*blkdst+16*blkdst];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al
	jnz	c5b
	lea	esi, [esi-3*16*blkdst-8*blkdst];; Restore source pointer
	end_timer 18

;; Do inverse FFT levels 1,2,3
;; On input the 32-byte cache lines hold these data values:
;;	0	4K	1	4K+1
;;	256	...
;;	...
;;	8K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	14K	1	14K+1
;;	256
;;	...
;;	14K-256

;; Do 16 seven_reals_last_unfft macros
;;	distance between fft data elements is 16

	start_timer 19
	mov	al, 8			;; Load loop counter
c1b:	two_seven_reals_last_unfft 8, 8*blkdst
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test loop counter
	jnc	c1b			;; Iterate if necessary
	IFDEF PFETCH
	prefetcht1 [esi-32+clm*32]
;	prefetcht1 [esi-32+clm*32+8*blkdst]
	prefetcht1 [esi-32+clm*32+16*blkdst]
;	prefetcht1 [esi-32+clm*32+24*blkdst]
	prefetcht1 [esi-32+clm*32+32*blkdst]
;	prefetcht1 [esi-32+clm*32+40*blkdst]
	prefetcht1 [esi-32+clm*32+48*blkdst]
	ENDIF
	loopclm	dl, clm, c1b
	lea	esi, [esi-clm*2*dist1+blkdst];; Next source pointer
	dec	al			;; Test loop counter
	jnz	c1b			;; Iterate if necessary
	lea	esi, [esi-8*blkdst]	;; Restore source pointer
	end_timer 19

;; Normalize these values

	start_timer 20
	mov	eax, NORMRTN		;; Addr of normalization routine
	call	eax
	sub	eax, eax
	end_timer 20

;; Have we been given permission to start the FFT on the result?
;; If so, do so now while the data is in the L2 cache.

	cmp	POSTFFT, 0		;; Test flag
	je	no_fft			;; Skip FFT code if flag not set
	cmp	edx, p2cl/128*65536+(128/clm-4/clm)*256;; Delay fft on 8 sets
	jg	no_fft			;; Jump if delaying FFT
	copy_3_words clm, 0

;; Do FFT levels 1,2,3
;;
;; On input the 32-byte cache lines hold these data values:
;;	0	14K	1	14K+1
;;	256
;;	...
;;	14K-256
;; On output the 32-byte cache lines hold these data values:
;;	0	4K	1	4K+1
;;	256	...
;;	...
;;	8K	...
;;	...

;; Do 16 seven_reals_first_fft macros
;;	distance between fft data elements is 16

b0b:	start_timer 21
	mov	ebx, DIST_TO_FFTSRCARG
	mov	al, 8			;; 8 iterations
b1b:	two_seven_reals_first_fft 8*blkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	add	dl, 256/2/clm		;; Test loop counter
	jnc	b1b			;; Iterate if necessary
	lea	esi, [esi-clm*2*dist1+blkdst];; Next source pointer
	dec	al			;; Test loop counter
	jnz	b1b			;; Iterate if necessary
	lea	esi, [esi-8*blkdst]	;; Restore source pointer
	end_timer 21

;; Do FFT levels 4,5
;;
;; On input the 32-byte cache lines hold these data values:
;;	0	4K	1	4K+1
;;	256	...
;;	...
;;	8K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	1K	1	1K+1
;;	256	...
;;	...
;;	2K	...
;;	...

;; Do 2 eight_reals_fft_2 macros
;;	distance between fft data elements is 4

	start_timer 22
	mov	al, 2			;; Load loop counter
b4b:	disp eight_reals_fft_2, 2*blkdst, 4*blkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test inner loop counter
	jnc	b4b			;; Iterate if necessary
	IFDEF PFETCH
;	prefetcht1 [esi-32+clm*32]
	prefetcht1 [esi-32+clm*32+2*blkdst]
;	prefetcht1 [esi-32+clm*32+4*blkdst]
	prefetcht1 [esi-32+clm*32+6*blkdst]
	ENDIF
	loopclm	dl, clm, b4b
	lea	esi, [esi-clm*2*dist1+blkdst];; Next source pointer
	dec	al
	jnz	b4b
	lea	esi, [esi-2*blkdst+8*blkdst];; Next source pointer

;; Do 12 four_complex_fft macros
;;	distance between fft data elements is 4

	mov	edi, sincos2		;; Load sin/cos pointer
	mov	eax, 3			;; 3 iterations of 4
b5b:	disp four_complex_fft, 4*blkdst, 8*blkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test inner loop counter
	jnc	b5b			;; Iterate if necessary
	IFDEF PFETCH
;	prefetcht1 [esi-32+clm*32]
	prefetcht1 [esi-32+clm*32+4*blkdst]
;	prefetcht1 [esi-32+clm*32+8*blkdst]
	prefetcht1 [esi-32+clm*32+12*blkdst]
	ENDIF
	loopclm	dl, clm, b5b
	lea	esi, [esi-clm*2*dist1+blkdst];; Next source pointer
	add	ah, 256/4		;; Test middle loop counter
	jnc	b5b			;; Iterate if necessary
	lea	esi, [esi-4*blkdst+16*blkdst];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al
	jnz	b5b
	lea	esi, [esi-3*16*blkdst-8*blkdst];; Restore source pointer
	end_timer 22

;; Do FFT levels 6,7
;;
;; On input the 32-byte cache lines hold these data values:
;;	0	1K	1	1K+1
;;	256	...
;;	...
;;	2K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	256	1	257
;;	512	...
;;	...

;; Do 1 four_real_four_semireal_fft macros
;;	distance between fft data elements is 1

	start_timer 23
b2b:	disp four_real_four_semireal_fft, blkdst, 2*blkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test inner loop counter
	jnc	b2b			;; Iterate if necessary
	IFDEF PFETCH
;	prefetcht1 [esi-32+clm*32]
	prefetcht1 [esi-32+clm*32+blkdst]
;	prefetcht1 [esi-32+clm*32+2*blkdst]
	prefetcht1 [esi-32+clm*32+3*blkdst]
	ENDIF
	loopclm	dl, clm, b2b
	lea	esi, [esi-clm*2*dist1+4*blkdst];; Next source pointer

;; Do 13 four_complex_fft macros
;;	distance between fft data elements is 1

	mov	edi, sincos1		;; Load sin/cos pointer
	mov	al, 13			;; 13 iterations
b3b:	disp four_complex_fft, blkdst, 2*blkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test inner loop counter
	jnc	b3b			;; Iterate if necessary
	IFDEF PFETCH
;	prefetcht1 [esi-32+clm*32]
	prefetcht1 [esi-32+clm*32+blkdst]
;	prefetcht1 [esi-32+clm*32+2*blkdst]
	prefetcht1 [esi-32+clm*32+3*blkdst]
	ENDIF
	loopclm	dl, clm, b3b
	lea	esi, [esi-clm*2*dist1+4*blkdst];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al
	jnz	b3b
	lea	esi, [esi-14*4*blkdst]	;; Restore source pointer
	end_timer 23

;; Work on next 4 sets of 32 values

no_fft:	lea	esi, [esi+clm*2*dist1]	;; Next source pointer
	dec	dh			;; Test loop counter
	jnz	short more		;; Do more FFTs
	lea	esi, [esi+64]		;; Skip pad every 128 cache lines (4KB)
	sub	edx, 65536		;; Test loop counter
	jz	short fftdn		;; Jump if done looping
	mov	dh, 128/clm		;; Restore cache line counter
more:	cmp	norm_ptr1, 0		;; What was the loop start point?
	jne	c0b			;; Do inverse-FFT/norm/FFT iteration
	jmp	b0b			;; Do another FFT-only iteration
fftdn:	lea	esi, [esi-p2cl*2*dist1-p2cl/128*64];; Restore source pointer
	cmp	esi, DESTARG		;; If esi was not restored, then we 
	jne	short fftdn1		;; just finished skipped postfft data
	sub	ebx, ebx
	mov	DIST_TO_FFTSRCARG, ebx
	cmp	norm_ptr1, 0		;; What was the loop start point?
	je	pass2			;; FFT-only, go do pass 2

;; Split the carries into high and low carries (so they do not exceed
;; the desired number of bits per FFT word.  Then add the carries back to the
;; FFT data.

	start_timer 28
	call	gw_carries
	end_timer 28

;; Have we been given permission to start the FFT on the result?
;; If so, set loop to FFT the first few cache lines now that the carries
;; have been added in.

	cmp	POSTFFT, 0		;; Test flag
	je	short fftdn1		;; Skip FFT code if flag not set
	mov	esi, DESTARG		;; Load FFT data address
	mov	edx, 65536+4/clm*256	;; Do 8 previously skipped cache lines
	mov	norm_ptr1, 0		;; Set flag so only forward FFT is done
	copy_4_words
	jmp	b0b

;; Now add in the FFT'ed carries, cleanup and return

fftdn1:	end_timer 1
	jmp	gw_finish_mult
	ENDM


;; Perform a 32K-element FFT.  This is done in two passes.  Pass 1
;; does 7 levels, pass 2 uses common code to perform the last 8 levels.

pass1levels7pfa8 MACRO pass2_macro, p2cl, clm
	LOCAL	b0b, b1b, b2b, b3b, b4b, b5b, c0b, c1b, c2b, c3b, c4b, c5b
	LOCAL	pass2, no_fft, more, fftdn, fftdn1

;; Call FFT code unless the FFT has already been started

	clear_timers
	cmp	DWORD PTR [esi-28][ebx], 0 ;; Test FFT-started flag
	jne	pass2			;; Jump is FFT already started
	mov	norm_ptr1, 0		;; Set FFT-ing flag
	mov	edx, p2cl/128*65536+128/clm*256;; Load loop counter
	jmp	b0b			;; Do the pass 1 fft

;; Do the last 8 FFT levels
;;	do 1 pass2_8_levels_real and 63 pass2_8_levels_complex

pass2:	start_timer 0
	CALLP	pass2_macro
	end_timer 0
	cmp	ffttype, 1		;; We're done if FFTing only
	je	gw_finish_fft

;; Do the inverse FFT

	start_timer 1
	fldz				;; Init SUMOUT
	fstp	SUMOUT
	mov	edx, norm_biglit_array	;; Addr of the big/little flags array
	mov	norm_ptr1, edx		;; Save ptr
	mov	edx, norm_col_mults	;; Addr of the column multipliers
	mov	norm_ptr2, edx		;; Save ptr
	mov	edx, p2cl/128*65536+128/clm*256;; Load loop counter

;; Do inverse FFT levels 6,7
;; On input the 32-byte cache lines hold these data values:
;;	0	256	1	257
;;	512	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	1K	1	1K+1
;;	256	...
;;	...
;;	2K	...
;;	...

;; Do 1 four_real_four_semireal_unfft macros
;;	distance between fft data elements is 1

c0b:	start_timer 17
c2b:	disp four_real_four_semireal_unfft, 8, blkdst, 2*blkdst
	lea	esi, [esi+dist1]	;; Next source pointer
	add	dl, 256/2/clm		;; Test inner loop counter
	jnc	c2b			;; Iterate if necessary
	lea	esi, [esi-clm*2*dist1+4*blkdst];; Next source pointer

;; Do 15 four_complex_unfft macros
;;	distance between fft data elements is 1

	mov	edi, sincos1		;; Load sin/cos pointer
	mov	al, 15			;; 15 iterations
c3b:	disp four_complex_unfft, 8, blkdst, 2*blkdst
	lea	esi, [esi+dist1]	;; Next source pointer
	add	dl, 256/2/clm		;; Test loop counter
	jnc	c3b			;; Iterate if necessary
	lea	esi, [esi-clm*2*dist1+4*blkdst];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al			;; Test outer loop counter
	jnz	c3b			;; Iterate if necessary
	lea	esi, [esi-16*4*blkdst]	;; Restore source pointer
	end_timer 17

;; Do inverse FFT levels 4,5
;; On input the 32-byte cache lines hold these data values:
;;	0	1K	1	1K+1
;;	256	...
;;	...
;;	2K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	4K	1	4K+1
;;	256	...
;;	...
;;	8K	...
;;	...

;; Do 4 four_real_four_semireal_unfft macros
;;	distance between fft data elements is 4

	start_timer 18
	mov	al, 4			;; Load loop counter
c4b:	disp four_real_four_semireal_unfft, 8, 4*blkdst, 8*blkdst
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test inner loop counter
	jnc	c4b			;; Iterate if necessary
	IFDEF PFETCH
	prefetcht1 [esi-32+clm*32]
;	prefetcht1 [esi-32+clm*32+4*blkdst]
	prefetcht1 [esi-32+clm*32+8*blkdst]
;	prefetcht1 [esi-32+clm*32+12*blkdst]
	ENDIF
	loopclm	dl, clm, c4b
	lea	esi, [esi-clm*2*dist1+blkdst];; Next source pointer
	dec	al
	jnz	c4b
	lea	esi, [esi-4*blkdst+16*blkdst];; Next source pointer

;; Do 12 four_complex_unfft macros
;;	distance between fft data elements is 4

	mov	edi, sincos2		;; Load sin/cos pointer
	mov	eax, 3			;; 3 iterations of 4
c5b:	disp four_complex_unfft, 8, 4*blkdst, 8*blkdst
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test inner loop counter
	jnc	c5b			;; Iterate if necessary
	IFDEF PFETCH
	prefetcht1 [esi-32+clm*32]
;	prefetcht1 [esi-32+clm*32+4*blkdst]
	prefetcht1 [esi-32+clm*32+8*blkdst]
;	prefetcht1 [esi-32+clm*32+12*blkdst]
	ENDIF
	loopclm	dl, clm, c5b
	lea	esi, [esi-clm*2*dist1+blkdst];; Next source pointer
	add	ah, 256/4		;; Test middle loop counter
	jnc	c5b			;; Iterate if necessary
	lea	esi, [esi-4*blkdst+16*blkdst];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al
	jnz	c5b
	lea	esi, [esi-4*16*blkdst]	;; Restore source pointer
	end_timer 18

;; Do inverse FFT levels 1,2,3
;; On input the 32-byte cache lines hold these data values:
;;	0	4K	1	4K+1
;;	256	...
;;	...
;;	8K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	16K	1	16K+1
;;	256
;;	...
;;	16K-256

;; Do 16 eight_reals_last_unfft macros
;;	distance between fft data elements is 16

	start_timer 19
	mov	al, 16			;; Load loop counter
c1b:	disp eight_reals_last_unfft, 8, 16*blkdst, 32*blkdst
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test loop counter
	jnc	c1b			;; Iterate if necessary
	IFDEF PFETCH
	prefetcht1 [esi-32+clm*32]
;	prefetcht1 [esi-32+clm*32+16*blkdst]
	prefetcht1 [esi-32+clm*32+32*blkdst]
;	prefetcht1 [esi-32+clm*32+48*blkdst]
	ENDIF
	loopclm	dl, clm, c1b
	lea	esi, [esi-clm*2*dist1+blkdst];; Next source pointer
	dec	al			;; Test loop counter
	jnz	c1b			;; Iterate if necessary
	lea	esi, [esi-16*blkdst]	;; Restore source pointer
	end_timer 19

;; Normalize these values

	start_timer 20
	mov	eax, NORMRTN		;; Addr of normalization routine
	call	eax
	sub	eax, eax
	end_timer 20

;; Have we been given permission to start the FFT on the result?
;; If so, do so now while the data is in the L2 cache.

	cmp	POSTFFT, 0		;; Test flag
	je	no_fft			;; Skip FFT code if flag not set
	cmp	edx, p2cl/128*65536+(128/clm-4/clm)*256;; Delay fft on 8 sets
	jg	no_fft			;; Jump if delaying FFT
	copy_3_words clm, 0

;; Do FFT levels 1,2,3
;;
;; On input the 32-byte cache lines hold these data values:
;;	0	16K	1	16K+1
;;	256
;;	...
;;	16K-256
;; On output the 32-byte cache lines hold these data values:
;;	0	4K	1	4K+1
;;	256	...
;;	...
;;	8K	...
;;	...

;; Do 16 eight_reals_first_fft macros
;;	distance between fft data elements is 16

b0b:	start_timer 21
	mov	ebx, DIST_TO_FFTSRCARG
	mov	al, 16			;; 16 iterations
b1b:	disp eight_reals_first_fft, 16*blkdst, 32*blkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	add	dl, 256/2/clm		;; Test loop counter
	jnc	b1b			;; Iterate if necessary
	lea	esi, [esi-clm*2*dist1+blkdst];; Next source pointer
	dec	al			;; Test loop counter
	jnz	b1b			;; Iterate if necessary
	lea	esi, [esi-16*blkdst]	;; Restore source pointer
	end_timer 21

;; Do FFT levels 4,5
;;
;; On input the 32-byte cache lines hold these data values:
;;	0	4K	1	4K+1
;;	256	...
;;	...
;;	8K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	1K	1	1K+1
;;	256	...
;;	...
;;	2K	...
;;	...

;; Do 4 four_real_four_semireal_fft macros
;;	distance between fft data elements is 4

	start_timer 22
	mov	al, 4			;; Load loop counter
b4b:	disp four_real_four_semireal_fft, 4*blkdst, 8*blkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test inner loop counter
	jnc	b4b			;; Iterate if necessary
	IFDEF PFETCH
;	prefetcht1 [esi-32+clm*32]
	prefetcht1 [esi-32+clm*32+4*blkdst]
;	prefetcht1 [esi-32+clm*32+8*blkdst]
	prefetcht1 [esi-32+clm*32+12*blkdst]
	ENDIF
	loopclm	dl, clm, b4b
	lea	esi, [esi-clm*2*dist1+blkdst];; Next source pointer
	dec	al
	jnz	b4b
	lea	esi, [esi-4*blkdst+16*blkdst];; Next source pointer

;; Do 12 four_complex_fft macros
;;	distance between fft data elements is 4

	mov	edi, sincos2		;; Load sin/cos pointer
	mov	eax, 3			;; 3 iterations of 4
b5b:	disp four_complex_fft, 4*blkdst, 8*blkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test inner loop counter
	jnc	b5b			;; Iterate if necessary
	IFDEF PFETCH
;	prefetcht1 [esi-32+clm*32]
	prefetcht1 [esi-32+clm*32+4*blkdst]
;	prefetcht1 [esi-32+clm*32+8*blkdst]
	prefetcht1 [esi-32+clm*32+12*blkdst]
	ENDIF
	loopclm	dl, clm, b5b
	lea	esi, [esi-clm*2*dist1+blkdst];; Next source pointer
	add	ah, 256/4		;; Test middle loop counter
	jnc	b5b			;; Iterate if necessary
	lea	esi, [esi-4*blkdst+16*blkdst];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al
	jnz	b5b
	lea	esi, [esi-4*16*blkdst]	;; Restore source pointer
	end_timer 22

;; Do FFT levels 6,7
;;
;; On input the 32-byte cache lines hold these data values:
;;	0	1K	1	1K+1
;;	256	...
;;	...
;;	2K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	256	1	257
;;	512	...
;;	...

;; Do 1 four_real_four_semireal_fft macros
;;	distance between fft data elements is 1

	start_timer 23
b2b:	disp four_real_four_semireal_fft, blkdst, 2*blkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test inner loop counter
	jnc	b2b			;; Iterate if necessary
	IFDEF PFETCH
;	prefetcht1 [esi-32+clm*32]
	prefetcht1 [esi-32+clm*32+blkdst]
;	prefetcht1 [esi-32+clm*32+2*blkdst]
	prefetcht1 [esi-32+clm*32+3*blkdst]
	ENDIF
	loopclm	dl, clm, b2b
	lea	esi, [esi-clm*2*dist1+4*blkdst];; Next source pointer

;; Do 15 four_complex_fft macros
;;	distance between fft data elements is 1

	mov	edi, sincos1		;; Load sin/cos pointer
	mov	al, 15			;; 15 iterations
b3b:	disp four_complex_fft, blkdst, 2*blkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test inner loop counter
	jnc	b3b			;; Iterate if necessary
	IFDEF PFETCH
;	prefetcht1 [esi-32+clm*32]
	prefetcht1 [esi-32+clm*32+blkdst]
;	prefetcht1 [esi-32+clm*32+2*blkdst]
	prefetcht1 [esi-32+clm*32+3*blkdst]
	ENDIF
	loopclm	dl, clm, b3b
	lea	esi, [esi-clm*2*dist1+4*blkdst];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al
	jnz	b3b
	lea	esi, [esi-16*4*blkdst]	;; Restore source pointer
	end_timer 23

;; Work on next 4 sets of 32 values

no_fft:	lea	esi, [esi+clm*2*dist1]	;; Next source pointer
	dec	dh			;; Test loop counter
	jnz	short more		;; Do more FFTs
	lea	esi, [esi+64]		;; Skip pad every 128 cache lines (4KB)
	sub	edx, 65536		;; Test loop counter
	jz	short fftdn		;; Jump if done looping
	mov	dh, 128/clm		;; Restore cache line counter
more:	cmp	norm_ptr1, 0		;; What was the loop start point?
	jne	c0b			;; Do inverse-FFT/norm/FFT iteration
	jmp	b0b			;; Do another FFT-only iteration
fftdn:	lea	esi, [esi-p2cl*2*dist1-p2cl/128*64];; Restore source pointer
	cmp	esi, DESTARG		;; If esi was not restored, then we 
	jne	short fftdn1		;; just finished skipped postfft data
	sub	ebx, ebx
	mov	DIST_TO_FFTSRCARG, ebx
	cmp	norm_ptr1, 0		;; What was the loop start point?
	je	pass2			;; FFT-only, go do pass 2

;; Split the carries into high and low carries (so they do not exceed
;; the desired number of bits per FFT word.  Then add the carries back to the
;; FFT data.

	start_timer 28
	call	gw_carries
	end_timer 28

;; Have we been given permission to start the FFT on the result?
;; If so, set loop to FFT the first few cache lines now that the carries
;; have been added in.

	cmp	POSTFFT, 0		;; Test flag
	je	short fftdn1		;; Skip FFT code if flag not set
	mov	esi, DESTARG		;; Load FFT data address
	mov	edx, 65536+4/clm*256	;; Do 8 previously skipped cache lines
	mov	norm_ptr1, 0		;; Set flag so only forward FFT is done
	copy_4_words
	jmp	b0b

;; Now add in the FFT'ed carries, cleanup and return

fftdn1:	end_timer 1
	jmp	gw_finish_mult
	ENDM


;; Perform a 32K-element negacyclic FFT.  This is done in two passes.  Pass 1
;; does 7 levels, pass 2 uses common code to perform the last 8 levels.

pass1levels7complex4 MACRO pass2_macro, p2cl, clm
	LOCAL	b0b, b1b, b3b, b5b, c0b, c1b, c3b, c5b
	LOCAL	pass2, no_fft, more, fftdn, fftdn1

;; Call FFT code unless the FFT has already been started

	clear_timers
	cmp	DWORD PTR [esi-28][ebx], 0 ;; Test FFT-started flag
	jne	pass2			;; Jump is FFT already started
	mov	norm_ptr1, 0		;; Set FFT-ing flag
	mov	edx, p2cl/128*65536+128/clm*256;; Load loop counter
	jmp	b0b			;; Do the pass 1 fft

;; Do the last 8 FFT levels
;;	do 64 pass2_8_levels_complex

pass2:	start_timer 0
	CALLP	pass2_macro
	end_timer 0
	cmp	ffttype, 1		;; We're done if FFTing only
	je	gw_finish_fft

;; Do the inverse FFT

	start_timer 1
	fldz				;; Init SUMOUT
	fstp	SUMOUT
	mov	edx, norm_biglit_array	;; Addr of the big/little flags array
	mov	norm_ptr1, edx		;; Save ptr
	mov	edx, norm_col_mults	;; Addr of the column multipliers
	mov	norm_ptr2, edx		;; Save ptr
	mov	edx, p2cl/128*65536+128/clm*256;; Load loop counter

;; Do inverse FFT levels 6,7
;; On input the 32-byte cache lines hold these data values:
;;	0	256	1	257
;;	512	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	1K	1	1K+1
;;	256	...
;;	...
;;	2K	...
;;	...

;; Do 16 four_complex_unfft macros
;;	distance between fft data elements is 1

c0b:	start_timer 17
	mov	edi, sincos1		;; Load sin/cos pointer
	mov	al, 16			;; 16 iterations
c3b:	disp four_complex_unfft, 8, blkdst, 2*blkdst
	lea	esi, [esi+dist1]	;; Next source pointer
	add	dl, 256/2/clm		;; Test loop counter
	jnc	c3b			;; Iterate if necessary
	lea	esi, [esi-clm*2*dist1+4*blkdst];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al			;; Test outer loop counter
	jnz	c3b			;; Iterate if necessary
	lea	esi, [esi-16*4*blkdst]	;; Restore source pointer
	end_timer 17

;; Do inverse FFT levels 4,5
;; On input the 32-byte cache lines hold these data values:
;;	0	1K	1	1K+1
;;	256	...
;;	...
;;	2K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	4K	1	4K+1
;;	256	...
;;	...
;;	8K	...
;;	...

;; Do 16 four_complex_unfft macros
;;	distance between fft data elements is 4

	start_timer 18
	mov	edi, sincos2		;; Load sin/cos pointer
	mov	eax, 4			;; 4 iterations of 4
c5b:	disp four_complex_unfft, 8, 4*blkdst, 8*blkdst
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test inner loop counter
	jnc	c5b			;; Iterate if necessary
	IFDEF PFETCH
	prefetcht1 [esi-32+clm*32]
;	prefetcht1 [esi-32+clm*32+4*blkdst]
	prefetcht1 [esi-32+clm*32+8*blkdst]
;	prefetcht1 [esi-32+clm*32+12*blkdst]
	ENDIF
	loopclm	dl, clm, c5b
	lea	esi, [esi-clm*2*dist1+blkdst];; Next source pointer
	add	ah, 256/4		;; Test middle loop counter
	jnc	c5b			;; Iterate if necessary
	lea	esi, [esi-4*blkdst+16*blkdst];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al
	jnz	c5b
	lea	esi, [esi-4*16*blkdst]	;; Restore source pointer
	end_timer 18

;; Do inverse FFT levels 1,2,3
;; On input the 32-byte cache lines hold these data values:
;;	0	4K	1	4K+1
;;	256	...
;;	...
;;	8K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	16K	1	16K+1
;;	256
;;	...
;;	16K-256

;; Do 16 four_complex_last_unfft macros
;;	distance between fft data elements is 16

	start_timer 19
	mov	edi, plus1_premults	;; Address of premultiplier table
	mov	al, 16			;; Load loop counter
c1b:	disp four_complex_last_unfft, 8, 16*blkdst, 32*blkdst
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test loop counter
	jnc	c1b			;; Iterate if necessary
	IFDEF PFETCH
	prefetcht1 [esi-32+clm*32]
;	prefetcht1 [esi-32+clm*32+16*blkdst]
	prefetcht1 [esi-32+clm*32+32*blkdst]
;	prefetcht1 [esi-32+clm*32+48*blkdst]
	ENDIF
	loopclm	dl, clm, c1b
	lea	esi, [esi-clm*2*dist1+blkdst];; Next source pointer
	lea	edi, [edi+64]		;; Next premultiplier pointer
	dec	al			;; Test loop counter
	jnz	c1b			;; Iterate if necessary
	lea	esi, [esi-16*blkdst]	;; Restore source pointer
	end_timer 19

;; Normalize these values

	start_timer 20
	mov	eax, NORMRTN		;; Addr of normalization routine
	call	eax
	sub	eax, eax
	end_timer 20

;; Have we been given permission to start the FFT on the result?
;; If so, do so now while the data is in the L2 cache.

	cmp	POSTFFT, 0		;; Test flag
	je	no_fft			;; Skip FFT code if flag not set
	cmp	edx, p2cl/128*65536+(128/clm-4/clm)*256;; Delay fft on 8 sets
	jg	no_fft			;; Jump if delaying FFT
	copy_3_words clm, 0

;; Do FFT levels 1,2,3
;;
;; On input the 32-byte cache lines hold these data values:
;;	0	16K	1	16K+1
;;	256
;;	...
;;	16K-256
;; On output the 32-byte cache lines hold these data values:
;;	0	4K	1	4K+1
;;	256	...
;;	...
;;	8K	...
;;	...

;; Do 16 four_complex_first_fft macros
;;	distance between fft data elements is 16

b0b:	start_timer 21
	mov	ebx, DIST_TO_FFTSRCARG
	mov	edi, plus1_premults	;; Address of premultiplier table
	mov	al, 16			;; 16 iterations
b1b:	disp four_complex_first_fft, 16*blkdst, 32*blkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	add	dl, 256/2/clm		;; Test loop counter
	jnc	b1b			;; Iterate if necessary
	lea	esi, [esi-clm*2*dist1+blkdst];; Next source pointer
	lea	edi, [edi+64]		;; Next premultiplier pointer
	dec	al			;; Test loop counter
	jnz	b1b			;; Iterate if necessary
	lea	esi, [esi-16*blkdst]	;; Restore source pointer
	end_timer 21

;; Do FFT levels 4,5
;;
;; On input the 32-byte cache lines hold these data values:
;;	0	4K	1	4K+1
;;	256	...
;;	...
;;	8K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	1K	1	1K+1
;;	256	...
;;	...
;;	2K	...
;;	...

;; Do 16 four_complex_fft macros
;;	distance between fft data elements is 4

	start_timer 22
	mov	edi, sincos2		;; Load sin/cos pointer
	mov	eax, 4			;; 4 iterations of 4
b5b:	disp four_complex_fft, 4*blkdst, 8*blkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test inner loop counter
	jnc	b5b			;; Iterate if necessary
	IFDEF PFETCH
;	prefetcht1 [esi-32+clm*32]
	prefetcht1 [esi-32+clm*32+4*blkdst]
;	prefetcht1 [esi-32+clm*32+8*blkdst]
	prefetcht1 [esi-32+clm*32+12*blkdst]
	ENDIF
	loopclm	dl, clm, b5b
	lea	esi, [esi-clm*2*dist1+blkdst];; Next source pointer
	add	ah, 256/4		;; Test middle loop counter
	jnc	b5b			;; Iterate if necessary
	lea	esi, [esi-4*blkdst+16*blkdst];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al
	jnz	b5b
	lea	esi, [esi-4*16*blkdst]	;; Restore source pointer
	end_timer 22

;; Do FFT levels 6,7
;;
;; On input the 32-byte cache lines hold these data values:
;;	0	1K	1	1K+1
;;	256	...
;;	...
;;	2K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	256	1	257
;;	512	...
;;	...

;; Do 16 four_complex_fft macros
;;	distance between fft data elements is 1

	start_timer 23
	mov	edi, sincos1		;; Load sin/cos pointer
	mov	al, 16			;; 16 iterations
b3b:	disp four_complex_fft, blkdst, 2*blkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test inner loop counter
	jnc	b3b			;; Iterate if necessary
	IFDEF PFETCH
;	prefetcht1 [esi-32+clm*32]
	prefetcht1 [esi-32+clm*32+blkdst]
;	prefetcht1 [esi-32+clm*32+2*blkdst]
	prefetcht1 [esi-32+clm*32+3*blkdst]
	ENDIF
	loopclm	dl, clm, b3b
	lea	esi, [esi-clm*2*dist1+4*blkdst];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al
	jnz	b3b
	lea	esi, [esi-16*4*blkdst]	;; Restore source pointer
	end_timer 23

;; Work on next 4 sets of 32 values

no_fft:	lea	esi, [esi+clm*2*dist1]	;; Next source pointer
	dec	dh			;; Test loop counter
	jnz	short more		;; Do more FFTs
	lea	esi, [esi+64]		;; Skip pad every 128 cache lines (4KB)
	sub	edx, 65536		;; Test loop counter
	jz	short fftdn		;; Jump if done looping
	mov	dh, 128/clm		;; Restore cache line counter
more:	cmp	norm_ptr1, 0		;; What was the loop start point?
	jne	c0b			;; Do inverse-FFT/norm/FFT iteration
	jmp	b0b			;; Do another FFT-only iteration
fftdn:	lea	esi, [esi-p2cl*2*dist1-p2cl/128*64];; Restore source pointer
	cmp	esi, DESTARG		;; If esi was not restored, then we 
	jne	short fftdn1		;; just finished skipped postfft data
	sub	ebx, ebx
	mov	DIST_TO_FFTSRCARG, ebx
	cmp	norm_ptr1, 0		;; What was the loop start point?
	je	pass2			;; FFT-only, go do pass 2

;; Split the carries into high and low carries (so they do not exceed
;; the desired number of bits per FFT word.  Then add the carries back to the
;; FFT data.

	start_timer 28
	call	gw_carries
	end_timer 28

;; Have we been given permission to start the FFT on the result?
;; If so, set loop to FFT the first few cache lines now that the carries
;; have been added in.

	cmp	POSTFFT, 0		;; Test flag
	je	short fftdn1		;; Skip FFT code if flag not set
	mov	esi, DESTARG		;; Load FFT data address
	mov	edx, 65536+4/clm*256	;; Do 8 previously skipped cache lines
	mov	norm_ptr1, 0		;; Set flag so only forward FFT is done
	copy_4_words
	jmp	b0b

;; Now add in the FFT'ed carries, cleanup and return

fftdn1:	end_timer 1
	jmp	gw_finish_mult
	ENDM


;; Perform a 40K-element FFT.  This is done in two passes.  Pass 1
;; does 8 levels, pass 2 uses common code to perform the last 8 levels.

pass1levels8pfa5 MACRO pass2_macro, p2cl, clm
	LOCAL	b0b, b1b, b2b, b3b, b4b, b5b, b6b, b7b
	LOCAL	c0b, c1b, c2b, c3b, c4b, c5b, c6b, c7b
	LOCAL	pass2, no_fft, more, fftdn, fftdn1

;; Call FFT code unless the FFT has already been started

	clear_timers
	cmp	DWORD PTR [esi-28][ebx], 0 ;; Test FFT-started flag
	jne	pass2			;; Jump is FFT already started
	mov	norm_ptr1, 0		;; Set FFT-ing flag
	mov	edx, p2cl/128*65536+128/clm*256;; Load loop counter
	jmp	b0b			;; Do the pass 1 fft

;; Do the last 8 FFT levels
;;	do 1 pass2_8_levels_real and 79 pass2_8_levels_complex

pass2:	start_timer 0
	CALLP	pass2_macro
	end_timer 0
	cmp	ffttype, 1		;; We're done if FFTing only
	je	gw_finish_fft

;; Do the inverse FFT

	start_timer 1
	fldz				;; Init SUMOUT
	fstp	SUMOUT
	mov	edx, norm_biglit_array	;; Addr of the big/little flags array
	mov	norm_ptr1, edx		;; Save ptr
	mov	edx, norm_col_mults	;; Addr of the column multipliers
	mov	norm_ptr2, edx		;; Save ptr
	mov	edx, p2cl/128*65536+128/clm*256;; Load loop counter

;; Do inverse FFT levels 7,8
;; On input the 32-byte cache lines hold these data values:
;;	0	256	1	257
;;	512	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	1K	1	1K+1
;;	256	...
;;	...
;;	2K	...
;;	...

;; Do 1 four_real_four_semireal_unfft macros
;;	distance between fft data elements is 1

c0b:	start_timer 16
c2b:	disp four_real_four_semireal_unfft, 8, blkdst, 2*blkdst
	lea	esi, [esi+dist1]	;; Next source pointer
	add	dl, 256/2/clm		;; Test inner loop counter
	jnc	c2b			;; Iterate if necessary
	lea	esi, [esi-clm*2*dist1+4*blkdst];; Next source pointer

;; Do 19 four_complex_unfft macros
;;	distance between fft data elements is 1

	mov	edi, sincos1		;; Load sin/cos pointer
	mov	al, 19			;; 19 iterations
c3b:	disp four_complex_unfft, 8, blkdst, 2*blkdst
	lea	esi, [esi+dist1]	;; Next source pointer
	add	dl, 256/2/clm		;; Test loop counter
	jnc	c3b			;; Iterate if necessary
	lea	esi, [esi-clm*2*dist1+4*blkdst];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al			;; Test outer loop counter
	jnz	c3b			;; Iterate if necessary
	lea	esi, [esi-20*4*blkdst]	;; Restore source pointer
	end_timer 16

;; Do inverse FFT levels 5,6
;; On input the 32-byte cache lines hold these data values:
;;	0	1K	1	1K+1
;;	256	...
;;	...
;;	2K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	4K	1	4K+1
;;	256	...
;;	...
;;	8K	...
;;	...

;; Do 4 four_real_four_semireal_unfft macros
;;	distance between fft data elements is 4

	start_timer 17
	mov	al, 4			;; Load loop counter
c4b:	disp four_real_four_semireal_unfft, 8, 4*blkdst, 8*blkdst
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test inner loop counter
	jnc	c4b			;; Iterate if necessary
	IFDEF PFETCH
	prefetcht1 [esi-32+clm*32]
;	prefetcht1 [esi-32+clm*32+4*blkdst]
	prefetcht1 [esi-32+clm*32+8*blkdst]
;	prefetcht1 [esi-32+clm*32+12*blkdst]
	ENDIF
	loopclm	dl, clm, c4b
	lea	esi, [esi-clm*2*dist1+blkdst];; Next source pointer
	dec	al
	jnz	c4b
	lea	esi, [esi-4*blkdst+16*blkdst];; Next source pointer

;; Do 16 four_complex_unfft macros
;;	distance between fft data elements is 4

	mov	edi, sincos2		;; Load sin/cos pointer
	mov	eax, 4			;; 4 iterations of 4
c5b:	disp four_complex_unfft, 8, 4*blkdst, 8*blkdst
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test inner loop counter
	jnc	c5b			;; Iterate if necessary
	IFDEF PFETCH
	prefetcht1 [esi-32+clm*32]
;	prefetcht1 [esi-32+clm*32+4*blkdst]
	prefetcht1 [esi-32+clm*32+8*blkdst]
;	prefetcht1 [esi-32+clm*32+12*blkdst]
	ENDIF
	loopclm	dl, clm, c5b
	lea	esi, [esi-clm*2*dist1+blkdst];; Next source pointer
	add	ah, 256/4		;; Test middle loop counter
	jnc	c5b			;; Iterate if necessary
	lea	esi, [esi-4*blkdst+16*blkdst];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al
	jnz	c5b
	lea	esi, [esi-5*16*blkdst]	;; Restore source pointer
	end_timer 17

;; Do inverse FFT levels 4
;; On input the 32-byte cache lines hold these data values:
;;	0	4K	1	4K+1
;;	256	...
;;	...
;;	8K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	8K	1	8K+1
;;	256	...
;;	...
;;	16K	...
;;	...

;; Do 4 eight_reals_unfft_1 macros
;;	distance between fft data elements is 16

	start_timer 18
	mov	al, 4			;; Load loop counter
c6b:	disp eight_reals_unfft_1, 4*blkdst, 8*blkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test inner loop counter
	jnc	c6b			;; Iterate if necessary
	IFDEF PFETCH
	prefetcht1 [esi-32+clm*32]
;	prefetcht1 [esi-32+clm*32+4*blkdst]
	prefetcht1 [esi-32+clm*32+8*blkdst]
;	prefetcht1 [esi-32+clm*32+12*blkdst]
	ENDIF
	loopclm	dl, clm, c6b
	lea	esi, [esi-clm*2*dist1+blkdst];; Next source pointer
	dec	al
	jnz	c6b
	lea	esi, [esi-4*blkdst+16*blkdst];; Next source pointer

;; Do 16 four_complex_unfft_1 macros
;;	distance between fft data elements is 16

	mov	edi, sincos3		;; Load sin/cos pointer
	mov	eax, 2			;; 2 iterations of 8
c7b:	disp four_complex_unfft_1, 8*blkdst, 8, 16*blkdst
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test inner loop counter
	jnc	c7b			;; Iterate if necessary
	IFDEF PFETCH
	prefetcht1 [esi-32+clm*32]
;	prefetcht1 [esi-32+clm*32+8*blkdst]
	prefetcht1 [esi-32+clm*32+16*blkdst]
;	prefetcht1 [esi-32+clm*32+24*blkdst]
	ENDIF
	loopclm	dl, clm, c7b
	lea	esi, [esi-clm*2*dist1+blkdst];; Next source pointer
	add	ah, 256/8		;; Test middle loop counter
	jnc	c7b			;; Iterate if necessary
	lea	esi, [esi-8*blkdst+32*blkdst];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al
	jnz	c7b
	lea	esi, [esi-2*32*blkdst-16*blkdst];; Restore source pointer
	end_timer 18

;; Do inverse FFT levels 1,2,3
;; On input the 32-byte cache lines hold these data values:
;;	0	8K	1	8K+1
;;	256	...
;;	...
;;	16K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	20K	1	20K+1
;;	256
;;	...
;;	20K-256

;; Do 32 five_reals_last_unfft macros
;;	distance between fft data elements is 32

	start_timer 19
	mov	al, 16			;; Load loop counter
c1b:	two_five_reals_last_unfft 8, 16*blkdst
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test loop counter
	jnc	c1b			;; Iterate if necessary
	IFDEF PFETCH
	prefetcht1 [esi-32+clm*32]
;	prefetcht1 [esi-32+clm*32+16*blkdst]
	prefetcht1 [esi-32+clm*32+32*blkdst]
;	prefetcht1 [esi-32+clm*32+48*blkdst]
	prefetcht1 [esi-32+clm*32+64*blkdst]
	ENDIF
	loopclm	dl, clm, c1b
	lea	esi, [esi-clm*2*dist1+blkdst];; Next source pointer
	dec	al			;; Test loop counter
	jnz	c1b			;; Iterate if necessary
	lea	esi, [esi-16*blkdst]	;; Restore source pointer
	end_timer 19

;; Normalize these values

	start_timer 20
	mov	eax, NORMRTN		;; Addr of normalization routine
	call	eax
	sub	eax, eax
	end_timer 20

;; Have we been given permission to start the FFT on the result?
;; If so, do so now while the data is in the L2 cache.

	cmp	POSTFFT, 0		;; Test flag
	je	no_fft			;; Skip FFT code if flag not set
	cmp	edx, p2cl/128*65536+(128/clm-4/clm)*256;; Delay fft on 8 sets
	jg	no_fft			;; Jump if delaying FFT
	copy_3_words clm, 0

;; Do FFT levels 1,2,3
;;
;; On input the 32-byte cache lines hold these data values:
;;	0	20K	1	20K+1
;;	256
;;	...
;;	20K-256
;; On output the 32-byte cache lines hold these data values:
;;	0	8K	1	8K+1
;;	256	...
;;	...
;;	16K	...
;;	...

;; Do 32 five_reals_first_fft macros
;;	distance between fft data elements is 32

b0b:	start_timer 21
	mov	ebx, DIST_TO_FFTSRCARG
	mov	al, 16			;; 16 iterations
b1b:	two_five_reals_first_fft 16*blkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	add	dl, 256/2/clm		;; Test loop counter
	jnc	b1b			;; Iterate if necessary
	lea	esi, [esi-clm*2*dist1+blkdst];; Next source pointer
	dec	al			;; Test loop counter
	jnz	b1b			;; Iterate if necessary
	lea	esi, [esi-16*blkdst]	;; Restore source pointer
	end_timer 21

;; Do FFT levels 4
;;
;; On input the 32-byte cache lines hold these data values:
;;	0	8K	1	8K+1
;;	256	...
;;	...
;;	16K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	4K	1	4K+1
;;	256	...
;;	...
;;	8K	...
;;	...

;; Do 4 eight_reals_fft_1 macros
;;	distance between fft data elements is 16

	start_timer 22
	mov	al, 4			;; Load loop counter
b6b:	disp eight_reals_fft_1, 4*blkdst, 8*blkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test inner loop counter
	jnc	b6b			;; Iterate if necessary
	IFDEF PFETCH
;	prefetcht1 [esi-32+clm*32]
	prefetcht1 [esi-32+clm*32+4*blkdst]
;	prefetcht1 [esi-32+clm*32+8*blkdst]
	prefetcht1 [esi-32+clm*32+12*blkdst]
	ENDIF
	loopclm	dl, clm, b6b
	lea	esi, [esi-clm*2*dist1+blkdst];; Next source pointer
	dec	al
	jnz	b6b
	lea	esi, [esi-4*blkdst+16*blkdst];; Next source pointer

;; Do 16 four_complex_fft_1 macros
;;	distance between fft data elements is 16

	mov	edi, sincos3		;; Load sin/cos pointer
	mov	eax, 2			;; 2 iterations of 8
b7b:	disp four_complex_fft_1, 8*blkdst, 16*blkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test inner loop counter
	jnc	b7b			;; Iterate if necessary
	IFDEF PFETCH
;	prefetcht1 [esi-32+clm*32]
	prefetcht1 [esi-32+clm*32+8*blkdst]
;	prefetcht1 [esi-32+clm*32+16*blkdst]
	prefetcht1 [esi-32+clm*32+24*blkdst]
	ENDIF
	loopclm	dl, clm, b7b
	lea	esi, [esi-clm*2*dist1+blkdst];; Next source pointer
	add	ah, 256/8		;; Test middle loop counter
	jnc	b7b			;; Iterate if necessary
	lea	esi, [esi-8*blkdst+32*blkdst];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al
	jnz	b7b
	lea	esi, [esi-2*32*blkdst-16*blkdst];; Restore source pointer
	end_timer 22

;; Do FFT levels 5,6
;;
;; On input the 32-byte cache lines hold these data values:
;;	0	4K	1	4K+1
;;	256	...
;;	...
;;	8K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	1K	1	1K+1
;;	256	...
;;	...
;;	2K	...
;;	...

;; Do 4 four_real_four_semireal_fft macros
;;	distance between fft data elements is 4

	start_timer 23
	mov	al, 4			;; Load loop counter
b4b:	disp four_real_four_semireal_fft, 4*blkdst, 8*blkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test inner loop counter
	jnc	b4b			;; Iterate if necessary
	IFDEF PFETCH
;	prefetcht1 [esi-32+clm*32]
	prefetcht1 [esi-32+clm*32+4*blkdst]
;	prefetcht1 [esi-32+clm*32+8*blkdst]
	prefetcht1 [esi-32+clm*32+12*blkdst]
	ENDIF
	loopclm	dl, clm, b4b
	lea	esi, [esi-clm*2*dist1+blkdst];; Next source pointer
	dec	al
	jnz	b4b
	lea	esi, [esi-4*blkdst+16*blkdst];; Next source pointer

;; Do 16 four_complex_fft macros
;;	distance between fft data elements is 4

	mov	edi, sincos2		;; Load sin/cos pointer
	mov	eax, 4			;; 4 iterations of 4
b5b:	disp four_complex_fft, 4*blkdst, 8*blkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test inner loop counter
	jnc	b5b			;; Iterate if necessary
	IFDEF PFETCH
;	prefetcht1 [esi-32+clm*32]
	prefetcht1 [esi-32+clm*32+4*blkdst]
;	prefetcht1 [esi-32+clm*32+8*blkdst]
	prefetcht1 [esi-32+clm*32+12*blkdst]
	ENDIF
	loopclm	dl, clm, b5b
	lea	esi, [esi-clm*2*dist1+blkdst];; Next source pointer
	add	ah, 256/4		;; Test middle loop counter
	jnc	b5b			;; Iterate if necessary
	lea	esi, [esi-4*blkdst+16*blkdst];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al
	jnz	b5b
	lea	esi, [esi-5*16*blkdst]	;; Restore source pointer
	end_timer 23

;; Do FFT levels 7,8
;;
;; On input the 32-byte cache lines hold these data values:
;;	0	1K	1	1K+1
;;	256	...
;;	...
;;	2K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	256	1	257
;;	512	...
;;	...

;; Do 1 four_real_four_semireal_fft macros
;;	distance between fft data elements is 1

	start_timer 24
b2b:	disp four_real_four_semireal_fft, blkdst, 2*blkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test inner loop counter
	jnc	b2b			;; Iterate if necessary
	IFDEF PFETCH
;	prefetcht1 [esi-32+clm*32]
	prefetcht1 [esi-32+clm*32+blkdst]
;	prefetcht1 [esi-32+clm*32+2*blkdst]
	prefetcht1 [esi-32+clm*32+3*blkdst]
	ENDIF
	loopclm	dl, clm, b2b
	lea	esi, [esi-clm*2*dist1+4*blkdst];; Next source pointer

;; Do 19 four_complex_fft macros
;;	distance between fft data elements is 1

	mov	edi, sincos1		;; Load sin/cos pointer
	mov	al, 19			;; 19 iterations
b3b:	disp four_complex_fft, blkdst, 2*blkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test inner loop counter
	jnc	b3b			;; Iterate if necessary
	IFDEF PFETCH
;	prefetcht1 [esi-32+clm*32]
	prefetcht1 [esi-32+clm*32+blkdst]
;	prefetcht1 [esi-32+clm*32+2*blkdst]
	prefetcht1 [esi-32+clm*32+3*blkdst]
	ENDIF
	loopclm	dl, clm, b3b
	lea	esi, [esi-clm*2*dist1+4*blkdst];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al
	jnz	b3b
	lea	esi, [esi-20*4*blkdst]	;; Restore source pointer
	end_timer 24

;; Work on next 4 sets of 32 values

no_fft:	lea	esi, [esi+clm*2*dist1]	;; Next source pointer
	dec	dh			;; Test loop counter
	jnz	short more		;; Do more FFTs
	lea	esi, [esi+64]		;; Skip pad every 128 cache lines (4KB)
	sub	edx, 65536		;; Test loop counter
	jz	short fftdn		;; Jump if done looping
	mov	dh, 128/clm		;; Restore cache line counter
more:	cmp	norm_ptr1, 0		;; What was the loop start point?
	jne	c0b			;; Do inverse-FFT/norm/FFT iteration
	jmp	b0b			;; Do another FFT-only iteration
fftdn:	lea	esi, [esi-p2cl*2*dist1-p2cl/128*64];; Restore source pointer
	cmp	esi, DESTARG		;; If esi was not restored, then we 
	jne	short fftdn1		;; just finished skipped postfft data
	sub	ebx, ebx
	mov	DIST_TO_FFTSRCARG, ebx
	cmp	norm_ptr1, 0		;; What was the loop start point?
	je	pass2			;; FFT-only, go do pass 2

;; Split the carries into high and low carries (so they do not exceed
;; the desired number of bits per FFT word.  Then add the carries back to the
;; FFT data.

	start_timer 28
	call	gw_carries
	end_timer 28

;; Have we been given permission to start the FFT on the result?
;; If so, set loop to FFT the first few cache lines now that the carries
;; have been added in.

	cmp	POSTFFT, 0		;; Test flag
	je	short fftdn1		;; Skip FFT code if flag not set
	mov	esi, DESTARG		;; Load FFT data address
	mov	edx, 65536+4/clm*256	;; Do 8 previously skipped cache lines
	mov	norm_ptr1, 0		;; Set flag so only forward FFT is done
	copy_4_words
	jmp	b0b

;; Now add in the FFT'ed carries, cleanup and return

fftdn1:	end_timer 1
	jmp	gw_finish_mult
	ENDM


;; Perform a 48K-element FFT.  This is done in two passes.  Pass 1
;; does 8 levels, pass 2 uses common code to perform the last 8 levels.

pass1levels8pfa6 MACRO pass2_macro, p2cl, clm
	LOCAL	b0b, b1b, b2b, b3b, b4b, b5b, b6b, b7b
	LOCAL	c0b, c1b, c2b, c3b, c4b, c5b, c6b, c7b
	LOCAL	pass2, no_fft, more, fftdn, fftdn1

;; Call FFT code unless the FFT has already been started

	clear_timers
	cmp	DWORD PTR [esi-28][ebx], 0 ;; Test FFT-started flag
	jne	pass2			;; Jump is FFT already started
	mov	norm_ptr1, 0		;; Set FFT-ing flag
	mov	edx, p2cl/128*65536+128/clm*256;; Load loop counter
	jmp	b0b			;; Do the pass 1 fft

;; Do the last 8 FFT levels
;;	do 1 pass2_8_levels_real and 95 pass2_8_levels_complex

pass2:	start_timer 0
	CALLP	pass2_macro
	end_timer 0
	cmp	ffttype, 1		;; We're done if FFTing only
	je	gw_finish_fft

;; Do the inverse FFT

	start_timer 1
	fldz				;; Init SUMOUT
	fstp	SUMOUT
	mov	edx, norm_biglit_array	;; Addr of the big/little flags array
	mov	norm_ptr1, edx		;; Save ptr
	mov	edx, norm_col_mults	;; Addr of the column multipliers
	mov	norm_ptr2, edx		;; Save ptr
	mov	edx, p2cl/128*65536+128/clm*256;; Load loop counter

;; Do inverse FFT levels 7,8
;; On input the 32-byte cache lines hold these data values:
;;	0	256	1	257
;;	512	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	1K	1	1K+1
;;	256	...
;;	...
;;	2K	...
;;	...

;; Do 1 four_real_four_semireal_unfft macros
;;	distance between fft data elements is 1

c0b:	start_timer 16
c2b:	disp four_real_four_semireal_unfft, 8, blkdst, 2*blkdst
	lea	esi, [esi+dist1]	;; Next source pointer
	add	dl, 256/2/clm		;; Test inner loop counter
	jnc	c2b			;; Iterate if necessary
	lea	esi, [esi-clm*2*dist1+4*blkdst];; Next source pointer

;; Do 23 four_complex_unfft macros
;;	distance between fft data elements is 1

	mov	edi, sincos1		;; Load sin/cos pointer
	mov	al, 23			;; 23 iterations
c3b:	disp four_complex_unfft, 8, blkdst, 2*blkdst
	lea	esi, [esi+dist1]	;; Next source pointer
	add	dl, 256/2/clm		;; Test loop counter
	jnc	c3b			;; Iterate if necessary
	lea	esi, [esi-clm*2*dist1+4*blkdst];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al			;; Test outer loop counter
	jnz	c3b			;; Iterate if necessary
	lea	esi, [esi-24*4*blkdst]	;; Restore source pointer
	end_timer 16

;; Do inverse FFT levels 5,6
;; On input the 32-byte cache lines hold these data values:
;;	0	1K	1	1K+1
;;	256	...
;;	...
;;	2K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	4K	1	4K+1
;;	256	...
;;	...
;;	8K	...
;;	...

;; Do 4 four_real_four_semireal_unfft macros
;;	distance between fft data elements is 4

	start_timer 17
	mov	al, 4			;; Load loop counter
c4b:	disp four_real_four_semireal_unfft, 8, 4*blkdst, 8*blkdst
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test inner loop counter
	jnc	c4b			;; Iterate if necessary
	IFDEF PFETCH
	prefetcht1 [esi-32+clm*32]
;	prefetcht1 [esi-32+clm*32+4*blkdst]
	prefetcht1 [esi-32+clm*32+8*blkdst]
;	prefetcht1 [esi-32+clm*32+12*blkdst]
	ENDIF
	loopclm	dl, clm, c4b
	lea	esi, [esi-clm*2*dist1+blkdst];; Next source pointer
	dec	al
	jnz	c4b
	lea	esi, [esi-4*blkdst+16*blkdst];; Next source pointer

;; Do 20 four_complex_unfft macros
;;	distance between fft data elements is 4

	mov	edi, sincos2		;; Load sin/cos pointer
	mov	eax, 5			;; 5 iterations of 4
c5b:	disp four_complex_unfft, 8, 4*blkdst, 8*blkdst
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test inner loop counter
	jnc	c5b			;; Iterate if necessary
	IFDEF PFETCH
	prefetcht1 [esi-32+clm*32]
;	prefetcht1 [esi-32+clm*32+4*blkdst]
	prefetcht1 [esi-32+clm*32+8*blkdst]
;	prefetcht1 [esi-32+clm*32+12*blkdst]
	ENDIF
	loopclm	dl, clm, c5b
	lea	esi, [esi-clm*2*dist1+blkdst];; Next source pointer
	add	ah, 256/4		;; Test middle loop counter
	jnc	c5b			;; Iterate if necessary
	lea	esi, [esi-4*blkdst+16*blkdst];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al
	jnz	c5b
	lea	esi, [esi-6*16*blkdst]	;; Restore source pointer
	end_timer 17

;; Do inverse FFT levels 4
;; On input the 32-byte cache lines hold these data values:
;;	0	4K	1	4K+1
;;	256	...
;;	...
;;	8K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	8K	1	8K+1
;;	256	...
;;	...
;;	16K	...
;;	...

;; Do 8 four_real_four_semireal_unfft_1 macros
;;	distance between fft data elements is 16

	start_timer 18
	mov	al, 8			;; Load loop counter
c6b:	disp four_real_four_semireal_unfft_1, 8*blkdst, 8, 16*blkdst
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test inner loop counter
	jnc	c6b			;; Iterate if necessary
	IFDEF PFETCH
	prefetcht1 [esi-32+clm*32]
;	prefetcht1 [esi-32+clm*32+8*blkdst]
	prefetcht1 [esi-32+clm*32+16*blkdst]
;	prefetcht1 [esi-32+clm*32+24*blkdst]
	ENDIF
	loopclm	dl, clm, c6b
	lea	esi, [esi-clm*2*dist1+blkdst];; Next source pointer
	dec	al
	jnz	c6b
	lea	esi, [esi-8*blkdst+32*blkdst];; Next source pointer

;; Do 16 four_complex_unfft_1 macros
;;	distance between fft data elements is 16

	mov	edi, sincos3		;; Load sin/cos pointer
	mov	eax, 2			;; 2 iterations of 8
c7b:	disp four_complex_unfft_1, 8*blkdst, 8, 16*blkdst
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test inner loop counter
	jnc	c7b			;; Iterate if necessary
	IFDEF PFETCH
	prefetcht1 [esi-32+clm*32]
;	prefetcht1 [esi-32+clm*32+8*blkdst]
	prefetcht1 [esi-32+clm*32+16*blkdst]
;	prefetcht1 [esi-32+clm*32+24*blkdst]
	ENDIF
	loopclm	dl, clm, c7b
	lea	esi, [esi-clm*2*dist1+blkdst];; Next source pointer
	add	ah, 256/8		;; Test middle loop counter
	jnc	c7b			;; Iterate if necessary
	lea	esi, [esi-8*blkdst+32*blkdst];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al
	jnz	c7b
	lea	esi, [esi-3*32*blkdst]	;; Restore source pointer
	end_timer 18

;; Do inverse FFT levels 1,2,3
;; On input the 32-byte cache lines hold these data values:
;;	0	8K	1	8K+1
;;	256	...
;;	...
;;	16K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	24K	1	24K+1
;;	256
;;	...
;;	24K-256

;; Do 32 six_reals_last_unfft macros
;;	distance between fft data elements is 32

	start_timer 19
	mov	al, 32			;; Load loop counter
c1b:	disp six_reals_last_unfft, 8, 32*blkdst, 64*blkdst
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test loop counter
	jnc	c1b			;; Iterate if necessary
	IFDEF PFETCH
	prefetcht1 [esi-32+clm*32]
;	prefetcht1 [esi-32+clm*32+32*blkdst]
	prefetcht1 [esi-32+clm*32+64*blkdst]
	ENDIF
	loopclm	dl, clm, c1b
	lea	esi, [esi-clm*2*dist1+blkdst];; Next source pointer
	dec	al			;; Test loop counter
	jnz	c1b			;; Iterate if necessary
	lea	esi, [esi-32*blkdst]	;; Restore source pointer
	end_timer 19

;; Normalize these values

	start_timer 20
	mov	eax, NORMRTN		;; Addr of normalization routine
	call	eax
	sub	eax, eax
	end_timer 20

;; Have we been given permission to start the FFT on the result?
;; If so, do so now while the data is in the L2 cache.

	cmp	POSTFFT, 0		;; Test flag
	je	no_fft			;; Skip FFT code if flag not set
	cmp	edx, p2cl/128*65536+(128/clm-4/clm)*256;; Delay fft on 8 sets
	jg	no_fft			;; Jump if delaying FFT
	copy_3_words clm, 0

;; Do FFT levels 1,2,3
;;
;; On input the 32-byte cache lines hold these data values:
;;	0	24K	1	24K+1
;;	256
;;	...
;;	24K-256
;; On output the 32-byte cache lines hold these data values:
;;	0	8K	1	8K+1
;;	256	...
;;	...
;;	16K	...
;;	...

;; Do 32 six_reals_first_fft macros
;;	distance between fft data elements is 32

b0b:	start_timer 21
	mov	ebx, DIST_TO_FFTSRCARG
	mov	al, 32			;; 32 iterations
b1b:	disp six_reals_first_fft, 32*blkdst, 64*blkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	add	dl, 256/2/clm		;; Test loop counter
	jnc	b1b			;; Iterate if necessary
	lea	esi, [esi-clm*2*dist1+blkdst];; Next source pointer
	dec	al			;; Test loop counter
	jnz	b1b			;; Iterate if necessary
	lea	esi, [esi-32*blkdst]	;; Restore source pointer
	end_timer 21

;; Do FFT levels 4
;;
;; On input the 32-byte cache lines hold these data values:
;;	0	8K	1	8K+1
;;	256	...
;;	...
;;	16K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	4K	1	4K+1
;;	256	...
;;	...
;;	8K	...
;;	...

;; Do 8 four_real_four_semireal_fft_1 macros
;;	distance between fft data elements is 16

	start_timer 22
	mov	al, 8			;; Load loop counter
b6b:	disp four_real_four_semireal_fft_1, 8*blkdst, 16*blkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test inner loop counter
	jnc	b6b			;; Iterate if necessary
	IFDEF PFETCH
;	prefetcht1 [esi-32+clm*32]
	prefetcht1 [esi-32+clm*32+8*blkdst]
;	prefetcht1 [esi-32+clm*32+16*blkdst]
	prefetcht1 [esi-32+clm*32+24*blkdst]
	ENDIF
	loopclm	dl, clm, b6b
	lea	esi, [esi-clm*2*dist1+blkdst];; Next source pointer
	dec	al
	jnz	b6b
	lea	esi, [esi-8*blkdst+32*blkdst];; Next source pointer

;; Do 16 four_complex_fft_1 macros
;;	distance between fft data elements is 16

	mov	edi, sincos3		;; Load sin/cos pointer
	mov	eax, 2			;; 2 iterations of 8
b7b:	disp four_complex_fft_1, 8*blkdst, 16*blkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test inner loop counter
	jnc	b7b			;; Iterate if necessary
	IFDEF PFETCH
;	prefetcht1 [esi-32+clm*32]
	prefetcht1 [esi-32+clm*32+8*blkdst]
;	prefetcht1 [esi-32+clm*32+16*blkdst]
	prefetcht1 [esi-32+clm*32+24*blkdst]
	ENDIF
	loopclm	dl, clm, b7b
	lea	esi, [esi-clm*2*dist1+blkdst];; Next source pointer
	add	ah, 256/8		;; Test middle loop counter
	jnc	b7b			;; Iterate if necessary
	lea	esi, [esi-8*blkdst+32*blkdst];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al
	jnz	b7b
	lea	esi, [esi-3*32*blkdst]	;; Restore source pointer
	end_timer 22

;; Do FFT levels 5,6
;;
;; On input the 32-byte cache lines hold these data values:
;;	0	4K	1	4K+1
;;	256	...
;;	...
;;	8K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	1K	1	1K+1
;;	256	...
;;	...
;;	2K	...
;;	...

;; Do 4 four_real_four_semireal_fft macros
;;	distance between fft data elements is 4

	start_timer 23
	mov	al, 4			;; Load loop counter
b4b:	disp four_real_four_semireal_fft, 4*blkdst, 8*blkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test inner loop counter
	jnc	b4b			;; Iterate if necessary
	IFDEF PFETCH
;	prefetcht1 [esi-32+clm*32]
	prefetcht1 [esi-32+clm*32+4*blkdst]
;	prefetcht1 [esi-32+clm*32+8*blkdst]
	prefetcht1 [esi-32+clm*32+12*blkdst]
	ENDIF
	loopclm	dl, clm, b4b
	lea	esi, [esi-clm*2*dist1+blkdst];; Next source pointer
	dec	al
	jnz	b4b
	lea	esi, [esi-4*blkdst+16*blkdst];; Next source pointer

;; Do 20 four_complex_fft macros
;;	distance between fft data elements is 4

	mov	edi, sincos2		;; Load sin/cos pointer
	mov	eax, 5			;; 5 iterations of 4
b5b:	disp four_complex_fft, 4*blkdst, 8*blkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test inner loop counter
	jnc	b5b			;; Iterate if necessary
	IFDEF PFETCH
;	prefetcht1 [esi-32+clm*32]
	prefetcht1 [esi-32+clm*32+4*blkdst]
;	prefetcht1 [esi-32+clm*32+8*blkdst]
	prefetcht1 [esi-32+clm*32+12*blkdst]
	ENDIF
	loopclm	dl, clm, b5b
	lea	esi, [esi-clm*2*dist1+blkdst];; Next source pointer
	add	ah, 256/4		;; Test middle loop counter
	jnc	b5b			;; Iterate if necessary
	lea	esi, [esi-4*blkdst+16*blkdst];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al
	jnz	b5b
	lea	esi, [esi-6*16*blkdst]	;; Restore source pointer
	end_timer 23

;; Do FFT levels 7,8
;;
;; On input the 32-byte cache lines hold these data values:
;;	0	1K	1	1K+1
;;	256	...
;;	...
;;	2K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	256	1	257
;;	512	...
;;	...

;; Do 1 four_real_four_semireal_fft macros
;;	distance between fft data elements is 1

	start_timer 24
b2b:	disp four_real_four_semireal_fft, blkdst, 2*blkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test inner loop counter
	jnc	b2b			;; Iterate if necessary
	IFDEF PFETCH
;	prefetcht1 [esi-32+clm*32]
	prefetcht1 [esi-32+clm*32+blkdst]
;	prefetcht1 [esi-32+clm*32+2*blkdst]
	prefetcht1 [esi-32+clm*32+3*blkdst]
	ENDIF
	loopclm	dl, clm, b2b
	lea	esi, [esi-clm*2*dist1+4*blkdst];; Next source pointer

;; Do 23 four_complex_fft macros
;;	distance between fft data elements is 1

	mov	edi, sincos1		;; Load sin/cos pointer
	mov	al, 23			;; 23 iterations
b3b:	disp four_complex_fft, blkdst, 2*blkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test inner loop counter
	jnc	b3b			;; Iterate if necessary
	IFDEF PFETCH
;	prefetcht1 [esi-32+clm*32]
	prefetcht1 [esi-32+clm*32+blkdst]
;	prefetcht1 [esi-32+clm*32+2*blkdst]
	prefetcht1 [esi-32+clm*32+3*blkdst]
	ENDIF
	loopclm	dl, clm, b3b
	lea	esi, [esi-clm*2*dist1+4*blkdst];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al
	jnz	b3b
	lea	esi, [esi-24*4*blkdst]	;; Restore source pointer
	end_timer 24

;; Work on next 4 sets of 32 values

no_fft:	lea	esi, [esi+clm*2*dist1]	;; Next source pointer
	dec	dh			;; Test loop counter
	jnz	short more		;; Do more FFTs
	lea	esi, [esi+64]		;; Skip pad every 128 cache lines (4KB)
	sub	edx, 65536		;; Test loop counter
	jz	short fftdn		;; Jump if done looping
	mov	dh, 128/clm		;; Restore cache line counter
more:	cmp	norm_ptr1, 0		;; What was the loop start point?
	jne	c0b			;; Do inverse-FFT/norm/FFT iteration
	jmp	b0b			;; Do another FFT-only iteration
fftdn:	lea	esi, [esi-p2cl*2*dist1-p2cl/128*64];; Restore source pointer
	cmp	esi, DESTARG		;; If esi was not restored, then we 
	jne	short fftdn1		;; just finished skipped postfft data
	sub	ebx, ebx
	mov	DIST_TO_FFTSRCARG, ebx
	cmp	norm_ptr1, 0		;; What was the loop start point?
	je	pass2			;; FFT-only, go do pass 2

;; Split the carries into high and low carries (so they do not exceed
;; the desired number of bits per FFT word.  Then add the carries back to the
;; FFT data.

	start_timer 28
	call	gw_carries
	end_timer 28

;; Have we been given permission to start the FFT on the result?
;; If so, set loop to FFT the first few cache lines now that the carries
;; have been added in.

	cmp	POSTFFT, 0		;; Test flag
	je	short fftdn1		;; Skip FFT code if flag not set
	mov	esi, DESTARG		;; Load FFT data address
	mov	edx, 65536+4/clm*256	;; Do 8 previously skipped cache lines
	mov	norm_ptr1, 0		;; Set flag so only forward FFT is done
	copy_4_words
	jmp	b0b

;; Now add in the FFT'ed carries, cleanup and return

fftdn1:	end_timer 1
	jmp	gw_finish_mult
	ENDM


;; Perform a 48K-element negacyclic FFT.  This is done in two passes.  Pass 1
;; does 8 levels, pass 2 uses common code to perform the last 8 levels.

pass1levels8complex3 MACRO pass2_macro, p2cl, clm
	LOCAL	b0b, b1b, b3b, b5b, b7b, c0b, c1b, c3b, c5b, c7b
	LOCAL	pass2, no_fft, more, fftdn, fftdn1

;; Call FFT code unless the FFT has already been started

	clear_timers
	cmp	DWORD PTR [esi-28][ebx], 0 ;; Test FFT-started flag
	jne	pass2			;; Jump is FFT already started
	mov	norm_ptr1, 0		;; Set FFT-ing flag
	mov	edx, p2cl/128*65536+128/clm*256;; Load loop counter
	jmp	b0b			;; Do the pass 1 fft

;; Do the last 8 FFT levels
;;	do 96 pass2_8_levels_complex

pass2:	start_timer 0
	CALLP	pass2_macro
	end_timer 0
	cmp	ffttype, 1		;; We're done if FFTing only
	je	gw_finish_fft

;; Do the inverse FFT

	start_timer 1
	fldz				;; Init SUMOUT
	fstp	SUMOUT
	mov	edx, norm_biglit_array	;; Addr of the big/little flags array
	mov	norm_ptr1, edx		;; Save ptr
	mov	edx, norm_col_mults	;; Addr of the column multipliers
	mov	norm_ptr2, edx		;; Save ptr
	mov	edx, p2cl/128*65536+128/clm*256;; Load loop counter

;; Do inverse FFT levels 7,8
;; On input the 32-byte cache lines hold these data values:
;;	0	256	1	257
;;	512	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	1K	1	1K+1
;;	256	...
;;	...
;;	2K	...
;;	...

;; Do 24 four_complex_unfft macros
;;	distance between fft data elements is 1

c0b:	start_timer 16
	mov	edi, sincos1		;; Load sin/cos pointer
	mov	al, 24			;; 24 iterations
c3b:	disp four_complex_unfft, 8, blkdst, 2*blkdst
	lea	esi, [esi+dist1]	;; Next source pointer
	add	dl, 256/2/clm		;; Test loop counter
	jnc	c3b			;; Iterate if necessary
	lea	esi, [esi-clm*2*dist1+4*blkdst];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al			;; Test outer loop counter
	jnz	c3b			;; Iterate if necessary
	lea	esi, [esi-24*4*blkdst]	;; Restore source pointer
	end_timer 16

;; Do inverse FFT levels 5,6
;; On input the 32-byte cache lines hold these data values:
;;	0	1K	1	1K+1
;;	256	...
;;	...
;;	2K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	4K	1	4K+1
;;	256	...
;;	...
;;	8K	...
;;	...

;; Do 24 four_complex_unfft macros
;;	distance between fft data elements is 4

	start_timer 17
	mov	edi, sincos2		;; Load sin/cos pointer
	mov	eax, 6			;; 6 iterations of 4
c5b:	disp four_complex_unfft, 8, 4*blkdst, 8*blkdst
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test inner loop counter
	jnc	c5b			;; Iterate if necessary
	IFDEF PFETCH
	prefetcht1 [esi-32+clm*32]
;	prefetcht1 [esi-32+clm*32+4*blkdst]
	prefetcht1 [esi-32+clm*32+8*blkdst]
;	prefetcht1 [esi-32+clm*32+12*blkdst]
	ENDIF
	loopclm	dl, clm, c5b
	lea	esi, [esi-clm*2*dist1+blkdst];; Next source pointer
	add	ah, 256/4		;; Test middle loop counter
	jnc	c5b			;; Iterate if necessary
	lea	esi, [esi-4*blkdst+16*blkdst];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al
	jnz	c5b
	lea	esi, [esi-6*16*blkdst]	;; Restore source pointer
	end_timer 17

;; Do inverse FFT levels 4
;; On input the 32-byte cache lines hold these data values:
;;	0	4K	1	4K+1
;;	256	...
;;	...
;;	8K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	8K	1	8K+1
;;	256	...
;;	...
;;	16K	...
;;	...

;; Do 24 four_complex_unfft_1 macros
;;	distance between fft data elements is 16

	start_timer 18
	mov	edi, sincos3		;; Load sin/cos pointer
	mov	eax, 3			;; 3 iterations of 8
c7b:	disp four_complex_unfft_1, 8*blkdst, 8, 16*blkdst
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test inner loop counter
	jnc	c7b			;; Iterate if necessary
	IFDEF PFETCH
	prefetcht1 [esi-32+clm*32]
;	prefetcht1 [esi-32+clm*32+8*blkdst]
	prefetcht1 [esi-32+clm*32+16*blkdst]
;	prefetcht1 [esi-32+clm*32+24*blkdst]
	ENDIF
	loopclm	dl, clm, c7b
	lea	esi, [esi-clm*2*dist1+blkdst];; Next source pointer
	add	ah, 256/8		;; Test middle loop counter
	jnc	c7b			;; Iterate if necessary
	lea	esi, [esi-8*blkdst+32*blkdst];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al
	jnz	c7b
	lea	esi, [esi-3*32*blkdst]	;; Restore source pointer
	end_timer 18

;; Do inverse FFT levels 1,2,3
;; On input the 32-byte cache lines hold these data values:
;;	0	8K	1	8K+1
;;	256	...
;;	...
;;	16K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	24K	1	24K+1
;;	256
;;	...
;;	24K-256

;; Do 32 three_complex_last_unfft macros
;;	distance between fft data elements is 32

	start_timer 19
	mov	edi, plus1_premults	;; Address of premultiplier table
	mov	al, 32			;; Load loop counter
c1b:	three_complex_last_unfft 8, 32*blkdst, 64*blkdst
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test loop counter
	jnc	c1b			;; Iterate if necessary
	IFDEF PFETCH
	prefetcht1 [esi-32+clm*32]
;	prefetcht1 [esi-32+clm*32+32*blkdst]
	prefetcht1 [esi-32+clm*32+64*blkdst]
	ENDIF
	loopclm	dl, clm, c1b
	lea	esi, [esi-clm*2*dist1+blkdst];; Next source pointer
	lea	edi, [edi+48]		;; Next premultiplier pointer
	dec	al			;; Test loop counter
	jnz	c1b			;; Iterate if necessary
	lea	esi, [esi-32*blkdst]	;; Restore source pointer
	end_timer 19

;; Normalize these values

	start_timer 20
	mov	eax, NORMRTN		;; Addr of normalization routine
	call	eax
	sub	eax, eax
	end_timer 20

;; Have we been given permission to start the FFT on the result?
;; If so, do so now while the data is in the L2 cache.

	cmp	POSTFFT, 0		;; Test flag
	je	no_fft			;; Skip FFT code if flag not set
	cmp	edx, p2cl/128*65536+(128/clm-4/clm)*256;; Delay fft on 8 sets
	jg	no_fft			;; Jump if delaying FFT
	copy_3_words clm, 0

;; Do FFT levels 1,2,3
;;
;; On input the 32-byte cache lines hold these data values:
;;	0	24K	1	24K+1
;;	256
;;	...
;;	24K-256
;; On output the 32-byte cache lines hold these data values:
;;	0	8K	1	8K+1
;;	256	...
;;	...
;;	16K	...
;;	...

;; Do 32 three_complex_first_fft macros
;;	distance between fft data elements is 32

b0b:	start_timer 21
	mov	ebx, DIST_TO_FFTSRCARG
	mov	edi, plus1_premults	;; Address of premultiplier table
	mov	al, 32			;; 32 iterations
b1b:	three_complex_first_fft 32*blkdst, 64*blkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	add	dl, 256/2/clm		;; Test loop counter
	jnc	b1b			;; Iterate if necessary
	lea	esi, [esi-clm*2*dist1+blkdst];; Next source pointer
	lea	edi, [edi+48]		;; Next premultiplier pointer
	dec	al			;; Test loop counter
	jnz	b1b			;; Iterate if necessary
	lea	esi, [esi-32*blkdst]	;; Restore source pointer
	end_timer 21

;; Do FFT levels 4
;;
;; On input the 32-byte cache lines hold these data values:
;;	0	8K	1	8K+1
;;	256	...
;;	...
;;	16K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	4K	1	4K+1
;;	256	...
;;	...
;;	8K	...
;;	...

;; Do 24 four_complex_fft_1 macros
;;	distance between fft data elements is 16

	start_timer 22
	mov	edi, sincos3		;; Load sin/cos pointer
	mov	eax, 3			;; 3 iterations of 8
b7b:	disp four_complex_fft_1, 8*blkdst, 16*blkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test inner loop counter
	jnc	b7b			;; Iterate if necessary
	IFDEF PFETCH
;	prefetcht1 [esi-32+clm*32]
	prefetcht1 [esi-32+clm*32+8*blkdst]
;	prefetcht1 [esi-32+clm*32+16*blkdst]
	prefetcht1 [esi-32+clm*32+24*blkdst]
	ENDIF
	loopclm	dl, clm, b7b
	lea	esi, [esi-clm*2*dist1+blkdst];; Next source pointer
	add	ah, 256/8		;; Test middle loop counter
	jnc	b7b			;; Iterate if necessary
	lea	esi, [esi-8*blkdst+32*blkdst];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al
	jnz	b7b
	lea	esi, [esi-3*32*blkdst]	;; Restore source pointer
	end_timer 22

;; Do FFT levels 5,6
;;
;; On input the 32-byte cache lines hold these data values:
;;	0	4K	1	4K+1
;;	256	...
;;	...
;;	8K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	1K	1	1K+1
;;	256	...
;;	...
;;	2K	...
;;	...

;; Do 24 four_complex_fft macros
;;	distance between fft data elements is 4

	start_timer 23
	mov	edi, sincos2		;; Load sin/cos pointer
	mov	eax, 6			;; 6 iterations of 4
b5b:	disp four_complex_fft, 4*blkdst, 8*blkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test inner loop counter
	jnc	b5b			;; Iterate if necessary
	IFDEF PFETCH
;	prefetcht1 [esi-32+clm*32]
	prefetcht1 [esi-32+clm*32+4*blkdst]
;	prefetcht1 [esi-32+clm*32+8*blkdst]
	prefetcht1 [esi-32+clm*32+12*blkdst]
	ENDIF
	loopclm	dl, clm, b5b
	lea	esi, [esi-clm*2*dist1+blkdst];; Next source pointer
	add	ah, 256/4		;; Test middle loop counter
	jnc	b5b			;; Iterate if necessary
	lea	esi, [esi-4*blkdst+16*blkdst];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al
	jnz	b5b
	lea	esi, [esi-6*16*blkdst]	;; Restore source pointer
	end_timer 23

;; Do FFT levels 7,8
;;
;; On input the 32-byte cache lines hold these data values:
;;	0	1K	1	1K+1
;;	256	...
;;	...
;;	2K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	256	1	257
;;	512	...
;;	...

;; Do 24 four_complex_fft macros
;;	distance between fft data elements is 1

	start_timer 24
	mov	edi, sincos1		;; Load sin/cos pointer
	mov	al, 24			;; 24 iterations
b3b:	disp four_complex_fft, blkdst, 2*blkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test inner loop counter
	jnc	b3b			;; Iterate if necessary
	IFDEF PFETCH
;	prefetcht1 [esi-32+clm*32]
	prefetcht1 [esi-32+clm*32+blkdst]
;	prefetcht1 [esi-32+clm*32+2*blkdst]
	prefetcht1 [esi-32+clm*32+3*blkdst]
	ENDIF
	loopclm	dl, clm, b3b
	lea	esi, [esi-clm*2*dist1+4*blkdst];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al
	jnz	b3b
	lea	esi, [esi-24*4*blkdst]	;; Restore source pointer
	end_timer 24

;; Work on next 4 sets of 32 values

no_fft:	lea	esi, [esi+clm*2*dist1]	;; Next source pointer
	dec	dh			;; Test loop counter
	jnz	short more		;; Do more FFTs
	lea	esi, [esi+64]		;; Skip pad every 128 cache lines (4KB)
	sub	edx, 65536		;; Test loop counter
	jz	short fftdn		;; Jump if done looping
	mov	dh, 128/clm		;; Restore cache line counter
more:	cmp	norm_ptr1, 0		;; What was the loop start point?
	jne	c0b			;; Do inverse-FFT/norm/FFT iteration
	jmp	b0b			;; Do another FFT-only iteration
fftdn:	lea	esi, [esi-p2cl*2*dist1-p2cl/128*64];; Restore source pointer
	cmp	esi, DESTARG		;; If esi was not restored, then we 
	jne	short fftdn1		;; just finished skipped postfft data
	sub	ebx, ebx
	mov	DIST_TO_FFTSRCARG, ebx
	cmp	norm_ptr1, 0		;; What was the loop start point?
	je	pass2			;; FFT-only, go do pass 2

;; Split the carries into high and low carries (so they do not exceed
;; the desired number of bits per FFT word.  Then add the carries back to the
;; FFT data.

	start_timer 28
	call	gw_carries
	end_timer 28

;; Have we been given permission to start the FFT on the result?
;; If so, set loop to FFT the first few cache lines now that the carries
;; have been added in.

	cmp	POSTFFT, 0		;; Test flag
	je	short fftdn1		;; Skip FFT code if flag not set
	mov	esi, DESTARG		;; Load FFT data address
	mov	edx, 65536+4/clm*256	;; Do 8 previously skipped cache lines
	mov	norm_ptr1, 0		;; Set flag so only forward FFT is done
	copy_4_words
	jmp	b0b

;; Now add in the FFT'ed carries, cleanup and return

fftdn1:	end_timer 1
	jmp	gw_finish_mult
	ENDM


;; Perform a 56K-element FFT.  This is done in two passes.  Pass 1
;; does 8 levels, pass 2 uses common code to perform the last 8 levels.

pass1levels8pfa7 MACRO pass2_macro, p2cl, clm
	LOCAL	b0b, b1b, b2b, b3b, b4b, b5b, b6b, b7b
	LOCAL	c0b, c1b, c2b, c3b, c4b, c5b, c6b, c7b
	LOCAL	pass2, no_fft, more, fftdn, fftdn1

;; Call FFT code unless the FFT has already been started

	clear_timers
	cmp	DWORD PTR [esi-28][ebx], 0 ;; Test FFT-started flag
	jne	pass2			;; Jump is FFT already started
	mov	norm_ptr1, 0		;; Set FFT-ing flag
	mov	edx, p2cl/128*65536+128/clm*256;; Load loop counter
	jmp	b0b			;; Do the pass 1 fft

;; Do the last 8 FFT levels
;;	do 1 pass2_8_levels_real and 111 pass2_8_levels_complex

pass2:	start_timer 0
	CALLP	pass2_macro
	end_timer 0
	cmp	ffttype, 1		;; We're done if FFTing only
	je	gw_finish_fft

;; Do the inverse FFT

	start_timer 1
	fldz				;; Init SUMOUT
	fstp	SUMOUT
	mov	edx, norm_biglit_array	;; Addr of the big/little flags array
	mov	norm_ptr1, edx		;; Save ptr
	mov	edx, norm_col_mults	;; Addr of the column multipliers
	mov	norm_ptr2, edx		;; Save ptr
	mov	edx, p2cl/128*65536+128/clm*256;; Load loop counter

;; Do inverse FFT levels 7,8
;; On input the 32-byte cache lines hold these data values:
;;	0	256	1	257
;;	512	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	1K	1	1K+1
;;	256	...
;;	...
;;	2K	...
;;	...

;; Do 1 four_real_four_semireal_unfft macros
;;	distance between fft data elements is 1

c0b:	start_timer 16
c2b:	disp four_real_four_semireal_unfft, 8, blkdst, 2*blkdst
	lea	esi, [esi+dist1]	;; Next source pointer
	add	dl, 256/2/clm		;; Test inner loop counter
	jnc	c2b			;; Iterate if necessary
	lea	esi, [esi-clm*2*dist1+4*blkdst];; Next source pointer

;; Do 27 four_complex_unfft macros
;;	distance between fft data elements is 1

	mov	edi, sincos1		;; Load sin/cos pointer
	mov	al, 27			;; 27 iterations
c3b:	disp four_complex_unfft, 8, blkdst, 2*blkdst
	lea	esi, [esi+dist1]	;; Next source pointer
	add	dl, 256/2/clm		;; Test loop counter
	jnc	c3b			;; Iterate if necessary
	lea	esi, [esi-clm*2*dist1+4*blkdst];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al			;; Test outer loop counter
	jnz	c3b			;; Iterate if necessary
	lea	esi, [esi-28*4*blkdst]	;; Restore source pointer
	end_timer 16

;; Do inverse FFT levels 5,6
;; On input the 32-byte cache lines hold these data values:
;;	0	1K	1	1K+1
;;	256	...
;;	...
;;	2K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	4K	1	4K+1
;;	256	...
;;	...
;;	8K	...
;;	...

;; Do 4 four_real_four_semireal_unfft macros
;;	distance between fft data elements is 4

	start_timer 17
	mov	al, 4			;; Load loop counter
c4b:	disp four_real_four_semireal_unfft, 8, 4*blkdst, 8*blkdst
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test inner loop counter
	jnc	c4b			;; Iterate if necessary
	IFDEF PFETCH
	prefetcht1 [esi-32+clm*32]
;	prefetcht1 [esi-32+clm*32+4*blkdst]
	prefetcht1 [esi-32+clm*32+8*blkdst]
;	prefetcht1 [esi-32+clm*32+12*blkdst]
	ENDIF
	loopclm	dl, clm, c4b
	lea	esi, [esi-clm*2*dist1+blkdst];; Next source pointer
	dec	al
	jnz	c4b
	lea	esi, [esi-4*blkdst+16*blkdst];; Next source pointer

;; Do 24 four_complex_unfft macros
;;	distance between fft data elements is 4

	mov	edi, sincos2		;; Load sin/cos pointer
	mov	eax, 6			;; 6 iterations of 4
c5b:	disp four_complex_unfft, 8, 4*blkdst, 8*blkdst
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test inner loop counter
	jnc	c5b			;; Iterate if necessary
	IFDEF PFETCH
	prefetcht1 [esi-32+clm*32]
;	prefetcht1 [esi-32+clm*32+4*blkdst]
	prefetcht1 [esi-32+clm*32+8*blkdst]
;	prefetcht1 [esi-32+clm*32+12*blkdst]
	ENDIF
	loopclm	dl, clm, c5b
	lea	esi, [esi-clm*2*dist1+blkdst];; Next source pointer
	add	ah, 256/4		;; Test middle loop counter
	jnc	c5b			;; Iterate if necessary
	lea	esi, [esi-4*blkdst+16*blkdst];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al
	jnz	c5b
	lea	esi, [esi-7*16*blkdst]	;; Restore source pointer
	end_timer 17

;; Do inverse FFT levels 4
;; On input the 32-byte cache lines hold these data values:
;;	0	4K	1	4K+1
;;	256	...
;;	...
;;	8K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	8K	1	8K+1
;;	256	...
;;	...
;;	16K	...
;;	...

;; Do 4 eight_reals_unfft_1 macros
;;	distance between fft data elements is 16

	start_timer 18
	mov	al, 4			;; Load loop counter
c6b:	disp eight_reals_unfft_1, 4*blkdst, 8*blkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test inner loop counter
	jnc	c6b			;; Iterate if necessary
	IFDEF PFETCH
	prefetcht1 [esi-32+clm*32]
;	prefetcht1 [esi-32+clm*32+4*blkdst]
	prefetcht1 [esi-32+clm*32+8*blkdst]
;	prefetcht1 [esi-32+clm*32+12*blkdst]
	ENDIF
	loopclm	dl, clm, c6b
	lea	esi, [esi-clm*2*dist1+blkdst];; Next source pointer
	dec	al
	jnz	c6b
	lea	esi, [esi-4*blkdst+16*blkdst];; Next source pointer

;; Do 24 four_complex_unfft_1 macros
;;	distance between fft data elements is 16

	mov	edi, sincos3		;; Load sin/cos pointer
	mov	eax, 3			;; 3 iterations of 8
c7b:	disp four_complex_unfft_1, 8*blkdst, 8, 16*blkdst
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test inner loop counter
	jnc	c7b			;; Iterate if necessary
	IFDEF PFETCH
	prefetcht1 [esi-32+clm*32]
;	prefetcht1 [esi-32+clm*32+8*blkdst]
	prefetcht1 [esi-32+clm*32+16*blkdst]
;	prefetcht1 [esi-32+clm*32+24*blkdst]
	ENDIF
	loopclm	dl, clm, c7b
	lea	esi, [esi-clm*2*dist1+blkdst];; Next source pointer
	add	ah, 256/8		;; Test middle loop counter
	jnc	c7b			;; Iterate if necessary
	lea	esi, [esi-8*blkdst+32*blkdst];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al
	jnz	c7b
	lea	esi, [esi-3*32*blkdst-16*blkdst];; Restore source pointer
	end_timer 18

;; Do inverse FFT levels 1,2,3
;; On input the 32-byte cache lines hold these data values:
;;	0	8K	1	8K+1
;;	256	...
;;	...
;;	16K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	28K	1	28K+1
;;	256
;;	...
;;	28K-256

;; Do 32 seven_reals_last_unfft macros
;;	distance between fft data elements is 32

	start_timer 19
	mov	al, 16			;; Load loop counter
c1b:	two_seven_reals_last_unfft 8, 16*blkdst
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test loop counter
	jnc	c1b			;; Iterate if necessary
	IFDEF PFETCH
	prefetcht1 [esi-32+clm*32]
;	prefetcht1 [esi-32+clm*32+16*blkdst]
	prefetcht1 [esi-32+clm*32+32*blkdst]
;	prefetcht1 [esi-32+clm*32+48*blkdst]
	prefetcht1 [esi-32+clm*32+64*blkdst]
;	prefetcht1 [esi-32+clm*32+80*blkdst]
	prefetcht1 [esi-32+clm*32+96*blkdst]
	ENDIF
	loopclm	dl, clm, c1b
	lea	esi, [esi-clm*2*dist1+blkdst];; Next source pointer
	dec	al			;; Test loop counter
	jnz	c1b			;; Iterate if necessary
	lea	esi, [esi-16*blkdst]	;; Restore source pointer
	end_timer 19

;; Normalize these values

	start_timer 20
	mov	eax, NORMRTN		;; Addr of normalization routine
	call	eax
	sub	eax, eax
	end_timer 20

;; Have we been given permission to start the FFT on the result?
;; If so, do so now while the data is in the L2 cache.

	cmp	POSTFFT, 0		;; Test flag
	je	no_fft			;; Skip FFT code if flag not set
	cmp	edx, p2cl/128*65536+(128/clm-4/clm)*256;; Delay fft on 8 sets
	jg	no_fft			;; Jump if delaying FFT
	copy_3_words clm, 0

;; Do FFT levels 1,2,3
;;
;; On input the 32-byte cache lines hold these data values:
;;	0	28K	1	28K+1
;;	256
;;	...
;;	28K-256
;; On output the 32-byte cache lines hold these data values:
;;	0	8K	1	8K+1
;;	256	...
;;	...
;;	16K	...
;;	...

;; Do 32 seven_reals_first_fft macros
;;	distance between fft data elements is 32

b0b:	start_timer 21
	mov	ebx, DIST_TO_FFTSRCARG
	mov	al, 16			;; 16 iterations
b1b:	two_seven_reals_first_fft 16*blkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	add	dl, 256/2/clm		;; Test loop counter
	jnc	b1b			;; Iterate if necessary
	lea	esi, [esi-clm*2*dist1+blkdst];; Next source pointer
	dec	al			;; Test loop counter
	jnz	b1b			;; Iterate if necessary
	lea	esi, [esi-16*blkdst]	;; Restore source pointer
	end_timer 21

;; Do FFT levels 4
;;
;; On input the 32-byte cache lines hold these data values:
;;	0	8K	1	8K+1
;;	256	...
;;	...
;;	16K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	4K	1	4K+1
;;	256	...
;;	...
;;	8K	...
;;	...

;; Do 4 eight_reals_fft_1 macros
;;	distance between fft data elements is 16

	start_timer 22
	mov	al, 4			;; Load loop counter
b6b:	disp eight_reals_fft_1, 4*blkdst, 8*blkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test inner loop counter
	jnc	b6b			;; Iterate if necessary
	IFDEF PFETCH
;	prefetcht1 [esi-32+clm*32]
	prefetcht1 [esi-32+clm*32+4*blkdst]
;	prefetcht1 [esi-32+clm*32+8*blkdst]
	prefetcht1 [esi-32+clm*32+12*blkdst]
	ENDIF
	loopclm	dl, clm, b6b
	lea	esi, [esi-clm*2*dist1+blkdst];; Next source pointer
	dec	al
	jnz	b6b
	lea	esi, [esi-4*blkdst+16*blkdst];; Next source pointer

;; Do 24 four_complex_fft_1 macros
;;	distance between fft data elements is 16

	mov	edi, sincos3		;; Load sin/cos pointer
	mov	eax, 3			;; 3 iterations of 8
b7b:	disp four_complex_fft_1, 8*blkdst, 16*blkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test inner loop counter
	jnc	b7b			;; Iterate if necessary
	IFDEF PFETCH
;	prefetcht1 [esi-32+clm*32]
	prefetcht1 [esi-32+clm*32+8*blkdst]
;	prefetcht1 [esi-32+clm*32+16*blkdst]
	prefetcht1 [esi-32+clm*32+24*blkdst]
	ENDIF
	loopclm	dl, clm, b7b
	lea	esi, [esi-clm*2*dist1+blkdst];; Next source pointer
	add	ah, 256/8		;; Test middle loop counter
	jnc	b7b			;; Iterate if necessary
	lea	esi, [esi-8*blkdst+32*blkdst];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al
	jnz	b7b
	lea	esi, [esi-3*32*blkdst-16*blkdst];; Restore source pointer
	end_timer 22

;; Do FFT levels 5,6
;;
;; On input the 32-byte cache lines hold these data values:
;;	0	4K	1	4K+1
;;	256	...
;;	...
;;	8K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	1K	1	1K+1
;;	256	...
;;	...
;;	2K	...
;;	...

;; Do 4 four_real_four_semireal_fft macros
;;	distance between fft data elements is 4

	start_timer 23
	mov	al, 4			;; Load loop counter
b4b:	disp four_real_four_semireal_fft, 4*blkdst, 8*blkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test inner loop counter
	jnc	b4b			;; Iterate if necessary
	IFDEF PFETCH
;	prefetcht1 [esi-32+clm*32]
	prefetcht1 [esi-32+clm*32+4*blkdst]
;	prefetcht1 [esi-32+clm*32+8*blkdst]
	prefetcht1 [esi-32+clm*32+12*blkdst]
	ENDIF
	loopclm	dl, clm, b4b
	lea	esi, [esi-clm*2*dist1+blkdst];; Next source pointer
	dec	al
	jnz	b4b
	lea	esi, [esi-4*blkdst+16*blkdst];; Next source pointer

;; Do 24 four_complex_fft macros
;;	distance between fft data elements is 4

	mov	edi, sincos2		;; Load sin/cos pointer
	mov	eax, 6			;; 6 iterations of 4
b5b:	disp four_complex_fft, 4*blkdst, 8*blkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test inner loop counter
	jnc	b5b			;; Iterate if necessary
	IFDEF PFETCH
;	prefetcht1 [esi-32+clm*32]
	prefetcht1 [esi-32+clm*32+4*blkdst]
;	prefetcht1 [esi-32+clm*32+8*blkdst]
	prefetcht1 [esi-32+clm*32+12*blkdst]
	ENDIF
	loopclm	dl, clm, b5b
	lea	esi, [esi-clm*2*dist1+blkdst];; Next source pointer
	add	ah, 256/4		;; Test middle loop counter
	jnc	b5b			;; Iterate if necessary
	lea	esi, [esi-4*blkdst+16*blkdst];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al
	jnz	b5b
	lea	esi, [esi-7*16*blkdst]	;; Restore source pointer
	end_timer 23

;; Do FFT levels 7,8
;;
;; On input the 32-byte cache lines hold these data values:
;;	0	1K	1	1K+1
;;	256	...
;;	...
;;	2K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	256	1	257
;;	512	...
;;	...

;; Do 1 four_real_four_semireal_fft macros
;;	distance between fft data elements is 1

	start_timer 24
b2b:	disp four_real_four_semireal_fft, blkdst, 2*blkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test inner loop counter
	jnc	b2b			;; Iterate if necessary
	IFDEF PFETCH
;	prefetcht1 [esi-32+clm*32]
	prefetcht1 [esi-32+clm*32+blkdst]
;	prefetcht1 [esi-32+clm*32+2*blkdst]
	prefetcht1 [esi-32+clm*32+3*blkdst]
	ENDIF
	loopclm	dl, clm, b2b
	lea	esi, [esi-clm*2*dist1+4*blkdst];; Next source pointer

;; Do 27 four_complex_fft macros
;;	distance between fft data elements is 1

	mov	edi, sincos1		;; Load sin/cos pointer
	mov	al, 27			;; 27 iterations
b3b:	disp four_complex_fft, blkdst, 2*blkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test inner loop counter
	jnc	b3b			;; Iterate if necessary
	IFDEF PFETCH
;	prefetcht1 [esi-32+clm*32]
	prefetcht1 [esi-32+clm*32+blkdst]
;	prefetcht1 [esi-32+clm*32+2*blkdst]
	prefetcht1 [esi-32+clm*32+3*blkdst]
	ENDIF
	loopclm	dl, clm, b3b
	lea	esi, [esi-clm*2*dist1+4*blkdst];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al
	jnz	b3b
	lea	esi, [esi-28*4*blkdst]	;; Restore source pointer
	end_timer 24

;; Work on next 4 sets of 32 values

no_fft:	lea	esi, [esi+clm*2*dist1]	;; Next source pointer
	dec	dh			;; Test loop counter
	jnz	short more		;; Do more FFTs
	lea	esi, [esi+64]		;; Skip pad every 128 cache lines (4KB)
	sub	edx, 65536		;; Test loop counter
	jz	short fftdn		;; Jump if done looping
	mov	dh, 128/clm		;; Restore cache line counter
more:	cmp	norm_ptr1, 0		;; What was the loop start point?
	jne	c0b			;; Do inverse-FFT/norm/FFT iteration
	jmp	b0b			;; Do another FFT-only iteration
fftdn:	lea	esi, [esi-p2cl*2*dist1-p2cl/128*64];; Restore source pointer
	cmp	esi, DESTARG		;; If esi was not restored, then we 
	jne	short fftdn1		;; just finished skipped postfft data
	sub	ebx, ebx
	mov	DIST_TO_FFTSRCARG, ebx
	cmp	norm_ptr1, 0		;; What was the loop start point?
	je	pass2			;; FFT-only, go do pass 2

;; Split the carries into high and low carries (so they do not exceed
;; the desired number of bits per FFT word.  Then add the carries back to the
;; FFT data.

	start_timer 28
	call	gw_carries
	end_timer 28

;; Have we been given permission to start the FFT on the result?
;; If so, set loop to FFT the first few cache lines now that the carries
;; have been added in.

	cmp	POSTFFT, 0		;; Test flag
	je	short fftdn1		;; Skip FFT code if flag not set
	mov	esi, DESTARG		;; Load FFT data address
	mov	edx, 65536+4/clm*256	;; Do 8 previously skipped cache lines
	mov	norm_ptr1, 0		;; Set flag so only forward FFT is done
	copy_4_words
	jmp	b0b

;; Now add in the FFT'ed carries, cleanup and return

fftdn1:	end_timer 1
	jmp	gw_finish_mult
	ENDM


;; Perform a 64K-element FFT.  This is done in two passes.  Pass 1
;; does 8 levels, pass 2 uses common code to perform the last 8 levels.

pass1levels8pfa8 MACRO pass2_macro, p2cl, clm
	LOCAL	b0b, b1b, b2b, b3b, b4b, b5b, b6b, b7b
	LOCAL	c0b, c1b, c2b, c3b, c4b, c5b, c6b, c7b
	LOCAL	pass2, no_fft, more, fftdn, fftdn1

;; Call FFT code unless the FFT has already been started

	clear_timers
	cmp	DWORD PTR [esi-28][ebx], 0 ;; Test FFT-started flag
	jne	pass2			;; Jump is FFT already started
	mov	norm_ptr1, 0		;; Set FFT-ing flag
	mov	edx, p2cl/128*65536+128/clm*256;; Load loop counter
	jmp	b0b			;; Do the pass 1 fft

;; Do the last 8 FFT levels
;;	do 1 pass2_8_levels_real and 127 pass2_8_levels_complex

pass2:	start_timer 0
	CALLP	pass2_macro
	end_timer 0
	cmp	ffttype, 1		;; We're done if FFTing only
	je	gw_finish_fft

;; Do the inverse FFT

	start_timer 1
	fldz				;; Init SUMOUT
	fstp	SUMOUT
	mov	edx, norm_biglit_array	;; Addr of the big/little flags array
	mov	norm_ptr1, edx		;; Save ptr
	mov	edx, norm_col_mults	;; Addr of the column multipliers
	mov	norm_ptr2, edx		;; Save ptr
	mov	edx, p2cl/128*65536+128/clm*256;; Load loop counter

;; Do inverse FFT levels 7,8
;; On input the 32-byte cache lines hold these data values:
;;	0	256	1	257
;;	512	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	1K	1	1K+1
;;	256	...
;;	...
;;	2K	...
;;	...

;; Do 1 four_real_four_semireal_unfft macros
;;	distance between fft data elements is 1

c0b:	start_timer 16
c2b:	disp four_real_four_semireal_unfft, 8, blkdst, 2*blkdst
	lea	esi, [esi+dist1]	;; Next source pointer
	add	dl, 256/2/clm		;; Test inner loop counter
	jnc	c2b			;; Iterate if necessary
	lea	esi, [esi-clm*2*dist1+4*blkdst];; Next source pointer

;; Do 31 four_complex_unfft macros
;;	distance between fft data elements is 1

	mov	edi, sincos1		;; Load sin/cos pointer
	mov	al, 31			;; 31 iterations
c3b:	disp four_complex_unfft, 8, blkdst, 2*blkdst
	lea	esi, [esi+dist1]	;; Next source pointer
	add	dl, 256/2/clm		;; Test loop counter
	jnc	c3b			;; Iterate if necessary
	lea	esi, [esi-clm*2*dist1+4*blkdst];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al			;; Test outer loop counter
	jnz	c3b			;; Iterate if necessary
	lea	esi, [esi-32*4*blkdst]	;; Restore source pointer
	end_timer 16

;; Do inverse FFT levels 5,6
;; On input the 32-byte cache lines hold these data values:
;;	0	1K	1	1K+1
;;	256	...
;;	...
;;	2K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	4K	1	4K+1
;;	256	...
;;	...
;;	8K	...
;;	...

;; Do 4 four_real_four_semireal_unfft macros
;;	distance between fft data elements is 4

	start_timer 17
	mov	al, 4			;; Load loop counter
c4b:	disp four_real_four_semireal_unfft, 8, 4*blkdst, 8*blkdst
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test inner loop counter
	jnc	c4b			;; Iterate if necessary
	IFDEF PFETCH
	prefetcht1 [esi-32+clm*32]
;	prefetcht1 [esi-32+clm*32+4*blkdst]
	prefetcht1 [esi-32+clm*32+8*blkdst]
;	prefetcht1 [esi-32+clm*32+12*blkdst]
	ENDIF
	loopclm	dl, clm, c4b
	lea	esi, [esi-clm*2*dist1+blkdst];; Next source pointer
	dec	al
	jnz	c4b
	lea	esi, [esi-4*blkdst+16*blkdst];; Next source pointer

;; Do 28 four_complex_unfft macros
;;	distance between fft data elements is 4

	mov	edi, sincos2		;; Load sin/cos pointer
	mov	eax, 7			;; 7 iterations of 4
c5b:	disp four_complex_unfft, 8, 4*blkdst, 8*blkdst
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test inner loop counter
	jnc	c5b			;; Iterate if necessary
	IFDEF PFETCH
	prefetcht1 [esi-32+clm*32]
;	prefetcht1 [esi-32+clm*32+4*blkdst]
	prefetcht1 [esi-32+clm*32+8*blkdst]
;	prefetcht1 [esi-32+clm*32+12*blkdst]
	ENDIF
	loopclm	dl, clm, c5b
	lea	esi, [esi-clm*2*dist1+blkdst];; Next source pointer
	add	ah, 256/4		;; Test middle loop counter
	jnc	c5b			;; Iterate if necessary
	lea	esi, [esi-4*blkdst+16*blkdst];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al
	jnz	c5b
	lea	esi, [esi-8*16*blkdst]	;; Restore source pointer
	end_timer 17

;; Do inverse FFT levels 4
;; On input the 32-byte cache lines hold these data values:
;;	0	4K	1	4K+1
;;	256	...
;;	...
;;	8K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	8K	1	8K+1
;;	256	...
;;	...
;;	16K	...
;;	...

;; Do 8 four_real_four_semireal_unfft_1 macros
;;	distance between fft data elements is 16

	start_timer 18
	mov	al, 8			;; Load loop counter
c6b:	disp four_real_four_semireal_unfft_1, 8*blkdst, 8, 16*blkdst
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test inner loop counter
	jnc	c6b			;; Iterate if necessary
	IFDEF PFETCH
	prefetcht1 [esi-32+clm*32]
;	prefetcht1 [esi-32+clm*32+8*blkdst]
	prefetcht1 [esi-32+clm*32+16*blkdst]
;	prefetcht1 [esi-32+clm*32+24*blkdst]
	ENDIF
	loopclm	dl, clm, c6b
	lea	esi, [esi-clm*2*dist1+blkdst];; Next source pointer
	dec	al
	jnz	c6b
	lea	esi, [esi-8*blkdst+32*blkdst];; Next source pointer

;; Do 24 four_complex_unfft_1 macros
;;	distance between fft data elements is 16

	mov	edi, sincos3		;; Load sin/cos pointer
	mov	eax, 3			;; 3 iterations of 8
c7b:	disp four_complex_unfft_1, 8*blkdst, 8, 16*blkdst
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test inner loop counter
	jnc	c7b			;; Iterate if necessary
	IFDEF PFETCH
	prefetcht1 [esi-32+clm*32]
;	prefetcht1 [esi-32+clm*32+8*blkdst]
	prefetcht1 [esi-32+clm*32+16*blkdst]
;	prefetcht1 [esi-32+clm*32+24*blkdst]
	ENDIF
	loopclm	dl, clm, c7b
	lea	esi, [esi-clm*2*dist1+blkdst];; Next source pointer
	add	ah, 256/8		;; Test middle loop counter
	jnc	c7b			;; Iterate if necessary
	lea	esi, [esi-8*blkdst+32*blkdst];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al
	jnz	c7b
	lea	esi, [esi-4*32*blkdst]	;; Restore source pointer
	end_timer 18

;; Do inverse FFT levels 1,2,3
;; On input the 32-byte cache lines hold these data values:
;;	0	8K	1	8K+1
;;	256	...
;;	...
;;	16K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	32K	1	32K+1
;;	256
;;	...
;;	32K-256

;; Do 32 eight_reals_last_unfft macros
;;	distance between fft data elements is 32

	start_timer 19
	mov	al, 32			;; Load loop counter
c1b:	disp eight_reals_last_unfft, 8, 32*blkdst, 64*blkdst
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test loop counter
	jnc	c1b			;; Iterate if necessary
	IFDEF PFETCH
	prefetcht1 [esi-32+clm*32]
;	prefetcht1 [esi-32+clm*32+32*blkdst]
	prefetcht1 [esi-32+clm*32+64*blkdst]
;	prefetcht1 [esi-32+clm*32+96*blkdst]
	ENDIF
	loopclm	dl, clm, c1b
	lea	esi, [esi-clm*2*dist1+blkdst];; Next source pointer
	dec	al			;; Test loop counter
	jnz	c1b			;; Iterate if necessary
	lea	esi, [esi-32*blkdst]	;; Restore source pointer
	end_timer 19

;; Normalize these values

	start_timer 20
	mov	eax, NORMRTN		;; Addr of normalization routine
	call	eax
	sub	eax, eax
	end_timer 20

;; Have we been given permission to start the FFT on the result?
;; If so, do so now while the data is in the L2 cache.

	cmp	POSTFFT, 0		;; Test flag
	je	no_fft			;; Skip FFT code if flag not set
	cmp	edx, p2cl/128*65536+(128/clm-4/clm)*256;; Delay fft on 8 sets
	jg	no_fft			;; Jump if delaying FFT
	copy_3_words clm, 0

;; Do FFT levels 1,2,3
;;
;; On input the 32-byte cache lines hold these data values:
;;	0	32K	1	32K+1
;;	256
;;	...
;;	32K-256
;; On output the 32-byte cache lines hold these data values:
;;	0	8K	1	8K+1
;;	256	...
;;	...
;;	16K	...
;;	...

;; Do 32 eight_reals_first_fft macros
;;	distance between fft data elements is 32

b0b:	start_timer 21
	mov	ebx, DIST_TO_FFTSRCARG
	mov	al, 32			;; 32 iterations
b1b:	disp eight_reals_first_fft, 32*blkdst, 64*blkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	add	dl, 256/2/clm		;; Test loop counter
	jnc	b1b			;; Iterate if necessary
	lea	esi, [esi-clm*2*dist1+blkdst];; Next source pointer
	dec	al			;; Test loop counter
	jnz	b1b			;; Iterate if necessary
	lea	esi, [esi-32*blkdst]	;; Restore source pointer
	end_timer 21

;; Do FFT levels 4
;;
;; On input the 32-byte cache lines hold these data values:
;;	0	8K	1	8K+1
;;	256	...
;;	...
;;	16K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	4K	1	4K+1
;;	256	...
;;	...
;;	8K	...
;;	...

;; Do 8 four_real_four_semireal_fft_1 macros
;;	distance between fft data elements is 16

	start_timer 22
	mov	al, 8			;; Load loop counter
b6b:	disp four_real_four_semireal_fft_1, 8*blkdst, 16*blkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test inner loop counter
	jnc	b6b			;; Iterate if necessary
	IFDEF PFETCH
;	prefetcht1 [esi-32+clm*32]
	prefetcht1 [esi-32+clm*32+8*blkdst]
;	prefetcht1 [esi-32+clm*32+16*blkdst]
	prefetcht1 [esi-32+clm*32+24*blkdst]
	ENDIF
	loopclm	dl, clm, b6b
	lea	esi, [esi-clm*2*dist1+blkdst];; Next source pointer
	dec	al
	jnz	b6b
	lea	esi, [esi-8*blkdst+32*blkdst];; Next source pointer

;; Do 24 four_complex_fft_1 macros
;;	distance between fft data elements is 16

	mov	edi, sincos3		;; Load sin/cos pointer
	mov	eax, 3			;; 3 iterations of 8
b7b:	disp four_complex_fft_1, 8*blkdst, 16*blkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test inner loop counter
	jnc	b7b			;; Iterate if necessary
	IFDEF PFETCH
;	prefetcht1 [esi-32+clm*32]
	prefetcht1 [esi-32+clm*32+8*blkdst]
;	prefetcht1 [esi-32+clm*32+16*blkdst]
	prefetcht1 [esi-32+clm*32+24*blkdst]
	ENDIF
	loopclm	dl, clm, b7b
	lea	esi, [esi-clm*2*dist1+blkdst];; Next source pointer
	add	ah, 256/8		;; Test middle loop counter
	jnc	b7b			;; Iterate if necessary
	lea	esi, [esi-8*blkdst+32*blkdst];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al
	jnz	b7b
	lea	esi, [esi-4*32*blkdst]	;; Restore source pointer
	end_timer 22

;; Do FFT levels 5,6
;;
;; On input the 32-byte cache lines hold these data values:
;;	0	4K	1	4K+1
;;	256	...
;;	...
;;	8K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	1K	1	1K+1
;;	256	...
;;	...
;;	2K	...
;;	...

;; Do 4 four_real_four_semireal_fft macros
;;	distance between fft data elements is 4

	start_timer 23
	mov	al, 4			;; Load loop counter
b4b:	disp four_real_four_semireal_fft, 4*blkdst, 8*blkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test inner loop counter
	jnc	b4b			;; Iterate if necessary
	IFDEF PFETCH
;	prefetcht1 [esi-32+clm*32]
	prefetcht1 [esi-32+clm*32+4*blkdst]
;	prefetcht1 [esi-32+clm*32+8*blkdst]
	prefetcht1 [esi-32+clm*32+12*blkdst]
	ENDIF
	loopclm	dl, clm, b4b
	lea	esi, [esi-clm*2*dist1+blkdst];; Next source pointer
	dec	al
	jnz	b4b
	lea	esi, [esi-4*blkdst+16*blkdst];; Next source pointer

;; Do 28 four_complex_fft macros
;;	distance between fft data elements is 4

	mov	edi, sincos2		;; Load sin/cos pointer
	mov	eax, 7			;; 7 iterations of 4
b5b:	disp four_complex_fft, 4*blkdst, 8*blkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test inner loop counter
	jnc	b5b			;; Iterate if necessary
	IFDEF PFETCH
;	prefetcht1 [esi-32+clm*32]
	prefetcht1 [esi-32+clm*32+4*blkdst]
;	prefetcht1 [esi-32+clm*32+8*blkdst]
	prefetcht1 [esi-32+clm*32+12*blkdst]
	ENDIF
	loopclm	dl, clm, b5b
	lea	esi, [esi-clm*2*dist1+blkdst];; Next source pointer
	add	ah, 256/4		;; Test middle loop counter
	jnc	b5b			;; Iterate if necessary
	lea	esi, [esi-4*blkdst+16*blkdst];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al
	jnz	b5b
	lea	esi, [esi-8*16*blkdst]	;; Restore source pointer
	end_timer 23

;; Do FFT levels 7,8
;;
;; On input the 32-byte cache lines hold these data values:
;;	0	1K	1	1K+1
;;	256	...
;;	...
;;	2K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	256	1	257
;;	512	...
;;	...

;; Do 1 four_real_four_semireal_fft macros
;;	distance between fft data elements is 1

	start_timer 24
b2b:	disp four_real_four_semireal_fft, blkdst, 2*blkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test inner loop counter
	jnc	b2b			;; Iterate if necessary
	IFDEF PFETCH
;	prefetcht1 [esi-32+clm*32]
	prefetcht1 [esi-32+clm*32+blkdst]
;	prefetcht1 [esi-32+clm*32+2*blkdst]
	prefetcht1 [esi-32+clm*32+3*blkdst]
	ENDIF
	loopclm	dl, clm, b2b
	lea	esi, [esi-clm*2*dist1+4*blkdst];; Next source pointer

;; Do 31 four_complex_fft macros
;;	distance between fft data elements is 1

	mov	edi, sincos1		;; Load sin/cos pointer
	mov	al, 31			;; 31 iterations
b3b:	disp four_complex_fft, blkdst, 2*blkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test inner loop counter
	jnc	b3b			;; Iterate if necessary
	IFDEF PFETCH
;	prefetcht1 [esi-32+clm*32]
	prefetcht1 [esi-32+clm*32+blkdst]
;	prefetcht1 [esi-32+clm*32+2*blkdst]
	prefetcht1 [esi-32+clm*32+3*blkdst]
	ENDIF
	loopclm	dl, clm, b3b
	lea	esi, [esi-clm*2*dist1+4*blkdst];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al
	jnz	b3b
	lea	esi, [esi-32*4*blkdst]	;; Restore source pointer
	end_timer 24

;; Work on next 4 sets of 32 values

no_fft:	lea	esi, [esi+clm*2*dist1]	;; Next source pointer
	dec	dh			;; Test loop counter
	jnz	short more		;; Do more FFTs
	lea	esi, [esi+64]		;; Skip pad every 128 cache lines (4KB)
	sub	edx, 65536		;; Test loop counter
	jz	short fftdn		;; Jump if done looping
	mov	dh, 128/clm		;; Restore cache line counter
more:	cmp	norm_ptr1, 0		;; What was the loop start point?
	jne	c0b			;; Do inverse-FFT/norm/FFT iteration
	jmp	b0b			;; Do another FFT-only iteration
fftdn:	lea	esi, [esi-p2cl*2*dist1-p2cl/128*64];; Restore source pointer
	cmp	esi, DESTARG		;; If esi was not restored, then we 
	jne	short fftdn1		;; just finished skipped postfft data
	sub	ebx, ebx
	mov	DIST_TO_FFTSRCARG, ebx
	cmp	norm_ptr1, 0		;; What was the loop start point?
	je	pass2			;; FFT-only, go do pass 2

;; Split the carries into high and low carries (so they do not exceed
;; the desired number of bits per FFT word.  Then add the carries back to the
;; FFT data.

	start_timer 28
	call	gw_carries
	end_timer 28

;; Have we been given permission to start the FFT on the result?
;; If so, set loop to FFT the first few cache lines now that the carries
;; have been added in.

	cmp	POSTFFT, 0		;; Test flag
	je	short fftdn1		;; Skip FFT code if flag not set
	mov	esi, DESTARG		;; Load FFT data address
	mov	edx, 65536+4/clm*256	;; Do 8 previously skipped cache lines
	mov	norm_ptr1, 0		;; Set flag so only forward FFT is done
	copy_4_words
	jmp	b0b

;; Now add in the FFT'ed carries, cleanup and return

fftdn1:	end_timer 1
	jmp	gw_finish_mult
	ENDM


;; Perform a 64K-element negacyclic FFT.  This is done in two passes.  Pass 1
;; does 8 levels, pass 2 uses common code to perform the last 8 levels.

pass1levels8complex4 MACRO pass2_macro, p2cl, clm
	LOCAL	b0b, b1b, b3b, b5b, b7b, c0b, c1b, c3b, c5b, c7b
	LOCAL	pass2, no_fft, more, fftdn, fftdn1

;; Call FFT code unless the FFT has already been started

	clear_timers
	cmp	DWORD PTR [esi-28][ebx], 0 ;; Test FFT-started flag
	jne	pass2			;; Jump is FFT already started
	mov	norm_ptr1, 0		;; Set FFT-ing flag
	mov	edx, p2cl/128*65536+128/clm*256;; Load loop counter
	jmp	b0b			;; Do the pass 1 fft

;; Do the last 8 FFT levels
;;	do 128 pass2_8_levels_complex

pass2:	start_timer 0
	CALLP	pass2_macro
	end_timer 0
	cmp	ffttype, 1		;; We're done if FFTing only
	je	gw_finish_fft

;; Do the inverse FFT

	start_timer 1
	fldz				;; Init SUMOUT
	fstp	SUMOUT
	mov	edx, norm_biglit_array	;; Addr of the big/little flags array
	mov	norm_ptr1, edx		;; Save ptr
	mov	edx, norm_col_mults	;; Addr of the column multipliers
	mov	norm_ptr2, edx		;; Save ptr
	mov	edx, p2cl/128*65536+128/clm*256;; Load loop counter

;; Do inverse FFT levels 7,8
;; On input the 32-byte cache lines hold these data values:
;;	0	256	1	257
;;	512	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	1K	1	1K+1
;;	256	...
;;	...
;;	2K	...
;;	...

;; Do 32 four_complex_unfft macros
;;	distance between fft data elements is 1

c0b:	start_timer 16
	mov	edi, sincos1		;; Load sin/cos pointer
	mov	al, 32			;; 32 iterations
c3b:	disp four_complex_unfft, 8, blkdst, 2*blkdst
	lea	esi, [esi+dist1]	;; Next source pointer
	add	dl, 256/2/clm		;; Test loop counter
	jnc	c3b			;; Iterate if necessary
	lea	esi, [esi-clm*2*dist1+4*blkdst];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al			;; Test outer loop counter
	jnz	c3b			;; Iterate if necessary
	lea	esi, [esi-32*4*blkdst]	;; Restore source pointer
	end_timer 16

;; Do inverse FFT levels 5,6
;; On input the 32-byte cache lines hold these data values:
;;	0	1K	1	1K+1
;;	256	...
;;	...
;;	2K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	4K	1	4K+1
;;	256	...
;;	...
;;	8K	...
;;	...

;; Do 32 four_complex_unfft macros
;;	distance between fft data elements is 4

	start_timer 17
	mov	edi, sincos2		;; Load sin/cos pointer
	mov	eax, 8			;; 8 iterations of 4
c5b:	disp four_complex_unfft, 8, 4*blkdst, 8*blkdst
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test inner loop counter
	jnc	c5b			;; Iterate if necessary
	IFDEF PFETCH
	prefetcht1 [esi-32+clm*32]
;	prefetcht1 [esi-32+clm*32+4*blkdst]
	prefetcht1 [esi-32+clm*32+8*blkdst]
;	prefetcht1 [esi-32+clm*32+12*blkdst]
	ENDIF
	loopclm	dl, clm, c5b
	lea	esi, [esi-clm*2*dist1+blkdst];; Next source pointer
	add	ah, 256/4		;; Test middle loop counter
	jnc	c5b			;; Iterate if necessary
	lea	esi, [esi-4*blkdst+16*blkdst];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al
	jnz	c5b
	lea	esi, [esi-8*16*blkdst]	;; Restore source pointer
	end_timer 17

;; Do inverse FFT levels 4
;; On input the 32-byte cache lines hold these data values:
;;	0	4K	1	4K+1
;;	256	...
;;	...
;;	8K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	8K	1	8K+1
;;	256	...
;;	...
;;	16K	...
;;	...

;; Do 32 four_complex_unfft_1 macros
;;	distance between fft data elements is 16

	start_timer 18
	mov	edi, sincos3		;; Load sin/cos pointer
	mov	eax, 4			;; 4 iterations of 8
c7b:	disp four_complex_unfft_1, 8*blkdst, 8, 16*blkdst
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test inner loop counter
	jnc	c7b			;; Iterate if necessary
	IFDEF PFETCH
	prefetcht1 [esi-32+clm*32]
;	prefetcht1 [esi-32+clm*32+8*blkdst]
	prefetcht1 [esi-32+clm*32+16*blkdst]
;	prefetcht1 [esi-32+clm*32+24*blkdst]
	ENDIF
	loopclm	dl, clm, c7b
	lea	esi, [esi-clm*2*dist1+blkdst];; Next source pointer
	add	ah, 256/8		;; Test middle loop counter
	jnc	c7b			;; Iterate if necessary
	lea	esi, [esi-8*blkdst+32*blkdst];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al
	jnz	c7b
	lea	esi, [esi-4*32*blkdst]	;; Restore source pointer
	end_timer 18

;; Do inverse FFT levels 1,2,3
;; On input the 32-byte cache lines hold these data values:
;;	0	8K	1	8K+1
;;	256	...
;;	...
;;	16K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	32K	1	32K+1
;;	256
;;	...
;;	32K-256

;; Do 32 four_complex_last_unfft macros
;;	distance between fft data elements is 32

	start_timer 19
	mov	edi, plus1_premults	;; Address of premultiplier table
	mov	al, 32			;; Load loop counter
c1b:	disp four_complex_last_unfft, 8, 32*blkdst, 64*blkdst
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test loop counter
	jnc	c1b			;; Iterate if necessary
	IFDEF PFETCH
	prefetcht1 [esi-32+clm*32]
;	prefetcht1 [esi-32+clm*32+32*blkdst]
	prefetcht1 [esi-32+clm*32+64*blkdst]
;	prefetcht1 [esi-32+clm*32+96*blkdst]
	ENDIF
	loopclm	dl, clm, c1b
	lea	esi, [esi-clm*2*dist1+blkdst];; Next source pointer
	lea	edi, [edi+64]		;; Next premultiplier pointer
	dec	al			;; Test loop counter
	jnz	c1b			;; Iterate if necessary
	lea	esi, [esi-32*blkdst]	;; Restore source pointer
	end_timer 19

;; Normalize these values

	start_timer 20
	mov	eax, NORMRTN		;; Addr of normalization routine
	call	eax
	sub	eax, eax
	end_timer 20

;; Have we been given permission to start the FFT on the result?
;; If so, do so now while the data is in the L2 cache.

	cmp	POSTFFT, 0		;; Test flag
	je	no_fft			;; Skip FFT code if flag not set
	cmp	edx, p2cl/128*65536+(128/clm-4/clm)*256;; Delay fft on 8 sets
	jg	no_fft			;; Jump if delaying FFT
	copy_3_words clm, 0

;; Do FFT levels 1,2,3
;;
;; On input the 32-byte cache lines hold these data values:
;;	0	32K	1	32K+1
;;	256
;;	...
;;	32K-256
;; On output the 32-byte cache lines hold these data values:
;;	0	8K	1	8K+1
;;	256	...
;;	...
;;	16K	...
;;	...

;; Do 32 four_complex_first_fft macros
;;	distance between fft data elements is 32

b0b:	start_timer 21
	mov	ebx, DIST_TO_FFTSRCARG
	mov	edi, plus1_premults	;; Address of premultiplier table
	mov	al, 32			;; 32 iterations
b1b:	disp four_complex_first_fft, 32*blkdst, 64*blkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	add	dl, 256/2/clm		;; Test loop counter
	jnc	b1b			;; Iterate if necessary
	lea	esi, [esi-clm*2*dist1+blkdst];; Next source pointer
	lea	edi, [edi+64]		;; Next premultiplier pointer
	dec	al			;; Test loop counter
	jnz	b1b			;; Iterate if necessary
	lea	esi, [esi-32*blkdst]	;; Restore source pointer
	end_timer 21

;; Do FFT levels 4
;;
;; On input the 32-byte cache lines hold these data values:
;;	0	8K	1	8K+1
;;	256	...
;;	...
;;	16K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	4K	1	4K+1
;;	256	...
;;	...
;;	8K	...
;;	...

;; Do 32 four_complex_fft_1 macros
;;	distance between fft data elements is 16

	start_timer 22
	mov	edi, sincos3		;; Load sin/cos pointer
	mov	eax, 4			;; 4 iterations of 8
b7b:	disp four_complex_fft_1, 8*blkdst, 16*blkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test inner loop counter
	jnc	b7b			;; Iterate if necessary
	IFDEF PFETCH
;	prefetcht1 [esi-32+clm*32]
	prefetcht1 [esi-32+clm*32+8*blkdst]
;	prefetcht1 [esi-32+clm*32+16*blkdst]
	prefetcht1 [esi-32+clm*32+24*blkdst]
	ENDIF
	loopclm	dl, clm, b7b
	lea	esi, [esi-clm*2*dist1+blkdst];; Next source pointer
	add	ah, 256/8		;; Test middle loop counter
	jnc	b7b			;; Iterate if necessary
	lea	esi, [esi-8*blkdst+32*blkdst];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al
	jnz	b7b
	lea	esi, [esi-4*32*blkdst]	;; Restore source pointer
	end_timer 22

;; Do FFT levels 5,6
;;
;; On input the 32-byte cache lines hold these data values:
;;	0	4K	1	4K+1
;;	256	...
;;	...
;;	8K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	1K	1	1K+1
;;	256	...
;;	...
;;	2K	...
;;	...

;; Do 32 four_complex_fft macros
;;	distance between fft data elements is 4

	start_timer 23
	mov	edi, sincos2		;; Load sin/cos pointer
	mov	eax, 8			;; 8 iterations of 4
b5b:	disp four_complex_fft, 4*blkdst, 8*blkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test inner loop counter
	jnc	b5b			;; Iterate if necessary
	IFDEF PFETCH
;	prefetcht1 [esi-32+clm*32]
	prefetcht1 [esi-32+clm*32+4*blkdst]
;	prefetcht1 [esi-32+clm*32+8*blkdst]
	prefetcht1 [esi-32+clm*32+12*blkdst]
	ENDIF
	loopclm	dl, clm, b5b
	lea	esi, [esi-clm*2*dist1+blkdst];; Next source pointer
	add	ah, 256/4		;; Test middle loop counter
	jnc	b5b			;; Iterate if necessary
	lea	esi, [esi-4*blkdst+16*blkdst];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al
	jnz	b5b
	lea	esi, [esi-8*16*blkdst]	;; Restore source pointer
	end_timer 23

;; Do FFT levels 7,8
;;
;; On input the 32-byte cache lines hold these data values:
;;	0	1K	1	1K+1
;;	256	...
;;	...
;;	2K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	256	1	257
;;	512	...
;;	...

;; Do 32 four_complex_fft macros
;;	distance between fft data elements is 1

	start_timer 24
	mov	edi, sincos1		;; Load sin/cos pointer
	mov	al, 32			;; 32 iterations
b3b:	disp four_complex_fft, blkdst, 2*blkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test inner loop counter
	jnc	b3b			;; Iterate if necessary
	IFDEF PFETCH
;	prefetcht1 [esi-32+clm*32]
	prefetcht1 [esi-32+clm*32+blkdst]
;	prefetcht1 [esi-32+clm*32+2*blkdst]
	prefetcht1 [esi-32+clm*32+3*blkdst]
	ENDIF
	loopclm	dl, clm, b3b
	lea	esi, [esi-clm*2*dist1+4*blkdst];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al
	jnz	b3b
	lea	esi, [esi-32*4*blkdst]	;; Restore source pointer
	end_timer 24

;; Work on next 4 sets of 32 values

no_fft:	lea	esi, [esi+clm*2*dist1]	;; Next source pointer
	dec	dh			;; Test loop counter
	jnz	short more		;; Do more FFTs
	lea	esi, [esi+64]		;; Skip pad every 128 cache lines (4KB)
	sub	edx, 65536		;; Test loop counter
	jz	short fftdn		;; Jump if done looping
	mov	dh, 128/clm		;; Restore cache line counter
more:	cmp	norm_ptr1, 0		;; What was the loop start point?
	jne	c0b			;; Do inverse-FFT/norm/FFT iteration
	jmp	b0b			;; Do another FFT-only iteration
fftdn:	lea	esi, [esi-p2cl*2*dist1-p2cl/128*64];; Restore source pointer
	cmp	esi, DESTARG		;; If esi was not restored, then we 
	jne	short fftdn1		;; just finished skipped postfft data
	sub	ebx, ebx
	mov	DIST_TO_FFTSRCARG, ebx
	cmp	norm_ptr1, 0		;; What was the loop start point?
	je	pass2			;; FFT-only, go do pass 2

;; Split the carries into high and low carries (so they do not exceed
;; the desired number of bits per FFT word.  Then add the carries back to the
;; FFT data.

	start_timer 28
	call	gw_carries
	end_timer 28

;; Have we been given permission to start the FFT on the result?
;; If so, set loop to FFT the first few cache lines now that the carries
;; have been added in.

	cmp	POSTFFT, 0		;; Test flag
	je	short fftdn1		;; Skip FFT code if flag not set
	mov	esi, DESTARG		;; Load FFT data address
	mov	edx, 65536+4/clm*256	;; Do 8 previously skipped cache lines
	mov	norm_ptr1, 0		;; Set flag so only forward FFT is done
	copy_4_words
	jmp	b0b

;; Now add in the FFT'ed carries, cleanup and return

fftdn1:	end_timer 1
	jmp	gw_finish_mult
	ENDM


;; Perform a 80K-element FFT.  This is done in two passes.  Pass 1
;; does 9 levels, pass 2 uses common code to perform the last 8 levels.

pass1levels9pfa5 MACRO pass2_macro, p2cl, clm
	LOCAL	b0b, b1b, b2b, b3b, b4b, b5b, b6b, b7b
	LOCAL	c0b, c1b, c2b, c3b, c4b, c5b, c6b, c7b
	LOCAL	pass2, no_fft, more, fftdn, fftdn1

;; Call FFT code unless the FFT has already been started

	clear_timers
	cmp	DWORD PTR [esi-28][ebx], 0 ;; Test FFT-started flag
	jne	pass2			;; Jump is FFT already started
	mov	norm_ptr1, 0		;; Set FFT-ing flag
	mov	edx, p2cl/128*65536+128/clm*256;; Load loop counter
	jmp	b0b			;; Do the pass 1 fft

;; Do the last 8 FFT levels
;;	do 1 pass2_8_levels_real and 159 pass2_8_levels_complex

pass2:	start_timer 0
	CALLP	pass2_macro
	end_timer 0
	cmp	ffttype, 1		;; We're done if FFTing only
	je	gw_finish_fft

;; Do the inverse FFT

	start_timer 1
	fldz				;; Init SUMOUT
	fstp	SUMOUT
	mov	edx, norm_biglit_array	;; Addr of the big/little flags array
	mov	norm_ptr1, edx		;; Save ptr
	mov	edx, norm_col_mults	;; Addr of the column multipliers
	mov	norm_ptr2, edx		;; Save ptr
	mov	edx, p2cl/128*65536+128/clm*256;; Load loop counter

;; Do inverse FFT levels 8,9
;; On input the 32-byte cache lines hold these data values:
;;	0	256	1	257
;;	512	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	1K	1	1K+1
;;	256	...
;;	...
;;	2K	...
;;	...

;; Do 1 four_real_four_semireal_unfft macros
;;	distance between fft data elements is 1

c0b:	start_timer 16
c2b:	disp four_real_four_semireal_unfft, 8, blkdst, 2*blkdst
	lea	esi, [esi+dist1]	;; Next source pointer
	add	dl, 256/2/clm		;; Test inner loop counter
	jnc	c2b			;; Iterate if necessary
	lea	esi, [esi-clm*2*dist1+4*blkdst];; Next source pointer

;; Do 39 four_complex_unfft macros
;;	distance between fft data elements is 1

	mov	edi, sincos1		;; Load sin/cos pointer
	mov	al, 39			;; 39 iterations
c3b:	disp four_complex_unfft, 8, blkdst, 2*blkdst
	lea	esi, [esi+dist1]	;; Next source pointer
	add	dl, 256/2/clm		;; Test loop counter
	jnc	c3b			;; Iterate if necessary
	lea	esi, [esi-clm*2*dist1+4*blkdst];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al			;; Test outer loop counter
	jnz	c3b			;; Iterate if necessary
	lea	esi, [esi-40*4*blkdst]	;; Restore source pointer
	end_timer 16

;; Do inverse FFT levels 6,7
;; On input the 32-byte cache lines hold these data values:
;;	0	1K	1	1K+1
;;	256	...
;;	...
;;	2K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	4K	1	4K+1
;;	256	...
;;	...
;;	8K	...
;;	...

;; Do 4 four_real_four_semireal_unfft macros
;;	distance between fft data elements is 4

	start_timer 17
	mov	al, 4			;; Load loop counter
c4b:	disp four_real_four_semireal_unfft, 8, 4*blkdst, 8*blkdst
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test inner loop counter
	jnc	c4b			;; Iterate if necessary
	IFDEF PFETCH
	prefetcht1 [esi-32+clm*32]
;	prefetcht1 [esi-32+clm*32+4*blkdst]
	prefetcht1 [esi-32+clm*32+8*blkdst]
;	prefetcht1 [esi-32+clm*32+12*blkdst]
	ENDIF
	loopclm	dl, clm, c4b
	lea	esi, [esi-clm*2*dist1+blkdst];; Next source pointer
	dec	al
	jnz	c4b
	lea	esi, [esi-4*blkdst+16*blkdst];; Next source pointer

;; Do 36 four_complex_unfft macros
;;	distance between fft data elements is 4

	mov	edi, sincos2		;; Load sin/cos pointer
	mov	eax, 9			;; 9 iterations of 4
c5b:	disp four_complex_unfft, 8, 4*blkdst, 8*blkdst
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test inner loop counter
	jnc	c5b			;; Iterate if necessary
	IFDEF PFETCH
	prefetcht1 [esi-32+clm*32]
;	prefetcht1 [esi-32+clm*32+4*blkdst]
	prefetcht1 [esi-32+clm*32+8*blkdst]
;	prefetcht1 [esi-32+clm*32+12*blkdst]
	ENDIF
	loopclm	dl, clm, c5b
	lea	esi, [esi-clm*2*dist1+blkdst];; Next source pointer
	add	ah, 256/4		;; Test middle loop counter
	jnc	c5b			;; Iterate if necessary
	lea	esi, [esi-4*blkdst+16*blkdst];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al
	jnz	c5b
	lea	esi, [esi-10*16*blkdst]	;; Restore source pointer
	end_timer 17

;; Do inverse FFT levels 4,5
;; On input the 32-byte cache lines hold these data values:
;;	0	4K	1	4K+1
;;	256	...
;;	...
;;	8K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	16K	1	16K+1
;;	256	...
;;	...
;;	32K	...
;;	...

;; Do 8 eight_reals_unfft_2 macros
;;	distance between fft data elements is 16

	start_timer 18
	mov	al, 8			;; Load loop counter
c6b:	disp eight_reals_unfft_2, 8*blkdst, 8, 16*blkdst
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test inner loop counter
	jnc	c6b			;; Iterate if necessary
	IFDEF PFETCH
	prefetcht1 [esi-32+clm*32]
;	prefetcht1 [esi-32+clm*32+8*blkdst]
	prefetcht1 [esi-32+clm*32+16*blkdst]
;	prefetcht1 [esi-32+clm*32+24*blkdst]
	ENDIF
	loopclm	dl, clm, c6b
	lea	esi, [esi-clm*2*dist1+blkdst];; Next source pointer
	dec	al
	jnz	c6b
	lea	esi, [esi-8*blkdst+32*blkdst];; Next source pointer

;; Do 32 four_complex_unfft macros
;;	distance between fft data elements is 16

	mov	edi, sincos3		;; Load sin/cos pointer
	mov	eax, 2			;; 2 iterations of 16
c7b:	disp four_complex_unfft, 8, 16*blkdst, 32*blkdst
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test inner loop counter
	jnc	c7b			;; Iterate if necessary
	IFDEF PFETCH
	prefetcht1 [esi-32+clm*32]
;	prefetcht1 [esi-32+clm*32+16*blkdst]
	prefetcht1 [esi-32+clm*32+32*blkdst]
;	prefetcht1 [esi-32+clm*32+48*blkdst]
	ENDIF
	loopclm	dl, clm, c7b
	lea	esi, [esi-clm*2*dist1+blkdst];; Next source pointer
	add	ah, 256/16		;; Test middle loop counter
	jnc	c7b			;; Iterate if necessary
	lea	esi, [esi-16*blkdst+64*blkdst];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al
	jnz	c7b
	lea	esi, [esi-2*64*blkdst-32*blkdst];; Restore source pointer
	end_timer 18

;; Do inverse FFT levels 1,2,3
;; On input the 32-byte cache lines hold these data values:
;;	0	16K	1	16K+1
;;	256	...
;;	...
;;	32K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	40K	1	40K+1
;;	256
;;	...
;;	40K-256

;; Do 64 five_reals_last_unfft macros
;;	distance between fft data elements is 64

	start_timer 19
	mov	al, 32			;; Load loop counter
c1b:	two_five_reals_last_unfft 8, 32*blkdst
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test loop counter
	jnc	c1b			;; Iterate if necessary
	IFDEF PFETCH
	prefetcht1 [esi-32+clm*32]
;	prefetcht1 [esi-32+clm*32+32*blkdst]
	prefetcht1 [esi-32+clm*32+64*blkdst]
;	prefetcht1 [esi-32+clm*32+96*blkdst]
	prefetcht1 [esi-32+clm*32+128*blkdst]
	ENDIF
	loopclm	dl, clm, c1b
	lea	esi, [esi-clm*2*dist1+blkdst];; Next source pointer
	dec	al			;; Test loop counter
	jnz	c1b			;; Iterate if necessary
	lea	esi, [esi-32*blkdst]	;; Restore source pointer
	end_timer 19

;; Normalize these values

	start_timer 20
	mov	eax, NORMRTN		;; Addr of normalization routine
	call	eax
	sub	eax, eax
	end_timer 20

;; Have we been given permission to start the FFT on the result?
;; If so, do so now while the data is in the L2 cache.

	cmp	POSTFFT, 0		;; Test flag
	je	no_fft			;; Skip FFT code if flag not set
	cmp	edx, p2cl/128*65536+(128/clm-4/clm)*256;; Delay fft on 8 sets
	jg	no_fft			;; Jump if delaying FFT
	copy_3_words clm, 0

;; Do FFT levels 1,2,3
;;
;; On input the 32-byte cache lines hold these data values:
;;	0	40K	1	40K+1
;;	256
;;	...
;;	40K-256
;; On output the 32-byte cache lines hold these data values:
;;	0	16K	1	16K+1
;;	256	...
;;	...
;;	32K	...
;;	...

;; Do 64 five_reals_first_fft macros
;;	distance between fft data elements is 64

b0b:	start_timer 21
	mov	ebx, DIST_TO_FFTSRCARG
	mov	al, 32			;; 32 iterations
b1b:	two_five_reals_first_fft 32*blkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	add	dl, 256/2/clm		;; Test loop counter
	jnc	b1b			;; Iterate if necessary
	lea	esi, [esi-clm*2*dist1+blkdst];; Next source pointer
	dec	al			;; Test loop counter
	jnz	b1b			;; Iterate if necessary
	lea	esi, [esi-32*blkdst]	;; Restore source pointer
	end_timer 21

;; Do FFT levels 4,5
;;
;; On input the 32-byte cache lines hold these data values:
;;	0	16K	1	16K+1
;;	256	...
;;	...
;;	32K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	4K	1	4K+1
;;	256	...
;;	...
;;	8K	...
;;	...

;; Do 8 eight_reals_fft_2 macros
;;	distance between fft data elements is 16

	start_timer 22
	mov	al, 8			;; Load loop counter
b6b:	disp eight_reals_fft_2, 8*blkdst, 16*blkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test inner loop counter
	jnc	b6b			;; Iterate if necessary
	IFDEF PFETCH
;	prefetcht1 [esi-32+clm*32]
	prefetcht1 [esi-32+clm*32+8*blkdst]
;	prefetcht1 [esi-32+clm*32+16*blkdst]
	prefetcht1 [esi-32+clm*32+24*blkdst]
	ENDIF
	loopclm	dl, clm, b6b
	lea	esi, [esi-clm*2*dist1+blkdst];; Next source pointer
	dec	al
	jnz	b6b
	lea	esi, [esi-8*blkdst+32*blkdst];; Next source pointer

;; Do 32 four_complex_fft macros
;;	distance between fft data elements is 16

	mov	edi, sincos3		;; Load sin/cos pointer
	mov	eax, 2			;; 2 iterations of 16
b7b:	disp four_complex_fft, 16*blkdst, 32*blkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test inner loop counter
	jnc	b7b			;; Iterate if necessary
	IFDEF PFETCH
;	prefetcht1 [esi-32+clm*32]
	prefetcht1 [esi-32+clm*32+16*blkdst]
;	prefetcht1 [esi-32+clm*32+32*blkdst]
	prefetcht1 [esi-32+clm*32+48*blkdst]
	ENDIF
	loopclm	dl, clm, b7b
	lea	esi, [esi-clm*2*dist1+blkdst];; Next source pointer
	add	ah, 256/16		;; Test middle loop counter
	jnc	b7b			;; Iterate if necessary
	lea	esi, [esi-16*blkdst+64*blkdst];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al
	jnz	b7b
	lea	esi, [esi-2*64*blkdst-32*blkdst];; Restore source pointer
	end_timer 22

;; Do FFT levels 6,7
;;
;; On input the 32-byte cache lines hold these data values:
;;	0	4K	1	4K+1
;;	256	...
;;	...
;;	8K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	1K	1	1K+1
;;	256	...
;;	...
;;	2K	...
;;	...

;; Do 4 four_real_four_semireal_fft macros
;;	distance between fft data elements is 4

	start_timer 23
	mov	al, 4			;; Load loop counter
b4b:	disp four_real_four_semireal_fft, 4*blkdst, 8*blkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test inner loop counter
	jnc	b4b			;; Iterate if necessary
	IFDEF PFETCH
;	prefetcht1 [esi-32+clm*32]
	prefetcht1 [esi-32+clm*32+4*blkdst]
;	prefetcht1 [esi-32+clm*32+8*blkdst]
	prefetcht1 [esi-32+clm*32+12*blkdst]
	ENDIF
	loopclm	dl, clm, b4b
	lea	esi, [esi-clm*2*dist1+blkdst];; Next source pointer
	dec	al
	jnz	b4b
	lea	esi, [esi-4*blkdst+16*blkdst];; Next source pointer

;; Do 36 four_complex_fft macros
;;	distance between fft data elements is 4

	mov	edi, sincos2		;; Load sin/cos pointer
	mov	eax, 9			;; 9 iterations of 4
b5b:	disp four_complex_fft, 4*blkdst, 8*blkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test inner loop counter
	jnc	b5b			;; Iterate if necessary
	IFDEF PFETCH
;	prefetcht1 [esi-32+clm*32]
	prefetcht1 [esi-32+clm*32+4*blkdst]
;	prefetcht1 [esi-32+clm*32+8*blkdst]
	prefetcht1 [esi-32+clm*32+12*blkdst]
	ENDIF
	loopclm	dl, clm, b5b
	lea	esi, [esi-clm*2*dist1+blkdst];; Next source pointer
	add	ah, 256/4		;; Test middle loop counter
	jnc	b5b			;; Iterate if necessary
	lea	esi, [esi-4*blkdst+16*blkdst];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al
	jnz	b5b
	lea	esi, [esi-10*16*blkdst]	;; Restore source pointer
	end_timer 23

;; Do FFT levels 8,9
;;
;; On input the 32-byte cache lines hold these data values:
;;	0	1K	1	1K+1
;;	256	...
;;	...
;;	2K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	256	1	257
;;	512	...
;;	...

;; Do 1 four_real_four_semireal_fft macros
;;	distance between fft data elements is 1

	start_timer 24
b2b:	disp four_real_four_semireal_fft, blkdst, 2*blkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test inner loop counter
	jnc	b2b			;; Iterate if necessary
	IFDEF PFETCH
;	prefetcht1 [esi-32+clm*32]
	prefetcht1 [esi-32+clm*32+blkdst]
;	prefetcht1 [esi-32+clm*32+2*blkdst]
	prefetcht1 [esi-32+clm*32+3*blkdst]
	ENDIF
	loopclm	dl, clm, b2b
	lea	esi, [esi-clm*2*dist1+4*blkdst];; Next source pointer

;; Do 39 four_complex_fft macros
;;	distance between fft data elements is 1

	mov	edi, sincos1		;; Load sin/cos pointer
	mov	al, 39			;; 39 iterations
b3b:	disp four_complex_fft, blkdst, 2*blkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test inner loop counter
	jnc	b3b			;; Iterate if necessary
	IFDEF PFETCH
;	prefetcht1 [esi-32+clm*32]
	prefetcht1 [esi-32+clm*32+blkdst]
;	prefetcht1 [esi-32+clm*32+2*blkdst]
	prefetcht1 [esi-32+clm*32+3*blkdst]
	ENDIF
	loopclm	dl, clm, b3b
	lea	esi, [esi-clm*2*dist1+4*blkdst];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al
	jnz	b3b
	lea	esi, [esi-40*4*blkdst]	;; Restore source pointer
	end_timer 24

;; Work on next 4 sets of 32 values

no_fft:	lea	esi, [esi+clm*2*dist1]	;; Next source pointer
	dec	dh			;; Test loop counter
	jnz	short more		;; Do more FFTs
	lea	esi, [esi+64]		;; Skip pad every 128 cache lines (4KB)
	sub	edx, 65536		;; Test loop counter
	jz	short fftdn		;; Jump if done looping
	mov	dh, 128/clm		;; Restore cache line counter
more:	cmp	norm_ptr1, 0		;; What was the loop start point?
	jne	c0b			;; Do inverse-FFT/norm/FFT iteration
	jmp	b0b			;; Do another FFT-only iteration
fftdn:	lea	esi, [esi-p2cl*2*dist1-p2cl/128*64];; Restore source pointer
	cmp	esi, DESTARG		;; If esi was not restored, then we 
	jne	short fftdn1		;; just finished skipped postfft data
	sub	ebx, ebx
	mov	DIST_TO_FFTSRCARG, ebx
	cmp	norm_ptr1, 0		;; What was the loop start point?
	je	pass2			;; FFT-only, go do pass 2

;; Split the carries into high and low carries (so they do not exceed
;; the desired number of bits per FFT word.  Then add the carries back to the
;; FFT data.

	start_timer 28
	call	gw_carries
	end_timer 28

;; Have we been given permission to start the FFT on the result?
;; If so, set loop to FFT the first few cache lines now that the carries
;; have been added in.

	cmp	POSTFFT, 0		;; Test flag
	je	short fftdn1		;; Skip FFT code if flag not set
	mov	esi, DESTARG		;; Load FFT data address
	mov	edx, 65536+4/clm*256	;; Do 8 previously skipped cache lines
	mov	norm_ptr1, 0		;; Set flag so only forward FFT is done
	copy_4_words
	jmp	b0b

;; Now add in the FFT'ed carries, cleanup and return

fftdn1:	end_timer 1
	jmp	gw_finish_mult
	ENDM


;; Perform a 96K-element FFT.  This is done in two passes.  Pass 1
;; does 9 levels, pass 2 uses common code to perform the last 8 levels.

pass1levels9pfa6 MACRO pass2_macro, p2cl, clm
	LOCAL	b0b, b1b, b2b, b3b, b4b, b5b, b6b, b7b
	LOCAL	c0b, c1b, c2b, c3b, c4b, c5b, c6b, c7b
	LOCAL	pass2, no_fft, more, fftdn, fftdn1

;; Call FFT code unless the FFT has already been started

	clear_timers
	cmp	DWORD PTR [esi-28][ebx], 0 ;; Test FFT-started flag
	jne	pass2			;; Jump is FFT already started
	mov	norm_ptr1, 0		;; Set FFT-ing flag
	mov	edx, p2cl/128*65536+128/clm*256;; Load loop counter
	jmp	b0b			;; Do the pass 1 fft

;; Do the last 8 FFT levels
;;	do 1 pass2_8_levels_real and 191 pass2_8_levels_complex

pass2:	start_timer 0
	CALLP	pass2_macro
	end_timer 0
	cmp	ffttype, 1		;; We're done if FFTing only
	je	gw_finish_fft

;; Do the inverse FFT

	start_timer 1
	fldz				;; Init SUMOUT
	fstp	SUMOUT
	mov	edx, norm_biglit_array	;; Addr of the big/little flags array
	mov	norm_ptr1, edx		;; Save ptr
	mov	edx, norm_col_mults	;; Addr of the column multipliers
	mov	norm_ptr2, edx		;; Save ptr
	mov	edx, p2cl/128*65536+128/clm*256;; Load loop counter

;; Do inverse FFT levels 8,9
;; On input the 32-byte cache lines hold these data values:
;;	0	256	1	257
;;	512	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	1K	1	1K+1
;;	256	...
;;	...
;;	2K	...
;;	...

;; Do 1 four_real_four_semireal_unfft macros
;;	distance between fft data elements is 1

c0b:	start_timer 16
c2b:	disp four_real_four_semireal_unfft, 8, blkdst, 2*blkdst
	lea	esi, [esi+dist1]	;; Next source pointer
	add	dl, 256/2/clm		;; Test inner loop counter
	jnc	c2b			;; Iterate if necessary
	lea	esi, [esi-clm*2*dist1+4*blkdst];; Next source pointer

;; Do 47 four_complex_unfft macros
;;	distance between fft data elements is 1

	mov	edi, sincos1		;; Load sin/cos pointer
	mov	al, 47			;; 47 iterations
c3b:	disp four_complex_unfft, 8, blkdst, 2*blkdst
	lea	esi, [esi+dist1]	;; Next source pointer
	add	dl, 256/2/clm		;; Test loop counter
	jnc	c3b			;; Iterate if necessary
	lea	esi, [esi-clm*2*dist1+4*blkdst];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al			;; Test outer loop counter
	jnz	c3b			;; Iterate if necessary
	lea	esi, [esi-48*4*blkdst]	;; Restore source pointer
	end_timer 16

;; Do inverse FFT levels 6,7
;; On input the 32-byte cache lines hold these data values:
;;	0	1K	1	1K+1
;;	256	...
;;	...
;;	2K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	4K	1	4K+1
;;	256	...
;;	...
;;	8K	...
;;	...

;; Do 4 four_real_four_semireal_unfft macros
;;	distance between fft data elements is 4

	start_timer 17
	mov	al, 4			;; Load loop counter
c4b:	disp four_real_four_semireal_unfft, 8, 4*blkdst, 8*blkdst
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test inner loop counter
	jnc	c4b			;; Iterate if necessary
	IFDEF PFETCH
	prefetcht1 [esi-32+clm*32]
;	prefetcht1 [esi-32+clm*32+4*blkdst]
	prefetcht1 [esi-32+clm*32+8*blkdst]
;	prefetcht1 [esi-32+clm*32+12*blkdst]
	ENDIF
	loopclm	dl, clm, c4b
	lea	esi, [esi-clm*2*dist1+blkdst];; Next source pointer
	dec	al
	jnz	c4b
	lea	esi, [esi-4*blkdst+16*blkdst];; Next source pointer

;; Do 44 four_complex_unfft macros
;;	distance between fft data elements is 4

	mov	edi, sincos2		;; Load sin/cos pointer
	mov	eax, 11			;; 11 iterations of 4
c5b:	disp four_complex_unfft, 8, 4*blkdst, 8*blkdst
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test inner loop counter
	jnc	c5b			;; Iterate if necessary
	IFDEF PFETCH
	prefetcht1 [esi-32+clm*32]
;	prefetcht1 [esi-32+clm*32+4*blkdst]
	prefetcht1 [esi-32+clm*32+8*blkdst]
;	prefetcht1 [esi-32+clm*32+12*blkdst]
	ENDIF
	loopclm	dl, clm, c5b
	lea	esi, [esi-clm*2*dist1+blkdst];; Next source pointer
	add	ah, 256/4		;; Test middle loop counter
	jnc	c5b			;; Iterate if necessary
	lea	esi, [esi-4*blkdst+16*blkdst];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al
	jnz	c5b
	lea	esi, [esi-12*16*blkdst]	;; Restore source pointer
	end_timer 17

;; Do inverse FFT levels 4,5
;; On input the 32-byte cache lines hold these data values:
;;	0	4K	1	4K+1
;;	256	...
;;	...
;;	8K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	16K	1	16K+1
;;	256	...
;;	...
;;	32K	...
;;	...

;; Do 16 four_real_four_semireal_unfft macros
;;	distance between fft data elements is 16

	start_timer 18
	mov	al, 16			;; Load loop counter
c6b:	disp four_real_four_semireal_unfft, 8, 16*blkdst, 32*blkdst
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test inner loop counter
	jnc	c6b			;; Iterate if necessary
	IFDEF PFETCH
	prefetcht1 [esi-32+clm*32]
;	prefetcht1 [esi-32+clm*32+16*blkdst]
	prefetcht1 [esi-32+clm*32+32*blkdst]
;	prefetcht1 [esi-32+clm*32+48*blkdst]
	ENDIF
	loopclm	dl, clm, c6b
	lea	esi, [esi-clm*2*dist1+blkdst];; Next source pointer
	dec	al
	jnz	c6b
	lea	esi, [esi-16*blkdst+64*blkdst];; Next source pointer

;; Do 32 four_complex_unfft macros
;;	distance between fft data elements is 16

	mov	edi, sincos3		;; Load sin/cos pointer
	mov	eax, 2			;; 2 iterations of 16
c7b:	disp four_complex_unfft, 8, 16*blkdst, 32*blkdst
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test inner loop counter
	jnc	c7b			;; Iterate if necessary
	IFDEF PFETCH
	prefetcht1 [esi-32+clm*32]
;	prefetcht1 [esi-32+clm*32+16*blkdst]
	prefetcht1 [esi-32+clm*32+32*blkdst]
;	prefetcht1 [esi-32+clm*32+48*blkdst]
	ENDIF
	loopclm	dl, clm, c7b
	lea	esi, [esi-clm*2*dist1+blkdst];; Next source pointer
	add	ah, 256/16		;; Test middle loop counter
	jnc	c7b			;; Iterate if necessary
	lea	esi, [esi-16*blkdst+64*blkdst];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al
	jnz	c7b
	lea	esi, [esi-3*64*blkdst]	;; Restore source pointer
	end_timer 18

;; Do inverse FFT levels 1,2,3
;; On input the 32-byte cache lines hold these data values:
;;	0	16K	1	16K+1
;;	256	...
;;	...
;;	32K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	48K	1	48K+1
;;	256
;;	...
;;	48K-256

;; Do 64 six_reals_last_unfft macros
;;	distance between fft data elements is 64

	start_timer 19
	mov	al, 64			;; Load loop counter
c1b:	disp six_reals_last_unfft, 8, 64*blkdst, 128*blkdst
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test loop counter
	jnc	c1b			;; Iterate if necessary
	IFDEF PFETCH
	prefetcht1 [esi-32+clm*32]
;	prefetcht1 [esi-32+clm*32+64*blkdst]
	prefetcht1 [esi-32+clm*32+128*blkdst]
;	prefetcht1 [esi-32+clm*32+192*blkdst]
	ENDIF
	loopclm	dl, clm, c1b
	lea	esi, [esi-clm*2*dist1+blkdst];; Next source pointer
	dec	al			;; Test loop counter
	jnz	c1b			;; Iterate if necessary
	lea	esi, [esi-64*blkdst]	;; Restore source pointer
	end_timer 19

;; Normalize these values

	start_timer 20
	mov	eax, NORMRTN		;; Addr of normalization routine
	call	eax
	sub	eax, eax
	end_timer 20

;; Have we been given permission to start the FFT on the result?
;; If so, do so now while the data is in the L2 cache.

	cmp	POSTFFT, 0		;; Test flag
	je	no_fft			;; Skip FFT code if flag not set
	cmp	edx, p2cl/128*65536+(128/clm-4/clm)*256;; Delay fft on 8 sets
	jg	no_fft			;; Jump if delaying FFT
	copy_3_words clm, 0

;; Do FFT levels 1,2,3
;;
;; On input the 32-byte cache lines hold these data values:
;;	0	48K	1	48K+1
;;	256
;;	...
;;	48K-256
;; On output the 32-byte cache lines hold these data values:
;;	0	16K	1	16K+1
;;	256	...
;;	...
;;	32K	...
;;	...

;; Do 64 six_reals_first_fft macros
;;	distance between fft data elements is 64

b0b:	start_timer 21
	mov	ebx, DIST_TO_FFTSRCARG
	mov	al, 64			;; 64 iterations
b1b:	disp six_reals_first_fft, 64*blkdst, 128*blkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	add	dl, 256/2/clm		;; Test loop counter
	jnc	b1b			;; Iterate if necessary
	lea	esi, [esi-clm*2*dist1+blkdst];; Next source pointer
	dec	al			;; Test loop counter
	jnz	b1b			;; Iterate if necessary
	lea	esi, [esi-64*blkdst]	;; Restore source pointer
	end_timer 21

;; Do FFT levels 4,5
;;
;; On input the 32-byte cache lines hold these data values:
;;	0	16K	1	16K+1
;;	256	...
;;	...
;;	32K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	4K	1	4K+1
;;	256	...
;;	...
;;	8K	...
;;	...

;; Do 16 four_real_four_semireal_fft macros
;;	distance between fft data elements is 16

	start_timer 22
	mov	al, 16			;; Load loop counter
b6b:	disp four_real_four_semireal_fft, 16*blkdst, 32*blkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test inner loop counter
	jnc	b6b			;; Iterate if necessary
	IFDEF PFETCH
;	prefetcht1 [esi-32+clm*32]
	prefetcht1 [esi-32+clm*32+16*blkdst]
;	prefetcht1 [esi-32+clm*32+32*blkdst]
	prefetcht1 [esi-32+clm*32+48*blkdst]
	ENDIF
	loopclm	dl, clm, b6b
	lea	esi, [esi-clm*2*dist1+blkdst];; Next source pointer
	dec	al
	jnz	b6b
	lea	esi, [esi-16*blkdst+64*blkdst];; Next source pointer

;; Do 32 four_complex_fft macros
;;	distance between fft data elements is 16

	mov	edi, sincos3		;; Load sin/cos pointer
	mov	eax, 2			;; 2 iterations of 16
b7b:	disp four_complex_fft, 16*blkdst, 32*blkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test inner loop counter
	jnc	b7b			;; Iterate if necessary
	IFDEF PFETCH
;	prefetcht1 [esi-32+clm*32]
	prefetcht1 [esi-32+clm*32+16*blkdst]
;	prefetcht1 [esi-32+clm*32+32*blkdst]
	prefetcht1 [esi-32+clm*32+48*blkdst]
	ENDIF
	loopclm	dl, clm, b7b
	lea	esi, [esi-clm*2*dist1+blkdst];; Next source pointer
	add	ah, 256/16		;; Test middle loop counter
	jnc	b7b			;; Iterate if necessary
	lea	esi, [esi-16*blkdst+64*blkdst];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al
	jnz	b7b
	lea	esi, [esi-3*64*blkdst]	;; Restore source pointer
	end_timer 22

;; Do FFT levels 6,7
;;
;; On input the 32-byte cache lines hold these data values:
;;	0	4K	1	4K+1
;;	256	...
;;	...
;;	8K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	1K	1	1K+1
;;	256	...
;;	...
;;	2K	...
;;	...

;; Do 4 four_real_four_semireal_fft macros
;;	distance between fft data elements is 4

	start_timer 23
	mov	al, 4			;; Load loop counter
b4b:	disp four_real_four_semireal_fft, 4*blkdst, 8*blkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test inner loop counter
	jnc	b4b			;; Iterate if necessary
	IFDEF PFETCH
;	prefetcht1 [esi-32+clm*32]
	prefetcht1 [esi-32+clm*32+4*blkdst]
;	prefetcht1 [esi-32+clm*32+8*blkdst]
	prefetcht1 [esi-32+clm*32+12*blkdst]
	ENDIF
	loopclm	dl, clm, b4b
	lea	esi, [esi-clm*2*dist1+blkdst];; Next source pointer
	dec	al
	jnz	b4b
	lea	esi, [esi-4*blkdst+16*blkdst];; Next source pointer

;; Do 44 four_complex_fft macros
;;	distance between fft data elements is 4

	mov	edi, sincos2		;; Load sin/cos pointer
	mov	eax, 11			;; 11 iterations of 4
b5b:	disp four_complex_fft, 4*blkdst, 8*blkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test inner loop counter
	jnc	b5b			;; Iterate if necessary
	IFDEF PFETCH
;	prefetcht1 [esi-32+clm*32]
	prefetcht1 [esi-32+clm*32+4*blkdst]
;	prefetcht1 [esi-32+clm*32+8*blkdst]
	prefetcht1 [esi-32+clm*32+12*blkdst]
	ENDIF
	loopclm	dl, clm, b5b
	lea	esi, [esi-clm*2*dist1+blkdst];; Next source pointer
	add	ah, 256/4		;; Test middle loop counter
	jnc	b5b			;; Iterate if necessary
	lea	esi, [esi-4*blkdst+16*blkdst];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al
	jnz	b5b
	lea	esi, [esi-12*16*blkdst]	;; Restore source pointer
	end_timer 23

;; Do FFT levels 8,9
;;
;; On input the 32-byte cache lines hold these data values:
;;	0	1K	1	1K+1
;;	256	...
;;	...
;;	2K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	256	1	257
;;	512	...
;;	...

;; Do 1 four_real_four_semireal_fft macros
;;	distance between fft data elements is 1

	start_timer 24
b2b:	disp four_real_four_semireal_fft, blkdst, 2*blkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test inner loop counter
	jnc	b2b			;; Iterate if necessary
	IFDEF PFETCH
;	prefetcht1 [esi-32+clm*32]
	prefetcht1 [esi-32+clm*32+blkdst]
;	prefetcht1 [esi-32+clm*32+2*blkdst]
	prefetcht1 [esi-32+clm*32+3*blkdst]
	ENDIF
	loopclm	dl, clm, b2b
	lea	esi, [esi-clm*2*dist1+4*blkdst];; Next source pointer

;; Do 47 four_complex_fft macros
;;	distance between fft data elements is 1

	mov	edi, sincos1		;; Load sin/cos pointer
	mov	al, 47			;; 47 iterations
b3b:	disp four_complex_fft, blkdst, 2*blkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test inner loop counter
	jnc	b3b			;; Iterate if necessary
	IFDEF PFETCH
;	prefetcht1 [esi-32+clm*32]
	prefetcht1 [esi-32+clm*32+blkdst]
;	prefetcht1 [esi-32+clm*32+2*blkdst]
	prefetcht1 [esi-32+clm*32+3*blkdst]
	ENDIF
	loopclm	dl, clm, b3b
	lea	esi, [esi-clm*2*dist1+4*blkdst];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al
	jnz	b3b
	lea	esi, [esi-48*4*blkdst]	;; Restore source pointer
	end_timer 24

;; Work on next 4 sets of 32 values

no_fft:	lea	esi, [esi+clm*2*dist1]	;; Next source pointer
	dec	dh			;; Test loop counter
	jnz	short more		;; Do more FFTs
	lea	esi, [esi+64]		;; Skip pad every 128 cache lines (4KB)
	sub	edx, 65536		;; Test loop counter
	jz	short fftdn		;; Jump if done looping
	mov	dh, 128/clm		;; Restore cache line counter
more:	cmp	norm_ptr1, 0		;; What was the loop start point?
	jne	c0b			;; Do inverse-FFT/norm/FFT iteration
	jmp	b0b			;; Do another FFT-only iteration
fftdn:	lea	esi, [esi-p2cl*2*dist1-p2cl/128*64];; Restore source pointer
	cmp	esi, DESTARG		;; If esi was not restored, then we 
	jne	short fftdn1		;; just finished skipped postfft data
	sub	ebx, ebx
	mov	DIST_TO_FFTSRCARG, ebx
	cmp	norm_ptr1, 0		;; What was the loop start point?
	je	pass2			;; FFT-only, go do pass 2

;; Split the carries into high and low carries (so they do not exceed
;; the desired number of bits per FFT word.  Then add the carries back to the
;; FFT data.

	start_timer 28
	call	gw_carries
	end_timer 28

;; Have we been given permission to start the FFT on the result?
;; If so, set loop to FFT the first few cache lines now that the carries
;; have been added in.

	cmp	POSTFFT, 0		;; Test flag
	je	short fftdn1		;; Skip FFT code if flag not set
	mov	esi, DESTARG		;; Load FFT data address
	mov	edx, 65536+4/clm*256	;; Do 8 previously skipped cache lines
	mov	norm_ptr1, 0		;; Set flag so only forward FFT is done
	copy_4_words
	jmp	b0b

;; Now add in the FFT'ed carries, cleanup and return

fftdn1:	end_timer 1
	jmp	gw_finish_mult
	ENDM


;; Perform a 96K-element negacyclic FFT.  This is done in two passes.  Pass 1
;; does 9 levels, pass 2 uses common code to perform the last 8 levels.

pass1levels9complex3 MACRO pass2_macro, p2cl, clm
	LOCAL	b0b, b1b, b3b, b5b, b7b, c0b, c1b, c3b, c5b, c7b
	LOCAL	pass2, no_fft, more, fftdn, fftdn1

;; Call FFT code unless the FFT has already been started

	clear_timers
	cmp	DWORD PTR [esi-28][ebx], 0 ;; Test FFT-started flag
	jne	pass2			;; Jump is FFT already started
	mov	norm_ptr1, 0		;; Set FFT-ing flag
	mov	edx, p2cl/128*65536+128/clm*256;; Load loop counter
	jmp	b0b			;; Do the pass 1 fft

;; Do the last 8 FFT levels
;;	do 192 pass2_8_levels_complex

pass2:	start_timer 0
	CALLP	pass2_macro
	end_timer 0
	cmp	ffttype, 1		;; We're done if FFTing only
	je	gw_finish_fft

;; Do the inverse FFT

	start_timer 1
	fldz				;; Init SUMOUT
	fstp	SUMOUT
	mov	edx, norm_biglit_array	;; Addr of the big/little flags array
	mov	norm_ptr1, edx		;; Save ptr
	mov	edx, norm_col_mults	;; Addr of the column multipliers
	mov	norm_ptr2, edx		;; Save ptr
	mov	edx, p2cl/128*65536+128/clm*256;; Load loop counter

;; Do inverse FFT levels 8,9
;; On input the 32-byte cache lines hold these data values:
;;	0	256	1	257
;;	512	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	1K	1	1K+1
;;	256	...
;;	...
;;	2K	...
;;	...

;; Do 48 four_complex_unfft macros
;;	distance between fft data elements is 1

c0b:	start_timer 16
	mov	edi, sincos1		;; Load sin/cos pointer
	mov	al, 48			;; 48 iterations
c3b:	disp four_complex_unfft, 8, blkdst, 2*blkdst
	lea	esi, [esi+dist1]	;; Next source pointer
	add	dl, 256/2/clm		;; Test loop counter
	jnc	c3b			;; Iterate if necessary
	lea	esi, [esi-clm*2*dist1+4*blkdst];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al			;; Test outer loop counter
	jnz	c3b			;; Iterate if necessary
	lea	esi, [esi-48*4*blkdst]	;; Restore source pointer
	end_timer 16

;; Do inverse FFT levels 6,7
;; On input the 32-byte cache lines hold these data values:
;;	0	1K	1	1K+1
;;	256	...
;;	...
;;	2K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	4K	1	4K+1
;;	256	...
;;	...
;;	8K	...
;;	...

;; Do 48 four_complex_unfft macros
;;	distance between fft data elements is 4

	start_timer 17
	mov	edi, sincos2		;; Load sin/cos pointer
	mov	eax, 12			;; 12 iterations of 4
c5b:	disp four_complex_unfft, 8, 4*blkdst, 8*blkdst
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test inner loop counter
	jnc	c5b			;; Iterate if necessary
	IFDEF PFETCH
	prefetcht1 [esi-32+clm*32]
;	prefetcht1 [esi-32+clm*32+4*blkdst]
	prefetcht1 [esi-32+clm*32+8*blkdst]
;	prefetcht1 [esi-32+clm*32+12*blkdst]
	ENDIF
	loopclm	dl, clm, c5b
	lea	esi, [esi-clm*2*dist1+blkdst];; Next source pointer
	add	ah, 256/4		;; Test middle loop counter
	jnc	c5b			;; Iterate if necessary
	lea	esi, [esi-4*blkdst+16*blkdst];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al
	jnz	c5b
	lea	esi, [esi-12*16*blkdst]	;; Restore source pointer
	end_timer 17

;; Do inverse FFT levels 4,5
;; On input the 32-byte cache lines hold these data values:
;;	0	4K	1	4K+1
;;	256	...
;;	...
;;	8K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	16K	1	16K+1
;;	256	...
;;	...
;;	32K	...
;;	...

;; Do 48 four_complex_unfft macros
;;	distance between fft data elements is 16

	start_timer 18
	mov	edi, sincos3		;; Load sin/cos pointer
	mov	eax, 3			;; 3 iterations of 16
c7b:	disp four_complex_unfft, 8, 16*blkdst, 32*blkdst
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test inner loop counter
	jnc	c7b			;; Iterate if necessary
	IFDEF PFETCH
	prefetcht1 [esi-32+clm*32]
;	prefetcht1 [esi-32+clm*32+16*blkdst]
	prefetcht1 [esi-32+clm*32+32*blkdst]
;	prefetcht1 [esi-32+clm*32+48*blkdst]
	ENDIF
	loopclm	dl, clm, c7b
	lea	esi, [esi-clm*2*dist1+blkdst];; Next source pointer
	add	ah, 256/16		;; Test middle loop counter
	jnc	c7b			;; Iterate if necessary
	lea	esi, [esi-16*blkdst+64*blkdst];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al
	jnz	c7b
	lea	esi, [esi-3*64*blkdst]	;; Restore source pointer
	end_timer 18

;; Do inverse FFT levels 1,2,3
;; On input the 32-byte cache lines hold these data values:
;;	0	16K	1	16K+1
;;	256	...
;;	...
;;	32K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	48K	1	48K+1
;;	256
;;	...
;;	48K-256

;; Do 64 three_complex_last_unfft macros
;;	distance between fft data elements is 64

	start_timer 19
	mov	edi, plus1_premults	;; Address of premultiplier table
	mov	al, 64			;; Load loop counter
c1b:	three_complex_last_unfft 8, 64*blkdst, 128*blkdst
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test loop counter
	jnc	c1b			;; Iterate if necessary
	IFDEF PFETCH
	prefetcht1 [esi-32+clm*32]
;	prefetcht1 [esi-32+clm*32+64*blkdst]
	prefetcht1 [esi-32+clm*32+128*blkdst]
	ENDIF
	loopclm	dl, clm, c1b
	lea	esi, [esi-clm*2*dist1+blkdst];; Next source pointer
	lea	edi, [edi+48]		;; Next premultiplier pointer
	dec	al			;; Test loop counter
	jnz	c1b			;; Iterate if necessary
	lea	esi, [esi-64*blkdst]	;; Restore source pointer
	end_timer 19

;; Normalize these values

	start_timer 20
	mov	eax, NORMRTN		;; Addr of normalization routine
	call	eax
	sub	eax, eax
	end_timer 20

;; Have we been given permission to start the FFT on the result?
;; If so, do so now while the data is in the L2 cache.

	cmp	POSTFFT, 0		;; Test flag
	je	no_fft			;; Skip FFT code if flag not set
	cmp	edx, p2cl/128*65536+(128/clm-4/clm)*256;; Delay fft on 8 sets
	jg	no_fft			;; Jump if delaying FFT
	copy_3_words clm, 0

;; Do FFT levels 1,2,3
;;
;; On input the 32-byte cache lines hold these data values:
;;	0	48K	1	48K+1
;;	256
;;	...
;;	48K-256
;; On output the 32-byte cache lines hold these data values:
;;	0	16K	1	16K+1
;;	256	...
;;	...
;;	32K	...
;;	...

;; Do 64 three_complex_first_fft macros
;;	distance between fft data elements is 64

b0b:	start_timer 21
	mov	ebx, DIST_TO_FFTSRCARG
	mov	edi, plus1_premults	;; Address of premultiplier table
	mov	al, 64			;; 64 iterations
b1b:	three_complex_first_fft 64*blkdst, 128*blkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	add	dl, 256/2/clm		;; Test loop counter
	jnc	b1b			;; Iterate if necessary
	lea	esi, [esi-clm*2*dist1+blkdst];; Next source pointer
	lea	edi, [edi+48]		;; Next premultiplier pointer
	dec	al			;; Test loop counter
	jnz	b1b			;; Iterate if necessary
	lea	esi, [esi-64*blkdst]	;; Restore source pointer
	end_timer 21

;; Do FFT levels 4,5
;;
;; On input the 32-byte cache lines hold these data values:
;;	0	16K	1	16K+1
;;	256	...
;;	...
;;	32K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	4K	1	4K+1
;;	256	...
;;	...
;;	8K	...
;;	...

;; Do 48 four_complex_fft macros
;;	distance between fft data elements is 16

	start_timer 22
	mov	edi, sincos3		;; Load sin/cos pointer
	mov	eax, 3			;; 3 iterations of 16
b7b:	disp four_complex_fft, 16*blkdst, 32*blkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test inner loop counter
	jnc	b7b			;; Iterate if necessary
	IFDEF PFETCH
;	prefetcht1 [esi-32+clm*32]
	prefetcht1 [esi-32+clm*32+16*blkdst]
;	prefetcht1 [esi-32+clm*32+32*blkdst]
	prefetcht1 [esi-32+clm*32+48*blkdst]
	ENDIF
	loopclm	dl, clm, b7b
	lea	esi, [esi-clm*2*dist1+blkdst];; Next source pointer
	add	ah, 256/16		;; Test middle loop counter
	jnc	b7b			;; Iterate if necessary
	lea	esi, [esi-16*blkdst+64*blkdst];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al
	jnz	b7b
	lea	esi, [esi-3*64*blkdst]	;; Restore source pointer
	end_timer 22

;; Do FFT levels 6,7
;;
;; On input the 32-byte cache lines hold these data values:
;;	0	4K	1	4K+1
;;	256	...
;;	...
;;	8K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	1K	1	1K+1
;;	256	...
;;	...
;;	2K	...
;;	...

;; Do 48 four_complex_fft macros
;;	distance between fft data elements is 4

	start_timer 23
	mov	edi, sincos2		;; Load sin/cos pointer
	mov	eax, 12			;; 12 iterations of 4
b5b:	disp four_complex_fft, 4*blkdst, 8*blkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test inner loop counter
	jnc	b5b			;; Iterate if necessary
	IFDEF PFETCH
;	prefetcht1 [esi-32+clm*32]
	prefetcht1 [esi-32+clm*32+4*blkdst]
;	prefetcht1 [esi-32+clm*32+8*blkdst]
	prefetcht1 [esi-32+clm*32+12*blkdst]
	ENDIF
	loopclm	dl, clm, b5b
	lea	esi, [esi-clm*2*dist1+blkdst];; Next source pointer
	add	ah, 256/4		;; Test middle loop counter
	jnc	b5b			;; Iterate if necessary
	lea	esi, [esi-4*blkdst+16*blkdst];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al
	jnz	b5b
	lea	esi, [esi-12*16*blkdst]	;; Restore source pointer
	end_timer 23

;; Do FFT levels 8,9
;;
;; On input the 32-byte cache lines hold these data values:
;;	0	1K	1	1K+1
;;	256	...
;;	...
;;	2K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	256	1	257
;;	512	...
;;	...

;; Do 48 four_complex_fft macros
;;	distance between fft data elements is 1

	start_timer 24
	mov	edi, sincos1		;; Load sin/cos pointer
	mov	al, 48			;; 48 iterations
b3b:	disp four_complex_fft, blkdst, 2*blkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test inner loop counter
	jnc	b3b			;; Iterate if necessary
	IFDEF PFETCH
;	prefetcht1 [esi-32+clm*32]
	prefetcht1 [esi-32+clm*32+blkdst]
;	prefetcht1 [esi-32+clm*32+2*blkdst]
	prefetcht1 [esi-32+clm*32+3*blkdst]
	ENDIF
	loopclm	dl, clm, b3b
	lea	esi, [esi-clm*2*dist1+4*blkdst];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al
	jnz	b3b
	lea	esi, [esi-48*4*blkdst]	;; Restore source pointer
	end_timer 24

;; Work on next 4 sets of 32 values

no_fft:	lea	esi, [esi+clm*2*dist1]	;; Next source pointer
	dec	dh			;; Test loop counter
	jnz	short more		;; Do more FFTs
	lea	esi, [esi+64]		;; Skip pad every 128 cache lines (4KB)
	sub	edx, 65536		;; Test loop counter
	jz	short fftdn		;; Jump if done looping
	mov	dh, 128/clm		;; Restore cache line counter
more:	cmp	norm_ptr1, 0		;; What was the loop start point?
	jne	c0b			;; Do inverse-FFT/norm/FFT iteration
	jmp	b0b			;; Do another FFT-only iteration
fftdn:	lea	esi, [esi-p2cl*2*dist1-p2cl/128*64];; Restore source pointer
	cmp	esi, DESTARG		;; If esi was not restored, then we 
	jne	short fftdn1		;; just finished skipped postfft data
	sub	ebx, ebx
	mov	DIST_TO_FFTSRCARG, ebx
	cmp	norm_ptr1, 0		;; What was the loop start point?
	je	pass2			;; FFT-only, go do pass 2

;; Split the carries into high and low carries (so they do not exceed
;; the desired number of bits per FFT word.  Then add the carries back to the
;; FFT data.

	start_timer 28
	call	gw_carries
	end_timer 28

;; Have we been given permission to start the FFT on the result?
;; If so, set loop to FFT the first few cache lines now that the carries
;; have been added in.

	cmp	POSTFFT, 0		;; Test flag
	je	short fftdn1		;; Skip FFT code if flag not set
	mov	esi, DESTARG		;; Load FFT data address
	mov	edx, 65536+4/clm*256	;; Do 8 previously skipped cache lines
	mov	norm_ptr1, 0		;; Set flag so only forward FFT is done
	copy_4_words
	jmp	b0b

;; Now add in the FFT'ed carries, cleanup and return

fftdn1:	end_timer 1
	jmp	gw_finish_mult
	ENDM


;; Perform a 112K-element FFT.  This is done in two passes.  Pass 1
;; does 9 levels, pass 2 uses common code to perform the last 8 levels.

pass1levels9pfa7 MACRO pass2_macro, p2cl, clm
	LOCAL	b0b, b1b, b2b, b3b, b4b, b5b, b6b, b7b
	LOCAL	c0b, c1b, c2b, c3b, c4b, c5b, c6b, c7b
	LOCAL	pass2, no_fft, more, fftdn, fftdn1

;; Call FFT code unless the FFT has already been started

	clear_timers
	cmp	DWORD PTR [esi-28][ebx], 0 ;; Test FFT-started flag
	jne	pass2			;; Jump is FFT already started
	mov	norm_ptr1, 0		;; Set FFT-ing flag
	mov	edx, p2cl/128*65536+128/clm*256;; Load loop counter
	jmp	b0b			;; Do the pass 1 fft

;; Do the last 8 FFT levels
;;	do 1 pass2_8_levels_real and 223 pass2_8_levels_complex

pass2:	start_timer 0
	CALLP	pass2_macro
	end_timer 0
	cmp	ffttype, 1		;; We're done if FFTing only
	je	gw_finish_fft

;; Do the inverse FFT

	start_timer 1
	fldz				;; Init SUMOUT
	fstp	SUMOUT
	mov	edx, norm_biglit_array	;; Addr of the big/little flags array
	mov	norm_ptr1, edx		;; Save ptr
	mov	edx, norm_col_mults	;; Addr of the column multipliers
	mov	norm_ptr2, edx		;; Save ptr
	mov	edx, p2cl/128*65536+128/clm*256;; Load loop counter

;; Do inverse FFT levels 8,9
;; On input the 32-byte cache lines hold these data values:
;;	0	256	1	257
;;	512	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	1K	1	1K+1
;;	256	...
;;	...
;;	2K	...
;;	...

;; Do 1 four_real_four_semireal_unfft macros
;;	distance between fft data elements is 1

c0b:	start_timer 16
c2b:	disp four_real_four_semireal_unfft, 8, blkdst, 2*blkdst
	lea	esi, [esi+dist1]	;; Next source pointer
	add	dl, 256/2/clm		;; Test inner loop counter
	jnc	c2b			;; Iterate if necessary
	lea	esi, [esi-clm*2*dist1+4*blkdst];; Next source pointer

;; Do 55 four_complex_unfft macros
;;	distance between fft data elements is 1

	mov	edi, sincos1		;; Load sin/cos pointer
	mov	al, 55			;; 55 iterations
c3b:	disp four_complex_unfft, 8, blkdst, 2*blkdst
	lea	esi, [esi+dist1]	;; Next source pointer
	add	dl, 256/2/clm		;; Test loop counter
	jnc	c3b			;; Iterate if necessary
	lea	esi, [esi-clm*2*dist1+4*blkdst];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al			;; Test outer loop counter
	jnz	c3b			;; Iterate if necessary
	lea	esi, [esi-56*4*blkdst]	;; Restore source pointer
	end_timer 16

;; Do inverse FFT levels 6,7
;; On input the 32-byte cache lines hold these data values:
;;	0	1K	1	1K+1
;;	256	...
;;	...
;;	2K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	4K	1	4K+1
;;	256	...
;;	...
;;	8K	...
;;	...

;; Do 4 four_real_four_semireal_unfft macros
;;	distance between fft data elements is 4

	start_timer 17
	mov	al, 4			;; Load loop counter
c4b:	disp four_real_four_semireal_unfft, 8, 4*blkdst, 8*blkdst
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test inner loop counter
	jnc	c4b			;; Iterate if necessary
	IFDEF PFETCH
	prefetcht1 [esi-32+clm*32]
;	prefetcht1 [esi-32+clm*32+4*blkdst]
	prefetcht1 [esi-32+clm*32+8*blkdst]
;	prefetcht1 [esi-32+clm*32+12*blkdst]
	ENDIF
	loopclm	dl, clm, c4b
	lea	esi, [esi-clm*2*dist1+blkdst];; Next source pointer
	dec	al
	jnz	c4b
	lea	esi, [esi-4*blkdst+16*blkdst];; Next source pointer

;; Do 52 four_complex_unfft macros
;;	distance between fft data elements is 4

	mov	edi, sincos2		;; Load sin/cos pointer
	mov	eax, 13			;; 13 iterations of 4
c5b:	disp four_complex_unfft, 8, 4*blkdst, 8*blkdst
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test inner loop counter
	jnc	c5b			;; Iterate if necessary
	IFDEF PFETCH
	prefetcht1 [esi-32+clm*32]
;	prefetcht1 [esi-32+clm*32+4*blkdst]
	prefetcht1 [esi-32+clm*32+8*blkdst]
;	prefetcht1 [esi-32+clm*32+12*blkdst]
	ENDIF
	loopclm	dl, clm, c5b
	lea	esi, [esi-clm*2*dist1+blkdst];; Next source pointer
	add	ah, 256/4		;; Test middle loop counter
	jnc	c5b			;; Iterate if necessary
	lea	esi, [esi-4*blkdst+16*blkdst];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al
	jnz	c5b
	lea	esi, [esi-14*16*blkdst]	;; Restore source pointer
	end_timer 17

;; Do inverse FFT levels 4,5
;; On input the 32-byte cache lines hold these data values:
;;	0	4K	1	4K+1
;;	256	...
;;	...
;;	8K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	16K	1	16K+1
;;	256	...
;;	...
;;	32K	...
;;	...

;; Do 8 eight_reals_unfft_2 macros
;;	distance between fft data elements is 16

	start_timer 18
	mov	al, 8			;; Load loop counter
c6b:	disp eight_reals_unfft_2, 8*blkdst, 8, 16*blkdst
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test inner loop counter
	jnc	c6b			;; Iterate if necessary
	IFDEF PFETCH
	prefetcht1 [esi-32+clm*32]
;	prefetcht1 [esi-32+clm*32+8*blkdst]
	prefetcht1 [esi-32+clm*32+16*blkdst]
;	prefetcht1 [esi-32+clm*32+24*blkdst]
	ENDIF
	loopclm	dl, clm, c6b
	lea	esi, [esi-clm*2*dist1+blkdst];; Next source pointer
	dec	al
	jnz	c6b
	lea	esi, [esi-8*blkdst+32*blkdst];; Next source pointer

;; Do 48 four_complex_unfft macros
;;	distance between fft data elements is 16

	mov	edi, sincos3		;; Load sin/cos pointer
	mov	eax, 3			;; 3 iterations of 16
c7b:	disp four_complex_unfft, 8, 16*blkdst, 32*blkdst
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test inner loop counter
	jnc	c7b			;; Iterate if necessary
	IFDEF PFETCH
	prefetcht1 [esi-32+clm*32]
;	prefetcht1 [esi-32+clm*32+16*blkdst]
	prefetcht1 [esi-32+clm*32+32*blkdst]
;	prefetcht1 [esi-32+clm*32+48*blkdst]
	ENDIF
	loopclm	dl, clm, c7b
	lea	esi, [esi-clm*2*dist1+blkdst];; Next source pointer
	add	ah, 256/16		;; Test middle loop counter
	jnc	c7b			;; Iterate if necessary
	lea	esi, [esi-16*blkdst+64*blkdst];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al
	jnz	c7b
	lea	esi, [esi-3*64*blkdst-32*blkdst];; Restore source pointer
	end_timer 18

;; Do inverse FFT levels 1,2,3
;; On input the 32-byte cache lines hold these data values:
;;	0	16K	1	16K+1
;;	256	...
;;	...
;;	32K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	56K	1	56K+1
;;	256
;;	...
;;	56K-256

;; Do 64 seven_reals_last_unfft macros
;;	distance between fft data elements is 64

	start_timer 19
	mov	al, 32			;; Load loop counter
c1b:	two_seven_reals_last_unfft 8, 32*blkdst
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test loop counter
	jnc	c1b			;; Iterate if necessary
	IFDEF PFETCH
	prefetcht1 [esi-32+clm*32]
;	prefetcht1 [esi-32+clm*32+32*blkdst]
	prefetcht1 [esi-32+clm*32+64*blkdst]
;	prefetcht1 [esi-32+clm*32+96*blkdst]
	prefetcht1 [esi-32+clm*32+128*blkdst]
;	prefetcht1 [esi-32+clm*32+160*blkdst]
	prefetcht1 [esi-32+clm*32+192*blkdst]
	ENDIF
	loopclm	dl, clm, c1b
	lea	esi, [esi-clm*2*dist1+blkdst];; Next source pointer
	dec	al			;; Test loop counter
	jnz	c1b			;; Iterate if necessary
	lea	esi, [esi-32*blkdst]	;; Restore source pointer
	end_timer 19

;; Normalize these values

	start_timer 20
	mov	eax, NORMRTN		;; Addr of normalization routine
	call	eax
	sub	eax, eax
	end_timer 20

;; Have we been given permission to start the FFT on the result?
;; If so, do so now while the data is in the L2 cache.

	cmp	POSTFFT, 0		;; Test flag
	je	no_fft			;; Skip FFT code if flag not set
	cmp	edx, p2cl/128*65536+(128/clm-4/clm)*256;; Delay fft on 8 sets
	jg	no_fft			;; Jump if delaying FFT
	copy_3_words clm, 0

;; Do FFT levels 1,2,3
;;
;; On input the 32-byte cache lines hold these data values:
;;	0	56K	1	56K+1
;;	256
;;	...
;;	56K-256
;; On output the 32-byte cache lines hold these data values:
;;	0	16K	1	16K+1
;;	256	...
;;	...
;;	32K	...
;;	...

;; Do 64 seven_reals_first_fft macros
;;	distance between fft data elements is 64

b0b:	start_timer 21
	mov	ebx, DIST_TO_FFTSRCARG
	mov	al, 32			;; 32 iterations
b1b:	two_seven_reals_first_fft 32*blkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	add	dl, 256/2/clm		;; Test loop counter
	jnc	b1b			;; Iterate if necessary
	lea	esi, [esi-clm*2*dist1+blkdst];; Next source pointer
	dec	al			;; Test loop counter
	jnz	b1b			;; Iterate if necessary
	lea	esi, [esi-32*blkdst]	;; Restore source pointer
	end_timer 21

;; Do FFT levels 4,5
;;
;; On input the 32-byte cache lines hold these data values:
;;	0	16K	1	16K+1
;;	256	...
;;	...
;;	32K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	4K	1	4K+1
;;	256	...
;;	...
;;	8K	...
;;	...

;; Do 8 eight_reals_fft_2 macros
;;	distance between fft data elements is 16

	start_timer 22
	mov	al, 8			;; Load loop counter
b6b:	disp eight_reals_fft_2, 8*blkdst, 16*blkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test inner loop counter
	jnc	b6b			;; Iterate if necessary
	IFDEF PFETCH
;	prefetcht1 [esi-32+clm*32]
	prefetcht1 [esi-32+clm*32+8*blkdst]
;	prefetcht1 [esi-32+clm*32+16*blkdst]
	prefetcht1 [esi-32+clm*32+24*blkdst]
	ENDIF
	loopclm	dl, clm, b6b
	lea	esi, [esi-clm*2*dist1+blkdst];; Next source pointer
	dec	al
	jnz	b6b
	lea	esi, [esi-8*blkdst+32*blkdst];; Next source pointer

;; Do 48 four_complex_fft macros
;;	distance between fft data elements is 16

	mov	edi, sincos3		;; Load sin/cos pointer
	mov	eax, 3			;; 3 iterations of 16
b7b:	disp four_complex_fft, 16*blkdst, 32*blkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test inner loop counter
	jnc	b7b			;; Iterate if necessary
	IFDEF PFETCH
;	prefetcht1 [esi-32+clm*32]
	prefetcht1 [esi-32+clm*32+16*blkdst]
;	prefetcht1 [esi-32+clm*32+32*blkdst]
	prefetcht1 [esi-32+clm*32+48*blkdst]
	ENDIF
	loopclm	dl, clm, b7b
	lea	esi, [esi-clm*2*dist1+blkdst];; Next source pointer
	add	ah, 256/16		;; Test middle loop counter
	jnc	b7b			;; Iterate if necessary
	lea	esi, [esi-16*blkdst+64*blkdst];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al
	jnz	b7b
	lea	esi, [esi-3*64*blkdst-32*blkdst];; Restore source pointer
	end_timer 22

;; Do FFT levels 6,7
;;
;; On input the 32-byte cache lines hold these data values:
;;	0	4K	1	4K+1
;;	256	...
;;	...
;;	8K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	1K	1	1K+1
;;	256	...
;;	...
;;	2K	...
;;	...

;; Do 4 four_real_four_semireal_fft macros
;;	distance between fft data elements is 4

	start_timer 23
	mov	al, 4			;; Load loop counter
b4b:	disp four_real_four_semireal_fft, 4*blkdst, 8*blkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test inner loop counter
	jnc	b4b			;; Iterate if necessary
	IFDEF PFETCH
;	prefetcht1 [esi-32+clm*32]
	prefetcht1 [esi-32+clm*32+4*blkdst]
;	prefetcht1 [esi-32+clm*32+8*blkdst]
	prefetcht1 [esi-32+clm*32+12*blkdst]
	ENDIF
	loopclm	dl, clm, b4b
	lea	esi, [esi-clm*2*dist1+blkdst];; Next source pointer
	dec	al
	jnz	b4b
	lea	esi, [esi-4*blkdst+16*blkdst];; Next source pointer

;; Do 52 four_complex_fft macros
;;	distance between fft data elements is 4

	mov	edi, sincos2		;; Load sin/cos pointer
	mov	eax, 13			;; 13 iterations of 4
b5b:	disp four_complex_fft, 4*blkdst, 8*blkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test inner loop counter
	jnc	b5b			;; Iterate if necessary
	IFDEF PFETCH
;	prefetcht1 [esi-32+clm*32]
	prefetcht1 [esi-32+clm*32+4*blkdst]
;	prefetcht1 [esi-32+clm*32+8*blkdst]
	prefetcht1 [esi-32+clm*32+12*blkdst]
	ENDIF
	loopclm	dl, clm, b5b
	lea	esi, [esi-clm*2*dist1+blkdst];; Next source pointer
	add	ah, 256/4		;; Test middle loop counter
	jnc	b5b			;; Iterate if necessary
	lea	esi, [esi-4*blkdst+16*blkdst];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al
	jnz	b5b
	lea	esi, [esi-14*16*blkdst]	;; Restore source pointer
	end_timer 23

;; Do FFT levels 8,9
;;
;; On input the 32-byte cache lines hold these data values:
;;	0	1K	1	1K+1
;;	256	...
;;	...
;;	2K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	256	1	257
;;	512	...
;;	...

;; Do 1 four_real_four_semireal_fft macros
;;	distance between fft data elements is 1

	start_timer 24
b2b:	disp four_real_four_semireal_fft, blkdst, 2*blkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test inner loop counter
	jnc	b2b			;; Iterate if necessary
	IFDEF PFETCH
;	prefetcht1 [esi-32+clm*32]
	prefetcht1 [esi-32+clm*32+blkdst]
;	prefetcht1 [esi-32+clm*32+2*blkdst]
	prefetcht1 [esi-32+clm*32+3*blkdst]
	ENDIF
	loopclm	dl, clm, b2b
	lea	esi, [esi-clm*2*dist1+4*blkdst];; Next source pointer

;; Do 55 four_complex_fft macros
;;	distance between fft data elements is 1

	mov	edi, sincos1		;; Load sin/cos pointer
	mov	al, 55			;; 55 iterations
b3b:	disp four_complex_fft, blkdst, 2*blkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test inner loop counter
	jnc	b3b			;; Iterate if necessary
	IFDEF PFETCH
;	prefetcht1 [esi-32+clm*32]
	prefetcht1 [esi-32+clm*32+blkdst]
;	prefetcht1 [esi-32+clm*32+2*blkdst]
	prefetcht1 [esi-32+clm*32+3*blkdst]
	ENDIF
	loopclm	dl, clm, b3b
	lea	esi, [esi-clm*2*dist1+4*blkdst];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al
	jnz	b3b
	lea	esi, [esi-56*4*blkdst]	;; Restore source pointer
	end_timer 24

;; Work on next 4 sets of 32 values

no_fft:	lea	esi, [esi+clm*2*dist1]	;; Next source pointer
	dec	dh			;; Test loop counter
	jnz	short more		;; Do more FFTs
	lea	esi, [esi+64]		;; Skip pad every 128 cache lines (4KB)
	sub	edx, 65536		;; Test loop counter
	jz	short fftdn		;; Jump if done looping
	mov	dh, 128/clm		;; Restore cache line counter
more:	cmp	norm_ptr1, 0		;; What was the loop start point?
	jne	c0b			;; Do inverse-FFT/norm/FFT iteration
	jmp	b0b			;; Do another FFT-only iteration
fftdn:	lea	esi, [esi-p2cl*2*dist1-p2cl/128*64];; Restore source pointer
	cmp	esi, DESTARG		;; If esi was not restored, then we 
	jne	short fftdn1		;; just finished skipped postfft data
	sub	ebx, ebx
	mov	DIST_TO_FFTSRCARG, ebx
	cmp	norm_ptr1, 0		;; What was the loop start point?
	je	pass2			;; FFT-only, go do pass 2

;; Split the carries into high and low carries (so they do not exceed
;; the desired number of bits per FFT word.  Then add the carries back to the
;; FFT data.

	start_timer 28
	call	gw_carries
	end_timer 28

;; Have we been given permission to start the FFT on the result?
;; If so, set loop to FFT the first few cache lines now that the carries
;; have been added in.

	cmp	POSTFFT, 0		;; Test flag
	je	short fftdn1		;; Skip FFT code if flag not set
	mov	esi, DESTARG		;; Load FFT data address
	mov	edx, 65536+4/clm*256	;; Do 8 previously skipped cache lines
	mov	norm_ptr1, 0		;; Set flag so only forward FFT is done
	copy_4_words
	jmp	b0b

;; Now add in the FFT'ed carries, cleanup and return

fftdn1:	end_timer 1
	jmp	gw_finish_mult
	ENDM


;; Perform a 128K-element FFT.  This is done in two passes.  Pass 1
;; does 9 levels, pass 2 uses common code to perform the last 8 levels.

pass1levels9pfa8 MACRO pass2_macro, p2cl, clm
	LOCAL	b0b, b1b, b2b, b3b, b4b, b5b, b6b, b7b
	LOCAL	c0b, c1b, c2b, c3b, c4b, c5b, c6b, c7b
	LOCAL	pass2, no_fft, more, fftdn, fftdn1

;; Call FFT code unless the FFT has already been started

	clear_timers
	cmp	DWORD PTR [esi-28][ebx], 0 ;; Test FFT-started flag
	jne	pass2			;; Jump is FFT already started
	mov	norm_ptr1, 0		;; Set FFT-ing flag
	mov	edx, p2cl/128*65536+128/clm*256;; Load loop counter
	jmp	b0b			;; Do the pass 1 fft

;; Do the last 8 FFT levels
;;	do 1 pass2_8_levels_real and 255 pass2_8_levels_complex

pass2:	start_timer 0
	CALLP	pass2_macro
	end_timer 0
	cmp	ffttype, 1		;; We're done if FFTing only
	je	gw_finish_fft

;; Do the inverse FFT

	start_timer 1
	fldz				;; Init SUMOUT
	fstp	SUMOUT
	mov	edx, norm_biglit_array	;; Addr of the big/little flags array
	mov	norm_ptr1, edx		;; Save ptr
	mov	edx, norm_col_mults	;; Addr of the column multipliers
	mov	norm_ptr2, edx		;; Save ptr
	mov	edx, p2cl/128*65536+128/clm*256;; Load loop counter

;; Do inverse FFT levels 8,9
;; On input the 32-byte cache lines hold these data values:
;;	0	256	1	257
;;	512	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	1K	1	1K+1
;;	256	...
;;	...
;;	2K	...
;;	...

;; Do 1 four_real_four_semireal_unfft macros
;;	distance between fft data elements is 1

c0b:	start_timer 16
c2b:	disp four_real_four_semireal_unfft, 8, blkdst, 2*blkdst
	lea	esi, [esi+dist1]	;; Next source pointer
	add	dl, 256/2/clm		;; Test inner loop counter
	jnc	c2b			;; Iterate if necessary
	lea	esi, [esi-clm*2*dist1+4*blkdst];; Next source pointer

;; Do 63 four_complex_unfft macros
;;	distance between fft data elements is 1

	mov	edi, sincos1		;; Load sin/cos pointer
	mov	al, 63			;; 63 iterations
c3b:	disp four_complex_unfft, 8, blkdst, 2*blkdst
	lea	esi, [esi+dist1]	;; Next source pointer
	add	dl, 256/2/clm		;; Test loop counter
	jnc	c3b			;; Iterate if necessary
	lea	esi, [esi-clm*2*dist1+4*blkdst];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al			;; Test outer loop counter
	jnz	c3b			;; Iterate if necessary
	lea	esi, [esi-64*4*blkdst]	;; Restore source pointer
	end_timer 16

;; Do inverse FFT levels 6,7
;; On input the 32-byte cache lines hold these data values:
;;	0	1K	1	1K+1
;;	256	...
;;	...
;;	2K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	4K	1	4K+1
;;	256	...
;;	...
;;	8K	...
;;	...

;; Do 4 four_real_four_semireal_unfft macros
;;	distance between fft data elements is 4

	start_timer 17
	mov	al, 4			;; Load loop counter
c4b:	disp four_real_four_semireal_unfft, 8, 4*blkdst, 8*blkdst
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test inner loop counter
	jnc	c4b			;; Iterate if necessary
	IFDEF PFETCH
	prefetcht1 [esi-32+clm*32]
;	prefetcht1 [esi-32+clm*32+4*blkdst]
	prefetcht1 [esi-32+clm*32+8*blkdst]
;	prefetcht1 [esi-32+clm*32+12*blkdst]
	ENDIF
	loopclm	dl, clm, c4b
	lea	esi, [esi-clm*2*dist1+blkdst];; Next source pointer
	dec	al
	jnz	c4b
	lea	esi, [esi-4*blkdst+16*blkdst];; Next source pointer

;; Do 60 four_complex_unfft macros
;;	distance between fft data elements is 4

	mov	edi, sincos2		;; Load sin/cos pointer
	mov	eax, 15			;; 15 iterations of 4
c5b:	disp four_complex_unfft, 8, 4*blkdst, 8*blkdst
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test inner loop counter
	jnc	c5b			;; Iterate if necessary
	IFDEF PFETCH
	prefetcht1 [esi-32+clm*32]
;	prefetcht1 [esi-32+clm*32+4*blkdst]
	prefetcht1 [esi-32+clm*32+8*blkdst]
;	prefetcht1 [esi-32+clm*32+12*blkdst]
	ENDIF
	loopclm	dl, clm, c5b
	lea	esi, [esi-clm*2*dist1+blkdst];; Next source pointer
	add	ah, 256/4		;; Test middle loop counter
	jnc	c5b			;; Iterate if necessary
	lea	esi, [esi-4*blkdst+16*blkdst];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al
	jnz	c5b
	lea	esi, [esi-16*16*blkdst]	;; Restore source pointer
	end_timer 17

;; Do inverse FFT levels 4,5
;; On input the 32-byte cache lines hold these data values:
;;	0	4K	1	4K+1
;;	256	...
;;	...
;;	8K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	16K	1	16K+1
;;	256	...
;;	...
;;	32K	...
;;	...

;; Do 16 four_real_four_semireal_unfft macros
;;	distance between fft data elements is 16

	start_timer 18
	mov	al, 16			;; Load loop counter
c6b:	disp four_real_four_semireal_unfft, 8, 16*blkdst, 32*blkdst
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test inner loop counter
	jnc	c6b			;; Iterate if necessary
	IFDEF PFETCH
	prefetcht1 [esi-32+clm*32]
;	prefetcht1 [esi-32+clm*32+16*blkdst]
	prefetcht1 [esi-32+clm*32+32*blkdst]
;	prefetcht1 [esi-32+clm*32+48*blkdst]
	ENDIF
	loopclm	dl, clm, c6b
	lea	esi, [esi-clm*2*dist1+blkdst];; Next source pointer
	dec	al
	jnz	c6b
	lea	esi, [esi-16*blkdst+64*blkdst];; Next source pointer

;; Do 48 four_complex_unfft macros
;;	distance between fft data elements is 16

	mov	edi, sincos3		;; Load sin/cos pointer
	mov	eax, 3			;; 3 iterations of 16
c7b:	disp four_complex_unfft, 8, 16*blkdst, 32*blkdst
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test inner loop counter
	jnc	c7b			;; Iterate if necessary
	IFDEF PFETCH
	prefetcht1 [esi-32+clm*32]
;	prefetcht1 [esi-32+clm*32+16*blkdst]
	prefetcht1 [esi-32+clm*32+32*blkdst]
;	prefetcht1 [esi-32+clm*32+48*blkdst]
	ENDIF
	loopclm	dl, clm, c7b
	lea	esi, [esi-clm*2*dist1+blkdst];; Next source pointer
	add	ah, 256/16		;; Test middle loop counter
	jnc	c7b			;; Iterate if necessary
	lea	esi, [esi-16*blkdst+64*blkdst];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al
	jnz	c7b
	lea	esi, [esi-4*64*blkdst]	;; Restore source pointer
	end_timer 18

;; Do inverse FFT levels 1,2,3
;; On input the 32-byte cache lines hold these data values:
;;	0	16K	1	16K+1
;;	256	...
;;	...
;;	32K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	64K	1	64K+1
;;	256
;;	...
;;	64K-256

;; Do 64 eight_reals_last_unfft macros
;;	distance between fft data elements is 64

	start_timer 19
	mov	al, 64			;; Load loop counter
c1b:	disp eight_reals_last_unfft, 8, 64*blkdst, 128*blkdst
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test loop counter
	jnc	c1b			;; Iterate if necessary
	IFDEF PFETCH
	prefetcht1 [esi-32+clm*32]
;	prefetcht1 [esi-32+clm*32+64*blkdst]
	prefetcht1 [esi-32+clm*32+128*blkdst]
;	prefetcht1 [esi-32+clm*32+192*blkdst]
	ENDIF
	loopclm	dl, clm, c1b
	lea	esi, [esi-clm*2*dist1+blkdst];; Next source pointer
	dec	al			;; Test loop counter
	jnz	c1b			;; Iterate if necessary
	lea	esi, [esi-64*blkdst]	;; Restore source pointer
	end_timer 19

;; Normalize these values

	start_timer 20
	mov	eax, NORMRTN		;; Addr of normalization routine
	call	eax
	sub	eax, eax
	end_timer 20

;; Have we been given permission to start the FFT on the result?
;; If so, do so now while the data is in the L2 cache.

	cmp	POSTFFT, 0		;; Test flag
	je	no_fft			;; Skip FFT code if flag not set
	cmp	edx, p2cl/128*65536+(128/clm-4/clm)*256;; Delay fft on 8 sets
	jg	no_fft			;; Jump if delaying FFT
	copy_3_words clm, 0

;; Do FFT levels 1,2,3
;;
;; On input the 32-byte cache lines hold these data values:
;;	0	64K	1	64K+1
;;	256
;;	...
;;	64K-256
;; On output the 32-byte cache lines hold these data values:
;;	0	16K	1	16K+1
;;	256	...
;;	...
;;	32K	...
;;	...

;; Do 64 eight_reals_first_fft macros
;;	distance between fft data elements is 64

b0b:	start_timer 21
	mov	ebx, DIST_TO_FFTSRCARG
	mov	al, 64			;; 64 iterations
b1b:	disp eight_reals_first_fft, 64*blkdst, 128*blkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	add	dl, 256/2/clm		;; Test loop counter
	jnc	b1b			;; Iterate if necessary
	lea	esi, [esi-clm*2*dist1+blkdst];; Next source pointer
	dec	al			;; Test loop counter
	jnz	b1b			;; Iterate if necessary
	lea	esi, [esi-64*blkdst]	;; Restore source pointer
	end_timer 21

;; Do FFT levels 4,5
;;
;; On input the 32-byte cache lines hold these data values:
;;	0	16K	1	16K+1
;;	256	...
;;	...
;;	32K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	4K	1	4K+1
;;	256	...
;;	...
;;	8K	...
;;	...

;; Do 16 four_real_four_semireal_fft macros
;;	distance between fft data elements is 16

	start_timer 22
	mov	al, 16			;; Load loop counter
b6b:	disp four_real_four_semireal_fft, 16*blkdst, 32*blkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test inner loop counter
	jnc	b6b			;; Iterate if necessary
	IFDEF PFETCH
;	prefetcht1 [esi-32+clm*32]
	prefetcht1 [esi-32+clm*32+16*blkdst]
;	prefetcht1 [esi-32+clm*32+32*blkdst]
	prefetcht1 [esi-32+clm*32+48*blkdst]
	ENDIF
	loopclm	dl, clm, b6b
	lea	esi, [esi-clm*2*dist1+blkdst];; Next source pointer
	dec	al
	jnz	b6b
	lea	esi, [esi-16*blkdst+64*blkdst];; Next source pointer

;; Do 48 four_complex_fft macros
;;	distance between fft data elements is 16

	mov	edi, sincos3		;; Load sin/cos pointer
	mov	eax, 3			;; 3 iterations of 16
b7b:	disp four_complex_fft, 16*blkdst, 32*blkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test inner loop counter
	jnc	b7b			;; Iterate if necessary
	IFDEF PFETCH
;	prefetcht1 [esi-32+clm*32]
	prefetcht1 [esi-32+clm*32+16*blkdst]
;	prefetcht1 [esi-32+clm*32+32*blkdst]
	prefetcht1 [esi-32+clm*32+48*blkdst]
	ENDIF
	loopclm	dl, clm, b7b
	lea	esi, [esi-clm*2*dist1+blkdst];; Next source pointer
	add	ah, 256/16		;; Test middle loop counter
	jnc	b7b			;; Iterate if necessary
	lea	esi, [esi-16*blkdst+64*blkdst];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al
	jnz	b7b
	lea	esi, [esi-4*64*blkdst]	;; Restore source pointer
	end_timer 22

;; Do FFT levels 6,7
;;
;; On input the 32-byte cache lines hold these data values:
;;	0	4K	1	4K+1
;;	256	...
;;	...
;;	8K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	1K	1	1K+1
;;	256	...
;;	...
;;	2K	...
;;	...

;; Do 4 four_real_four_semireal_fft macros
;;	distance between fft data elements is 4

	start_timer 23
	mov	al, 4			;; Load loop counter
b4b:	disp four_real_four_semireal_fft, 4*blkdst, 8*blkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test inner loop counter
	jnc	b4b			;; Iterate if necessary
	IFDEF PFETCH
;	prefetcht1 [esi-32+clm*32]
	prefetcht1 [esi-32+clm*32+4*blkdst]
;	prefetcht1 [esi-32+clm*32+8*blkdst]
	prefetcht1 [esi-32+clm*32+12*blkdst]
	ENDIF
	loopclm	dl, clm, b4b
	lea	esi, [esi-clm*2*dist1+blkdst];; Next source pointer
	dec	al
	jnz	b4b
	lea	esi, [esi-4*blkdst+16*blkdst];; Next source pointer

;; Do 60 four_complex_fft macros
;;	distance between fft data elements is 4

	mov	edi, sincos2		;; Load sin/cos pointer
	mov	eax, 15			;; 15 iterations of 4
b5b:	disp four_complex_fft, 4*blkdst, 8*blkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test inner loop counter
	jnc	b5b			;; Iterate if necessary
	IFDEF PFETCH
;	prefetcht1 [esi-32+clm*32]
	prefetcht1 [esi-32+clm*32+4*blkdst]
;	prefetcht1 [esi-32+clm*32+8*blkdst]
	prefetcht1 [esi-32+clm*32+12*blkdst]
	ENDIF
	loopclm	dl, clm, b5b
	lea	esi, [esi-clm*2*dist1+blkdst];; Next source pointer
	add	ah, 256/4		;; Test middle loop counter
	jnc	b5b			;; Iterate if necessary
	lea	esi, [esi-4*blkdst+16*blkdst];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al
	jnz	b5b
	lea	esi, [esi-16*16*blkdst]	;; Restore source pointer
	end_timer 23

;; Do FFT levels 8,9
;;
;; On input the 32-byte cache lines hold these data values:
;;	0	1K	1	1K+1
;;	256	...
;;	...
;;	2K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	256	1	257
;;	512	...
;;	...

;; Do 1 four_real_four_semireal_fft macros
;;	distance between fft data elements is 1

	start_timer 24
b2b:	disp four_real_four_semireal_fft, blkdst, 2*blkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test inner loop counter
	jnc	b2b			;; Iterate if necessary
	IFDEF PFETCH
;	prefetcht1 [esi-32+clm*32]
	prefetcht1 [esi-32+clm*32+blkdst]
;	prefetcht1 [esi-32+clm*32+2*blkdst]
	prefetcht1 [esi-32+clm*32+3*blkdst]
	ENDIF
	loopclm	dl, clm, b2b
	lea	esi, [esi-clm*2*dist1+4*blkdst];; Next source pointer

;; Do 63 four_complex_fft macros
;;	distance between fft data elements is 1

	mov	edi, sincos1		;; Load sin/cos pointer
	mov	al, 63			;; 63 iterations
b3b:	disp four_complex_fft, blkdst, 2*blkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test inner loop counter
	jnc	b3b			;; Iterate if necessary
	IFDEF PFETCH
;	prefetcht1 [esi-32+clm*32]
	prefetcht1 [esi-32+clm*32+blkdst]
;	prefetcht1 [esi-32+clm*32+2*blkdst]
	prefetcht1 [esi-32+clm*32+3*blkdst]
	ENDIF
	loopclm	dl, clm, b3b
	lea	esi, [esi-clm*2*dist1+4*blkdst];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al
	jnz	b3b
	lea	esi, [esi-64*4*blkdst]	;; Restore source pointer
	end_timer 24

;; Work on next 4 sets of 32 values

no_fft:	lea	esi, [esi+clm*2*dist1]	;; Next source pointer
	dec	dh			;; Test loop counter
	jnz	short more		;; Do more FFTs
	lea	esi, [esi+64]		;; Skip pad every 128 cache lines (4KB)
	sub	edx, 65536		;; Test loop counter
	jz	short fftdn		;; Jump if done looping
	mov	dh, 128/clm		;; Restore cache line counter
more:	cmp	norm_ptr1, 0		;; What was the loop start point?
	jne	c0b			;; Do inverse-FFT/norm/FFT iteration
	jmp	b0b			;; Do another FFT-only iteration
fftdn:	lea	esi, [esi-p2cl*2*dist1-p2cl/128*64];; Restore source pointer
	cmp	esi, DESTARG		;; If esi was not restored, then we 
	jne	short fftdn1		;; just finished skipped postfft data
	sub	ebx, ebx
	mov	DIST_TO_FFTSRCARG, ebx
	cmp	norm_ptr1, 0		;; What was the loop start point?
	je	pass2			;; FFT-only, go do pass 2

;; Split the carries into high and low carries (so they do not exceed
;; the desired number of bits per FFT word.  Then add the carries back to the
;; FFT data.

	start_timer 28
	call	gw_carries
	end_timer 28

;; Have we been given permission to start the FFT on the result?
;; If so, set loop to FFT the first few cache lines now that the carries
;; have been added in.

	cmp	POSTFFT, 0		;; Test flag
	je	short fftdn1		;; Skip FFT code if flag not set
	mov	esi, DESTARG		;; Load FFT data address
	mov	edx, 65536+4/clm*256	;; Do 8 previously skipped cache lines
	mov	norm_ptr1, 0		;; Set flag so only forward FFT is done
	copy_4_words
	jmp	b0b

;; Now add in the FFT'ed carries, cleanup and return

fftdn1:	end_timer 1
	jmp	gw_finish_mult
	ENDM


;; Perform a 128K-element negacyclic FFT.  This is done in two passes.  Pass 1
;; does 9 levels, pass 2 uses common code to perform the last 8 levels.

pass1levels9complex4 MACRO pass2_macro, p2cl, clm
	LOCAL	b0b, b1b, b3b, b5b, b7b, c0b, c1b, c3b, c5b, c7b
	LOCAL	pass2, no_fft, more, fftdn, fftdn1

;; Call FFT code unless the FFT has already been started

	clear_timers
	cmp	DWORD PTR [esi-28][ebx], 0 ;; Test FFT-started flag
	jne	pass2			;; Jump is FFT already started
	mov	norm_ptr1, 0		;; Set FFT-ing flag
	mov	edx, p2cl/128*65536+128/clm*256;; Load loop counter
	jmp	b0b			;; Do the pass 1 fft

;; Do the last 8 FFT levels
;;	do 256 pass2_8_levels_complex

pass2:	start_timer 0
	CALLP	pass2_macro
	end_timer 0
	cmp	ffttype, 1		;; We're done if FFTing only
	je	gw_finish_fft

;; Do the inverse FFT

	start_timer 1
	fldz				;; Init SUMOUT
	fstp	SUMOUT
	mov	edx, norm_biglit_array	;; Addr of the big/little flags array
	mov	norm_ptr1, edx		;; Save ptr
	mov	edx, norm_col_mults	;; Addr of the column multipliers
	mov	norm_ptr2, edx		;; Save ptr
	mov	edx, p2cl/128*65536+128/clm*256;; Load loop counter

;; Do inverse FFT levels 8,9
;; On input the 32-byte cache lines hold these data values:
;;	0	256	1	257
;;	512	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	1K	1	1K+1
;;	256	...
;;	...
;;	2K	...
;;	...

;; Do 64 four_complex_unfft macros
;;	distance between fft data elements is 1

c0b:	start_timer 16
	mov	edi, sincos1		;; Load sin/cos pointer
	mov	al, 64			;; 64 iterations
c3b:	disp four_complex_unfft, 8, blkdst, 2*blkdst
	lea	esi, [esi+dist1]	;; Next source pointer
	add	dl, 256/2/clm		;; Test loop counter
	jnc	c3b			;; Iterate if necessary
	lea	esi, [esi-clm*2*dist1+4*blkdst];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al			;; Test outer loop counter
	jnz	c3b			;; Iterate if necessary
	lea	esi, [esi-64*4*blkdst]	;; Restore source pointer
	end_timer 16

;; Do inverse FFT levels 6,7
;; On input the 32-byte cache lines hold these data values:
;;	0	1K	1	1K+1
;;	256	...
;;	...
;;	2K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	4K	1	4K+1
;;	256	...
;;	...
;;	8K	...
;;	...

;; Do 64 four_complex_unfft macros
;;	distance between fft data elements is 4

	start_timer 17
	mov	edi, sincos2		;; Load sin/cos pointer
	mov	eax, 16			;; 16 iterations of 4
c5b:	disp four_complex_unfft, 8, 4*blkdst, 8*blkdst
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test inner loop counter
	jnc	c5b			;; Iterate if necessary
	IFDEF PFETCH
	prefetcht1 [esi-32+clm*32]
;	prefetcht1 [esi-32+clm*32+4*blkdst]
	prefetcht1 [esi-32+clm*32+8*blkdst]
;	prefetcht1 [esi-32+clm*32+12*blkdst]
	ENDIF
	loopclm	dl, clm, c5b
	lea	esi, [esi-clm*2*dist1+blkdst];; Next source pointer
	add	ah, 256/4		;; Test middle loop counter
	jnc	c5b			;; Iterate if necessary
	lea	esi, [esi-4*blkdst+16*blkdst];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al
	jnz	c5b
	lea	esi, [esi-16*16*blkdst]	;; Restore source pointer
	end_timer 17

;; Do inverse FFT levels 4,5
;; On input the 32-byte cache lines hold these data values:
;;	0	4K	1	4K+1
;;	256	...
;;	...
;;	8K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	16K	1	16K+1
;;	256	...
;;	...
;;	32K	...
;;	...

;; Do 64 four_complex_unfft macros
;;	distance between fft data elements is 16

	start_timer 18
	mov	edi, sincos3		;; Load sin/cos pointer
	mov	eax, 4			;; 4 iterations of 16
c7b:	disp four_complex_unfft, 8, 16*blkdst, 32*blkdst
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test inner loop counter
	jnc	c7b			;; Iterate if necessary
	IFDEF PFETCH
	prefetcht1 [esi-32+clm*32]
;	prefetcht1 [esi-32+clm*32+16*blkdst]
	prefetcht1 [esi-32+clm*32+32*blkdst]
;	prefetcht1 [esi-32+clm*32+48*blkdst]
	ENDIF
	loopclm	dl, clm, c7b
	lea	esi, [esi-clm*2*dist1+blkdst];; Next source pointer
	add	ah, 256/16		;; Test middle loop counter
	jnc	c7b			;; Iterate if necessary
	lea	esi, [esi-16*blkdst+64*blkdst];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al
	jnz	c7b
	lea	esi, [esi-4*64*blkdst]	;; Restore source pointer
	end_timer 18

;; Do inverse FFT levels 1,2,3
;; On input the 32-byte cache lines hold these data values:
;;	0	16K	1	16K+1
;;	256	...
;;	...
;;	32K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	64K	1	64K+1
;;	256
;;	...
;;	64K-256

;; Do 64 four_complex_last_unfft macros
;;	distance between fft data elements is 64

	start_timer 19
	mov	edi, plus1_premults	;; Address of premultiplier table
	mov	al, 64			;; Load loop counter
c1b:	disp four_complex_last_unfft, 8, 64*blkdst, 128*blkdst
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test loop counter
	jnc	c1b			;; Iterate if necessary
	IFDEF PFETCH
	prefetcht1 [esi-32+clm*32]
;	prefetcht1 [esi-32+clm*32+64*blkdst]
	prefetcht1 [esi-32+clm*32+128*blkdst]
;	prefetcht1 [esi-32+clm*32+192*blkdst]
	ENDIF
	loopclm	dl, clm, c1b
	lea	esi, [esi-clm*2*dist1+blkdst];; Next source pointer
	lea	edi, [edi+64]		;; Next premultiplier pointer
	dec	al			;; Test loop counter
	jnz	c1b			;; Iterate if necessary
	lea	esi, [esi-64*blkdst]	;; Restore source pointer
	end_timer 19

;; Normalize these values

	start_timer 20
	mov	eax, NORMRTN		;; Addr of normalization routine
	call	eax
	sub	eax, eax
	end_timer 20

;; Have we been given permission to start the FFT on the result?
;; If so, do so now while the data is in the L2 cache.

	cmp	POSTFFT, 0		;; Test flag
	je	no_fft			;; Skip FFT code if flag not set
	cmp	edx, p2cl/128*65536+(128/clm-4/clm)*256;; Delay fft on 8 sets
	jg	no_fft			;; Jump if delaying FFT
	copy_3_words clm, 0

;; Do FFT levels 1,2,3
;;
;; On input the 32-byte cache lines hold these data values:
;;	0	64K	1	64K+1
;;	256
;;	...
;;	64K-256
;; On output the 32-byte cache lines hold these data values:
;;	0	16K	1	16K+1
;;	256	...
;;	...
;;	32K	...
;;	...

;; Do 64 four_complex_first_fft macros
;;	distance between fft data elements is 64

b0b:	start_timer 21
	mov	ebx, DIST_TO_FFTSRCARG
	mov	edi, plus1_premults	;; Address of premultiplier table
	mov	al, 64			;; 64 iterations
b1b:	disp four_complex_first_fft, 64*blkdst, 128*blkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	add	dl, 256/2/clm		;; Test loop counter
	jnc	b1b			;; Iterate if necessary
	lea	esi, [esi-clm*2*dist1+blkdst];; Next source pointer
	lea	edi, [edi+64]		;; Next premultiplier pointer
	dec	al			;; Test loop counter
	jnz	b1b			;; Iterate if necessary
	lea	esi, [esi-64*blkdst]	;; Restore source pointer
	end_timer 21

;; Do FFT levels 4,5
;;
;; On input the 32-byte cache lines hold these data values:
;;	0	16K	1	16K+1
;;	256	...
;;	...
;;	32K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	4K	1	4K+1
;;	256	...
;;	...
;;	8K	...
;;	...

;; Do 64 four_complex_fft macros
;;	distance between fft data elements is 16

	start_timer 22
	mov	edi, sincos3		;; Load sin/cos pointer
	mov	eax, 4			;; 4 iterations of 16
b7b:	disp four_complex_fft, 16*blkdst, 32*blkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test inner loop counter
	jnc	b7b			;; Iterate if necessary
	IFDEF PFETCH
;	prefetcht1 [esi-32+clm*32]
	prefetcht1 [esi-32+clm*32+16*blkdst]
;	prefetcht1 [esi-32+clm*32+32*blkdst]
	prefetcht1 [esi-32+clm*32+48*blkdst]
	ENDIF
	loopclm	dl, clm, b7b
	lea	esi, [esi-clm*2*dist1+blkdst];; Next source pointer
	add	ah, 256/16		;; Test middle loop counter
	jnc	b7b			;; Iterate if necessary
	lea	esi, [esi-16*blkdst+64*blkdst];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al
	jnz	b7b
	lea	esi, [esi-4*64*blkdst]	;; Restore source pointer
	end_timer 22

;; Do FFT levels 6,7
;;
;; On input the 32-byte cache lines hold these data values:
;;	0	4K	1	4K+1
;;	256	...
;;	...
;;	8K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	1K	1	1K+1
;;	256	...
;;	...
;;	2K	...
;;	...

;; Do 64 four_complex_fft macros
;;	distance between fft data elements is 4

	start_timer 23
	mov	edi, sincos2		;; Load sin/cos pointer
	mov	eax, 16			;; 16 iterations of 4
b5b:	disp four_complex_fft, 4*blkdst, 8*blkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test inner loop counter
	jnc	b5b			;; Iterate if necessary
	IFDEF PFETCH
;	prefetcht1 [esi-32+clm*32]
	prefetcht1 [esi-32+clm*32+4*blkdst]
;	prefetcht1 [esi-32+clm*32+8*blkdst]
	prefetcht1 [esi-32+clm*32+12*blkdst]
	ENDIF
	loopclm	dl, clm, b5b
	lea	esi, [esi-clm*2*dist1+blkdst];; Next source pointer
	add	ah, 256/4		;; Test middle loop counter
	jnc	b5b			;; Iterate if necessary
	lea	esi, [esi-4*blkdst+16*blkdst];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al
	jnz	b5b
	lea	esi, [esi-16*16*blkdst]	;; Restore source pointer
	end_timer 23

;; Do FFT levels 8,9
;;
;; On input the 32-byte cache lines hold these data values:
;;	0	1K	1	1K+1
;;	256	...
;;	...
;;	2K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	256	1	257
;;	512	...
;;	...

;; Do 64 four_complex_fft macros
;;	distance between fft data elements is 1

	start_timer 24
	mov	edi, sincos1		;; Load sin/cos pointer
	mov	al, 64			;; 64 iterations
b3b:	disp four_complex_fft, blkdst, 2*blkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test inner loop counter
	jnc	b3b			;; Iterate if necessary
	IFDEF PFETCH
;	prefetcht1 [esi-32+clm*32]
	prefetcht1 [esi-32+clm*32+blkdst]
;	prefetcht1 [esi-32+clm*32+2*blkdst]
	prefetcht1 [esi-32+clm*32+3*blkdst]
	ENDIF
	loopclm	dl, clm, b3b
	lea	esi, [esi-clm*2*dist1+4*blkdst];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al
	jnz	b3b
	lea	esi, [esi-64*4*blkdst]	;; Restore source pointer
	end_timer 24

;; Work on next 4 sets of 32 values

no_fft:	lea	esi, [esi+clm*2*dist1]	;; Next source pointer
	dec	dh			;; Test loop counter
	jnz	short more		;; Do more FFTs
	lea	esi, [esi+64]		;; Skip pad every 128 cache lines (4KB)
	sub	edx, 65536		;; Test loop counter
	jz	short fftdn		;; Jump if done looping
	mov	dh, 128/clm		;; Restore cache line counter
more:	cmp	norm_ptr1, 0		;; What was the loop start point?
	jne	c0b			;; Do inverse-FFT/norm/FFT iteration
	jmp	b0b			;; Do another FFT-only iteration
fftdn:	lea	esi, [esi-p2cl*2*dist1-p2cl/128*64];; Restore source pointer
	cmp	esi, DESTARG		;; If esi was not restored, then we 
	jne	short fftdn1		;; just finished skipped postfft data
	sub	ebx, ebx
	mov	DIST_TO_FFTSRCARG, ebx
	cmp	norm_ptr1, 0		;; What was the loop start point?
	je	pass2			;; FFT-only, go do pass 2

;; Split the carries into high and low carries (so they do not exceed
;; the desired number of bits per FFT word.  Then add the carries back to the
;; FFT data.

	start_timer 28
	call	gw_carries
	end_timer 28

;; Have we been given permission to start the FFT on the result?
;; If so, set loop to FFT the first few cache lines now that the carries
;; have been added in.

	cmp	POSTFFT, 0		;; Test flag
	je	short fftdn1		;; Skip FFT code if flag not set
	mov	esi, DESTARG		;; Load FFT data address
	mov	edx, 65536+4/clm*256	;; Do 8 previously skipped cache lines
	mov	norm_ptr1, 0		;; Set flag so only forward FFT is done
	copy_4_words
	jmp	b0b

;; Now add in the FFT'ed carries, cleanup and return

fftdn1:	end_timer 1
	jmp	gw_finish_mult
	ENDM


;; Perform a 160K-element FFT.  This is done in two passes.  Pass 1
;; does 10 levels, pass 2 uses common code to perform the last 8 levels.

pass1levels10pfa5 MACRO pass2_macro, p2cl, clm
	LOCAL	b0b, b1b, b2b, b3b, b4b, b5b, b6b, b7b, b8b, b9b
	LOCAL	c0b, c1b, c2b, c3b, c4b, c5b, c6b, c7b, c8b, c9b
	LOCAL	pass2, no_fft, more, fftdn, fftdn1

;; Call FFT code unless the FFT has already been started

	clear_timers
	cmp	DWORD PTR [esi-28][ebx], 0 ;; Test FFT-started flag
	jne	pass2			;; Jump is FFT already started
	mov	norm_ptr1, 0		;; Set FFT-ing flag
	mov	edx, p2cl/128*65536+128/clm*256;; Load loop counter
	jmp	b0b			;; Do the pass 1 fft

;; Do the last 8 FFT levels
;;	do 1 pass2_8_levels_real and 319 pass2_8_levels_complex

pass2:	start_timer 0
	CALLP	pass2_macro
	end_timer 0
	cmp	ffttype, 1		;; We're done if FFTing only
	je	gw_finish_fft

;; Do the inverse FFT

	start_timer 1
	fldz				;; Init SUMOUT
	fstp	SUMOUT
	mov	edx, norm_biglit_array	;; Addr of the big/little flags array
	mov	norm_ptr1, edx		;; Save ptr
	mov	edx, norm_col_mults	;; Addr of the column multipliers
	mov	norm_ptr2, edx		;; Save ptr
	mov	edx, p2cl/128*65536+128/clm*256;; Load loop counter

;; Do inverse FFT levels 9,10
;; On input the 32-byte cache lines hold these data values:
;;	0	256	1	257
;;	512	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	1K	1	1K+1
;;	256	...
;;	...
;;	2K	...
;;	...

;; Do 1 four_real_four_semireal_unfft macros
;;	distance between fft data elements is 1

c0b:	start_timer 15
c2b:	disp four_real_four_semireal_unfft, 8, blkdst, 2*blkdst
	lea	esi, [esi+dist1]	;; Next source pointer
	add	dl, 256/2/clm		;; Test inner loop counter
	jnc	c2b			;; Iterate if necessary
	lea	esi, [esi-clm*2*dist1+4*blkdst];; Next source pointer

;; Do 79 four_complex_unfft macros
;;	distance between fft data elements is 1

	mov	edi, sincos1		;; Load sin/cos pointer
	mov	al, 79			;; 79 iterations
c3b:	disp four_complex_unfft, 8, blkdst, 2*blkdst
	lea	esi, [esi+dist1]	;; Next source pointer
	add	dl, 256/2/clm		;; Test loop counter
	jnc	c3b			;; Iterate if necessary
	lea	esi, [esi-clm*2*dist1+4*blkdst];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al			;; Test outer loop counter
	jnz	c3b			;; Iterate if necessary
	lea	esi, [esi-80*4*blkdst]	;; Restore source pointer
	end_timer 15

;; Do inverse FFT levels 7,8
;; On input the 32-byte cache lines hold these data values:
;;	0	1K	1	1K+1
;;	256	...
;;	...
;;	2K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	4K	1	4K+1
;;	256	...
;;	...
;;	8K	...
;;	...

;; Do 4 four_real_four_semireal_unfft macros
;;	distance between fft data elements is 4

	start_timer 16
	mov	al, 4			;; Load loop counter
c4b:	disp four_real_four_semireal_unfft, 8, 4*blkdst, 8*blkdst
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test inner loop counter
	jnc	c4b			;; Iterate if necessary
	IFDEF PFETCH
	prefetcht1 [esi-32+clm*32]
;	prefetcht1 [esi-32+clm*32+4*blkdst]
	prefetcht1 [esi-32+clm*32+8*blkdst]
;	prefetcht1 [esi-32+clm*32+12*blkdst]
	ENDIF
	loopclm	dl, clm, c4b
	lea	esi, [esi-clm*2*dist1+blkdst];; Next source pointer
	dec	al
	jnz	c4b
	lea	esi, [esi-4*blkdst+16*blkdst];; Next source pointer

;; Do 76 four_complex_unfft macros
;;	distance between fft data elements is 4

	mov	edi, sincos2		;; Load sin/cos pointer
	mov	eax, 19			;; 19 iterations of 4
c5b:	disp four_complex_unfft, 8, 4*blkdst, 8*blkdst
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test inner loop counter
	jnc	c5b			;; Iterate if necessary
	IFDEF PFETCH
	prefetcht1 [esi-32+clm*32]
;	prefetcht1 [esi-32+clm*32+4*blkdst]
	prefetcht1 [esi-32+clm*32+8*blkdst]
;	prefetcht1 [esi-32+clm*32+12*blkdst]
	ENDIF
	loopclm	dl, clm, c5b
	lea	esi, [esi-clm*2*dist1+blkdst];; Next source pointer
	add	ah, 256/4		;; Test middle loop counter
	jnc	c5b			;; Iterate if necessary
	lea	esi, [esi-4*blkdst+16*blkdst];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al
	jnz	c5b
	lea	esi, [esi-20*16*blkdst]	;; Restore source pointer
	end_timer 16

;; Do inverse FFT levels 5,6
;; On input the 32-byte cache lines hold these data values:
;;	0	4K	1	4K+1
;;	256	...
;;	...
;;	8K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	16K	1	16K+1
;;	256	...
;;	...
;;	32K	...
;;	...

;; Do 16 four_real_four_semireal_unfft macros
;;	distance between fft data elements is 16

	start_timer 17
	mov	al, 16			;; Load loop counter
c6b:	disp four_real_four_semireal_unfft, 8, 16*blkdst, 32*blkdst
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test inner loop counter
	jnc	c6b			;; Iterate if necessary
	IFDEF PFETCH
	prefetcht1 [esi-32+clm*32]
;	prefetcht1 [esi-32+clm*32+16*blkdst]
	prefetcht1 [esi-32+clm*32+32*blkdst]
;	prefetcht1 [esi-32+clm*32+48*blkdst]
	ENDIF
	loopclm	dl, clm, c6b
	lea	esi, [esi-clm*2*dist1+blkdst];; Next source pointer
	dec	al
	jnz	c6b
	lea	esi, [esi-16*blkdst+64*blkdst];; Next source pointer

;; Do 64 four_complex_unfft macros
;;	distance between fft data elements is 16

	mov	edi, sincos3		;; Load sin/cos pointer
	mov	eax, 4			;; 4 iterations of 16
c7b:	disp four_complex_unfft, 8, 16*blkdst, 32*blkdst
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test inner loop counter
	jnc	c7b			;; Iterate if necessary
	IFDEF PFETCH
	prefetcht1 [esi-32+clm*32]
;	prefetcht1 [esi-32+clm*32+16*blkdst]
	prefetcht1 [esi-32+clm*32+32*blkdst]
;	prefetcht1 [esi-32+clm*32+48*blkdst]
	ENDIF
	loopclm	dl, clm, c7b
	lea	esi, [esi-clm*2*dist1+blkdst];; Next source pointer
	add	ah, 256/16		;; Test middle loop counter
	jnc	c7b			;; Iterate if necessary
	lea	esi, [esi-16*blkdst+64*blkdst];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al
	jnz	c7b
	lea	esi, [esi-5*64*blkdst]	;; Restore source pointer
	end_timer 17

;; Do inverse FFT levels 4
;; On input the 32-byte cache lines hold these data values:
;;	0	16K	1	16K+1
;;	256	...
;;	...
;;	32K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	32K	1	32K+1
;;	256	...
;;	...
;;	64K	...
;;	...

;; Do 16 eight_reals_unfft_1 macros
;;	distance between fft data elements is 64

	start_timer 18
	mov	al, 16			;; Load loop counter
c8b:	disp eight_reals_unfft_1, 16*blkdst, 32*blkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test inner loop counter
	jnc	c8b			;; Iterate if necessary
	IFDEF PFETCH
	prefetcht1 [esi-32+clm*32]
;	prefetcht1 [esi-32+clm*32+16*blkdst]
	prefetcht1 [esi-32+clm*32+32*blkdst]
;	prefetcht1 [esi-32+clm*32+48*blkdst]
	ENDIF
	loopclm	dl, clm, c8b
	lea	esi, [esi-clm*2*dist1+blkdst];; Next source pointer
	dec	al
	jnz	c8b
	lea	esi, [esi-16*blkdst+64*blkdst];; Next source pointer

;; Do 64 four_complex_unfft_1 macros
;;	distance between fft data elements is 64

	mov	edi, sincos4		;; Load sin/cos pointer
	mov	eax, 2			;; 2 iterations of 32
c9b:	disp four_complex_unfft_1, 32*blkdst, 8, 64*blkdst
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test inner loop counter
	jnc	c9b			;; Iterate if necessary
	IFDEF PFETCH
	prefetcht1 [esi-32+clm*32]
;	prefetcht1 [esi-32+clm*32+32*blkdst]
	prefetcht1 [esi-32+clm*32+64*blkdst]
;	prefetcht1 [esi-32+clm*32+96*blkdst]
	ENDIF
	loopclm	dl, clm, c9b
	lea	esi, [esi-clm*2*dist1+blkdst];; Next source pointer
	add	ah, 256/32		;; Test middle loop counter
	jnc	c9b			;; Iterate if necessary
	lea	esi, [esi-32*blkdst+128*blkdst];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al
	jnz	c9b
	lea	esi, [esi-2*128*blkdst-64*blkdst];; Restore source pointer
	end_timer 18

;; Do inverse FFT levels 1,2,3
;; On input the 32-byte cache lines hold these data values:
;;	0	32K	1	32K+1
;;	256	...
;;	...
;;	64K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	80K	1	80K+1
;;	256
;;	...
;;	80K-256

;; Do 128 five_reals_last_unfft macros
;;	distance between fft data elements is 128

	start_timer 19
	mov	al, 64			;; Load loop counter
c1b:	two_five_reals_last_unfft 8, 64*blkdst
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test loop counter
	jnc	c1b			;; Iterate if necessary
	IFDEF PFETCH
	prefetcht1 [esi-32+clm*32]
;	prefetcht1 [esi-32+clm*32+64*blkdst]
	prefetcht1 [esi-32+clm*32+128*blkdst]
;	prefetcht1 [esi-32+clm*32+192*blkdst]
	prefetcht1 [esi-32+clm*32+256*blkdst]
	ENDIF
	loopclm	dl, clm, c1b
	lea	esi, [esi-clm*2*dist1+blkdst];; Next source pointer
	dec	al			;; Test loop counter
	jnz	c1b			;; Iterate if necessary
	lea	esi, [esi-64*blkdst]	;; Restore source pointer
	end_timer 19

;; Normalize these values

	start_timer 20
	mov	eax, NORMRTN		;; Addr of normalization routine
	call	eax
	sub	eax, eax
	end_timer 20

;; Have we been given permission to start the FFT on the result?
;; If so, do so now while the data is in the L2 cache.

	cmp	POSTFFT, 0		;; Test flag
	je	no_fft			;; Skip FFT code if flag not set
	cmp	edx, p2cl/128*65536+(128/clm-4/clm)*256;; Delay fft on 8 sets
	jg	no_fft			;; Jump if delaying FFT
	copy_3_words clm, 0

;; Do FFT levels 1,2,3
;;
;; On input the 32-byte cache lines hold these data values:
;;	0	80K	1	80K+1
;;	256
;;	...
;;	80K-256
;; On output the 32-byte cache lines hold these data values:
;;	0	32K	1	32K+1
;;	256	...
;;	...
;;	64K	...
;;	...

;; Do 128 five_reals_first_fft macros
;;	distance between fft data elements is 128

b0b:	start_timer 21
	mov	ebx, DIST_TO_FFTSRCARG
	mov	al, 64			;; 64 iterations
b1b:	two_five_reals_first_fft 64*blkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	add	dl, 256/2/clm		;; Test loop counter
	jnc	b1b			;; Iterate if necessary
	lea	esi, [esi-clm*2*dist1+blkdst];; Next source pointer
	dec	al			;; Test loop counter
	jnz	b1b			;; Iterate if necessary
	lea	esi, [esi-64*blkdst]	;; Restore source pointer
	end_timer 21

;; Do FFT levels 4
;;
;; On input the 32-byte cache lines hold these data values:
;;	0	32K	1	32K+1
;;	256	...
;;	...
;;	64K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	16K	1	16K+1
;;	256	...
;;	...
;;	32K	...
;;	...

;; Do 16 eight_reals_fft_1 macros
;;	distance between fft data elements is 64

	start_timer 22
	mov	al, 16			;; Load loop counter
b8b:	disp eight_reals_fft_1, 16*blkdst, 32*blkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test inner loop counter
	jnc	b8b			;; Iterate if necessary
	IFDEF PFETCH
;	prefetcht1 [esi-32+clm*32]
	prefetcht1 [esi-32+clm*32+16*blkdst]
;	prefetcht1 [esi-32+clm*32+32*blkdst]
	prefetcht1 [esi-32+clm*32+48*blkdst]
	ENDIF
	loopclm	dl, clm, b8b
	lea	esi, [esi-clm*2*dist1+blkdst];; Next source pointer
	dec	al
	jnz	b8b
	lea	esi, [esi-16*blkdst+64*blkdst];; Next source pointer

;; Do 64 four_complex_fft_1 macros
;;	distance between fft data elements is 64

	mov	edi, sincos4		;; Load sin/cos pointer
	mov	eax, 2			;; 2 iterations of 32
b9b:	disp four_complex_fft_1, 32*blkdst, 64*blkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test inner loop counter
	jnc	b9b			;; Iterate if necessary
	IFDEF PFETCH
;	prefetcht1 [esi-32+clm*32]
	prefetcht1 [esi-32+clm*32+32*blkdst]
;	prefetcht1 [esi-32+clm*32+64*blkdst]
	prefetcht1 [esi-32+clm*32+96*blkdst]
	ENDIF
	loopclm	dl, clm, b9b
	lea	esi, [esi-clm*2*dist1+blkdst];; Next source pointer
	add	ah, 256/32		;; Test middle loop counter
	jnc	b9b			;; Iterate if necessary
	lea	esi, [esi-32*blkdst+128*blkdst];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al
	jnz	b9b
	lea	esi, [esi-2*128*blkdst-64*blkdst];; Restore source pointer
	end_timer 22

;; Do FFT levels 5,6
;;
;; On input the 32-byte cache lines hold these data values:
;;	0	16K	1	16K+1
;;	256	...
;;	...
;;	32K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	4K	1	4K+1
;;	256	...
;;	...
;;	8K	...
;;	...

;; Do 16 four_real_four_semireal_fft macros
;;	distance between fft data elements is 16

	start_timer 23
	mov	al, 16			;; Load loop counter
b6b:	disp four_real_four_semireal_fft, 16*blkdst, 32*blkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test inner loop counter
	jnc	b6b			;; Iterate if necessary
	IFDEF PFETCH
;	prefetcht1 [esi-32+clm*32]
	prefetcht1 [esi-32+clm*32+16*blkdst]
;	prefetcht1 [esi-32+clm*32+32*blkdst]
	prefetcht1 [esi-32+clm*32+48*blkdst]
	ENDIF
	loopclm	dl, clm, b6b
	lea	esi, [esi-clm*2*dist1+blkdst];; Next source pointer
	dec	al
	jnz	b6b
	lea	esi, [esi-16*blkdst+64*blkdst];; Next source pointer

;; Do 64 four_complex_fft macros
;;	distance between fft data elements is 16

	mov	edi, sincos3		;; Load sin/cos pointer
	mov	eax, 4			;; 4 iterations of 16
b7b:	disp four_complex_fft, 16*blkdst, 32*blkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test inner loop counter
	jnc	b7b			;; Iterate if necessary
	IFDEF PFETCH
;	prefetcht1 [esi-32+clm*32]
	prefetcht1 [esi-32+clm*32+16*blkdst]
;	prefetcht1 [esi-32+clm*32+32*blkdst]
	prefetcht1 [esi-32+clm*32+48*blkdst]
	ENDIF
	loopclm	dl, clm, b7b
	lea	esi, [esi-clm*2*dist1+blkdst];; Next source pointer
	add	ah, 256/16		;; Test middle loop counter
	jnc	b7b			;; Iterate if necessary
	lea	esi, [esi-16*blkdst+64*blkdst];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al
	jnz	b7b
	lea	esi, [esi-5*64*blkdst]	;; Restore source pointer
	end_timer 23

;; Do FFT levels 7,8
;;
;; On input the 32-byte cache lines hold these data values:
;;	0	4K	1	4K+1
;;	256	...
;;	...
;;	8K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	1K	1	1K+1
;;	256	...
;;	...
;;	2K	...
;;	...

;; Do 4 four_real_four_semireal_fft macros
;;	distance between fft data elements is 4

	start_timer 24
	mov	al, 4			;; Load loop counter
b4b:	disp four_real_four_semireal_fft, 4*blkdst, 8*blkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test inner loop counter
	jnc	b4b			;; Iterate if necessary
	IFDEF PFETCH
;	prefetcht1 [esi-32+clm*32]
	prefetcht1 [esi-32+clm*32+4*blkdst]
;	prefetcht1 [esi-32+clm*32+8*blkdst]
	prefetcht1 [esi-32+clm*32+12*blkdst]
	ENDIF
	loopclm	dl, clm, b4b
	lea	esi, [esi-clm*2*dist1+blkdst];; Next source pointer
	dec	al
	jnz	b4b
	lea	esi, [esi-4*blkdst+16*blkdst];; Next source pointer

;; Do 76 four_complex_fft macros
;;	distance between fft data elements is 4

	mov	edi, sincos2		;; Load sin/cos pointer
	mov	eax, 19			;; 19 iterations of 4
b5b:	disp four_complex_fft, 4*blkdst, 8*blkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test inner loop counter
	jnc	b5b			;; Iterate if necessary
	IFDEF PFETCH
;	prefetcht1 [esi-32+clm*32]
	prefetcht1 [esi-32+clm*32+4*blkdst]
;	prefetcht1 [esi-32+clm*32+8*blkdst]
	prefetcht1 [esi-32+clm*32+12*blkdst]
	ENDIF
	loopclm	dl, clm, b5b
	lea	esi, [esi-clm*2*dist1+blkdst];; Next source pointer
	add	ah, 256/4		;; Test middle loop counter
	jnc	b5b			;; Iterate if necessary
	lea	esi, [esi-4*blkdst+16*blkdst];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al
	jnz	b5b
	lea	esi, [esi-20*16*blkdst]	;; Restore source pointer
	end_timer 24

;; Do FFT levels 9,10
;;
;; On input the 32-byte cache lines hold these data values:
;;	0	1K	1	1K+1
;;	256	...
;;	...
;;	2K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	256	1	257
;;	512	...
;;	...

;; Do 1 four_real_four_semireal_fft macros
;;	distance between fft data elements is 1

	start_timer 25
b2b:	disp four_real_four_semireal_fft, blkdst, 2*blkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test inner loop counter
	jnc	b2b			;; Iterate if necessary
	IFDEF PFETCH
;	prefetcht1 [esi-32+clm*32]
	prefetcht1 [esi-32+clm*32+blkdst]
;	prefetcht1 [esi-32+clm*32+2*blkdst]
	prefetcht1 [esi-32+clm*32+3*blkdst]
	ENDIF
	loopclm	dl, clm, b2b
	lea	esi, [esi-clm*2*dist1+4*blkdst];; Next source pointer

;; Do 79 four_complex_fft macros
;;	distance between fft data elements is 1

	mov	edi, sincos1		;; Load sin/cos pointer
	mov	al, 79			;; 79 iterations
b3b:	disp four_complex_fft, blkdst, 2*blkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test inner loop counter
	jnc	b3b			;; Iterate if necessary
	IFDEF PFETCH
;	prefetcht1 [esi-32+clm*32]
	prefetcht1 [esi-32+clm*32+blkdst]
;	prefetcht1 [esi-32+clm*32+2*blkdst]
	prefetcht1 [esi-32+clm*32+3*blkdst]
	ENDIF
	loopclm	dl, clm, b3b
	lea	esi, [esi-clm*2*dist1+4*blkdst];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al
	jnz	b3b
	lea	esi, [esi-80*4*blkdst]	;; Restore source pointer
	end_timer 25

;; Work on next 4 sets of 32 values

no_fft:	lea	esi, [esi+clm*2*dist1]	;; Next source pointer
	dec	dh			;; Test loop counter
	jnz	short more		;; Do more FFTs
	lea	esi, [esi+64]		;; Skip pad every 128 cache lines (4KB)
	sub	edx, 65536		;; Test loop counter
	jz	short fftdn		;; Jump if done looping
	mov	dh, 128/clm		;; Restore cache line counter
more:	cmp	norm_ptr1, 0		;; What was the loop start point?
	jne	c0b			;; Do inverse-FFT/norm/FFT iteration
	jmp	b0b			;; Do another FFT-only iteration
fftdn:	lea	esi, [esi-p2cl*2*dist1-p2cl/128*64];; Restore source pointer
	cmp	esi, DESTARG		;; If esi was not restored, then we 
	jne	short fftdn1		;; just finished skipped postfft data
	sub	ebx, ebx
	mov	DIST_TO_FFTSRCARG, ebx
	cmp	norm_ptr1, 0		;; What was the loop start point?
	je	pass2			;; FFT-only, go do pass 2

;; Split the carries into high and low carries (so they do not exceed
;; the desired number of bits per FFT word.  Then add the carries back to the
;; FFT data.

	start_timer 28
	call	gw_carries
	end_timer 28

;; Have we been given permission to start the FFT on the result?
;; If so, set loop to FFT the first few cache lines now that the carries
;; have been added in.

	cmp	POSTFFT, 0		;; Test flag
	je	short fftdn1		;; Skip FFT code if flag not set
	mov	esi, DESTARG		;; Load FFT data address
	mov	edx, 65536+4/clm*256	;; Do 8 previously skipped cache lines
	mov	norm_ptr1, 0		;; Set flag so only forward FFT is done
	copy_4_words
	jmp	b0b

;; Now add in the FFT'ed carries, cleanup and return

fftdn1:	end_timer 1
	jmp	gw_finish_mult
	ENDM


;; Perform a 192K-element FFT.  This is done in two passes.  Pass 1
;; does 10 levels, pass 2 uses common code to perform the last 8 levels.

pass1levels10pfa6 MACRO pass2_macro, p2cl, clm
	LOCAL	b0b, b1b, b2b, b3b, b4b, b5b, b6b, b7b, b8b, b9b
	LOCAL	c0b, c1b, c2b, c3b, c4b, c5b, c6b, c7b, c8b, c9b
	LOCAL	pass2, no_fft, more, fftdn, fftdn1

;; Call FFT code unless the FFT has already been started

	clear_timers
	cmp	DWORD PTR [esi-28][ebx], 0 ;; Test FFT-started flag
	jne	pass2			;; Jump is FFT already started
	mov	norm_ptr1, 0		;; Set FFT-ing flag
	mov	edx, p2cl/128*65536+128/clm*256;; Load loop counter
	jmp	b0b			;; Do the pass 1 fft

;; Do the last 8 FFT levels
;;	do 1 pass2_8_levels_real and 383 pass2_8_levels_complex

pass2:	start_timer 0
	CALLP	pass2_macro
	end_timer 0
	cmp	ffttype, 1		;; We're done if FFTing only
	je	gw_finish_fft

;; Do the inverse FFT

	start_timer 1
	fldz				;; Init SUMOUT
	fstp	SUMOUT
	mov	edx, norm_biglit_array	;; Addr of the big/little flags array
	mov	norm_ptr1, edx		;; Save ptr
	mov	edx, norm_col_mults	;; Addr of the column multipliers
	mov	norm_ptr2, edx		;; Save ptr
	mov	edx, p2cl/128*65536+128/clm*256;; Load loop counter

;; Do inverse FFT levels 9,10
;; On input the 32-byte cache lines hold these data values:
;;	0	256	1	257
;;	512	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	1K	1	1K+1
;;	256	...
;;	...
;;	2K	...
;;	...

;; Do 1 four_real_four_semireal_unfft macros
;;	distance between fft data elements is 1

c0b:	start_timer 15
c2b:	disp four_real_four_semireal_unfft, 8, blkdst, 2*blkdst
	lea	esi, [esi+dist1]	;; Next source pointer
	add	dl, 256/2/clm		;; Test inner loop counter
	jnc	c2b			;; Iterate if necessary
	lea	esi, [esi-clm*2*dist1+4*blkdst];; Next source pointer

;; Do 95 four_complex_unfft macros
;;	distance between fft data elements is 1

	mov	edi, sincos1		;; Load sin/cos pointer
	mov	al, 95			;; 95 iterations
c3b:	disp four_complex_unfft, 8, blkdst, 2*blkdst
	lea	esi, [esi+dist1]	;; Next source pointer
	add	dl, 256/2/clm		;; Test loop counter
	jnc	c3b			;; Iterate if necessary
	lea	esi, [esi-clm*2*dist1+4*blkdst];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al			;; Test outer loop counter
	jnz	c3b			;; Iterate if necessary
	lea	esi, [esi-96*4*blkdst]	;; Restore source pointer
	end_timer 15

;; Do inverse FFT levels 7,8
;; On input the 32-byte cache lines hold these data values:
;;	0	1K	1	1K+1
;;	256	...
;;	...
;;	2K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	4K	1	4K+1
;;	256	...
;;	...
;;	8K	...
;;	...

;; Do 4 four_real_four_semireal_unfft macros
;;	distance between fft data elements is 4

	start_timer 16
	mov	al, 4			;; Load loop counter
c4b:	disp four_real_four_semireal_unfft, 8, 4*blkdst, 8*blkdst
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test inner loop counter
	jnc	c4b			;; Iterate if necessary
	IFDEF PFETCH
	prefetcht1 [esi-32+clm*32]
;	prefetcht1 [esi-32+clm*32+4*blkdst]
	prefetcht1 [esi-32+clm*32+8*blkdst]
;	prefetcht1 [esi-32+clm*32+12*blkdst]
	ENDIF
	loopclm	dl, clm, c4b
	lea	esi, [esi-clm*2*dist1+blkdst];; Next source pointer
	dec	al
	jnz	c4b
	lea	esi, [esi-4*blkdst+16*blkdst];; Next source pointer

;; Do 92 four_complex_unfft macros
;;	distance between fft data elements is 4

	mov	edi, sincos2		;; Load sin/cos pointer
	mov	eax, 23			;; 23 iterations of 4
c5b:	disp four_complex_unfft, 8, 4*blkdst, 8*blkdst
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test inner loop counter
	jnc	c5b			;; Iterate if necessary
	IFDEF PFETCH
	prefetcht1 [esi-32+clm*32]
;	prefetcht1 [esi-32+clm*32+4*blkdst]
	prefetcht1 [esi-32+clm*32+8*blkdst]
;	prefetcht1 [esi-32+clm*32+12*blkdst]
	ENDIF
	loopclm	dl, clm, c5b
	lea	esi, [esi-clm*2*dist1+blkdst];; Next source pointer
	add	ah, 256/4		;; Test middle loop counter
	jnc	c5b			;; Iterate if necessary
	lea	esi, [esi-4*blkdst+16*blkdst];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al
	jnz	c5b
	lea	esi, [esi-24*16*blkdst]	;; Restore source pointer
	end_timer 16

;; Do inverse FFT levels 5,6
;; On input the 32-byte cache lines hold these data values:
;;	0	4K	1	4K+1
;;	256	...
;;	...
;;	8K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	16K	1	16K+1
;;	256	...
;;	...
;;	32K	...
;;	...

;; Do 16 four_real_four_semireal_unfft macros
;;	distance between fft data elements is 16

	start_timer 17
	mov	al, 16			;; Load loop counter
c6b:	disp four_real_four_semireal_unfft, 8, 16*blkdst, 32*blkdst
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test inner loop counter
	jnc	c6b			;; Iterate if necessary
	IFDEF PFETCH
	prefetcht1 [esi-32+clm*32]
;	prefetcht1 [esi-32+clm*32+16*blkdst]
	prefetcht1 [esi-32+clm*32+32*blkdst]
;	prefetcht1 [esi-32+clm*32+48*blkdst]
	ENDIF
	loopclm	dl, clm, c6b
	lea	esi, [esi-clm*2*dist1+blkdst];; Next source pointer
	dec	al
	jnz	c6b
	lea	esi, [esi-16*blkdst+64*blkdst];; Next source pointer

;; Do 80 four_complex_unfft macros
;;	distance between fft data elements is 16

	mov	edi, sincos3		;; Load sin/cos pointer
	mov	eax, 5			;; 5 iterations of 16
c7b:	disp four_complex_unfft, 8, 16*blkdst, 32*blkdst
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test inner loop counter
	jnc	c7b			;; Iterate if necessary
	IFDEF PFETCH
	prefetcht1 [esi-32+clm*32]
;	prefetcht1 [esi-32+clm*32+16*blkdst]
	prefetcht1 [esi-32+clm*32+32*blkdst]
;	prefetcht1 [esi-32+clm*32+48*blkdst]
	ENDIF
	loopclm	dl, clm, c7b
	lea	esi, [esi-clm*2*dist1+blkdst];; Next source pointer
	add	ah, 256/16		;; Test middle loop counter
	jnc	c7b			;; Iterate if necessary
	lea	esi, [esi-16*blkdst+64*blkdst];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al
	jnz	c7b
	lea	esi, [esi-6*64*blkdst]	;; Restore source pointer
	end_timer 17

;; Do inverse FFT levels 4
;; On input the 32-byte cache lines hold these data values:
;;	0	16K	1	16K+1
;;	256	...
;;	...
;;	32K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	32K	1	32K+1
;;	256	...
;;	...
;;	64K	...
;;	...

;; Do 32 four_real_four_semireal_unfft_1 macros
;;	distance between fft data elements is 64

	start_timer 18
	mov	al, 32			;; Load loop counter
c8b:	disp four_real_four_semireal_unfft_1, 32*blkdst, 8, 64*blkdst
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test inner loop counter
	jnc	c8b			;; Iterate if necessary
	IFDEF PFETCH
	prefetcht1 [esi-32+clm*32]
;	prefetcht1 [esi-32+clm*32+32*blkdst]
	prefetcht1 [esi-32+clm*32+64*blkdst]
;	prefetcht1 [esi-32+clm*32+96*blkdst]
	ENDIF
	loopclm	dl, clm, c8b
	lea	esi, [esi-clm*2*dist1+blkdst];; Next source pointer
	dec	al
	jnz	c8b
	lea	esi, [esi-32*blkdst+128*blkdst];; Next source pointer

;; Do 64 four_complex_unfft_1 macros
;;	distance between fft data elements is 64

	mov	edi, sincos4		;; Load sin/cos pointer
	mov	eax, 2			;; 2 iterations of 32
c9b:	disp four_complex_unfft_1, 32*blkdst, 8, 64*blkdst
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test inner loop counter
	jnc	c9b			;; Iterate if necessary
	IFDEF PFETCH
	prefetcht1 [esi-32+clm*32]
;	prefetcht1 [esi-32+clm*32+32*blkdst]
	prefetcht1 [esi-32+clm*32+64*blkdst]
;	prefetcht1 [esi-32+clm*32+96*blkdst]
	ENDIF
	loopclm	dl, clm, c9b
	lea	esi, [esi-clm*2*dist1+blkdst];; Next source pointer
	add	ah, 256/32		;; Test middle loop counter
	jnc	c9b			;; Iterate if necessary
	lea	esi, [esi-32*blkdst+128*blkdst];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al
	jnz	c9b
	lea	esi, [esi-3*128*blkdst]	;; Restore source pointer
	end_timer 18

;; Do inverse FFT levels 1,2,3
;; On input the 32-byte cache lines hold these data values:
;;	0	32K	1	32K+1
;;	256	...
;;	...
;;	64K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	96K	1	96K+1
;;	256
;;	...
;;	96K-256

;; Do 128 six_reals_last_unfft macros
;;	distance between fft data elements is 128

	start_timer 19
	mov	eax, 128*256		;; Load loop counter
c1b:	disp six_reals_last_unfft, 8, 128*blkdst, 256*blkdst
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test loop counter
	jnc	c1b			;; Iterate if necessary
	IFDEF PFETCH
	prefetcht1 [esi-32+clm*32]
;	prefetcht1 [esi-32+clm*32+128*blkdst]
	prefetcht1 [esi-32+clm*32+256*blkdst]
	ENDIF
	loopclm	dl, clm, c1b
	lea	esi, [esi-clm*2*dist1+blkdst];; Next source pointer
	sub	eax, 256		;; Test loop counter
	jnz	c1b			;; Iterate if necessary
	lea	esi, [esi-128*blkdst]	;; Restore source pointer
	end_timer 19

;; Normalize these values

	start_timer 20
	mov	eax, NORMRTN		;; Addr of normalization routine
	call	eax
	sub	eax, eax
	end_timer 20

;; Have we been given permission to start the FFT on the result?
;; If so, do so now while the data is in the L2 cache.

	cmp	POSTFFT, 0		;; Test flag
	je	no_fft			;; Skip FFT code if flag not set
	cmp	edx, p2cl/128*65536+(128/clm-4/clm)*256;; Delay fft on 8 sets
	jg	no_fft			;; Jump if delaying FFT
	copy_3_words clm, 0

;; Do FFT levels 1,2,3
;;
;; On input the 32-byte cache lines hold these data values:
;;	0	96K	1	96K+1
;;	256
;;	...
;;	96K-256
;; On output the 32-byte cache lines hold these data values:
;;	0	32K	1	32K+1
;;	256	...
;;	...
;;	64K	...
;;	...

;; Do 128 six_reals_first_fft macros
;;	distance between fft data elements is 128

b0b:	start_timer 21
	mov	ebx, DIST_TO_FFTSRCARG
	mov	al, 128			;; 128 iterations
b1b:	disp six_reals_first_fft, 128*blkdst, 256*blkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	add	dl, 256/2/clm		;; Test loop counter
	jnc	b1b			;; Iterate if necessary
	lea	esi, [esi-clm*2*dist1+blkdst];; Next source pointer
	dec	al			;; Test loop counter
	jnz	b1b			;; Iterate if necessary
	lea	esi, [esi-128*blkdst]	;; Restore source pointer
	end_timer 21

;; Do FFT levels 4
;;
;; On input the 32-byte cache lines hold these data values:
;;	0	32K	1	32K+1
;;	256	...
;;	...
;;	64K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	16K	1	16K+1
;;	256	...
;;	...
;;	32K	...
;;	...

;; Do 32 four_real_four_semireal_fft_1 macros
;;	distance between fft data elements is 64

	start_timer 22
	mov	al, 32			;; Load loop counter
b8b:	disp four_real_four_semireal_fft_1, 32*blkdst, 64*blkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test inner loop counter
	jnc	b8b			;; Iterate if necessary
	IFDEF PFETCH
;	prefetcht1 [esi-32+clm*32]
	prefetcht1 [esi-32+clm*32+32*blkdst]
;	prefetcht1 [esi-32+clm*32+64*blkdst]
	prefetcht1 [esi-32+clm*32+96*blkdst]
	ENDIF
	loopclm	dl, clm, b8b
	lea	esi, [esi-clm*2*dist1+blkdst];; Next source pointer
	dec	al
	jnz	b8b
	lea	esi, [esi-32*blkdst+128*blkdst];; Next source pointer

;; Do 64 four_complex_fft_1 macros
;;	distance between fft data elements is 64

	mov	edi, sincos4		;; Load sin/cos pointer
	mov	eax, 2			;; 2 iterations of 32
b9b:	disp four_complex_fft_1, 32*blkdst, 64*blkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test inner loop counter
	jnc	b9b			;; Iterate if necessary
	IFDEF PFETCH
;	prefetcht1 [esi-32+clm*32]
	prefetcht1 [esi-32+clm*32+32*blkdst]
;	prefetcht1 [esi-32+clm*32+64*blkdst]
	prefetcht1 [esi-32+clm*32+96*blkdst]
	ENDIF
	loopclm	dl, clm, b9b
	lea	esi, [esi-clm*2*dist1+blkdst];; Next source pointer
	add	ah, 256/32		;; Test middle loop counter
	jnc	b9b			;; Iterate if necessary
	lea	esi, [esi-32*blkdst+128*blkdst];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al
	jnz	b9b
	lea	esi, [esi-3*128*blkdst]	;; Restore source pointer
	end_timer 22

;; Do FFT levels 5,6
;;
;; On input the 32-byte cache lines hold these data values:
;;	0	16K	1	16K+1
;;	256	...
;;	...
;;	32K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	4K	1	4K+1
;;	256	...
;;	...
;;	8K	...
;;	...

;; Do 16 four_real_four_semireal_fft macros
;;	distance between fft data elements is 16

	start_timer 23
	mov	al, 16			;; Load loop counter
b6b:	disp four_real_four_semireal_fft, 16*blkdst, 32*blkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test inner loop counter
	jnc	b6b			;; Iterate if necessary
	IFDEF PFETCH
;	prefetcht1 [esi-32+clm*32]
	prefetcht1 [esi-32+clm*32+16*blkdst]
;	prefetcht1 [esi-32+clm*32+32*blkdst]
	prefetcht1 [esi-32+clm*32+48*blkdst]
	ENDIF
	loopclm	dl, clm, b6b
	lea	esi, [esi-clm*2*dist1+blkdst];; Next source pointer
	dec	al
	jnz	b6b
	lea	esi, [esi-16*blkdst+64*blkdst];; Next source pointer

;; Do 80 four_complex_fft macros
;;	distance between fft data elements is 16

	mov	edi, sincos3		;; Load sin/cos pointer
	mov	eax, 5			;; 5 iterations of 16
b7b:	disp four_complex_fft, 16*blkdst, 32*blkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test inner loop counter
	jnc	b7b			;; Iterate if necessary
	IFDEF PFETCH
;	prefetcht1 [esi-32+clm*32]
	prefetcht1 [esi-32+clm*32+16*blkdst]
;	prefetcht1 [esi-32+clm*32+32*blkdst]
	prefetcht1 [esi-32+clm*32+48*blkdst]
	ENDIF
	loopclm	dl, clm, b7b
	lea	esi, [esi-clm*2*dist1+blkdst];; Next source pointer
	add	ah, 256/16		;; Test middle loop counter
	jnc	b7b			;; Iterate if necessary
	lea	esi, [esi-16*blkdst+64*blkdst];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al
	jnz	b7b
	lea	esi, [esi-6*64*blkdst]	;; Restore source pointer
	end_timer 23

;; Do FFT levels 7,8
;;
;; On input the 32-byte cache lines hold these data values:
;;	0	4K	1	4K+1
;;	256	...
;;	...
;;	8K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	1K	1	1K+1
;;	256	...
;;	...
;;	2K	...
;;	...

;; Do 4 four_real_four_semireal_fft macros
;;	distance between fft data elements is 4

	start_timer 24
	mov	al, 4			;; Load loop counter
b4b:	disp four_real_four_semireal_fft, 4*blkdst, 8*blkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test inner loop counter
	jnc	b4b			;; Iterate if necessary
	IFDEF PFETCH
;	prefetcht1 [esi-32+clm*32]
	prefetcht1 [esi-32+clm*32+4*blkdst]
;	prefetcht1 [esi-32+clm*32+8*blkdst]
	prefetcht1 [esi-32+clm*32+12*blkdst]
	ENDIF
	loopclm	dl, clm, b4b
	lea	esi, [esi-clm*2*dist1+blkdst];; Next source pointer
	dec	al
	jnz	b4b
	lea	esi, [esi-4*blkdst+16*blkdst];; Next source pointer

;; Do 92 four_complex_fft macros
;;	distance between fft data elements is 4

	mov	edi, sincos2		;; Load sin/cos pointer
	mov	eax, 23			;; 23 iterations of 4
b5b:	disp four_complex_fft, 4*blkdst, 8*blkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test inner loop counter
	jnc	b5b			;; Iterate if necessary
	IFDEF PFETCH
;	prefetcht1 [esi-32+clm*32]
	prefetcht1 [esi-32+clm*32+4*blkdst]
;	prefetcht1 [esi-32+clm*32+8*blkdst]
	prefetcht1 [esi-32+clm*32+12*blkdst]
	ENDIF
	loopclm	dl, clm, b5b
	lea	esi, [esi-clm*2*dist1+blkdst];; Next source pointer
	add	ah, 256/4		;; Test middle loop counter
	jnc	b5b			;; Iterate if necessary
	lea	esi, [esi-4*blkdst+16*blkdst];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al
	jnz	b5b
	lea	esi, [esi-24*16*blkdst]	;; Restore source pointer
	end_timer 24

;; Do FFT levels 9,10
;;
;; On input the 32-byte cache lines hold these data values:
;;	0	1K	1	1K+1
;;	256	...
;;	...
;;	2K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	256	1	257
;;	512	...
;;	...

;; Do 1 four_real_four_semireal_fft macros
;;	distance between fft data elements is 1

	start_timer 25
b2b:	disp four_real_four_semireal_fft, blkdst, 2*blkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test inner loop counter
	jnc	b2b			;; Iterate if necessary
	IFDEF PFETCH
;	prefetcht1 [esi-32+clm*32]
	prefetcht1 [esi-32+clm*32+blkdst]
;	prefetcht1 [esi-32+clm*32+2*blkdst]
	prefetcht1 [esi-32+clm*32+3*blkdst]
	ENDIF
	loopclm	dl, clm, b2b
	lea	esi, [esi-clm*2*dist1+4*blkdst];; Next source pointer

;; Do 95 four_complex_fft macros
;;	distance between fft data elements is 1

	mov	edi, sincos1		;; Load sin/cos pointer
	mov	al, 95			;; 95 iterations
b3b:	disp four_complex_fft, blkdst, 2*blkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test inner loop counter
	jnc	b3b			;; Iterate if necessary
	IFDEF PFETCH
;	prefetcht1 [esi-32+clm*32]
	prefetcht1 [esi-32+clm*32+blkdst]
;	prefetcht1 [esi-32+clm*32+2*blkdst]
	prefetcht1 [esi-32+clm*32+3*blkdst]
	ENDIF
	loopclm	dl, clm, b3b
	lea	esi, [esi-clm*2*dist1+4*blkdst];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al
	jnz	b3b
	lea	esi, [esi-96*4*blkdst]	;; Restore source pointer
	end_timer 25

;; Work on next 4 sets of 32 values

no_fft:	lea	esi, [esi+clm*2*dist1]	;; Next source pointer
	dec	dh			;; Test loop counter
	jnz	short more		;; Do more FFTs
	lea	esi, [esi+64]		;; Skip pad every 128 cache lines (4KB)
	sub	edx, 65536		;; Test loop counter
	jz	short fftdn		;; Jump if done looping
	mov	dh, 128/clm		;; Restore cache line counter
more:	cmp	norm_ptr1, 0		;; What was the loop start point?
	jne	c0b			;; Do inverse-FFT/norm/FFT iteration
	jmp	b0b			;; Do another FFT-only iteration
fftdn:	lea	esi, [esi-p2cl*2*dist1-p2cl/128*64];; Restore source pointer
	cmp	esi, DESTARG		;; If esi was not restored, then we 
	jne	short fftdn1		;; just finished skipped postfft data
	sub	ebx, ebx
	mov	DIST_TO_FFTSRCARG, ebx
	cmp	norm_ptr1, 0		;; What was the loop start point?
	je	pass2			;; FFT-only, go do pass 2

;; Split the carries into high and low carries (so they do not exceed
;; the desired number of bits per FFT word.  Then add the carries back to the
;; FFT data.

	start_timer 28
	call	gw_carries
	end_timer 28

;; Have we been given permission to start the FFT on the result?
;; If so, set loop to FFT the first few cache lines now that the carries
;; have been added in.

	cmp	POSTFFT, 0		;; Test flag
	je	short fftdn1		;; Skip FFT code if flag not set
	mov	esi, DESTARG		;; Load FFT data address
	mov	edx, 65536+4/clm*256	;; Do 8 previously skipped cache lines
	mov	norm_ptr1, 0		;; Set flag so only forward FFT is done
	copy_4_words
	jmp	b0b

;; Now add in the FFT'ed carries, cleanup and return

fftdn1:	end_timer 1
	jmp	gw_finish_mult
	ENDM


;; Perform a 192K-element negacyclic FFT.  This is done in two passes.  Pass 1
;; does 10 levels, pass 2 uses common code to perform the last 8 levels.

pass1levels10complex3 MACRO pass2_macro, p2cl, clm
	LOCAL	b0b, b1b, b3b, b5b, b7b, b9b, c0b, c1b, c3b, c5b, c7b, c9b
	LOCAL	pass2, no_fft, more, fftdn, fftdn1

;; Call FFT code unless the FFT has already been started

	clear_timers
	cmp	DWORD PTR [esi-28][ebx], 0 ;; Test FFT-started flag
	jne	pass2			;; Jump is FFT already started
	mov	norm_ptr1, 0		;; Set FFT-ing flag
	mov	edx, p2cl/128*65536+128/clm*256;; Load loop counter
	jmp	b0b			;; Do the pass 1 fft

;; Do the last 8 FFT levels
;;	do 384 pass2_8_levels_complex

pass2:	start_timer 0
	CALLP	pass2_macro
	end_timer 0
	cmp	ffttype, 1		;; We're done if FFTing only
	je	gw_finish_fft

;; Do the inverse FFT

	start_timer 1
	fldz				;; Init SUMOUT
	fstp	SUMOUT
	mov	edx, norm_biglit_array	;; Addr of the big/little flags array
	mov	norm_ptr1, edx		;; Save ptr
	mov	edx, norm_col_mults	;; Addr of the column multipliers
	mov	norm_ptr2, edx		;; Save ptr
	mov	edx, p2cl/128*65536+128/clm*256;; Load loop counter

;; Do inverse FFT levels 9,10
;; On input the 32-byte cache lines hold these data values:
;;	0	256	1	257
;;	512	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	1K	1	1K+1
;;	256	...
;;	...
;;	2K	...
;;	...

;; Do 96 four_complex_unfft macros
;;	distance between fft data elements is 1

c0b:	start_timer 15
	mov	edi, sincos1		;; Load sin/cos pointer
	mov	al, 96			;; 96 iterations
c3b:	disp four_complex_unfft, 8, blkdst, 2*blkdst
	lea	esi, [esi+dist1]	;; Next source pointer
	add	dl, 256/2/clm		;; Test loop counter
	jnc	c3b			;; Iterate if necessary
	lea	esi, [esi-clm*2*dist1+4*blkdst];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al			;; Test outer loop counter
	jnz	c3b			;; Iterate if necessary
	lea	esi, [esi-96*4*blkdst]	;; Restore source pointer
	end_timer 15

;; Do inverse FFT levels 7,8
;; On input the 32-byte cache lines hold these data values:
;;	0	1K	1	1K+1
;;	256	...
;;	...
;;	2K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	4K	1	4K+1
;;	256	...
;;	...
;;	8K	...
;;	...

;; Do 96 four_complex_unfft macros
;;	distance between fft data elements is 4

	start_timer 16
	mov	edi, sincos2		;; Load sin/cos pointer
	mov	eax, 24			;; 24 iterations of 4
c5b:	disp four_complex_unfft, 8, 4*blkdst, 8*blkdst
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test inner loop counter
	jnc	c5b			;; Iterate if necessary
	IFDEF PFETCH
	prefetcht1 [esi-32+clm*32]
;	prefetcht1 [esi-32+clm*32+4*blkdst]
	prefetcht1 [esi-32+clm*32+8*blkdst]
;	prefetcht1 [esi-32+clm*32+12*blkdst]
	ENDIF
	loopclm	dl, clm, c5b
	lea	esi, [esi-clm*2*dist1+blkdst];; Next source pointer
	add	ah, 256/4		;; Test middle loop counter
	jnc	c5b			;; Iterate if necessary
	lea	esi, [esi-4*blkdst+16*blkdst];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al
	jnz	c5b
	lea	esi, [esi-24*16*blkdst]	;; Restore source pointer
	end_timer 16

;; Do inverse FFT levels 5,6
;; On input the 32-byte cache lines hold these data values:
;;	0	4K	1	4K+1
;;	256	...
;;	...
;;	8K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	16K	1	16K+1
;;	256	...
;;	...
;;	32K	...
;;	...

;; Do 96 four_complex_unfft macros
;;	distance between fft data elements is 16

	start_timer 17
	mov	edi, sincos3		;; Load sin/cos pointer
	mov	eax, 6			;; 6 iterations of 16
c7b:	disp four_complex_unfft, 8, 16*blkdst, 32*blkdst
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test inner loop counter
	jnc	c7b			;; Iterate if necessary
	IFDEF PFETCH
	prefetcht1 [esi-32+clm*32]
;	prefetcht1 [esi-32+clm*32+16*blkdst]
	prefetcht1 [esi-32+clm*32+32*blkdst]
;	prefetcht1 [esi-32+clm*32+48*blkdst]
	ENDIF
	loopclm	dl, clm, c7b
	lea	esi, [esi-clm*2*dist1+blkdst];; Next source pointer
	add	ah, 256/16		;; Test middle loop counter
	jnc	c7b			;; Iterate if necessary
	lea	esi, [esi-16*blkdst+64*blkdst];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al
	jnz	c7b
	lea	esi, [esi-6*64*blkdst]	;; Restore source pointer
	end_timer 17

;; Do inverse FFT levels 4
;; On input the 32-byte cache lines hold these data values:
;;	0	16K	1	16K+1
;;	256	...
;;	...
;;	32K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	32K	1	32K+1
;;	256	...
;;	...
;;	64K	...
;;	...

;; Do 96 four_complex_unfft_1 macros
;;	distance between fft data elements is 64

	start_timer 18
	mov	edi, sincos4		;; Load sin/cos pointer
	mov	eax, 3			;; 3 iterations of 32
c9b:	disp four_complex_unfft_1, 32*blkdst, 8, 64*blkdst
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test inner loop counter
	jnc	c9b			;; Iterate if necessary
	IFDEF PFETCH
	prefetcht1 [esi-32+clm*32]
;	prefetcht1 [esi-32+clm*32+32*blkdst]
	prefetcht1 [esi-32+clm*32+64*blkdst]
;	prefetcht1 [esi-32+clm*32+96*blkdst]
	ENDIF
	loopclm	dl, clm, c9b
	lea	esi, [esi-clm*2*dist1+blkdst];; Next source pointer
	add	ah, 256/32		;; Test middle loop counter
	jnc	c9b			;; Iterate if necessary
	lea	esi, [esi-32*blkdst+128*blkdst];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al
	jnz	c9b
	lea	esi, [esi-3*128*blkdst]	;; Restore source pointer
	end_timer 18

;; Do inverse FFT levels 1,2,3
;; On input the 32-byte cache lines hold these data values:
;;	0	32K	1	32K+1
;;	256	...
;;	...
;;	64K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	128K	1	128K+1
;;	256
;;	...
;;	128K-256

;; Do 128 three_complex_last_unfft macros
;;	distance between fft data elements is 128

	start_timer 19
	mov	edi, plus1_premults	;; Address of premultiplier table
	mov	eax, 128*256		;; Load loop counter
c1b:	three_complex_last_unfft 8, 128*blkdst, 256*blkdst
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test loop counter
	jnc	c1b			;; Iterate if necessary
	IFDEF PFETCH
	prefetcht1 [esi-32+clm*32]
;	prefetcht1 [esi-32+clm*32+128*blkdst]
	prefetcht1 [esi-32+clm*32+256*blkdst]
	ENDIF
	loopclm	dl, clm, c1b
	lea	esi, [esi-clm*2*dist1+blkdst];; Next source pointer
	lea	edi, [edi+48]		;; Next premultiplier pointer
	sub	eax, 256		;; Test loop counter
	jnz	c1b			;; Iterate if necessary
	lea	esi, [esi-128*blkdst]	;; Restore source pointer
	end_timer 19

;; Normalize these values

	start_timer 20
	mov	eax, NORMRTN		;; Addr of normalization routine
	call	eax
	sub	eax, eax
	end_timer 20

;; Have we been given permission to start the FFT on the result?
;; If so, do so now while the data is in the L2 cache.

	cmp	POSTFFT, 0		;; Test flag
	je	no_fft			;; Skip FFT code if flag not set
	cmp	edx, p2cl/128*65536+(128/clm-4/clm)*256;; Delay fft on 8 sets
	jg	no_fft			;; Jump if delaying FFT
	copy_3_words clm, 0

;; Do FFT levels 1,2,3
;;
;; On input the 32-byte cache lines hold these data values:
;;	0	128K	1	128K+1
;;	256
;;	...
;;	128K-256
;; On output the 32-byte cache lines hold these data values:
;;	0	32K	1	32K+1
;;	256	...
;;	...
;;	64K	...
;;	...

;; Do 128 three_complex_first_fft macros
;;	distance between fft data elements is 128

b0b:	start_timer 21
	mov	ebx, DIST_TO_FFTSRCARG
	mov	edi, plus1_premults	;; Address of premultiplier table
	mov	al, 128			;; 128 iterations
b1b:	three_complex_first_fft 128*blkdst, 256*blkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	add	dl, 256/2/clm		;; Test loop counter
	jnc	b1b			;; Iterate if necessary
	lea	esi, [esi-clm*2*dist1+blkdst];; Next source pointer
	lea	edi, [edi+48]		;; Next premultiplier pointer
	dec	al			;; Test loop counter
	jnz	b1b			;; Iterate if necessary
	lea	esi, [esi-128*blkdst]	;; Restore source pointer
	end_timer 21

;; Do FFT levels 4
;;
;; On input the 32-byte cache lines hold these data values:
;;	0	32K	1	32K+1
;;	256	...
;;	...
;;	64K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	16K	1	16K+1
;;	256	...
;;	...
;;	32K	...
;;	...

;; Do 96 four_complex_fft_1 macros
;;	distance between fft data elements is 64

	start_timer 22
	mov	edi, sincos4		;; Load sin/cos pointer
	mov	eax, 3			;; 3 iterations of 32
b9b:	disp four_complex_fft_1, 32*blkdst, 64*blkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test inner loop counter
	jnc	b9b			;; Iterate if necessary
	IFDEF PFETCH
;	prefetcht1 [esi-32+clm*32]
	prefetcht1 [esi-32+clm*32+32*blkdst]
;	prefetcht1 [esi-32+clm*32+64*blkdst]
	prefetcht1 [esi-32+clm*32+96*blkdst]
	ENDIF
	loopclm	dl, clm, b9b
	lea	esi, [esi-clm*2*dist1+blkdst];; Next source pointer
	add	ah, 256/32		;; Test middle loop counter
	jnc	b9b			;; Iterate if necessary
	lea	esi, [esi-32*blkdst+128*blkdst];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al
	jnz	b9b
	lea	esi, [esi-3*128*blkdst]	;; Restore source pointer
	end_timer 22

;; Do FFT levels 5,6
;;
;; On input the 32-byte cache lines hold these data values:
;;	0	16K	1	16K+1
;;	256	...
;;	...
;;	32K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	4K	1	4K+1
;;	256	...
;;	...
;;	8K	...
;;	...

;; Do 96 four_complex_fft macros
;;	distance between fft data elements is 16

	start_timer 23
	mov	edi, sincos3		;; Load sin/cos pointer
	mov	eax, 6			;; 6 iterations of 16
b7b:	disp four_complex_fft, 16*blkdst, 32*blkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test inner loop counter
	jnc	b7b			;; Iterate if necessary
	IFDEF PFETCH
;	prefetcht1 [esi-32+clm*32]
	prefetcht1 [esi-32+clm*32+16*blkdst]
;	prefetcht1 [esi-32+clm*32+32*blkdst]
	prefetcht1 [esi-32+clm*32+48*blkdst]
	ENDIF
	loopclm	dl, clm, b7b
	lea	esi, [esi-clm*2*dist1+blkdst];; Next source pointer
	add	ah, 256/16		;; Test middle loop counter
	jnc	b7b			;; Iterate if necessary
	lea	esi, [esi-16*blkdst+64*blkdst];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al
	jnz	b7b
	lea	esi, [esi-6*64*blkdst]	;; Restore source pointer
	end_timer 23

;; Do FFT levels 7,8
;;
;; On input the 32-byte cache lines hold these data values:
;;	0	4K	1	4K+1
;;	256	...
;;	...
;;	8K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	1K	1	1K+1
;;	256	...
;;	...
;;	2K	...
;;	...

;; Do 96 four_complex_fft macros
;;	distance between fft data elements is 4

	start_timer 24
	mov	edi, sincos2		;; Load sin/cos pointer
	mov	eax, 24			;; 24 iterations of 4
b5b:	disp four_complex_fft, 4*blkdst, 8*blkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test inner loop counter
	jnc	b5b			;; Iterate if necessary
	IFDEF PFETCH
;	prefetcht1 [esi-32+clm*32]
	prefetcht1 [esi-32+clm*32+4*blkdst]
;	prefetcht1 [esi-32+clm*32+8*blkdst]
	prefetcht1 [esi-32+clm*32+12*blkdst]
	ENDIF
	loopclm	dl, clm, b5b
	lea	esi, [esi-clm*2*dist1+blkdst];; Next source pointer
	add	ah, 256/4		;; Test middle loop counter
	jnc	b5b			;; Iterate if necessary
	lea	esi, [esi-4*blkdst+16*blkdst];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al
	jnz	b5b
	lea	esi, [esi-24*16*blkdst]	;; Restore source pointer
	end_timer 24

;; Do FFT levels 9,10
;;
;; On input the 32-byte cache lines hold these data values:
;;	0	1K	1	1K+1
;;	256	...
;;	...
;;	2K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	256	1	257
;;	512	...
;;	...

;; Do 96 four_complex_fft macros
;;	distance between fft data elements is 1

	start_timer 25
	mov	edi, sincos1		;; Load sin/cos pointer
	mov	al, 96			;; 96 iterations
b3b:	disp four_complex_fft, blkdst, 2*blkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test inner loop counter
	jnc	b3b			;; Iterate if necessary
	IFDEF PFETCH
;	prefetcht1 [esi-32+clm*32]
	prefetcht1 [esi-32+clm*32+blkdst]
;	prefetcht1 [esi-32+clm*32+2*blkdst]
	prefetcht1 [esi-32+clm*32+3*blkdst]
	ENDIF
	loopclm	dl, clm, b3b
	lea	esi, [esi-clm*2*dist1+4*blkdst];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al
	jnz	b3b
	lea	esi, [esi-96*4*blkdst]	;; Restore source pointer
	end_timer 25

;; Work on next 4 sets of 32 values

no_fft:	lea	esi, [esi+clm*2*dist1]	;; Next source pointer
	dec	dh			;; Test loop counter
	jnz	short more		;; Do more FFTs
	lea	esi, [esi+64]		;; Skip pad every 128 cache lines (4KB)
	sub	edx, 65536		;; Test loop counter
	jz	short fftdn		;; Jump if done looping
	mov	dh, 128/clm		;; Restore cache line counter
more:	cmp	norm_ptr1, 0		;; What was the loop start point?
	jne	c0b			;; Do inverse-FFT/norm/FFT iteration
	jmp	b0b			;; Do another FFT-only iteration
fftdn:	lea	esi, [esi-p2cl*2*dist1-p2cl/128*64];; Restore source pointer
	cmp	esi, DESTARG		;; If esi was not restored, then we 
	jne	short fftdn1		;; just finished skipped postfft data
	sub	ebx, ebx
	mov	DIST_TO_FFTSRCARG, ebx
	cmp	norm_ptr1, 0		;; What was the loop start point?
	je	pass2			;; FFT-only, go do pass 2

;; Split the carries into high and low carries (so they do not exceed
;; the desired number of bits per FFT word.  Then add the carries back to the
;; FFT data.

	start_timer 28
	call	gw_carries
	end_timer 28

;; Have we been given permission to start the FFT on the result?
;; If so, set loop to FFT the first few cache lines now that the carries
;; have been added in.

	cmp	POSTFFT, 0		;; Test flag
	je	short fftdn1		;; Skip FFT code if flag not set
	mov	esi, DESTARG		;; Load FFT data address
	mov	edx, 65536+4/clm*256	;; Do 8 previously skipped cache lines
	mov	norm_ptr1, 0		;; Set flag so only forward FFT is done
	copy_4_words
	jmp	b0b

;; Now add in the FFT'ed carries, cleanup and return

fftdn1:	end_timer 1
	jmp	gw_finish_mult
	ENDM


;; Perform a 224K-element FFT.  This is done in two passes.  Pass 1
;; does 10 levels, pass 2 uses common code to perform the last 8 levels.

pass1levels10pfa7 MACRO pass2_macro, p2cl, clm
	LOCAL	b0b, b1b, b2b, b3b, b4b, b5b, b6b, b7b, b8b, b9b
	LOCAL	c0b, c1b, c2b, c3b, c4b, c5b, c6b, c7b, c8b, c9b
	LOCAL	pass2, no_fft, more, fftdn, fftdn1

;; Call FFT code unless the FFT has already been started

	clear_timers
	cmp	DWORD PTR [esi-28][ebx], 0 ;; Test FFT-started flag
	jne	pass2			;; Jump is FFT already started
	mov	norm_ptr1, 0		;; Set FFT-ing flag
	mov	edx, p2cl/128*65536+128/clm*256;; Load loop counter
	jmp	b0b			;; Do the pass 1 fft

;; Do the last 8 FFT levels
;;	do 1 pass2_8_levels_real and 447 pass2_8_levels_complex

pass2:	start_timer 0
	CALLP	pass2_macro
	end_timer 0
	cmp	ffttype, 1		;; We're done if FFTing only
	je	gw_finish_fft

;; Do the inverse FFT

	start_timer 1
	fldz				;; Init SUMOUT
	fstp	SUMOUT
	mov	edx, norm_biglit_array	;; Addr of the big/little flags array
	mov	norm_ptr1, edx		;; Save ptr
	mov	edx, norm_col_mults	;; Addr of the column multipliers
	mov	norm_ptr2, edx		;; Save ptr
	mov	edx, p2cl/128*65536+128/clm*256;; Load loop counter

;; Do inverse FFT levels 9,10
;; On input the 32-byte cache lines hold these data values:
;;	0	256	1	257
;;	512	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	1K	1	1K+1
;;	256	...
;;	...
;;	2K	...
;;	...

;; Do 1 four_real_four_semireal_unfft macros
;;	distance between fft data elements is 1

c0b:	start_timer 15
c2b:	disp four_real_four_semireal_unfft, 8, blkdst, 2*blkdst
	lea	esi, [esi+dist1]	;; Next source pointer
	add	dl, 256/2/clm		;; Test inner loop counter
	jnc	c2b			;; Iterate if necessary
	lea	esi, [esi-clm*2*dist1+4*blkdst];; Next source pointer

;; Do 111 four_complex_unfft macros
;;	distance between fft data elements is 1

	mov	edi, sincos1		;; Load sin/cos pointer
	mov	al, 111			;; 111 iterations
c3b:	disp four_complex_unfft, 8, blkdst, 2*blkdst
	lea	esi, [esi+dist1]	;; Next source pointer
	add	dl, 256/2/clm		;; Test loop counter
	jnc	c3b			;; Iterate if necessary
	lea	esi, [esi-clm*2*dist1+4*blkdst];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al			;; Test outer loop counter
	jnz	c3b			;; Iterate if necessary
	lea	esi, [esi-112*4*blkdst]	;; Restore source pointer
	end_timer 15

;; Do inverse FFT levels 7,8
;; On input the 32-byte cache lines hold these data values:
;;	0	1K	1	1K+1
;;	256	...
;;	...
;;	2K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	4K	1	4K+1
;;	256	...
;;	...
;;	8K	...
;;	...

;; Do 4 four_real_four_semireal_unfft macros
;;	distance between fft data elements is 4

	start_timer 16
	mov	al, 4			;; Load loop counter
c4b:	disp four_real_four_semireal_unfft, 8, 4*blkdst, 8*blkdst
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test inner loop counter
	jnc	c4b			;; Iterate if necessary
	IFDEF PFETCH
	prefetcht1 [esi-32+clm*32]
;	prefetcht1 [esi-32+clm*32+4*blkdst]
	prefetcht1 [esi-32+clm*32+8*blkdst]
;	prefetcht1 [esi-32+clm*32+12*blkdst]
	ENDIF
	loopclm	dl, clm, c4b
	lea	esi, [esi-clm*2*dist1+blkdst];; Next source pointer
	dec	al
	jnz	c4b
	lea	esi, [esi-4*blkdst+16*blkdst];; Next source pointer

;; Do 108 four_complex_unfft macros
;;	distance between fft data elements is 4

	mov	edi, sincos2		;; Load sin/cos pointer
	mov	eax, 27			;; 27 iterations of 4
c5b:	disp four_complex_unfft, 8, 4*blkdst, 8*blkdst
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test inner loop counter
	jnc	c5b			;; Iterate if necessary
	IFDEF PFETCH
	prefetcht1 [esi-32+clm*32]
;	prefetcht1 [esi-32+clm*32+4*blkdst]
	prefetcht1 [esi-32+clm*32+8*blkdst]
;	prefetcht1 [esi-32+clm*32+12*blkdst]
	ENDIF
	loopclm	dl, clm, c5b
	lea	esi, [esi-clm*2*dist1+blkdst];; Next source pointer
	add	ah, 256/4		;; Test middle loop counter
	jnc	c5b			;; Iterate if necessary
	lea	esi, [esi-4*blkdst+16*blkdst];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al
	jnz	c5b
	lea	esi, [esi-28*16*blkdst]	;; Restore source pointer
	end_timer 16

;; Do inverse FFT levels 5,6
;; On input the 32-byte cache lines hold these data values:
;;	0	4K	1	4K+1
;;	256	...
;;	...
;;	8K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	16K	1	16K+1
;;	256	...
;;	...
;;	32K	...
;;	...

;; Do 16 four_real_four_semireal_unfft macros
;;	distance between fft data elements is 16

	start_timer 17
	mov	al, 16			;; Load loop counter
c6b:	disp four_real_four_semireal_unfft, 8, 16*blkdst, 32*blkdst
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test inner loop counter
	jnc	c6b			;; Iterate if necessary
	IFDEF PFETCH
	prefetcht1 [esi-32+clm*32]
;	prefetcht1 [esi-32+clm*32+16*blkdst]
	prefetcht1 [esi-32+clm*32+32*blkdst]
;	prefetcht1 [esi-32+clm*32+48*blkdst]
	ENDIF
	loopclm	dl, clm, c6b
	lea	esi, [esi-clm*2*dist1+blkdst];; Next source pointer
	dec	al
	jnz	c6b
	lea	esi, [esi-16*blkdst+64*blkdst];; Next source pointer

;; Do 96 four_complex_unfft macros
;;	distance between fft data elements is 16

	mov	edi, sincos3		;; Load sin/cos pointer
	mov	eax, 6			;; 6 iterations of 16
c7b:	disp four_complex_unfft, 8, 16*blkdst, 32*blkdst
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test inner loop counter
	jnc	c7b			;; Iterate if necessary
	IFDEF PFETCH
	prefetcht1 [esi-32+clm*32]
;	prefetcht1 [esi-32+clm*32+16*blkdst]
	prefetcht1 [esi-32+clm*32+32*blkdst]
;	prefetcht1 [esi-32+clm*32+48*blkdst]
	ENDIF
	loopclm	dl, clm, c7b
	lea	esi, [esi-clm*2*dist1+blkdst];; Next source pointer
	add	ah, 256/16		;; Test middle loop counter
	jnc	c7b			;; Iterate if necessary
	lea	esi, [esi-16*blkdst+64*blkdst];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al
	jnz	c7b
	lea	esi, [esi-7*64*blkdst]	;; Restore source pointer
	end_timer 17

;; Do inverse FFT levels 4
;; On input the 32-byte cache lines hold these data values:
;;	0	16K	1	16K+1
;;	256	...
;;	...
;;	32K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	32K	1	32K+1
;;	256	...
;;	...
;;	64K	...
;;	...

;; Do 16 eight_reals_unfft_1 macros
;;	distance between fft data elements is 64

	start_timer 18
	mov	al, 16			;; Load loop counter
c8b:	disp eight_reals_unfft_1, 16*blkdst, 32*blkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test inner loop counter
	jnc	c8b			;; Iterate if necessary
	IFDEF PFETCH
	prefetcht1 [esi-32+clm*32]
;	prefetcht1 [esi-32+clm*32+16*blkdst]
	prefetcht1 [esi-32+clm*32+32*blkdst]
;	prefetcht1 [esi-32+clm*32+48*blkdst]
	ENDIF
	loopclm	dl, clm, c8b
	lea	esi, [esi-clm*2*dist1+blkdst];; Next source pointer
	dec	al
	jnz	c8b
	lea	esi, [esi-16*blkdst+64*blkdst];; Next source pointer

;; Do 96 four_complex_unfft_1 macros
;;	distance between fft data elements is 64

	mov	edi, sincos4		;; Load sin/cos pointer
	mov	eax, 3			;; 3 iterations of 32
c9b:	disp four_complex_unfft_1, 32*blkdst, 8, 64*blkdst
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test inner loop counter
	jnc	c9b			;; Iterate if necessary
	IFDEF PFETCH
	prefetcht1 [esi-32+clm*32]
;	prefetcht1 [esi-32+clm*32+32*blkdst]
	prefetcht1 [esi-32+clm*32+64*blkdst]
;	prefetcht1 [esi-32+clm*32+96*blkdst]
	ENDIF
	loopclm	dl, clm, c9b
	lea	esi, [esi-clm*2*dist1+blkdst];; Next source pointer
	add	ah, 256/32		;; Test middle loop counter
	jnc	c9b			;; Iterate if necessary
	lea	esi, [esi-32*blkdst+128*blkdst];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al
	jnz	c9b
	lea	esi, [esi-3*128*blkdst-64*blkdst];; Restore source pointer
	end_timer 18

;; Do inverse FFT levels 1,2,3
;; On input the 32-byte cache lines hold these data values:
;;	0	32K	1	32K+1
;;	256	...
;;	...
;;	64K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	112K	1	112K+1
;;	256
;;	...
;;	112K-256

;; Do 128 seven_reals_last_unfft macros
;;	distance between fft data elements is 128

	start_timer 19
	mov	al, 64			;; Load loop counter
c1b:	two_seven_reals_last_unfft 8, 64*blkdst
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test loop counter
	jnc	c1b			;; Iterate if necessary
	IFDEF PFETCH
	prefetcht1 [esi-32+clm*32]
;	prefetcht1 [esi-32+clm*32+64*blkdst]
	prefetcht1 [esi-32+clm*32+128*blkdst]
;	prefetcht1 [esi-32+clm*32+192*blkdst]
	prefetcht1 [esi-32+clm*32+256*blkdst]
;	prefetcht1 [esi-32+clm*32+320*blkdst]
	prefetcht1 [esi-32+clm*32+384*blkdst]
	ENDIF
	loopclm	dl, clm, c1b
	lea	esi, [esi-clm*2*dist1+blkdst];; Next source pointer
	dec	al			;; Test loop counter
	jnz	c1b			;; Iterate if necessary
	lea	esi, [esi-64*blkdst]	;; Restore source pointer
	end_timer 19

;; Normalize these values

	start_timer 20
	mov	eax, NORMRTN		;; Addr of normalization routine
	call	eax
	sub	eax, eax
	end_timer 20

;; Have we been given permission to start the FFT on the result?
;; If so, do so now while the data is in the L2 cache.

	cmp	POSTFFT, 0		;; Test flag
	je	no_fft			;; Skip FFT code if flag not set
	cmp	edx, p2cl/128*65536+(128/clm-4/clm)*256;; Delay fft on 8 sets
	jg	no_fft			;; Jump if delaying FFT
	copy_3_words clm, 0

;; Do FFT levels 1,2,3
;;
;; On input the 32-byte cache lines hold these data values:
;;	0	112K	1	112K+1
;;	256
;;	...
;;	112K-256
;; On output the 32-byte cache lines hold these data values:
;;	0	32K	1	32K+1
;;	256	...
;;	...
;;	64K	...
;;	...

;; Do 128 seven_reals_first_fft macros
;;	distance between fft data elements is 128

b0b:	start_timer 21
	mov	ebx, DIST_TO_FFTSRCARG
	mov	al, 64			;; 64 iterations
b1b:	two_seven_reals_first_fft 64*blkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	add	dl, 256/2/clm		;; Test loop counter
	jnc	b1b			;; Iterate if necessary
	lea	esi, [esi-clm*2*dist1+blkdst];; Next source pointer
	dec	al			;; Test loop counter
	jnz	b1b			;; Iterate if necessary
	lea	esi, [esi-64*blkdst]	;; Restore source pointer
	end_timer 21

;; Do FFT levels 4
;;
;; On input the 32-byte cache lines hold these data values:
;;	0	32K	1	32K+1
;;	256	...
;;	...
;;	64K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	16K	1	16K+1
;;	256	...
;;	...
;;	32K	...
;;	...

;; Do 16 eight_reals_fft_1 macros
;;	distance between fft data elements is 64

	start_timer 22
	mov	al, 16			;; Load loop counter
b8b:	disp eight_reals_fft_1, 16*blkdst, 32*blkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test inner loop counter
	jnc	b8b			;; Iterate if necessary
	IFDEF PFETCH
;	prefetcht1 [esi-32+clm*32]
	prefetcht1 [esi-32+clm*32+16*blkdst]
;	prefetcht1 [esi-32+clm*32+32*blkdst]
	prefetcht1 [esi-32+clm*32+48*blkdst]
	ENDIF
	loopclm	dl, clm, b8b
	lea	esi, [esi-clm*2*dist1+blkdst];; Next source pointer
	dec	al
	jnz	b8b
	lea	esi, [esi-16*blkdst+64*blkdst];; Next source pointer

;; Do 96 four_complex_fft_1 macros
;;	distance between fft data elements is 64

	mov	edi, sincos4		;; Load sin/cos pointer
	mov	eax, 3			;; 3 iterations of 32
b9b:	disp four_complex_fft_1, 32*blkdst, 64*blkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test inner loop counter
	jnc	b9b			;; Iterate if necessary
	IFDEF PFETCH
;	prefetcht1 [esi-32+clm*32]
	prefetcht1 [esi-32+clm*32+32*blkdst]
;	prefetcht1 [esi-32+clm*32+64*blkdst]
	prefetcht1 [esi-32+clm*32+96*blkdst]
	ENDIF
	loopclm	dl, clm, b9b
	lea	esi, [esi-clm*2*dist1+blkdst];; Next source pointer
	add	ah, 256/32		;; Test middle loop counter
	jnc	b9b			;; Iterate if necessary
	lea	esi, [esi-32*blkdst+128*blkdst];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al
	jnz	b9b
	lea	esi, [esi-3*128*blkdst-64*blkdst];; Restore source pointer
	end_timer 22

;; Do FFT levels 5,6
;;
;; On input the 32-byte cache lines hold these data values:
;;	0	16K	1	16K+1
;;	256	...
;;	...
;;	32K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	4K	1	4K+1
;;	256	...
;;	...
;;	8K	...
;;	...

;; Do 16 four_real_four_semireal_fft macros
;;	distance between fft data elements is 16

	start_timer 23
	mov	al, 16			;; Load loop counter
b6b:	disp four_real_four_semireal_fft, 16*blkdst, 32*blkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test inner loop counter
	jnc	b6b			;; Iterate if necessary
	IFDEF PFETCH
;	prefetcht1 [esi-32+clm*32]
	prefetcht1 [esi-32+clm*32+16*blkdst]
;	prefetcht1 [esi-32+clm*32+32*blkdst]
	prefetcht1 [esi-32+clm*32+48*blkdst]
	ENDIF
	loopclm	dl, clm, b6b
	lea	esi, [esi-clm*2*dist1+blkdst];; Next source pointer
	dec	al
	jnz	b6b
	lea	esi, [esi-16*blkdst+64*blkdst];; Next source pointer

;; Do 96 four_complex_fft macros
;;	distance between fft data elements is 16

	mov	edi, sincos3		;; Load sin/cos pointer
	mov	eax, 6			;; 6 iterations of 16
b7b:	disp four_complex_fft, 16*blkdst, 32*blkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test inner loop counter
	jnc	b7b			;; Iterate if necessary
	IFDEF PFETCH
;	prefetcht1 [esi-32+clm*32]
	prefetcht1 [esi-32+clm*32+16*blkdst]
;	prefetcht1 [esi-32+clm*32+32*blkdst]
	prefetcht1 [esi-32+clm*32+48*blkdst]
	ENDIF
	loopclm	dl, clm, b7b
	lea	esi, [esi-clm*2*dist1+blkdst];; Next source pointer
	add	ah, 256/16		;; Test middle loop counter
	jnc	b7b			;; Iterate if necessary
	lea	esi, [esi-16*blkdst+64*blkdst];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al
	jnz	b7b
	lea	esi, [esi-7*64*blkdst]	;; Restore source pointer
	end_timer 23

;; Do FFT levels 7,8
;;
;; On input the 32-byte cache lines hold these data values:
;;	0	4K	1	4K+1
;;	256	...
;;	...
;;	8K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	1K	1	1K+1
;;	256	...
;;	...
;;	2K	...
;;	...

;; Do 4 four_real_four_semireal_fft macros
;;	distance between fft data elements is 4

	start_timer 24
	mov	al, 4			;; Load loop counter
b4b:	disp four_real_four_semireal_fft, 4*blkdst, 8*blkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test inner loop counter
	jnc	b4b			;; Iterate if necessary
	IFDEF PFETCH
;	prefetcht1 [esi-32+clm*32]
	prefetcht1 [esi-32+clm*32+4*blkdst]
;	prefetcht1 [esi-32+clm*32+8*blkdst]
	prefetcht1 [esi-32+clm*32+12*blkdst]
	ENDIF
	loopclm	dl, clm, b4b
	lea	esi, [esi-clm*2*dist1+blkdst];; Next source pointer
	dec	al
	jnz	b4b
	lea	esi, [esi-4*blkdst+16*blkdst];; Next source pointer

;; Do 108 four_complex_fft macros
;;	distance between fft data elements is 4

	mov	edi, sincos2		;; Load sin/cos pointer
	mov	eax, 27			;; 27 iterations of 4
b5b:	disp four_complex_fft, 4*blkdst, 8*blkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test inner loop counter
	jnc	b5b			;; Iterate if necessary
	IFDEF PFETCH
;	prefetcht1 [esi-32+clm*32]
	prefetcht1 [esi-32+clm*32+4*blkdst]
;	prefetcht1 [esi-32+clm*32+8*blkdst]
	prefetcht1 [esi-32+clm*32+12*blkdst]
	ENDIF
	loopclm	dl, clm, b5b
	lea	esi, [esi-clm*2*dist1+blkdst];; Next source pointer
	add	ah, 256/4		;; Test middle loop counter
	jnc	b5b			;; Iterate if necessary
	lea	esi, [esi-4*blkdst+16*blkdst];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al
	jnz	b5b
	lea	esi, [esi-28*16*blkdst]	;; Restore source pointer
	end_timer 24

;; Do FFT levels 9,10
;;
;; On input the 32-byte cache lines hold these data values:
;;	0	1K	1	1K+1
;;	256	...
;;	...
;;	2K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	256	1	257
;;	512	...
;;	...

;; Do 1 four_real_four_semireal_fft macros
;;	distance between fft data elements is 1

	start_timer 25
b2b:	disp four_real_four_semireal_fft, blkdst, 2*blkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test inner loop counter
	jnc	b2b			;; Iterate if necessary
	IFDEF PFETCH
;	prefetcht1 [esi-32+clm*32]
	prefetcht1 [esi-32+clm*32+blkdst]
;	prefetcht1 [esi-32+clm*32+2*blkdst]
	prefetcht1 [esi-32+clm*32+3*blkdst]
	ENDIF
	loopclm	dl, clm, b2b
	lea	esi, [esi-clm*2*dist1+4*blkdst];; Next source pointer

;; Do 111 four_complex_fft macros
;;	distance between fft data elements is 1

	mov	edi, sincos1		;; Load sin/cos pointer
	mov	al, 111			;; 111 iterations
b3b:	disp four_complex_fft, blkdst, 2*blkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test inner loop counter
	jnc	b3b			;; Iterate if necessary
	IFDEF PFETCH
;	prefetcht1 [esi-32+clm*32]
	prefetcht1 [esi-32+clm*32+blkdst]
;	prefetcht1 [esi-32+clm*32+2*blkdst]
	prefetcht1 [esi-32+clm*32+3*blkdst]
	ENDIF
	loopclm	dl, clm, b3b
	lea	esi, [esi-clm*2*dist1+4*blkdst];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al
	jnz	b3b
	lea	esi, [esi-112*4*blkdst]	;; Restore source pointer
	end_timer 25

;; Work on next 4 sets of 32 values

no_fft:	lea	esi, [esi+clm*2*dist1]	;; Next source pointer
	dec	dh			;; Test loop counter
	jnz	short more		;; Do more FFTs
	lea	esi, [esi+64]		;; Skip pad every 128 cache lines (4KB)
	sub	edx, 65536		;; Test loop counter
	jz	short fftdn		;; Jump if done looping
	mov	dh, 128/clm		;; Restore cache line counter
more:	cmp	norm_ptr1, 0		;; What was the loop start point?
	jne	c0b			;; Do inverse-FFT/norm/FFT iteration
	jmp	b0b			;; Do another FFT-only iteration
fftdn:	lea	esi, [esi-p2cl*2*dist1-p2cl/128*64];; Restore source pointer
	cmp	esi, DESTARG		;; If esi was not restored, then we 
	jne	short fftdn1		;; just finished skipped postfft data
	sub	ebx, ebx
	mov	DIST_TO_FFTSRCARG, ebx
	cmp	norm_ptr1, 0		;; What was the loop start point?
	je	pass2			;; FFT-only, go do pass 2

;; Split the carries into high and low carries (so they do not exceed
;; the desired number of bits per FFT word.  Then add the carries back to the
;; FFT data.

	start_timer 28
	call	gw_carries
	end_timer 28

;; Have we been given permission to start the FFT on the result?
;; If so, set loop to FFT the first few cache lines now that the carries
;; have been added in.

	cmp	POSTFFT, 0		;; Test flag
	je	short fftdn1		;; Skip FFT code if flag not set
	mov	esi, DESTARG		;; Load FFT data address
	mov	edx, 65536+4/clm*256	;; Do 8 previously skipped cache lines
	mov	norm_ptr1, 0		;; Set flag so only forward FFT is done
	copy_4_words
	jmp	b0b

;; Now add in the FFT'ed carries, cleanup and return

fftdn1:	end_timer 1
	jmp	gw_finish_mult
	ENDM


;; Perform a 256K-element FFT.  This is done in two passes.  Pass 1
;; does 10 levels, pass 2 uses common code to perform the last 8 levels.

pass1levels10pfa8 MACRO pass2_macro, p2cl, clm
	LOCAL	b0b, b1b, b2b, b3b, b4b, b5b, b6b, b7b, b8b, b9b
	LOCAL	c0b, c1b, c2b, c3b, c4b, c5b, c6b, c7b, c8b, c9b
	LOCAL	pass2, no_fft, more, fftdn, fftdn1

;; Call FFT code unless the FFT has already been started

	clear_timers
	cmp	DWORD PTR [esi-28][ebx], 0 ;; Test FFT-started flag
	jne	pass2			;; Jump is FFT already started
	mov	norm_ptr1, 0		;; Set FFT-ing flag
	mov	edx, p2cl/128*65536+128/clm*256;; Load loop counter
	jmp	b0b			;; Do the pass 1 fft

;; Do the last 8 FFT levels
;;	do 1 pass2_8_levels_real and 511 pass2_8_levels_complex

pass2:	start_timer 0
	CALLP	pass2_macro
	end_timer 0
	cmp	ffttype, 1		;; We're done if FFTing only
	je	gw_finish_fft

;; Do the inverse FFT

	start_timer 1
	fldz				;; Init SUMOUT
	fstp	SUMOUT
	mov	edx, norm_biglit_array	;; Addr of the big/little flags array
	mov	norm_ptr1, edx		;; Save ptr
	mov	edx, norm_col_mults	;; Addr of the column multipliers
	mov	norm_ptr2, edx		;; Save ptr
	mov	edx, p2cl/128*65536+128/clm*256;; Load loop counter

;; Do inverse FFT levels 9,10
;; On input the 32-byte cache lines hold these data values:
;;	0	256	1	257
;;	512	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	1K	1	1K+1
;;	256	...
;;	...
;;	2K	...
;;	...

;; Do 1 four_real_four_semireal_unfft macros
;;	distance between fft data elements is 1

c0b:	start_timer 15
c2b:	disp four_real_four_semireal_unfft, 8, blkdst, 2*blkdst
	lea	esi, [esi+dist1]	;; Next source pointer
	add	dl, 256/2/clm		;; Test inner loop counter
	jnc	c2b			;; Iterate if necessary
	lea	esi, [esi-clm*2*dist1+4*blkdst];; Next source pointer

;; Do 127 four_complex_unfft macros
;;	distance between fft data elements is 1

	mov	edi, sincos1		;; Load sin/cos pointer
	mov	al, 127			;; 127 iterations
c3b:	disp four_complex_unfft, 8, blkdst, 2*blkdst
	lea	esi, [esi+dist1]	;; Next source pointer
	add	dl, 256/2/clm		;; Test loop counter
	jnc	c3b			;; Iterate if necessary
	lea	esi, [esi-clm*2*dist1+4*blkdst];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al			;; Test outer loop counter
	jnz	c3b			;; Iterate if necessary
	lea	esi, [esi-128*4*blkdst]	;; Restore source pointer
	end_timer 15

;; Do inverse FFT levels 7,8
;; On input the 32-byte cache lines hold these data values:
;;	0	1K	1	1K+1
;;	256	...
;;	...
;;	2K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	4K	1	4K+1
;;	256	...
;;	...
;;	8K	...
;;	...

;; Do 4 four_real_four_semireal_unfft macros
;;	distance between fft data elements is 4

	start_timer 16
	mov	al, 4			;; Load loop counter
c4b:	disp four_real_four_semireal_unfft, 8, 4*blkdst, 8*blkdst
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test inner loop counter
	jnc	c4b			;; Iterate if necessary
	IFDEF PFETCH
	prefetcht1 [esi-32+clm*32]
;	prefetcht1 [esi-32+clm*32+4*blkdst]
	prefetcht1 [esi-32+clm*32+8*blkdst]
;	prefetcht1 [esi-32+clm*32+12*blkdst]
	ENDIF
	loopclm	dl, clm, c4b
	lea	esi, [esi-clm*2*dist1+blkdst];; Next source pointer
	dec	al
	jnz	c4b
	lea	esi, [esi-4*blkdst+16*blkdst];; Next source pointer

;; Do 124 four_complex_unfft macros
;;	distance between fft data elements is 4

	mov	edi, sincos2		;; Load sin/cos pointer
	mov	eax, 31			;; 31 iterations of 4
c5b:	disp four_complex_unfft, 8, 4*blkdst, 8*blkdst
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test inner loop counter
	jnc	c5b			;; Iterate if necessary
	IFDEF PFETCH
	prefetcht1 [esi-32+clm*32]
;	prefetcht1 [esi-32+clm*32+4*blkdst]
	prefetcht1 [esi-32+clm*32+8*blkdst]
;	prefetcht1 [esi-32+clm*32+12*blkdst]
	ENDIF
	loopclm	dl, clm, c5b
	lea	esi, [esi-clm*2*dist1+blkdst];; Next source pointer
	add	ah, 256/4		;; Test middle loop counter
	jnc	c5b			;; Iterate if necessary
	lea	esi, [esi-4*blkdst+16*blkdst];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al
	jnz	c5b
	lea	esi, [esi-32*16*blkdst]	;; Restore source pointer
	end_timer 16

;; Do inverse FFT levels 5,6
;; On input the 32-byte cache lines hold these data values:
;;	0	4K	1	4K+1
;;	256	...
;;	...
;;	8K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	16K	1	16K+1
;;	256	...
;;	...
;;	32K	...
;;	...

;; Do 16 four_real_four_semireal_unfft macros
;;	distance between fft data elements is 16

	start_timer 17
	mov	al, 16			;; Load loop counter
c6b:	disp four_real_four_semireal_unfft, 8, 16*blkdst, 32*blkdst
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test inner loop counter
	jnc	c6b			;; Iterate if necessary
	IFDEF PFETCH
	prefetcht1 [esi-32+clm*32]
;	prefetcht1 [esi-32+clm*32+16*blkdst]
	prefetcht1 [esi-32+clm*32+32*blkdst]
;	prefetcht1 [esi-32+clm*32+48*blkdst]
	ENDIF
	loopclm	dl, clm, c6b
	lea	esi, [esi-clm*2*dist1+blkdst];; Next source pointer
	dec	al
	jnz	c6b
	lea	esi, [esi-16*blkdst+64*blkdst];; Next source pointer

;; Do 112 four_complex_unfft macros
;;	distance between fft data elements is 16

	mov	edi, sincos3		;; Load sin/cos pointer
	mov	eax, 7			;; 7 iterations of 16
c7b:	disp four_complex_unfft, 8, 16*blkdst, 32*blkdst
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test inner loop counter
	jnc	c7b			;; Iterate if necessary
	IFDEF PFETCH
	prefetcht1 [esi-32+clm*32]
;	prefetcht1 [esi-32+clm*32+16*blkdst]
	prefetcht1 [esi-32+clm*32+32*blkdst]
;	prefetcht1 [esi-32+clm*32+48*blkdst]
	ENDIF
	loopclm	dl, clm, c7b
	lea	esi, [esi-clm*2*dist1+blkdst];; Next source pointer
	add	ah, 256/16		;; Test middle loop counter
	jnc	c7b			;; Iterate if necessary
	lea	esi, [esi-16*blkdst+64*blkdst];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al
	jnz	c7b
	lea	esi, [esi-8*64*blkdst]	;; Restore source pointer
	end_timer 17

;; Do inverse FFT levels 4
;; On input the 32-byte cache lines hold these data values:
;;	0	16K	1	16K+1
;;	256	...
;;	...
;;	32K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	32K	1	32K+1
;;	256	...
;;	...
;;	64K	...
;;	...

;; Do 32 four_real_four_semireal_unfft_1 macros
;;	distance between fft data elements is 64

	start_timer 18
	mov	al, 32			;; Load loop counter
c8b:	disp four_real_four_semireal_unfft_1, 32*blkdst, 8, 64*blkdst
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test inner loop counter
	jnc	c8b			;; Iterate if necessary
	IFDEF PFETCH
	prefetcht1 [esi-32+clm*32]
;	prefetcht1 [esi-32+clm*32+32*blkdst]
	prefetcht1 [esi-32+clm*32+64*blkdst]
;	prefetcht1 [esi-32+clm*32+96*blkdst]
	ENDIF
	loopclm	dl, clm, c8b
	lea	esi, [esi-clm*2*dist1+blkdst];; Next source pointer
	dec	al
	jnz	c8b
	lea	esi, [esi-32*blkdst+128*blkdst];; Next source pointer

;; Do 96 four_complex_unfft_1 macros
;;	distance between fft data elements is 64

	mov	edi, sincos4		;; Load sin/cos pointer
	mov	eax, 3			;; 3 iterations of 32
c9b:	disp four_complex_unfft_1, 32*blkdst, 8, 64*blkdst
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test inner loop counter
	jnc	c9b			;; Iterate if necessary
	IFDEF PFETCH
	prefetcht1 [esi-32+clm*32]
;	prefetcht1 [esi-32+clm*32+32*blkdst]
	prefetcht1 [esi-32+clm*32+64*blkdst]
;	prefetcht1 [esi-32+clm*32+96*blkdst]
	ENDIF
	loopclm	dl, clm, c9b
	lea	esi, [esi-clm*2*dist1+blkdst];; Next source pointer
	add	ah, 256/32		;; Test middle loop counter
	jnc	c9b			;; Iterate if necessary
	lea	esi, [esi-32*blkdst+128*blkdst];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al
	jnz	c9b
	lea	esi, [esi-4*128*blkdst]	;; Restore source pointer
	end_timer 18

;; Do inverse FFT levels 1,2,3
;; On input the 32-byte cache lines hold these data values:
;;	0	32K	1	32K+1
;;	256	...
;;	...
;;	64K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	128K	1	128K+1
;;	256
;;	...
;;	128K-256

;; Do 128 eight_reals_last_unfft macros
;;	distance between fft data elements is 128

	start_timer 19
	mov	eax, 128*256		;; Load loop counter
c1b:	disp eight_reals_last_unfft, 8, 128*blkdst, 256*blkdst
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test loop counter
	jnc	c1b			;; Iterate if necessary
	IFDEF PFETCH
	prefetcht1 [esi-32+clm*32]
;	prefetcht1 [esi-32+clm*32+128*blkdst]
	prefetcht1 [esi-32+clm*32+256*blkdst]
;	prefetcht1 [esi-32+clm*32+384*blkdst]
	ENDIF
	loopclm	dl, clm, c1b
	lea	esi, [esi-clm*2*dist1+blkdst];; Next source pointer
	sub	eax, 256		;; Test loop counter
	jnz	c1b			;; Iterate if necessary
	lea	esi, [esi-128*blkdst]	;; Restore source pointer
	end_timer 19

;; Normalize these values

	start_timer 20
	mov	eax, NORMRTN		;; Addr of normalization routine
	call	eax
	sub	eax, eax
	end_timer 20

;; Have we been given permission to start the FFT on the result?
;; If so, do so now while the data is in the L2 cache.

	cmp	POSTFFT, 0		;; Test flag
	je	no_fft			;; Skip FFT code if flag not set
	cmp	edx, p2cl/128*65536+(128/clm-4/clm)*256;; Delay fft on 8 sets
	jg	no_fft			;; Jump if delaying FFT
	copy_3_words clm, 0

;; Do FFT levels 1,2,3
;;
;; On input the 32-byte cache lines hold these data values:
;;	0	128K	1	128K+1
;;	256
;;	...
;;	128K-256
;; On output the 32-byte cache lines hold these data values:
;;	0	32K	1	32K+1
;;	256	...
;;	...
;;	64K	...
;;	...

;; Do 128 eight_reals_first_fft macros
;;	distance between fft data elements is 128

b0b:	start_timer 21
	mov	ebx, DIST_TO_FFTSRCARG
	mov	al, 128			;; 128 iterations
b1b:	disp eight_reals_first_fft, 128*blkdst, 256*blkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	add	dl, 256/2/clm		;; Test loop counter
	jnc	b1b			;; Iterate if necessary
	lea	esi, [esi-clm*2*dist1+blkdst];; Next source pointer
	dec	al			;; Test loop counter
	jnz	b1b			;; Iterate if necessary
	lea	esi, [esi-128*blkdst]	;; Restore source pointer
	end_timer 21

;; Do FFT levels 4
;;
;; On input the 32-byte cache lines hold these data values:
;;	0	32K	1	32K+1
;;	256	...
;;	...
;;	64K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	16K	1	16K+1
;;	256	...
;;	...
;;	32K	...
;;	...

;; Do 32 four_real_four_semireal_fft_1 macros
;;	distance between fft data elements is 64

	start_timer 22
	mov	al, 32			;; Load loop counter
b8b:	disp four_real_four_semireal_fft_1, 32*blkdst, 64*blkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test inner loop counter
	jnc	b8b			;; Iterate if necessary
	IFDEF PFETCH
;	prefetcht1 [esi-32+clm*32]
	prefetcht1 [esi-32+clm*32+32*blkdst]
;	prefetcht1 [esi-32+clm*32+64*blkdst]
	prefetcht1 [esi-32+clm*32+96*blkdst]
	ENDIF
	loopclm	dl, clm, b8b
	lea	esi, [esi-clm*2*dist1+blkdst];; Next source pointer
	dec	al
	jnz	b8b
	lea	esi, [esi-32*blkdst+128*blkdst];; Next source pointer

;; Do 96 four_complex_fft_1 macros
;;	distance between fft data elements is 64

	mov	edi, sincos4		;; Load sin/cos pointer
	mov	eax, 3			;; 3 iterations of 32
b9b:	disp four_complex_fft_1, 32*blkdst, 64*blkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test inner loop counter
	jnc	b9b			;; Iterate if necessary
	IFDEF PFETCH
;	prefetcht1 [esi-32+clm*32]
	prefetcht1 [esi-32+clm*32+32*blkdst]
;	prefetcht1 [esi-32+clm*32+64*blkdst]
	prefetcht1 [esi-32+clm*32+96*blkdst]
	ENDIF
	loopclm	dl, clm, b9b
	lea	esi, [esi-clm*2*dist1+blkdst];; Next source pointer
	add	ah, 256/32		;; Test middle loop counter
	jnc	b9b			;; Iterate if necessary
	lea	esi, [esi-32*blkdst+128*blkdst];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al
	jnz	b9b
	lea	esi, [esi-4*128*blkdst]	;; Restore source pointer
	end_timer 22

;; Do FFT levels 5,6
;;
;; On input the 32-byte cache lines hold these data values:
;;	0	16K	1	16K+1
;;	256	...
;;	...
;;	32K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	4K	1	4K+1
;;	256	...
;;	...
;;	8K	...
;;	...

;; Do 16 four_real_four_semireal_fft macros
;;	distance between fft data elements is 16

	start_timer 23
	mov	al, 16			;; Load loop counter
b6b:	disp four_real_four_semireal_fft, 16*blkdst, 32*blkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test inner loop counter
	jnc	b6b			;; Iterate if necessary
	IFDEF PFETCH
;	prefetcht1 [esi-32+clm*32]
	prefetcht1 [esi-32+clm*32+16*blkdst]
;	prefetcht1 [esi-32+clm*32+32*blkdst]
	prefetcht1 [esi-32+clm*32+48*blkdst]
	ENDIF
	loopclm	dl, clm, b6b
	lea	esi, [esi-clm*2*dist1+blkdst];; Next source pointer
	dec	al
	jnz	b6b
	lea	esi, [esi-16*blkdst+64*blkdst];; Next source pointer

;; Do 112 four_complex_fft macros
;;	distance between fft data elements is 16

	mov	edi, sincos3		;; Load sin/cos pointer
	mov	eax, 7			;; 7 iterations of 16
b7b:	disp four_complex_fft, 16*blkdst, 32*blkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test inner loop counter
	jnc	b7b			;; Iterate if necessary
	IFDEF PFETCH
;	prefetcht1 [esi-32+clm*32]
	prefetcht1 [esi-32+clm*32+16*blkdst]
;	prefetcht1 [esi-32+clm*32+32*blkdst]
	prefetcht1 [esi-32+clm*32+48*blkdst]
	ENDIF
	loopclm	dl, clm, b7b
	lea	esi, [esi-clm*2*dist1+blkdst];; Next source pointer
	add	ah, 256/16		;; Test middle loop counter
	jnc	b7b			;; Iterate if necessary
	lea	esi, [esi-16*blkdst+64*blkdst];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al
	jnz	b7b
	lea	esi, [esi-8*64*blkdst]	;; Restore source pointer
	end_timer 23

;; Do FFT levels 7,8
;;
;; On input the 32-byte cache lines hold these data values:
;;	0	4K	1	4K+1
;;	256	...
;;	...
;;	8K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	1K	1	1K+1
;;	256	...
;;	...
;;	2K	...
;;	...

;; Do 4 four_real_four_semireal_fft macros
;;	distance between fft data elements is 4

	start_timer 24
	mov	al, 4			;; Load loop counter
b4b:	disp four_real_four_semireal_fft, 4*blkdst, 8*blkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test inner loop counter
	jnc	b4b			;; Iterate if necessary
	IFDEF PFETCH
;	prefetcht1 [esi-32+clm*32]
	prefetcht1 [esi-32+clm*32+4*blkdst]
;	prefetcht1 [esi-32+clm*32+8*blkdst]
	prefetcht1 [esi-32+clm*32+12*blkdst]
	ENDIF
	loopclm	dl, clm, b4b
	lea	esi, [esi-clm*2*dist1+blkdst];; Next source pointer
	dec	al
	jnz	b4b
	lea	esi, [esi-4*blkdst+16*blkdst];; Next source pointer

;; Do 124 four_complex_fft macros
;;	distance between fft data elements is 4

	mov	edi, sincos2		;; Load sin/cos pointer
	mov	eax, 31			;; 31 iterations of 4
b5b:	disp four_complex_fft, 4*blkdst, 8*blkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test inner loop counter
	jnc	b5b			;; Iterate if necessary
	IFDEF PFETCH
;	prefetcht1 [esi-32+clm*32]
	prefetcht1 [esi-32+clm*32+4*blkdst]
;	prefetcht1 [esi-32+clm*32+8*blkdst]
	prefetcht1 [esi-32+clm*32+12*blkdst]
	ENDIF
	loopclm	dl, clm, b5b
	lea	esi, [esi-clm*2*dist1+blkdst];; Next source pointer
	add	ah, 256/4		;; Test middle loop counter
	jnc	b5b			;; Iterate if necessary
	lea	esi, [esi-4*blkdst+16*blkdst];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al
	jnz	b5b
	lea	esi, [esi-32*16*blkdst]	;; Restore source pointer
	end_timer 24

;; Do FFT levels 9,10
;;
;; On input the 32-byte cache lines hold these data values:
;;	0	1K	1	1K+1
;;	256	...
;;	...
;;	2K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	256	1	257
;;	512	...
;;	...

;; Do 1 four_real_four_semireal_fft macros
;;	distance between fft data elements is 1

	start_timer 25
b2b:	disp four_real_four_semireal_fft, blkdst, 2*blkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test inner loop counter
	jnc	b2b			;; Iterate if necessary
	IFDEF PFETCH
;	prefetcht1 [esi-32+clm*32]
	prefetcht1 [esi-32+clm*32+blkdst]
;	prefetcht1 [esi-32+clm*32+2*blkdst]
	prefetcht1 [esi-32+clm*32+3*blkdst]
	ENDIF
	loopclm	dl, clm, b2b
	lea	esi, [esi-clm*2*dist1+4*blkdst];; Next source pointer

;; Do 127 four_complex_fft macros
;;	distance between fft data elements is 1

	mov	edi, sincos1		;; Load sin/cos pointer
	mov	al, 127			;; 127 iterations
b3b:	disp four_complex_fft, blkdst, 2*blkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test inner loop counter
	jnc	b3b			;; Iterate if necessary
	IFDEF PFETCH
;	prefetcht1 [esi-32+clm*32]
	prefetcht1 [esi-32+clm*32+blkdst]
;	prefetcht1 [esi-32+clm*32+2*blkdst]
	prefetcht1 [esi-32+clm*32+3*blkdst]
	ENDIF
	loopclm	dl, clm, b3b
	lea	esi, [esi-clm*2*dist1+4*blkdst];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al
	jnz	b3b
	lea	esi, [esi-128*4*blkdst]	;; Restore source pointer
	end_timer 25

;; Work on next 4 sets of 32 values

no_fft:	lea	esi, [esi+clm*2*dist1]	;; Next source pointer
	dec	dh			;; Test loop counter
	jnz	short more		;; Do more FFTs
	lea	esi, [esi+64]		;; Skip pad every 128 cache lines (4KB)
	sub	edx, 65536		;; Test loop counter
	jz	short fftdn		;; Jump if done looping
	mov	dh, 128/clm		;; Restore cache line counter
more:	cmp	norm_ptr1, 0		;; What was the loop start point?
	jne	c0b			;; Do inverse-FFT/norm/FFT iteration
	jmp	b0b			;; Do another FFT-only iteration
fftdn:	lea	esi, [esi-p2cl*2*dist1-p2cl/128*64];; Restore source pointer
	cmp	esi, DESTARG		;; If esi was not restored, then we 
	jne	short fftdn1		;; just finished skipped postfft data
	sub	ebx, ebx
	mov	DIST_TO_FFTSRCARG, ebx
	cmp	norm_ptr1, 0		;; What was the loop start point?
	je	pass2			;; FFT-only, go do pass 2

;; Split the carries into high and low carries (so they do not exceed
;; the desired number of bits per FFT word.  Then add the carries back to the
;; FFT data.

	start_timer 28
	call	gw_carries
	end_timer 28

;; Have we been given permission to start the FFT on the result?
;; If so, set loop to FFT the first few cache lines now that the carries
;; have been added in.

	cmp	POSTFFT, 0		;; Test flag
	je	short fftdn1		;; Skip FFT code if flag not set
	mov	esi, DESTARG		;; Load FFT data address
	mov	edx, 65536+4/clm*256	;; Do 8 previously skipped cache lines
	mov	norm_ptr1, 0		;; Set flag so only forward FFT is done
	copy_4_words
	jmp	b0b

;; Now add in the FFT'ed carries, cleanup and return

fftdn1:	end_timer 1
	jmp	gw_finish_mult
	ENDM


;; Perform a 256K-element negacyclic FFT.  This is done in two passes.  Pass 1
;; does 10 levels, pass 2 uses common code to perform the last 8 levels.

pass1levels10complex4 MACRO pass2_macro, p2cl, clm
	LOCAL	b0b, b1b, b3b, b5b, b7b, b9b, c0b, c1b, c3b, c5b, c7b, c9b
	LOCAL	pass2, no_fft, more, fftdn, fftdn1

;; Call FFT code unless the FFT has already been started

	clear_timers
	cmp	DWORD PTR [esi-28][ebx], 0 ;; Test FFT-started flag
	jne	pass2			;; Jump is FFT already started
	mov	norm_ptr1, 0		;; Set FFT-ing flag
	mov	edx, p2cl/128*65536+128/clm*256;; Load loop counter
	jmp	b0b			;; Do the pass 1 fft

;; Do the last 8 FFT levels
;;	do 512 pass2_8_levels_complex

pass2:	start_timer 0
	CALLP	pass2_macro
	end_timer 0
	cmp	ffttype, 1		;; We're done if FFTing only
	je	gw_finish_fft

;; Do the inverse FFT

	start_timer 1
	fldz				;; Init SUMOUT
	fstp	SUMOUT
	mov	edx, norm_biglit_array	;; Addr of the big/little flags array
	mov	norm_ptr1, edx		;; Save ptr
	mov	edx, norm_col_mults	;; Addr of the column multipliers
	mov	norm_ptr2, edx		;; Save ptr
	mov	edx, p2cl/128*65536+128/clm*256;; Load loop counter

;; Do inverse FFT levels 9,10
;; On input the 32-byte cache lines hold these data values:
;;	0	256	1	257
;;	512	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	1K	1	1K+1
;;	256	...
;;	...
;;	2K	...
;;	...

;; Do 128 four_complex_unfft macros
;;	distance between fft data elements is 1

c0b:	start_timer 15
	mov	edi, sincos1		;; Load sin/cos pointer
	mov	al, 128			;; 128 iterations
c3b:	disp four_complex_unfft, 8, blkdst, 2*blkdst
	lea	esi, [esi+dist1]	;; Next source pointer
	add	dl, 256/2/clm		;; Test loop counter
	jnc	c3b			;; Iterate if necessary
	lea	esi, [esi-clm*2*dist1+4*blkdst];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al			;; Test outer loop counter
	jnz	c3b			;; Iterate if necessary
	lea	esi, [esi-128*4*blkdst]	;; Restore source pointer
	end_timer 15

;; Do inverse FFT levels 7,8
;; On input the 32-byte cache lines hold these data values:
;;	0	1K	1	1K+1
;;	256	...
;;	...
;;	2K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	4K	1	4K+1
;;	256	...
;;	...
;;	8K	...
;;	...

;; Do 128 four_complex_unfft macros
;;	distance between fft data elements is 4

	start_timer 16
	mov	edi, sincos2		;; Load sin/cos pointer
	mov	eax, 32			;; 32 iterations of 4
c5b:	disp four_complex_unfft, 8, 4*blkdst, 8*blkdst
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test inner loop counter
	jnc	c5b			;; Iterate if necessary
	IFDEF PFETCH
	prefetcht1 [esi-32+clm*32]
;	prefetcht1 [esi-32+clm*32+4*blkdst]
	prefetcht1 [esi-32+clm*32+8*blkdst]
;	prefetcht1 [esi-32+clm*32+12*blkdst]
	ENDIF
	loopclm	dl, clm, c5b
	lea	esi, [esi-clm*2*dist1+blkdst];; Next source pointer
	add	ah, 256/4		;; Test middle loop counter
	jnc	c5b			;; Iterate if necessary
	lea	esi, [esi-4*blkdst+16*blkdst];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al
	jnz	c5b
	lea	esi, [esi-32*16*blkdst]	;; Restore source pointer
	end_timer 16

;; Do inverse FFT levels 5,6
;; On input the 32-byte cache lines hold these data values:
;;	0	4K	1	4K+1
;;	256	...
;;	...
;;	8K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	16K	1	16K+1
;;	256	...
;;	...
;;	32K	...
;;	...

;; Do 128 four_complex_unfft macros
;;	distance between fft data elements is 16

	start_timer 17
	mov	edi, sincos3		;; Load sin/cos pointer
	mov	eax, 8			;; 8 iterations of 16
c7b:	disp four_complex_unfft, 8, 16*blkdst, 32*blkdst
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test inner loop counter
	jnc	c7b			;; Iterate if necessary
	IFDEF PFETCH
	prefetcht1 [esi-32+clm*32]
;	prefetcht1 [esi-32+clm*32+16*blkdst]
	prefetcht1 [esi-32+clm*32+32*blkdst]
;	prefetcht1 [esi-32+clm*32+48*blkdst]
	ENDIF
	loopclm	dl, clm, c7b
	lea	esi, [esi-clm*2*dist1+blkdst];; Next source pointer
	add	ah, 256/16		;; Test middle loop counter
	jnc	c7b			;; Iterate if necessary
	lea	esi, [esi-16*blkdst+64*blkdst];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al
	jnz	c7b
	lea	esi, [esi-8*64*blkdst]	;; Restore source pointer
	end_timer 17

;; Do inverse FFT levels 4
;; On input the 32-byte cache lines hold these data values:
;;	0	16K	1	16K+1
;;	256	...
;;	...
;;	32K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	32K	1	32K+1
;;	256	...
;;	...
;;	64K	...
;;	...

;; Do 128 four_complex_unfft_1 macros
;;	distance between fft data elements is 64

	start_timer 18
	mov	edi, sincos4		;; Load sin/cos pointer
	mov	eax, 4			;; 4 iterations of 32
c9b:	disp four_complex_unfft_1, 32*blkdst, 8, 64*blkdst
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test inner loop counter
	jnc	c9b			;; Iterate if necessary
	IFDEF PFETCH
	prefetcht1 [esi-32+clm*32]
;	prefetcht1 [esi-32+clm*32+32*blkdst]
	prefetcht1 [esi-32+clm*32+64*blkdst]
;	prefetcht1 [esi-32+clm*32+96*blkdst]
	ENDIF
	loopclm	dl, clm, c9b
	lea	esi, [esi-clm*2*dist1+blkdst];; Next source pointer
	add	ah, 256/32		;; Test middle loop counter
	jnc	c9b			;; Iterate if necessary
	lea	esi, [esi-32*blkdst+128*blkdst];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al
	jnz	c9b
	lea	esi, [esi-4*128*blkdst]	;; Restore source pointer
	end_timer 18

;; Do inverse FFT levels 1,2,3
;; On input the 32-byte cache lines hold these data values:
;;	0	32K	1	32K+1
;;	256	...
;;	...
;;	64K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	128K	1	128K+1
;;	256
;;	...
;;	128K-256

;; Do 128 four_complex_last_unfft macros
;;	distance between fft data elements is 128

	start_timer 19
	mov	edi, plus1_premults	;; Address of premultiplier table
	mov	eax, 128*256		;; Load loop counter
c1b:	disp four_complex_last_unfft, 8, 128*blkdst, 256*blkdst
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test loop counter
	jnc	c1b			;; Iterate if necessary
	IFDEF PFETCH
	prefetcht1 [esi-32+clm*32]
;	prefetcht1 [esi-32+clm*32+128*blkdst]
	prefetcht1 [esi-32+clm*32+256*blkdst]
;	prefetcht1 [esi-32+clm*32+384*blkdst]
	ENDIF
	loopclm	dl, clm, c1b
	lea	esi, [esi-clm*2*dist1+blkdst];; Next source pointer
	lea	edi, [edi+64]		;; Next premultiplier pointer
	sub	eax, 256		;; Test loop counter
	jnz	c1b			;; Iterate if necessary
	lea	esi, [esi-128*blkdst]	;; Restore source pointer
	end_timer 19

;; Normalize these values

	start_timer 20
	mov	eax, NORMRTN		;; Addr of normalization routine
	call	eax
	sub	eax, eax
	end_timer 20

;; Have we been given permission to start the FFT on the result?
;; If so, do so now while the data is in the L2 cache.

	cmp	POSTFFT, 0		;; Test flag
	je	no_fft			;; Skip FFT code if flag not set
	cmp	edx, p2cl/128*65536+(128/clm-4/clm)*256;; Delay fft on 8 sets
	jg	no_fft			;; Jump if delaying FFT
	copy_3_words clm, 0

;; Do FFT levels 1,2,3
;;
;; On input the 32-byte cache lines hold these data values:
;;	0	128K	1	128K+1
;;	256
;;	...
;;	128K-256
;; On output the 32-byte cache lines hold these data values:
;;	0	32K	1	32K+1
;;	256	...
;;	...
;;	64K	...
;;	...

;; Do 128 four_complex_first_fft macros
;;	distance between fft data elements is 128

b0b:	start_timer 21
	mov	ebx, DIST_TO_FFTSRCARG
	mov	edi, plus1_premults	;; Address of premultiplier table
	mov	al, 128			;; 128 iterations
b1b:	disp four_complex_first_fft, 128*blkdst, 256*blkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	add	dl, 256/2/clm		;; Test loop counter
	jnc	b1b			;; Iterate if necessary
	lea	esi, [esi-clm*2*dist1+blkdst];; Next source pointer
	lea	edi, [edi+64]		;; Next premultiplier pointer
	dec	al			;; Test loop counter
	jnz	b1b			;; Iterate if necessary
	lea	esi, [esi-128*blkdst]	;; Restore source pointer
	end_timer 21

;; Do FFT levels 4
;;
;; On input the 32-byte cache lines hold these data values:
;;	0	32K	1	32K+1
;;	256	...
;;	...
;;	64K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	16K	1	16K+1
;;	256	...
;;	...
;;	32K	...
;;	...

;; Do 128 four_complex_fft_1 macros
;;	distance between fft data elements is 64

	start_timer 22
	mov	edi, sincos4		;; Load sin/cos pointer
	mov	eax, 4			;; 4 iterations of 32
b9b:	disp four_complex_fft_1, 32*blkdst, 64*blkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test inner loop counter
	jnc	b9b			;; Iterate if necessary
	IFDEF PFETCH
;	prefetcht1 [esi-32+clm*32]
	prefetcht1 [esi-32+clm*32+32*blkdst]
;	prefetcht1 [esi-32+clm*32+64*blkdst]
	prefetcht1 [esi-32+clm*32+96*blkdst]
	ENDIF
	loopclm	dl, clm, b9b
	lea	esi, [esi-clm*2*dist1+blkdst];; Next source pointer
	add	ah, 256/32		;; Test middle loop counter
	jnc	b9b			;; Iterate if necessary
	lea	esi, [esi-32*blkdst+128*blkdst];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al
	jnz	b9b
	lea	esi, [esi-4*128*blkdst]	;; Restore source pointer
	end_timer 22

;; Do FFT levels 5,6
;;
;; On input the 32-byte cache lines hold these data values:
;;	0	16K	1	16K+1
;;	256	...
;;	...
;;	32K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	4K	1	4K+1
;;	256	...
;;	...
;;	8K	...
;;	...

;; Do 128 four_complex_fft macros
;;	distance between fft data elements is 16

	start_timer 23
	mov	edi, sincos3		;; Load sin/cos pointer
	mov	eax, 8			;; 8 iterations of 16
b7b:	disp four_complex_fft, 16*blkdst, 32*blkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test inner loop counter
	jnc	b7b			;; Iterate if necessary
	IFDEF PFETCH
;	prefetcht1 [esi-32+clm*32]
	prefetcht1 [esi-32+clm*32+16*blkdst]
;	prefetcht1 [esi-32+clm*32+32*blkdst]
	prefetcht1 [esi-32+clm*32+48*blkdst]
	ENDIF
	loopclm	dl, clm, b7b
	lea	esi, [esi-clm*2*dist1+blkdst];; Next source pointer
	add	ah, 256/16		;; Test middle loop counter
	jnc	b7b			;; Iterate if necessary
	lea	esi, [esi-16*blkdst+64*blkdst];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al
	jnz	b7b
	lea	esi, [esi-8*64*blkdst]	;; Restore source pointer
	end_timer 23

;; Do FFT levels 7,8
;;
;; On input the 32-byte cache lines hold these data values:
;;	0	4K	1	4K+1
;;	256	...
;;	...
;;	8K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	1K	1	1K+1
;;	256	...
;;	...
;;	2K	...
;;	...

;; Do 128 four_complex_fft macros
;;	distance between fft data elements is 4

	start_timer 24
	mov	edi, sincos2		;; Load sin/cos pointer
	mov	eax, 32			;; 32 iterations of 4
b5b:	disp four_complex_fft, 4*blkdst, 8*blkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test inner loop counter
	jnc	b5b			;; Iterate if necessary
	IFDEF PFETCH
;	prefetcht1 [esi-32+clm*32]
	prefetcht1 [esi-32+clm*32+4*blkdst]
;	prefetcht1 [esi-32+clm*32+8*blkdst]
	prefetcht1 [esi-32+clm*32+12*blkdst]
	ENDIF
	loopclm	dl, clm, b5b
	lea	esi, [esi-clm*2*dist1+blkdst];; Next source pointer
	add	ah, 256/4		;; Test middle loop counter
	jnc	b5b			;; Iterate if necessary
	lea	esi, [esi-4*blkdst+16*blkdst];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	dec	al
	jnz	b5b
	lea	esi, [esi-32*16*blkdst]	;; Restore source pointer
	end_timer 24

;; Do FFT levels 9,10
;;
;; On input the 32-byte cache lines hold these data values:
;;	0	1K	1	1K+1
;;	256	...
;;	...
;;	2K	...
;;	...
;; On output the 32-byte cache lines hold these data values:
;;	0	256	1	257
;;	512	...
;;	...

;; Do 128 four_complex_fft macros
;;	distance between fft data elements is 1

	start_timer 25
	mov	edi, sincos1		;; Load sin/cos pointer
	mov	eax, 128*256		;; 128 iterations
b3b:	disp four_complex_fft, blkdst, 2*blkdst, 8
	lea	esi, [esi+dist1]	;; Next source pointer
	add	al, 256/2		;; Test inner loop counter
	jnc	b3b			;; Iterate if necessary
	IFDEF PFETCH
;	prefetcht1 [esi-32+clm*32]
	prefetcht1 [esi-32+clm*32+blkdst]
;	prefetcht1 [esi-32+clm*32+2*blkdst]
	prefetcht1 [esi-32+clm*32+3*blkdst]
	ENDIF
	loopclm	dl, clm, b3b
	lea	esi, [esi-clm*2*dist1+4*blkdst];; Next source pointer
	lea	edi, [edi+SCD]		;; Next sine/cosine pointer
	sub	eax, 256
	jnz	b3b
	lea	esi, [esi-128*4*blkdst]	;; Restore source pointer
	end_timer 25

;; Work on next 4 sets of 32 values

no_fft:	lea	esi, [esi+clm*2*dist1]	;; Next source pointer
	dec	dh			;; Test loop counter
	jnz	short more		;; Do more FFTs
	lea	esi, [esi+64]		;; Skip pad every 128 cache lines (4KB)
	sub	edx, 65536		;; Test loop counter
	jz	short fftdn		;; Jump if done looping
	mov	dh, 128/clm		;; Restore cache line counter
more:	cmp	norm_ptr1, 0		;; What was the loop start point?
	jne	c0b			;; Do inverse-FFT/norm/FFT iteration
	jmp	b0b			;; Do another FFT-only iteration
fftdn:	lea	esi, [esi-p2cl*2*dist1-p2cl/128*64];; Restore source pointer
	cmp	esi, DESTARG		;; If esi was not restored, then we
	jne	short fftdn1		;; just finished skipped postfft data
	sub	ebx, ebx
	mov	DIST_TO_FFTSRCARG, ebx
	cmp	norm_ptr1, 0		;; What was the loop start point?
	je	pass2			;; FFT-only, go do pass 2

;; Split the carries into high and low carries (so they do not exceed
;; the desired number of bits per FFT word.  Then add the carries back to the
;; FFT data.

	start_timer 28
	call	gw_carries
	end_timer 28

;; Have we been given permission to start the FFT on the result?
;; If so, set loop to FFT the first few cache lines now that the carries
;; have been added in.

	cmp	POSTFFT, 0		;; Test flag
	je	short fftdn1		;; Skip FFT code if flag not set
	mov	esi, DESTARG		;; Load FFT data address
	mov	edx, 65536+4/clm*256	;; Do 8 previously skipped cache lines
	mov	norm_ptr1, 0		;; Set flag so only forward FFT is done
	copy_4_words
	jmp	b0b

;; Now add in the FFT'ed carries, cleanup and return

fftdn1:	end_timer 1
	jmp	gw_finish_mult
	ENDM

;;
;; Utility macros
;;

;; Loop clm times. Uses top bit(s) of specified register.

loopclm	MACRO reg, clm, label
	IF	clm GE 2
	add	reg, 256/clm
	jnc	label
	ENDIF
	ENDM

