; Copyright 2011-2024 - Mersenne Research, Inc.  All rights reserved
; Author:  George Woltman
; Email: woltman@alum.mit.edu
;

; ********************************************************
; ********************************************************
; ********************  FFT MACROS  **********************
; ********************************************************
; ********************************************************

;; These ffts are all done in a single pass.  On entry the memory layout is
;; quite simple.  It is assumed each cache line is 64 bytes or 8 doubles.

;; These macros support 4 "types".  Type 1 if forward FFT only.
;; Type 2 is forward FFT, square, inverse FFT.  Type 3 is forward
;; FFT, multiply, inverse FFT.  Type 4 is multiply, and inverse FFT.

;; rsi = destination pointer
;; DIST_TO_FFTSRCARG = rbx = distance from rsi to source (zero for types 1, 2)
;; DIST_TO_MULSRCARG = distance from rsi to multiplier (zero for types 2, 3)

;; August 2024, tried loop merging optimization for FMA3 CPUs only.  Little success.
;; 16-complex-first-fft was slower on Haswell, slightly faster on KabyLake
;; 16-complex-last-fft was slower on Haswell, and slower on KabyLake
IF (@INSTR(,%yarch,<FMA3>) NE 0)
	OLDWAY = 1
ELSE
	OLDWAY = 1
ENDIF

;;;
;;;  First some utility macros
;;;

;; Adjust pointers for two destination multiplies.  Normally, pass_1_wake_up_threads does this.

ytwo_dest_adjust MACRO
	mov	rsi, DESTARG
x64	add	rsi, DEST2ARG
x64	mov	DESTARG, rsi
	ENDM

;;;
;;;  Now we have one macro for each FFT length
;;;


;; Perform a 32-element FFT.

yfft32 MACRO type
	LOCAL	ymid2, ymid3, ymid4, yfinish_unfft

;; Check for a multiply with pre-FFTed inputs

	cmp	ffttype, 4
	je	ymid4

;; Do FFT levels 1,2,3
;; On input the 64-byte cache lines hold these data values:
;;	0	1	2	3	16	17	18	19
;;	4	...
;;	8	...
;;	12	...
;; On output the 64-byte cache lines hold these data values:
;;	0	1	2	3	4	5	6	7
;;	8	...
;;	16	...
;;	24	...
;; These are swizzled to:
;;	0	8	16	24	4	12	20	28
;;	1	...
;;	2	...
;;	3	...

;; Do 4 eight_reals_fft operations
;;	distance between fft data elements is 4
;;	do 1 macro processing 32 data values

	start_timer 4
	mov	rdi, sincos1		;; Load sin/cos data ptr
	yr4_fs4cl_eight_reals_fft_preload
	yr4_fs4cl_eight_reals_fft rsi, 0, 64, 2*64, rdi, 0, 1
	end_timer 4

;; Do FFT levels 4,5
;; On input and output the 64-byte cache lines hold these data values:
;;	0	8	16	24	4	12	20	28
;;	1	...
;;	2	...
;;	3	...

;; Do 1 eight_reals_fft operation
;; Do 3 four_complex_fft operations
;;	distance between fft data elements is 1
;;	do 1 macro processing 32 data values

	cmp	ffttype, 2
	je	ymid2
	ja	ymid3

	start_timer 5
;;	yr4_4cl_eight_reals_four_complex_fft_final rsi, 0, 64, 2*64
	yr4_dispatch call, yreal_fft_final, 64, 2*64
	bump	rsi, -4*64
	end_timer 5
	yfft_1_ret

ymid2:	start_timer 5
;;	yr4_4cl_eight_reals_four_complex_with_square rsi, 0, 64, 2*64
	yr4_dispatch call, yreal_square, 64, 2*64
	bump	rsi, -4*64
	end_timer 5
	jmp	yfinish_unfft

ymid3:	start_timer 5
x64	mov	r8, DEST2ARG
	mov	rbp, DIST_TO_MULSRCARG
;;	yr4_4cl_eight_reals_four_complex_with_mult rsi, 0, 64, 2*64
	yr4_dispatch call, yreal_mult, 64, 2*64
x64	add	rsi, r8
	bump	rsi, -4*64
	end_timer 5
	jmp	yfinish_unfft

ymid4:	start_timer 5
	mov	rbp, DIST_TO_MULSRCARG
;;	yr4_4cl_eight_reals_four_complex_with_mulf rsi, 0, 64, 2*64
	yr4_dispatch call, yreal_mulf, 64, 2*64
	bump	rsi, -4*64
	end_timer 5

yfinish_unfft:

;; Do inverse FFT levels 1,2,3
;; On input the 64-byte cache lines hold these data values:
;;	0	8	16	24	4	12	20	28
;;	1	...
;;	2	...
;;	3	...
;; These are swizzled to:
;;	0	1	2	3	4	5	6	7
;;	8	...
;;	16	...
;;	24	...
;; On output the 64-byte cache lines hold these data values:
;;	0	1	2	3	16	17	18	19
;;	4	...
;;	8	...
;;	12	...

;; Do 4 eight_reals_unfft operations
;;	distance between fft data elements is 4
;;	do 1 macro processing 32 data values

	start_timer 6
	mov	rdi, sincos1		;; Load sin/cos data ptr
	yr4_s4cl_eight_reals_unfft_preload
	yr4_s4cl_eight_reals_unfft rsi, 0, 64, 2*64, rdi, 0, 1
	end_timer 6

	ytwo_dest_adjust
	yfft_3_ret
	ENDM

;; Perform a 32-element negacyclic FFT

yfft32p MACRO type
	LOCAL	ymid2p, ymid3p, ymid4p, yfinish_unfft

;; Do a multiply with pre-FFTed inputs

	cmp	ffttype, 4
	je	ymid4p

;; Do FFT levels 1,2
;; On input the 64-byte cache lines hold these data values:
;;	0	1	2	3	16	17	18	19
;;	4	...
;;	8	...
;;	12	...
;; On output the 64-byte cache lines hold these data values:
;;	0	1	2	3	4	5	6	7
;;	8	...
;;	16	...
;;	24	...
;; These are swizzled to:
;;	0	8	16	24	4	12	20	28
;;	1	...
;;	2	...
;;	3	...

;; Do 4 four_complex_first_fft operations
;;	distance between fft data elements is 4
;;	do 1 macro processing 32 data values

	start_timer 4
	mov	rdi, sincos1		;; Load sin/cos pointer
	yr4_fs4cl_four_complex_first_djbfft_preload
	yr4_fs4cl_four_complex_first_djbfft rsi, 0, 64, 2*64, rdi, 0, 1
	end_timer 4

;; Do FFT levels 4,5
;; On input and output the 64-byte cache lines hold these data values:
;;	0	8	16	24	4	12	20	28
;;	1	...
;;	2	...
;;	3	...

;; Do 4 four_complex_fft operations
;;	distance between fft data elements is 1
;;	do 1 macro processing 32 data values

	cmp	ffttype, 2
	je	ymid2p
	ja	ymid3p

	start_timer 5
	yr4_4cl_four_complex_fft_final rsi, 0, 64, 2*64
	end_timer 5
	yfft_1_ret

ymid2p:	start_timer 5
	yr4_4cl_four_complex_with_square_preload
	yr4_4cl_four_complex_with_square rsi, 0, 64, 2*64, 1
	end_timer 5
	jmp	yfinish_unfft

ymid3p:	start_timer 5
x64	mov	r8, DEST2ARG
	mov	rbp, DIST_TO_MULSRCARG
	yr4_4cl_four_complex_with_mult_preload
	yr4_4cl_four_complex_with_mult rsi, 0, 64, 2*64
x64	add	rsi, r8
	end_timer 5
	jmp	yfinish_unfft

ymid4p:	start_timer 5
	mov	rbp, DIST_TO_MULSRCARG
	yr4_4cl_four_complex_with_mulf_preload
	yr4_4cl_four_complex_with_mulf rsi, 0, 64, 2*64
	end_timer 5

;; Do inverse FFT levels 1,2,3
;; On input the 64-byte cache lines hold these data values:
;;	0	8	16	24	4	12	20	28
;;	1	...
;;	2	...
;;	3	...
;; These are swizzled to:
;;	0	1	2	3	4	5	6	7
;;	8	...
;;	16	...
;;	24	...
;; On output the 64-byte cache lines hold these data values:
;;	0	1	2	3	16	17	18	19
;;	4	...
;;	8	...
;;	12	...

;; Do 4 four_complex_last_unfft operations
;;	distance between fft data elements is 4
;;	do 1 macro processing 32 data values

yfinish_unfft:
	start_timer 6
	mov	rdi, sincos1		;; Load sin/cos pointer
	yr4_s4cl_four_complex_last_unfft_preload
	yr4_s4cl_four_complex_last_unfft rsi, 0, 64, 2*64, rdi, 0, 1
	end_timer 6

	ytwo_dest_adjust
	yfft_3_ret
	ENDM


;; Perform a 64-element FFT.

yfft64 MACRO type
	LOCAL	ymid2, ymid3, ymid4, yfinish_unfft
	LOCAL	b1b, c1b

;; Check for a multiply with pre-FFTed inputs

	cmp	ffttype, 4
	je	ymid4

;; Do FFT levels 1,2,3
;; On input the 64-byte cache lines hold these data values:
;;	0	2	4	6	32	+2	+2	+2
;;	1	...
;;	8	...
;;	16	...
;;	24	...
;; On output the 64-byte cache lines hold these data values:
;;	0	2	4	6	8	+2	+2	+2
;;	1
;;	16	...
;;	32	...
;;	48	...
;; These are swizzled to:
;;	0	+16	+16	+16	8	+16	+16	+16
;;	1	...
;;	...
;;	7	...

;; Do 8 eight_reals_fft operations
;;	distance between fft data elements is 8
;;	do 2 macros each processing 32 data values

	start_timer 4
	mov	rdi, sincos1		;; Load sin/cos data ptr
	yloop_init  2			;; 2 iterations
	yr4_fs4cl_eight_reals_fft_preload
b1b:	yr4_fs4cl_eight_reals_fft rsi, 64, 2*64, 4*64, rdi, YMM_SCD3, 2
	yloop	2, b1b, -2*64		;; Test loop counter, restore source pointer
	end_timer 4

;; Do FFT levels 4,5,6
;; On input and output the 64-byte cache lines hold these data values:
;;	0	+16	+16	+16	8	+16	+16	+16
;;	1	...
;;	...
;;	7	...

;; Do 1 sixteen_reals_fft operation
;; Do 3 eight_complex_fft operations
;;	distance between fft data elements is 1
;;	do 1 macro processing 64 data values

	cmp	ffttype, 2
	je	ymid2
	ja	ymid3

	start_timer 5
;;	yr8_8cl_sixteen_reals_eight_complex_fft_final rsi, 0, 64, 2*64, 4*64
	yr8_dispatch call, y8real_fft_final, 64, 2*64, 4*64
	bump	rsi, -8*64
	end_timer 5
	yfft_1_ret

ymid2:	start_timer 5
;;	yr8_8cl_sixteen_reals_eight_complex_with_square rsi, 0, 64, 2*64, 4*64
	yr8_dispatch call, y8real_square, 64, 2*64, 4*64
	bump	rsi, -8*64
	end_timer 5
	jmp	yfinish_unfft

ymid3:	start_timer 5
x64	mov	r8, DEST2ARG
	mov	rbp, DIST_TO_MULSRCARG
;;	yr8_8cl_sixteen_reals_eight_complex_with_mult rsi, 0, 64, 2*64, 4*64
	yr8_dispatch call, y8real_mult, 64, 2*64, 4*64
x64	add	rsi, r8
	bump	rsi, -8*64
	end_timer 5
	jmp	yfinish_unfft

ymid4:	start_timer 5
	mov	rbp, DIST_TO_MULSRCARG
;;	yr8_8cl_sixteen_reals_eight_complex_with_mulf rsi, 0, 64, 2*64, 4*64
	yr8_dispatch call, y8real_mulf, 64, 2*64, 4*64
	bump	rsi, -8*64
	end_timer 5

yfinish_unfft:

;; Do inverse FFT levels 1,2,3
;; On input the 64-byte cache lines hold these data values:
;;	0	+16	+16	+16	8	+16	+16	+16
;;	1	...
;;	...
;;	7	...
;; These are swizzled to:
;;	0	+2	+2	+2	8	+2	+2	+2
;;	1	...
;;	16	...
;;	32	...
;;	48	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+2	+2	+2	32	+2	+2	+2
;;	1	...
;;	8	...
;;	16	...
;;	24	...

;; Do 8 eight_reals_unfft operations
;;	distance between fft data elements is 8
;;	do 2 macros each processing 32 data values

	start_timer 6
	mov	rdi, sincos1		;; Load sin/cos data ptr
	yloop_init  2			;; 2 iterations
	yr4_s4cl_eight_reals_unfft_preload
c1b:	yr4_s4cl_eight_reals_unfft rsi, 64, 2*64, 4*64, rdi, YMM_SCD3, 2
	yloop	2, c1b			;; Test loop counter
	end_timer 6

	ytwo_dest_adjust
	yfft_3_ret
	ENDM

;; Perform a 64-element negacyclic FFT

yfft64p MACRO type
	LOCAL	ymid2p, ymid3p, ymid4p, yfinish_unfft
	LOCAL	b1b, c1b

;; Do a multiply with pre-FFTed inputs

	cmp	ffttype, 4
	je	ymid4p

;; Do FFT levels 1,2
;; On input the 64-byte cache lines hold these data values:
;;	0	2	4	6	32	+2	+2	+2
;;	1	...
;;	8	...
;;	16	...
;;	24	...
;; On output the 64-byte cache lines hold these data values:
;;	0	2	4	6	8	+2	+2	+2
;;	1
;;	16	...
;;	32	...
;;	48	...
;; These are swizzled to:
;;	0	+16	+16	+16	8	+16	+16	+16
;;	1	...
;;	...
;;	7	...

;; Do 8 four_complex_first_fft operations
;;	distance between fft data elements is 8
;;	do 2 macros each processing 32 data values

	start_timer 4
	mov	rdi, sincos1		;; Load sin/cos pointer
	yloop_init  2			;; 2 iterations
	yr4_fs4cl_four_complex_first_djbfft_preload
b1b:	yr4_fs4cl_four_complex_first_djbfft rsi, 64, 2*64, 4*64, rdi, YMM_SCD4, 2
	yloop	2, b1b, -2*64		;; Test loop counter, restore source pointer
	end_timer 4

;; Do FFT levels 4,5,6
;; On input and output the 64-byte cache lines hold these data values:
;;	0	+16	+16	+16	8	+16	+16	+16
;;	1	...
;;	...
;;	7	...

;; Do 4 eight_complex_fft operations
;;	distance between fft data elements is 1
;;	do 1 macro processing 64 data values

	cmp	ffttype, 2
	je	ymid2p
	ja	ymid3p

	start_timer 5
	yr8_8cl_eight_complex_fft_final rsi, 0, 64, 2*64, 4*64
	end_timer 5
	yfft_1_ret

ymid2p:	start_timer 5
	yr8_8cl_eight_complex_with_square_preload
	yr8_8cl_eight_complex_with_square rsi, 0, 64, 2*64, 4*64
	end_timer 5
	jmp	yfinish_unfft

ymid3p:	start_timer 5
x64	mov	r8, DEST2ARG
	mov	rbp, DIST_TO_MULSRCARG
	yr8_8cl_eight_complex_with_mult_preload
	yr8_8cl_eight_complex_with_mult rsi, 0, 64, 2*64, 4*64
x64	add	rsi, r8
	end_timer 5
	jmp	yfinish_unfft

ymid4p:	start_timer 5
	mov	rbp, DIST_TO_MULSRCARG
	yr8_8cl_eight_complex_with_mulf_preload
	yr8_8cl_eight_complex_with_mulf rsi, 0, 64, 2*64, 4*64
	end_timer 5

;; Do inverse FFT levels 1,2,3
;; On input the 64-byte cache lines hold these data values:
;;	0	+16	+16	+16	8	+16	+16	+16
;;	1	...
;;	...
;;	7	...
;; These are swizzled to:
;;	0	+2	+2	+2	8	+2	+2	+2
;;	1	...
;;	16	...
;;	32	...
;;	48	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+2	+2	+2	32	+2	+2	+2
;;	1	...
;;	8	...
;;	16	...
;;	24	...

;; Do 8 four_complex_last_unfft operations
;;	distance between fft data elements is 8
;;	do 2 macros each processing 32 data values

yfinish_unfft:
	start_timer 6
	mov	rdi, sincos1		;; Load sin/cos pointer
	yloop_init  2			;; 2 iterations
	yr4_s4cl_four_complex_last_unfft_preload
c1b:	yr4_s4cl_four_complex_last_unfft rsi, 64, 2*64, 4*64, rdi, YMM_SCD4, 2
	yloop	2, c1b			;; Test loop counter
	end_timer 6

	ytwo_dest_adjust
	yfft_3_ret
	ENDM

;; Perform a 96-element FFT.

yfft96 MACRO type
	LOCAL	ymid2, ymid3, ymid4, yfinish_unfft
	LOCAL	b1b, b2b, b3a, b3b, b3c, b3d, c1b, c2b

;; Do a multiply with pre-FFTed inputs

	cmp	ffttype, 4
	je	ymid4

;; Do FFT levels 1,2,3
;; On input the 64-byte cache lines hold these data values:
;;	0	3	6	9	48	+3	+3	+3
;;	1
;;	2
;;	12	...
;;	24	...
;;	36	...
;; On output the 64-byte cache lines hold these data values:
;;	0	3	6	9	12	+3	+3	+3
;;	1
;;	2
;;	24	...
;;	48	...
;;	72	...
;; These are swizzled to:
;;	0	24	48	72	12	+24	+24	+24
;;	1
;;	...
;;	11

;; Do 12 eight_reals_fft operations
;;	distance between fft data elements is 12
;;	do 3 macros each processing 32 data values

	start_timer 4
	mov	rdi, sincos1		;; Load sin/cos pointer
	bump	rdi, 4*YMM_SCD2
	yloop_init  3			;; 3 iterations
	yr4_fs4cl_eight_reals_fft_preload
b1b:	yr4_fs4cl_eight_reals_fft rsi, 64, 3*64, 6*64, rdi, YMM_SCD3, 3
	yloop	3, b1b, -3*64		;; Test loop counter, restore source pointer
	end_timer 4

;; Do FFT levels 4,5
;; On input the 64-byte cache lines hold these data values:
;;	0	24	48	72	12	+24	+24	+24
;;	1
;;	...
;;	11
;; On output the 64-byte cache lines hold these data values:
;;	0	24	48	72	4	+24	+24	+24
;;	1
;;	...
;;	3
;;	8	...
;;	16	...

;; Do 4 six_reals_fft operations
;; Do 12 three_complex_fft operations
;;	distance between fft data elements is 4
;;	do 4 macros each processing 24 data values

	start_timer 5
	bump	rdi, -3*YMM_SCD3-4*YMM_SCD2 ;; Load sin/cos pointer
	yloop_init  4			;; 4 iterations
	yr3_3cl_six_reals_three_complex_djbfft_preload
b2b:	yr3_3cl_six_reals_three_complex_djbfft rsi, 64, 4*64, rdi, YMM_SCD2, 4
	yloop	4, b2b, -4*64		;; Test loop counter, restore source pointer
	end_timer 5

;; Do FFT levels 6,7
;; On input and output the 64-byte cache lines hold these data values:
;;	0	24	48	72	4	+24	+24	+24
;;	1
;;	...
;;	3
;;	8	...
;;	16	...

;; Do 1 eight_reals_fft operation
;; Do 11 four_complex_fft operations
;;	distance between fft data elements is 1
;;	do 3 macros each processing 32 data values

	cmp	ffttype, 2
	je	ymid2
	ja	ymid3

	start_timer 6
;;	yr4_4cl_eight_reals_four_complex_fft_final rsi, 4*64, 64, 2*64
	yr4_dispatch call, yreal_fft_final, 64, 2*64
	yloop_init  2			;; 2 more iterations
b3a:	yr4_4cl_four_complex_fft_final rsi, 4*64, 64, 2*64
	yloop	2, b3a			;; Test loop counter
	end_timer 6
	yfft_1_ret

ymid2:	start_timer 6
;;	yr4_4cl_eight_reals_four_complex_with_square rsi, 4*64, 64, 2*64
	yr4_dispatch call, yreal_square, 64, 2*64
	yloop_init  2			;; 2 more iterations
	yr4_4cl_four_complex_with_square_preload
b3b:	yr4_4cl_four_complex_with_square rsi, 4*64, 64, 2*64, 2
	yloop	2, b3b			;; Test loop counter
	end_timer 6
	jmp	yfinish_unfft

ymid3:	start_timer 6
x64	mov	r8, DEST2ARG
	mov	rbp, DIST_TO_MULSRCARG
;;	yr4_4cl_eight_reals_four_complex_with_mult rsi, 4*64, 64, 2*64
	yr4_dispatch call, yreal_mult, 64, 2*64
	yloop_init  2			;; 2 more iterations
	yr4_4cl_four_complex_with_mult_preload
b3c:	yr4_4cl_four_complex_with_mult rsi, 4*64, 64, 2*64
	yloop	2, b3c			;; Test loop counter
x64	add	rsi, r8
	end_timer 6
	jmp	yfinish_unfft

ymid4:	start_timer 6
	mov	rbp, DIST_TO_MULSRCARG
;;	yr4_4cl_eight_reals_four_complex_with_mulf rsi, 4*64, 64, 2*64
	yr4_dispatch call, yreal_mulf, 64, 2*64
	yloop_init  2			;; 2 more iterations
	yr4_4cl_four_complex_with_mulf_preload
b3d:	yr4_4cl_four_complex_with_mulf rsi, 4*64, 64, 2*64
	yloop	2, b3d			;; Test loop counter
	end_timer 6
yfinish_unfft:
	bump	rsi, -3*4*64		;; Restore source pointer

;; Do inverse FFT levels 4,5
;; On input the 64-byte cache lines hold these data values:
;;	0	24	48	72	4	+24	+24	+24
;;	1
;;	...
;;	3
;;	8	...
;;	16	...
;; On output the 64-byte cache lines hold these data values:
;;	0	24	48	72	12	+24	+24	+24
;;	1
;;	...
;;	11

;; Do 4 six_reals_unfft operations
;; Do 12 three_complex_unfft operations
;;	distance between fft data elements is 4
;;	do 4 macros each processing 24 data values

	start_timer 7
	mov	rdi, sincos1		;; Load sin/cos pointer
	yloop_init  4			;; 4 iterations
	yr3_3cl_six_reals_three_complex_djbunfft_preload
c2b:	yr3_3cl_six_reals_three_complex_djbunfft rsi, 64, 4*64, rdi, YMM_SCD2, 4
	yloop	4, c2b, -4*64		;; Test loop counter, restore source pointer
	end_timer 7

;; Do inverse FFT levels 1,2,3
;; On input the 64-byte cache lines hold these data values:
;;	0	24	48	72	12	+24	+24	+24
;;	1
;;	...
;;	11
;; These are swizzled to:
;;	0	3	6	9	12	+3	+3	+3
;;	1	...
;;	2
;;	24	...
;;	48	...
;;	72	...
;; On output the 64-byte cache lines hold these data values:
;;	0	3	6	9	48	+3	+3	+3
;;	1	...
;;	2
;;	12	...
;;	24	...
;;	36	...

;; Do 12 eight_reals_unfft operations
;;	distance between fft data elements is 12
;;	do 3 macros each processing 32 data values

	start_timer 8
	yloop_init  3			;; 3 iterations
	yr4_s4cl_eight_reals_unfft_preload
c1b:	yr4_s4cl_eight_reals_unfft rsi, 64, 3*64, 6*64, rdi, YMM_SCD3, 3
	yloop	3, c1b			;; Test loop counter
	end_timer 8

	ytwo_dest_adjust
	yfft_3_ret
	ENDM

;; Perform a 96-element negacyclic FFT

yfft96p MACRO type
	LOCAL	ymid2p, ymid3p, ymid4p, yfinish_unfft
	LOCAL	b1b, b2b, b3a, b3b, b3c, b3d, c1b, c2b

;; Do a multiply with pre-FFTed inputs

	cmp	ffttype, 4
	je	ymid4p

;; Do FFT levels 1,2,3
;; On input the 64-byte cache lines hold these data values:
;;	0	3	6	9	48	+3	+3	+3
;;	1
;;	2
;;	12	...
;;	24	...
;;	36	...
;; On output the 64-byte cache lines hold these data values:
;;	0	3	6	9	12	+3	+3	+3
;;	1
;;	2
;;	24	...
;;	48	...
;;	72	...
;; These are swizzled to:
;;	0	24	48	72	12	+24	+24	+24
;;	1
;;	...
;;	11

;; Do 12 four_complex_first_fft operations
;;	distance between fft data elements is 12
;;	do 3 macros each processing 32 data values

	start_timer 4
	mov	rdi, sincos1		;; Load sin/cos pointer
	bump	rdi, 4*YMM_SCD1/4
	yloop_init  3			;; 3 iterations
	yr4_fs4cl_four_complex_first_djbfft_preload
b1b:	yr4_fs4cl_four_complex_first_djbfft rsi, 64, 3*64, 6*64, rdi, YMM_SCD4, 3
	yloop	3, b1b, -3*64		;; Test loop counter, restore source pointer
	end_timer 4

;; Do FFT levels 4,5
;; On input the 64-byte cache lines hold these data values:
;;	0	24	48	72	12	+24	+24	+24
;;	1
;;	...
;;	11
;; On output the 64-byte cache lines hold these data values:
;;	0	24	48	72	4	+24	+24	+24
;;	1
;;	...
;;	3
;;	8	...
;;	16	...

;; Do 16 three_complex_fft operations
;;	distance between fft data elements is 4
;;	do 4 macros each processing 24 data values

	start_timer 5
	bump	rdi, -3*YMM_SCD4-4*YMM_SCD1/4 ;; Load sin/cos pointer
	yloop_init  4			;; 4 iterations
	yr3_b3cl_three_complex_djbfft_preload
b2b:	yr3_b3cl_three_complex_djbfft rsi, 64, 4*64, rdi, YMM_SCD1/4, 4
	yloop	4, b2b, -4*64		;; Test loop counter, restore source pointer
	end_timer 5

;; Do FFT levels 6,7
;; On input and output the 64-byte cache lines hold these data values:
;;	0	24	48	72	4	+24	+24	+24
;;	1
;;	...
;;	3
;;	8	...
;;	16	...

;; Do 12 four_complex_fft operations
;;	distance between fft data elements is 1
;;	do 3 macros each processing 32 data values

	cmp	ffttype, 2
	je	ymid2p
	ja	ymid3p

	start_timer 6
	yloop_init  3			;; 3 iterations
b3a:	yr4_4cl_four_complex_fft_final rsi, 4*64, 64, 2*64
	yloop	3, b3a			;; Test loop counter
	end_timer 6
	yfft_1_ret

ymid2p:	start_timer 6
	yloop_init  3			;; 3 iterations
	yr4_4cl_four_complex_with_square_preload
b3b:	yr4_4cl_four_complex_with_square rsi, 4*64, 64, 2*64, 3
	yloop	3, b3b			;; Test loop counter
	end_timer 6
	jmp	yfinish_unfft

ymid3p:	start_timer 6
x64	mov	r8, DEST2ARG
	mov	rbp, DIST_TO_MULSRCARG
	yloop_init  3			;; 3 iterations
	yr4_4cl_four_complex_with_mult_preload
b3c:	yr4_4cl_four_complex_with_mult rsi, 4*64, 64, 2*64
	yloop	3, b3c			;; Test loop counter
x64	add	rsi, r8
	end_timer 6
	jmp	yfinish_unfft

ymid4p:	start_timer 6
	mov	rbp, DIST_TO_MULSRCARG
	yloop_init  3			;; 3 iterations
	yr4_4cl_four_complex_with_mulf_preload
b3d:	yr4_4cl_four_complex_with_mulf rsi, 4*64, 64, 2*64
	yloop	3, b3d			;; Test loop counter
	end_timer 6
yfinish_unfft:
	bump	rsi, -3*4*64		;; Restore source pointer

;; Do inverse FFT levels 4,5
;; On input the 64-byte cache lines hold these data values:
;;	0	24	48	72	4	+24	+24	+24
;;	1
;;	...
;;	3
;;	8	...
;;	16	...
;; On output the 64-byte cache lines hold these data values:
;;	0	24	48	72	12	+24	+24	+24
;;	1
;;	...
;;	11

;; Do 16 three_complex_unfft operations
;;	distance between fft data elements is 4
;;	do 4 macros each processing 24 data values

	start_timer 7
	mov	rdi, sincos1		;; Load sin/cos pointer
	yloop_init  4			;; 4 iterations
	yr3_b3cl_three_complex_djbunfft_preload
c2b:	yr3_b3cl_three_complex_djbunfft rsi, 64, 4*64, rdi, YMM_SCD1/4, 4
	yloop	4, c2b, -4*64		;; Test loop counter, restore source pointer
	end_timer 7

;; Do inverse FFT levels 1,2,3
;; On input the 64-byte cache lines hold these data values:
;;	0	24	48	72	12	+24	+24	+24
;;	1
;;	...
;;	11
;; These are swizzled to:
;;	0	3	6	9	12	+3	+3	+3
;;	1	...
;;	2
;;	24	...
;;	48	...
;;	72	...
;; On output the 64-byte cache lines hold these data values:
;;	0	3	6	9	48	+3	+3	+3
;;	1	...
;;	2
;;	12	...
;;	24	...
;;	36	...

;; Do 12 four_complex_last_unfft operations
;;	distance between fft data elements is 12
;;	do 3 macros each processing 32 data values

	start_timer 8
	yloop_init  3			;; 3 iterations
	yr4_s4cl_four_complex_last_unfft_preload
c1b:	yr4_s4cl_four_complex_last_unfft rsi, 64, 3*64, 6*64, rdi, YMM_SCD4, 3
	yloop	3, c1b			;; Test loop counter
	end_timer 8

	ytwo_dest_adjust
	yfft_3_ret
	ENDM


;; Perform a 128-element FFT.

yfft128 MACRO type
	LOCAL	ymid2, ymid3, ymid4, yfinish_unfft
	LOCAL	b1b, b2b, b3a, b3b, b3c, b3d, c1b, c2b

;; Do a multiply with pre-FFTed inputs

	cmp	ffttype, 4
	je	ymid4

;; Do FFT levels 1,2,3
;; On input the 64-byte cache lines hold these data values:
;;	0	4	8	12	64	+4	+4	+4
;;	1
;;	...
;;	3
;;	16	...
;;	32	...
;;	48	...
;; On output the 64-byte cache lines hold these data values:
;;	0	4	8	12	16	+4	+4	+4
;;	1
;;	...
;;	3
;;	32	...
;;	64	...
;;	96	...
;; These are swizzled to:
;;	0	32	64	96	16	+32	+32	+32
;;	1
;;	...
;;	15

;; Do 16 eight_reals_fft operations
;;	distance between fft data elements is 16
;;	do 4 macros each processing 32 data values

	start_timer 4
	mov	rdi, sincos1		;; Load sin/cos pointer
	bump	rdi, 4*YMM_SCD3
	yloop_init  4			;; 4 iterations
	yr4_fs4cl_eight_reals_fft_preload
b1b:	yr4_fs4cl_eight_reals_fft rsi, 64, 4*64, 8*64, rdi, YMM_SCD3, 4
	yloop	4, b1b, -4*64		;; Test loop counter, restore source pointer
	end_timer 4

;; Do FFT levels 4,5
;; On input the 64-byte cache lines hold these data values:
;;	0	32	64	96	16	+32	+32	+32
;;	1
;;	...
;;	15
;; On output the 64-byte cache lines hold these data values:
;;	0	32	64	96	4	+32	+32	+32
;;	1
;;	...
;;	3
;;	8	...
;;	16	...
;;	24	...

;; Do 4 eight_reals_fft operations
;; Do 12 four_complex_fft operations
;;	distance between fft data elements is 4
;;	do 4 macros each processing 32 data values

	start_timer 5
	bump	rdi, -4*YMM_SCD3-4*YMM_SCD3 ;; Load sin/cos pointer
	yloop_init  4			;; 4 iterations
	yr4_4cl_eight_reals_four_complex_djbfft_preload
b2b:	yr4_4cl_eight_reals_four_complex_djbfft rsi, 64, 4*64, 8*64, rdi, YMM_SCD3, 4
	yloop	4, b2b, -4*64		;; Test loop counter, restore source pointer
	end_timer 5

;; Do FFT levels 6,7
;; On input and output the 64-byte cache lines hold these data values:
;;	0	32	64	96	4	+32	+32	+32
;;	1
;;	...
;;	3
;;	8	...
;;	16	...
;;	24	...

;; Do 1 eight_reals_fft operation
;; Do 15 four_complex_fft operations
;;	distance between fft data elements is 1
;;	do 4 macros each processing 32 data values

	cmp	ffttype, 2
	je	ymid2
	ja	ymid3

	start_timer 6
;;	yr4_4cl_eight_reals_four_complex_fft_final rsi, 4*64, 64, 2*64
	yr4_dispatch call, yreal_fft_final, 64, 2*64
	yloop_init  4, 1, 4		;; 3 more iterations
b3a:	yr4_4cl_four_complex_fft_final rsi, 4*64, 64, 2*64
	yloop	4, b3a			;; Test loop counter
	end_timer 6
	yfft_1_ret

ymid2:	start_timer 6
;;	yr4_4cl_eight_reals_four_complex_with_square rsi, 4*64, 64, 2*64
	yr4_dispatch call, yreal_square, 64, 2*64
	yloop_init  4, 1, 4		;; 3 more iterations
	yr4_4cl_four_complex_with_square_preload
b3b:	yr4_4cl_four_complex_with_square rsi, 4*64, 64, 2*64, 1
	yloop	4, b3b			;; Test loop counter
	end_timer 6
	jmp	yfinish_unfft

ymid3:	start_timer 6
x64	mov	r8, DEST2ARG
	mov	rbp, DIST_TO_MULSRCARG
;;	yr4_4cl_eight_reals_four_complex_with_mult rsi, 4*64, 64, 2*64
	yr4_dispatch call, yreal_mult, 64, 2*64
	yloop_init  4, 1, 4		;; 3 more iterations
	yr4_4cl_four_complex_with_mult_preload
b3c:	yr4_4cl_four_complex_with_mult rsi, 4*64, 64, 2*64
	yloop	4, b3c			;; Test loop counter
x64	add	rsi, r8
	end_timer 6
	jmp	yfinish_unfft

ymid4:	start_timer 6
	mov	rbp, DIST_TO_MULSRCARG
;;	yr4_4cl_eight_reals_four_complex_with_mulf rsi, 4*64, 64, 2*64
	yr4_dispatch call, yreal_mulf, 64, 2*64
	yloop_init  4, 1, 4		;; 3 more iterations
	yr4_4cl_four_complex_with_mulf_preload
b3d:	yr4_4cl_four_complex_with_mulf rsi, 4*64, 64, 2*64
	yloop	4, b3d			;; Test loop counter
	end_timer 6
yfinish_unfft:
	bump	rsi, -4*4*64		;; Restore source pointer

;; Do inverse FFT levels 4,5
;; On input the 64-byte cache lines hold these data values:
;;	0	32	64	96	4	+32	+32	+32
;;	1
;;	...
;;	3
;;	8	...
;;	16	...
;;	24	...
;; On output the 64-byte cache lines hold these data values:
;;	0	32	64	96	16	+32	+32	+32
;;	1
;;	...
;;	15

;; Do 4 eight_reals_unfft operations
;; Do 12 four_complex_unfft operations
;;	distance between fft data elements is 4
;;	do 4 macros each processing 32 data values

	start_timer 7
	mov	rdi, sincos1		;; Load sin/cos pointer
	yloop_init  4			;; 4 iterations
	yr4_4cl_eight_reals_four_complex_djbunfft_preload
c2b:	yr4_4cl_eight_reals_four_complex_djbunfft rsi, 64, 4*64, 8*64, rdi, YMM_SCD3, 4
	yloop	4, c2b, -4*64		;; Test loop counter, restore source pointer
	end_timer 7

;; Do inverse FFT levels 1,2,3
;; On input the 64-byte cache lines hold these data values:
;;	0	32	64	96	16	+32	+32	+32
;;	1
;;	...
;;	15
;; These are swizzled to:
;;	0	4	8	12	16	+4	+4	+4
;;	1	...
;;	...
;;	3
;;	32	...
;;	64	...
;;	96	...
;; On output the 64-byte cache lines hold these data values:
;;	0	4	8	12	64	+4	+4	+4
;;	1	...
;;	...
;;	3
;;	16	...
;;	32	...
;;	48	...

;; Do 16 eight_reals_unfft operations
;;	distance between fft data elements is 16
;;	do 4 macros each processing 32 data values

	start_timer 8
	yloop_init  4			;; 4 iterations
	yr4_s4cl_eight_reals_unfft_preload
c1b:	yr4_s4cl_eight_reals_unfft rsi, 64, 4*64, 8*64, rdi, YMM_SCD3, 4
	yloop	4, c1b			;; Test loop counter
	end_timer 8

	ytwo_dest_adjust
	yfft_3_ret
	ENDM

;; Perform a 128-element negacyclic FFT

yfft128p MACRO type
	LOCAL	ymid2p, ymid3p, ymid4p, yfinish_unfft
	LOCAL	b1b, b2b, b3a, b3b, b3c, b3d, c1b, c2b

;; Do a multiply with pre-FFTed inputs

	cmp	ffttype, 4
	je	ymid4p

;; Do FFT levels 1,2,3
;; On input the 64-byte cache lines hold these data values:
;;	0	4	8	12	64	+4	+4	+4
;;	1
;;	...
;;	3
;;	16	...
;;	32	...
;;	48	...
;; On output the 64-byte cache lines hold these data values:
;;	0	4	8	12	16	+4	+4	+4
;;	1
;;	...
;;	3
;;	32	...
;;	64	...
;;	96	...
;; These are swizzled to:
;;	0	32	64	96	16	+32	+32	+32
;;	1
;;	...
;;	15

;; Do 16 four_complex_first_fft operations
;;	distance between fft data elements is 16
;;	do 4 macros each processing 32 data values

	start_timer 4
	mov	rdi, sincos1		;; Load sin/cos pointer
	bump	rdi, 4*YMM_SCD2/4
	yloop_init  4			;; 4 iterations
	yr4_fs4cl_four_complex_first_djbfft_preload
b1b:	yr4_fs4cl_four_complex_first_djbfft rsi, 64, 4*64, 8*64, rdi, YMM_SCD4, 4
	yloop	4, b1b, -4*64		;; Test loop counter, restore source pointer
	end_timer 4

;; Do FFT levels 4,5
;; On input the 64-byte cache lines hold these data values:
;;	0	32	64	96	16	+32	+32	+32
;;	1
;;	...
;;	15
;; On output the 64-byte cache lines hold these data values:
;;	0	32	64	96	4	+32	+32	+32
;;	1
;;	...
;;	3
;;	8	...
;;	16	...
;;	24	...

;; Do 16 four_complex_fft operations
;;	distance between fft data elements is 4
;;	do 4 macros each processing 32 data values

	start_timer 5
	bump	rdi, -4*YMM_SCD4-4*YMM_SCD2/4 ;; Load sin/cos pointer
	yloop_init  4			;; 4 iterations
	yr4_b4cl_four_complex_djbfft_preload
b2b:	yr4_b4cl_four_complex_djbfft rsi, 64, 4*64, 8*64, rdi, YMM_SCD2/4, 4
	yloop	4, b2b, -4*64		;; Test loop counter, restore source pointer
	end_timer 5

;; Do FFT levels 6,7
;; On input and output the 64-byte cache lines hold these data values:
;;	0	32	64	96	4	+32	+32	+32
;;	1
;;	...
;;	3
;;	8	...
;;	16	...
;;	24	...

;; Do 16 four_complex_fft operations
;;	distance between fft data elements is 1
;;	do 4 macros each processing 32 data values

	cmp	ffttype, 2
	je	ymid2p
	ja	ymid3p

	start_timer 6
	yloop_init  4
b3a:	yr4_4cl_four_complex_fft_final rsi, 4*64, 64, 2*64
	yloop	4, b3a			;; Test loop counter
	end_timer 6
	yfft_1_ret

ymid2p:	start_timer 6
	yloop_init  4			;; 4 iterations
	yr4_4cl_four_complex_with_square_preload
b3b:	yr4_4cl_four_complex_with_square rsi, 4*64, 64, 2*64, 4
	yloop	4, b3b			;; Test loop counter
	end_timer 6
	jmp	yfinish_unfft

ymid3p:	start_timer 6
x64	mov	r8, DEST2ARG
	mov	rbp, DIST_TO_MULSRCARG
	yloop_init  4			;; 4 iterations
	yr4_4cl_four_complex_with_mult_preload
b3c:	yr4_4cl_four_complex_with_mult rsi, 4*64, 64, 2*64
	yloop	4, b3c			;; Test loop counter
x64	add	rsi, r8
	end_timer 6
	jmp	yfinish_unfft

ymid4p:	start_timer 6
	mov	rbp, DIST_TO_MULSRCARG
	yloop_init  4			;; 4 iterations
	yr4_4cl_four_complex_with_mulf_preload
b3d:	yr4_4cl_four_complex_with_mulf rsi, 4*64, 64, 2*64
	yloop	4, b3d			;; Test loop counter
	end_timer 6
yfinish_unfft:
	bump	rsi, -4*4*64		;; Restore source pointer

;; Do inverse FFT levels 4,5
;; On input the 64-byte cache lines hold these data values:
;;	0	32	64	96	4	+32	+32	+32
;;	1
;;	...
;;	3
;;	8	...
;;	16	...
;;	24	...
;; On output the 64-byte cache lines hold these data values:
;;	0	32	64	96	16	+32	+32	+32
;;	1
;;	...
;;	15

;; Do 16 four_complex_unfft operations
;;	distance between fft data elements is 4
;;	do 4 macros each processing 32 data values

	start_timer 7
	mov	rdi, sincos1		;; Load sin/cos pointer
	yloop_init  4			;; 4 iterations
	yr4_b4cl_four_complex_djbunfft_preload
c2b:	yr4_b4cl_four_complex_djbunfft rsi, 64, 4*64, 8*64, rdi, YMM_SCD2/4, 4
	yloop	4, c2b, -4*64		;; Test loop counter, restore source pointer
	end_timer 7

;; Do inverse FFT levels 1,2,3
;; On input the 64-byte cache lines hold these data values:
;;	0	32	64	96	16	+32	+32	+32
;;	1
;;	...
;;	15
;; These are swizzled to:
;;	0	4	8	12	16	+4	+4	+4
;;	1	...
;;	...
;;	3
;;	32	...
;;	64	...
;;	96	...
;; On output the 64-byte cache lines hold these data values:
;;	0	4	8	12	64	+4	+4	+4
;;	1	...
;;	...
;;	3
;;	16	...
;;	32	...
;;	48	...

;; Do 16 four_complex_last_unfft operations
;;	distance between fft data elements is 16
;;	do 4 macros each processing 32 data values

	start_timer 8
	yloop_init  4			;; 4 iterations
	yr4_s4cl_four_complex_last_unfft_preload
c1b:	yr4_s4cl_four_complex_last_unfft rsi, 64, 4*64, 8*64, rdi, YMM_SCD4, 4
	yloop	4, c1b			;; Test loop counter
	end_timer 8

	ytwo_dest_adjust
	yfft_3_ret
	ENDM


;; Perform a 160-element FFT.

yfft160 MACRO type
	LOCAL	ymid2, ymid3, ymid4, yfinish_unfft
	LOCAL	b1b, b2b, b3a, b3b, b3c, b3d, c1b, c2b

;; Do a multiply with pre-FFTed inputs

	cmp	ffttype, 4
	je	ymid4

;; Do FFT levels 1,2,3
;; On input the 64-byte cache lines hold these data values:
;;	0	5	10	15	80	+5	+5	+5
;;	1
;;	...
;;	4
;;	20	...
;;	40	...
;;	60	...
;; On output the 64-byte cache lines hold these data values:
;;	0	5	10	15	20	+5	+5	+5
;;	1
;;	...
;;	4
;;	40	...
;;	80	...
;;	120	...
;; These are swizzled to:
;;	0	40	80	120	20	+40	+40	+40
;;	1
;;	...
;;	19

;; Do 20 eight_reals_fft operations
;;	distance between fft data elements is 20
;;	do 5 macros each processing 32 data values

	start_timer 4
	mov	rdi, sincos1		;; Load sin/cos pointer
	bump	rdi, 4*YMM_SCD4
	yloop_init  5			;; 5 iterations
	yr4_fs4cl_eight_reals_fft_preload
b1b:	yr4_fs4cl_eight_reals_fft rsi, 64, 5*64, 10*64, rdi, YMM_SCD3, 5
	yloop	5, b1b, -5*64		;; Test loop counter, restore source pointer
	end_timer 4

;; Do FFT levels 4,5
;; On input the 64-byte cache lines hold these data values:
;;	0	40	80	120	20	+40	+40	+40
;;	1
;;	...
;;	19
;; On output the 64-byte cache lines hold these data values:
;;	0	+40	+40	+40	4	+40	+40	+40
;;	1
;;	...
;;	3
;;	8	...
;;	...
;;	32	...

;; Do 4 ten_reals_fft operations
;; Do 12 five_complex_fft operations
;;	distance between fft data elements is 4
;;	do 4 macros each processing 40 data values

	start_timer 5
	bump	rdi, -5*YMM_SCD3-4*YMM_SCD4 ;; Load sin/cos pointer
	yloop_init  4			;; 4 iterations
	yr5_5cl_ten_reals_five_complex_djbfft_preload
b2b:	yr5_5cl_ten_reals_five_complex_djbfft rsi, 64, 4*64, rdi, YMM_SCD4, 4
	yloop	4, b2b, -4*64		;; Test loop counter, restore source pointer
	end_timer 5

;; Do FFT levels 6,7
;; On input and output the 64-byte cache lines hold these data values:
;;	0	+40	+40	+40	4	+40	+40	+40
;;	1
;;	...
;;	3
;;	8	...
;;	...
;;	32	...

;; Do 1 eight_reals_fft operation
;; Do 19 four_complex_fft operations
;;	distance between fft data elements is 1
;;	do 5 macros each processing 32 data values

	cmp	ffttype, 2
	je	ymid2
	ja	ymid3

	start_timer 6
;;	yr4_4cl_eight_reals_four_complex_fft_final rsi, 4*64, 64, 2*64
	yr4_dispatch call, yreal_fft_final, 64, 2*64
	yloop_init  4			;; 4 more iterations
b3a:	yr4_4cl_four_complex_fft_final rsi, 4*64, 64, 2*64
	yloop	4, b3a			;; Test loop counter
	end_timer 6
	yfft_1_ret

ymid2:	start_timer 6
;;	yr4_4cl_eight_reals_four_complex_with_square rsi, 4*64, 64, 2*64
	yr4_dispatch call, yreal_square, 64, 2*64
	yloop_init  4			;; 4 more iterations
	yr4_4cl_four_complex_with_square_preload
b3b:	yr4_4cl_four_complex_with_square rsi, 4*64, 64, 2*64, 4
	yloop	4, b3b			;; Test loop counter
	end_timer 6
	jmp	yfinish_unfft

ymid3:	start_timer 6
x64	mov	r8, DEST2ARG
	mov	rbp, DIST_TO_MULSRCARG
;;	yr4_4cl_eight_reals_four_complex_with_mult rsi, 4*64, 64, 2*64
	yr4_dispatch call, yreal_mult, 64, 2*64
	yloop_init  4			;; 4 more iterations
	yr4_4cl_four_complex_with_mult_preload
b3c:	yr4_4cl_four_complex_with_mult rsi, 4*64, 64, 2*64
	yloop	4, b3c			;; Test loop counter
x64	add	rsi, r8
	end_timer 6
	jmp	yfinish_unfft

ymid4:	start_timer 6
	mov	rbp, DIST_TO_MULSRCARG
;;	yr4_4cl_eight_reals_four_complex_with_mulf rsi, 4*64, 64, 2*64
	yr4_dispatch call, yreal_mulf, 64, 2*64
	yloop_init  4			;; 4 more iterations
	yr4_4cl_four_complex_with_mulf_preload
b3d:	yr4_4cl_four_complex_with_mulf rsi, 4*64, 64, 2*64
	yloop	4, b3d			;; Test loop counter
	end_timer 6
yfinish_unfft:
	bump	rsi, -5*4*64		;; Restore source pointer

;; Do inverse FFT levels 4,5
;; On input the 64-byte cache lines hold these data values:
;;	0	+40	+40	+40	4	+40	+40	+40
;;	1
;;	...
;;	3
;;	8	...
;;	...
;;	32	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+40	+40	+40	20	+40	+40	+40
;;	1
;;	...
;;	19

;; Do 4 five_reals_unfft operations
;; Do 12 five_complex_unfft operations
;;	distance between fft data elements is 4
;;	do 4 macros each processing 40 data values

	start_timer 7
	mov	rdi, sincos1		;; Load sin/cos pointer
	yloop_init  4			;; 4 iterations
	yr5_5cl_ten_reals_five_complex_djbunfft_preload
c2b:	yr5_5cl_ten_reals_five_complex_djbunfft rsi, 64, 4*64, rdi, YMM_SCD4, 4
	yloop	4, c2b, -4*64		;; Test loop counter, restore source pointer
	end_timer 7

;; Do inverse FFT levels 1,2,3
;; On input the 64-byte cache lines hold these data values:
;;	0	+40	+40	+40	20	+40	+40	+40
;;	1
;;	...
;;	19
;; These are swizzled to:
;;	0	5	10	15	20	+5	+5	+5
;;	1	...
;;	...
;;	4
;;	40	...
;;	80	...
;;	120	...
;; On output the 64-byte cache lines hold these data values:
;;	0	5	10	15	80	+5	+5	+5
;;	1	...
;;	...
;;	4
;;	20	...
;;	40	...
;;	60	...

;; Do 20 eight_reals_unfft operations
;;	distance between fft data elements is 20
;;	do 5 macros each processing 32 data values

	start_timer 8
	yloop_init  5			;; 5 iterations
	yr4_s4cl_eight_reals_unfft_preload
c1b:	yr4_s4cl_eight_reals_unfft rsi, 64, 5*64, 10*64, rdi, YMM_SCD3, 5
	yloop	5, c1b			;; Test loop counter
	end_timer 8

	ytwo_dest_adjust
	yfft_3_ret
	ENDM

;; Perform a 160-element negacyclic FFT

yfft160p MACRO type
	LOCAL	ymid2p, ymid3p, ymid4p, yfinish_unfft
	LOCAL	b1b, b2b, b3a, b3b, b3c, b3d, c1b, c2b

;; Do a multiply with pre-FFTed inputs

	cmp	ffttype, 4
	je	ymid4p

;; Do FFT levels 1,2,3
;; On input the 64-byte cache lines hold these data values:
;;	0	5	10	15	80	+5	+5	+5
;;	1
;;	...
;;	4
;;	20	...
;;	40	...
;;	60	...
;; On output the 64-byte cache lines hold these data values:
;;	0	5	10	15	20	+5	+5	+5
;;	1
;;	...
;;	4
;;	40	...
;;	80	...
;;	120	...
;; These are swizzled to:
;;	0	40	80	120	20	+40	+40	+40
;;	1
;;	...
;;	19

;; Do 20 four_complex_first_fft operations
;;	distance between fft data elements is 20
;;	do 5 macros each processing 32 data values

	start_timer 4
	mov	rdi, sincos1		;; Load sin/cos pointer
	bump	rdi, 4*YMM_SCD2/4
	yloop_init  5			;; 5 iterations
	yr4_fs4cl_four_complex_first_djbfft_preload
b1b:	yr4_fs4cl_four_complex_first_djbfft rsi, 64, 5*64, 10*64, rdi, YMM_SCD4, 5
	yloop	5, b1b, -5*64		;; Test loop counter, restore source pointer
	end_timer 4

;; Do FFT levels 4,5
;; On input the 64-byte cache lines hold these data values:
;;	0	40	80	120	20	+40	+40	+40
;;	1
;;	...
;;	19
;; On output the 64-byte cache lines hold these data values:
;;	0	+40	+40	+40	4	+40	+40	+40
;;	1
;;	...
;;	3
;;	8	...
;;	...
;;	32	...

;; Do 16 five_complex_fft operations
;;	distance between fft data elements is 4
;;	do 4 macros each processing 40 data values

	start_timer 5
	bump	rdi, -5*YMM_SCD4-4*YMM_SCD2/4 ;; Load sin/cos pointer
	yloop_init  4			;; 4 iterations
	yr5_b5cl_five_complex_djbfft_preload
b2b:	yr5_b5cl_five_complex_djbfft rsi, 64, 4*64, rdi, YMM_SCD2/4, 4
	yloop	4, b2b, -4*64		;; Test loop counter, restore source pointer
	end_timer 5

;; Do FFT levels 6,7
;; On input and output the 64-byte cache lines hold these data values:
;;	0	+40	+40	+40	4	+40	+40	+40
;;	1
;;	...
;;	3
;;	8	...
;;	...
;;	32	...

;; Do 20 four_complex_fft operations
;;	distance between fft data elements is 1
;;	do 5 macros each processing 32 data values

	cmp	ffttype, 2
	je	ymid2p
	ja	ymid3p

	start_timer 6
	yloop_init  5			;; 5 iterations
b3a:	yr4_4cl_four_complex_fft_final rsi, 4*64, 64, 2*64
	yloop	5, b3a			;; Test loop counter
	end_timer 6
	yfft_1_ret

ymid2p:	start_timer 6
	yloop_init  5			;; 5 iterations
	yr4_4cl_four_complex_with_square_preload
b3b:	yr4_4cl_four_complex_with_square rsi, 4*64, 64, 2*64, 5
	yloop	5, b3b			;; Test loop counter
	end_timer 6
	jmp	yfinish_unfft

ymid3p:	start_timer 6
x64	mov	r8, DEST2ARG
	mov	rbp, DIST_TO_MULSRCARG
	yloop_init  5			;; 5 iterations
	yr4_4cl_four_complex_with_mult_preload
b3c:	yr4_4cl_four_complex_with_mult rsi, 4*64, 64, 2*64
	yloop	5, b3c			;; Test loop counter
x64	add	rsi, r8
	end_timer 6
	jmp	yfinish_unfft

ymid4p:	start_timer 6
	mov	rbp, DIST_TO_MULSRCARG
	yloop_init  5			;; 5 iterations
	yr4_4cl_four_complex_with_mulf_preload
b3d:	yr4_4cl_four_complex_with_mulf rsi, 4*64, 64, 2*64
	yloop	5, b3d			;; Test loop counter
	end_timer 6
yfinish_unfft:
	bump	rsi, -5*4*64		;; Restore source pointer

;; Do inverse FFT levels 4,5
;; On input the 64-byte cache lines hold these data values:
;;	0	+40	+40	+40	4	+40	+40	+40
;;	1
;;	...
;;	3
;;	8	...
;;	...
;;	32	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+40	+40	+40	20	+40	+40	+40
;;	1
;;	...
;;	19

;; Do 16 five_complex_unfft operations
;;	distance between fft data elements is 4
;;	do 4 macros each processing 40 data values

	start_timer 7
	mov	rdi, sincos1		;; Load sin/cos pointer
	yloop_init  4			;; 4 iterations
	yr5_b5cl_five_complex_djbunfft_preload
c2b:	yr5_b5cl_five_complex_djbunfft rsi, 64, 4*64, rdi, YMM_SCD2/4, 4
	yloop	4, c2b, -4*64		;; Test loop counter, restore source pointer
	end_timer 7

;; Do inverse FFT levels 1,2,3
;; On input the 64-byte cache lines hold these data values:
;;	0	+40	+40	+40	20	+40	+40	+40
;;	1
;;	...
;;	19
;; These are swizzled to:
;;	0	5	10	15	20	+5	+5	+5
;;	1	...
;;	...
;;	4
;;	40	...
;;	80	...
;;	120	...
;; On output the 64-byte cache lines hold these data values:
;;	0	5	10	15	80	+5	+5	+5
;;	1	...
;;	...
;;	4
;;	20	...
;;	40	...
;;	60	...

;; Do 20 four_complex_last_unfft operations
;;	distance between fft data elements is 20
;;	do 5 macros each processing 32 data values

	start_timer 8
	yloop_init  5			;; 5 iterations
	yr4_s4cl_four_complex_last_unfft_preload
c1b:	yr4_s4cl_four_complex_last_unfft rsi, 64, 5*64, 10*64, rdi, YMM_SCD4, 5
	yloop	5, c1b			;; Test loop counter
	end_timer 8

	ytwo_dest_adjust
	yfft_3_ret
	ENDM


;; Perform a 192-element FFT.

yfft192 MACRO type
	LOCAL	ymid2, ymid3, ymid4, yfinish_unfft
	LOCAL	b1b, b2b, b3a, b3b, b3c, b3d, c1b, c2b

;; Do a multiply with pre-FFTed inputs

	cmp	ffttype, 4
	je	ymid4

;; Do FFT levels 1,2,3
;; On input the 64-byte cache lines hold these data values:
;;	0	+6	+6	+6	96	+6	+6	+6
;;	1
;;	...
;;	5
;;	24	...
;;	48	...
;;	72	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+6	+6	+6	24	+6	+6	+6
;;	1
;;	...
;;	5
;;	48	...
;;	96	...
;;	144	...
;; These are swizzled to:
;;	0	+48	+48	+48	24	+48	+48	+48
;;	1
;;	...
;;	23

;; Do 24 eight_reals_fft operations
;;	distance between fft data elements is 24
;;	do 6 macros each processing 32 data values

	start_timer 4
	mov	rdi, sincos1		;; Load sin/cos pointer
	bump	rdi, 8*YMM_SCD2
	yloop_init  6			;; 6 iterations
	yr4_fs4cl_eight_reals_fft_preload
b1b:	yr4_fs4cl_eight_reals_fft rsi, 64, 6*64, 12*64, rdi, YMM_SCD3, 6
	yloop	6, b1b, -6*64		;; Test loop counter, restore source pointer
	end_timer 4

;; Do FFT levels 4,5
;; On input the 64-byte cache lines hold these data values:
;;	0	+48	+48	+48	24	+48	+48	+48
;;	1
;;	...
;;	23
;; On output the 64-byte cache lines hold these data values:
;;	0	+48	+48	+48	8	+48	+48	+48
;;	1
;;	...
;;	7
;;	16	...
;;	32	...

;; Do 8 six_reals_fft operations
;; Do 24 three_complex_fft operations
;;	distance between fft data elements is 8
;;	do 8 macros each processing 24 data values

	start_timer 5
	bump	rdi, -6*YMM_SCD3-8*YMM_SCD2 ;; Load sin/cos pointer
	yloop_init  8			;; 8 iterations
	yr3_3cl_six_reals_three_complex_djbfft_preload
b2b:	yr3_3cl_six_reals_three_complex_djbfft rsi, 64, 8*64, rdi, YMM_SCD2, 8
	yloop	8, b2b, -8*64		;; Test loop counter, restore source pointer
	end_timer 5

;; Do FFT levels 6,7
;; On input and output the 64-byte cache lines hold these data values:
;;	0	+48	+48	+48	8	+48	+48	+48
;;	1
;;	...
;;	7
;;	16	...
;;	32	...

;; Do 1 sixteen_reals_fft operation
;; Do 11 eignt_complex_fft operations
;;	distance between fft data elements is 1
;;	do 3 macros each processing 64 data values

	cmp	ffttype, 2
	je	ymid2
	ja	ymid3

	start_timer 6
;;	yr8_8cl_sixteen_reals_eight_complex_fft_final rsi, 8*64, 64, 2*64, 4*64
	yr8_dispatch call, y8real_fft_final, 64, 2*64, 4*64
	yloop_init  2			;; 2 more iterations
b3a:	yr8_8cl_eight_complex_fft_final rsi, 8*64, 64, 2*64, 4*64
	yloop	2, b3a			;; Test loop counter
	end_timer 6
	yfft_1_ret

ymid2:	start_timer 6
;;	yr8_8cl_sixteen_reals_eight_complex_with_square rsi, 8*64, 64, 2*64, 4*64
	yr8_dispatch call, y8real_square, 64, 2*64, 4*64
	yloop_init  2			;; 2 more iterations
	yr8_8cl_eight_complex_with_square_preload
b3b:	yr8_8cl_eight_complex_with_square rsi, 8*64, 64, 2*64, 4*64
	yloop	2, b3b			;; Test loop counter
	end_timer 6
	jmp	yfinish_unfft

ymid3:	start_timer 6
x64	mov	r8, DEST2ARG
	mov	rbp, DIST_TO_MULSRCARG
;;	yr8_8cl_sixteen_reals_eight_complex_with_mult rsi, 8*64, 64, 2*64, 4*64
	yr8_dispatch call, y8real_mult, 64, 2*64, 4*64
	yloop_init  2			;; 2 more iterations
	yr8_8cl_eight_complex_with_mult_preload
b3c:	yr8_8cl_eight_complex_with_mult rsi, 8*64, 64, 2*64, 4*64
	yloop	2, b3c			;; Test loop counter
x64	add	rsi, r8
	end_timer 6
	jmp	yfinish_unfft

ymid4:	start_timer 6
	mov	rbp, DIST_TO_MULSRCARG
;;	yr8_8cl_sixteen_reals_eight_complex_with_mulf rsi, 8*64, 64, 2*64, 4*64
	yr8_dispatch call, y8real_mulf, 64, 2*64, 4*64
	yloop_init  2			;; 2 more iterations
	yr8_8cl_eight_complex_with_mulf_preload
b3d:	yr8_8cl_eight_complex_with_mulf rsi, 8*64, 64, 2*64, 4*64
	yloop	2, b3d			;; Test loop counter
	end_timer 6
yfinish_unfft:
	bump	rsi, -3*8*64		;; Restore source pointer

;; Do inverse FFT levels 4,5
;; On input the 64-byte cache lines hold these data values:
;;	0	+48	+48	+48	8	+48	+48	+48
;;	1
;;	...
;;	7
;;	16	...
;;	32	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+48	+48	+48	24	+48	+48	+48
;;	1
;;	...
;;	23

;; Do 8 six_reals_unfft operations
;; Do 24 three_complex_unfft operations
;;	distance between fft data elements is 8
;;	do 8 macros each processing 24 data values

	start_timer 7
	mov	rdi, sincos1		;; Load sin/cos pointer
	yloop_init  8			;; 8 iterations
	yr3_3cl_six_reals_three_complex_djbunfft_preload
c2b:	yr3_3cl_six_reals_three_complex_djbunfft rsi, 64, 8*64, rdi, YMM_SCD2, 8
	yloop	8, c2b, -8*64		;; Test loop counter, restore source pointer
	end_timer 7

;; Do inverse FFT levels 1,2,3
;; On input the 64-byte cache lines hold these data values:
;;	0	+48	+48	+48	24	+48	+48	+48
;;	1
;;	...
;;	23
;; These are swizzled to:
;;	0	+6	+6	+6	24	+6	+6	+6
;;	1	...
;;	...
;;	5
;;	48	...
;;	96	...
;;	144	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+6	+6	+6	96	+6	+6	+6
;;	1	...
;;	...
;;	5
;;	24	...
;;	48	...
;;	72	...

;; Do 24 eight_reals_unfft operations
;;	distance between fft data elements is 24
;;	do 6 macros each processing 32 data values

	start_timer 8
	yloop_init  6			;; 6 iterations
	yr4_s4cl_eight_reals_unfft_preload
c1b:	yr4_s4cl_eight_reals_unfft rsi, 64, 6*64, 12*64, rdi, YMM_SCD3, 6
	yloop	6, c1b			;; Test loop counter
	end_timer 8

	ytwo_dest_adjust
	yfft_3_ret
	ENDM

;; Perform a 192-element negacyclic FFT

yfft192p MACRO type
	LOCAL	ymid2p, ymid3p, ymid4p, yfinish_unfft
	LOCAL	b1b, b2b, b3a, b3b, b3c, b3d, c1b, c2b

;; Do a multiply with pre-FFTed inputs

	cmp	ffttype, 4
	je	ymid4p

;; Do FFT levels 1,2,3
;; On input the 64-byte cache lines hold these data values:
;;	0	+6	+6	+6	96	+6	+6	+6
;;	1
;;	...
;;	5
;;	24	...
;;	48	...
;;	72	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+6	+6	+6	24	+6	+6	+6
;;	1
;;	...
;;	5
;;	48	...
;;	96	...
;;	144	...
;; These are swizzled to:
;;	0	+48	+48	+48	24	+48	+48	+48
;;	1
;;	...
;;	23

;; Do 24 four_complex_first_fft operations
;;	distance between fft data elements is 24
;;	do 6 macros each processing 32 data values

	start_timer 4
	mov	rdi, sincos1		;; Load sin/cos pointer
	bump	rdi, 8*YMM_SCD1/4
	yloop_init  6			;; 6 iterations
	yr4_fs4cl_four_complex_first_djbfft_preload
b1b:	yr4_fs4cl_four_complex_first_djbfft rsi, 64, 6*64, 12*64, rdi, YMM_SCD4, 6
	yloop	6, b1b, -6*64		;; Test loop counter, restore source pointer
	end_timer 4

;; Do FFT levels 4,5
;; On input the 64-byte cache lines hold these data values:
;;	0	+48	+48	+48	24	+48	+48	+48
;;	1
;;	...
;;	23
;; On output the 64-byte cache lines hold these data values:
;;	0	+48	+48	+48	8	+48	+48	+48
;;	1
;;	...
;;	7
;;	16	...
;;	32	...

;; Do 32 three_complex_fft operations
;;	distance between fft data elements is 8
;;	do 8 macros each processing 24 data values

	start_timer 5
	bump	rdi, -6*YMM_SCD4-8*YMM_SCD1/4 ;; Load sin/cos pointer
	yloop_init  8			;; 8 iterations
	yr3_b3cl_three_complex_djbfft_preload
b2b:	yr3_b3cl_three_complex_djbfft rsi, 64, 8*64, rdi, YMM_SCD1/4, 8
	yloop	8, b2b, -8*64		;; Test loop counter, restore source pointer
	end_timer 5

;; Do FFT levels 6,7
;; On input and output the 64-byte cache lines hold these data values:
;;	0	+48	+48	+48	8	+48	+48	+48
;;	1
;;	...
;;	7
;;	16	...
;;	32	...

;; Do 12 eight_complex_fft operations
;;	distance between fft data elements is 1
;;	do 3 macros each processing 64 data values

	cmp	ffttype, 2
	je	ymid2p
	ja	ymid3p

	start_timer 6
	yloop_init  3			;; 3 iterations
b3a:	yr8_8cl_eight_complex_fft_final rsi, 8*64, 64, 2*64, 4*64
	yloop	3, b3a			;; Test loop counter
	end_timer 6
	yfft_1_ret

ymid2p:	start_timer 6
	yloop_init  3			;; 3 iterations
	yr8_8cl_eight_complex_with_square_preload
b3b:	yr8_8cl_eight_complex_with_square rsi, 8*64, 64, 2*64, 4*64
	yloop	3, b3b			;; Test loop counter
	end_timer 6
	jmp	yfinish_unfft

ymid3p:	start_timer 6
x64	mov	r8, DEST2ARG
	mov	rbp, DIST_TO_MULSRCARG
	yloop_init  3			;; 3 iterations
	yr8_8cl_eight_complex_with_mult_preload
b3c:	yr8_8cl_eight_complex_with_mult rsi, 8*64, 64, 2*64, 4*64
	yloop	3, b3c			;; Test loop counter
x64	add	rsi, r8
	end_timer 6
	jmp	yfinish_unfft

ymid4p:	start_timer 6
	mov	rbp, DIST_TO_MULSRCARG
	yloop_init  3			;; 3 iterations
	yr8_8cl_eight_complex_with_mulf_preload
b3d:	yr8_8cl_eight_complex_with_mulf rsi, 8*64, 64, 2*64, 4*64
	yloop	3, b3d			;; Test loop counter
	end_timer 6
yfinish_unfft:
	bump	rsi, -3*8*64		;; Restore source pointer

;; Do inverse FFT levels 4,5
;; On input the 64-byte cache lines hold these data values:
;;	0	+48	+48	+48	8	+48	+48	+48
;;	1
;;	...
;;	7
;;	16	...
;;	32	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+48	+48	+48	24	+48	+48	+48
;;	1
;;	...
;;	23

;; Do 32 three_complex_unfft operations
;;	distance between fft data elements is 8
;;	do 8 macros each processing 24 data values

	start_timer 7
	mov	rdi, sincos1		;; Load sin/cos pointer
	yloop_init  8			;; 8 iterations
	yr3_b3cl_three_complex_djbunfft_preload
c2b:	yr3_b3cl_three_complex_djbunfft rsi, 64, 8*64, rdi, YMM_SCD1/4, 8
	yloop	8, c2b, -8*64		;; Test loop counter, restore source pointer
	end_timer 7

;; Do inverse FFT levels 1,2,3
;; On input the 64-byte cache lines hold these data values:
;;	0	+48	+48	+48	24	+48	+48	+48
;;	1
;;	...
;;	23
;; These are swizzled to:
;;	0	+6	+6	+6	24	+6	+6	+6
;;	1	...
;;	...
;;	5
;;	48	...
;;	96	...
;;	144	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+6	+6	+6	96	+6	+6	+6
;;	1	...
;;	...
;;	5
;;	24	...
;;	48	...
;;	72	...

;; Do 24 four_complex_last_unfft operations
;;	distance between fft data elements is 24
;;	do 6 macros each processing 32 data values

	start_timer 8
	yloop_init  6			;; 6 iterations
	yr4_s4cl_four_complex_last_unfft_preload
c1b:	yr4_s4cl_four_complex_last_unfft rsi, 64, 6*64, 12*64, rdi, YMM_SCD4, 6
	yloop	6, c1b			;; Test loop counter
	end_timer 8

	ytwo_dest_adjust
	yfft_3_ret
	ENDM


;; Perform a 256-element FFT.

yfft256 MACRO type
	LOCAL	ymid2, ymid3, ymid4, yfinish_unfft
	LOCAL	b1b, b2b, b3a, b3b, b3c, b3d, c1b, c2b

;; Do a multiply with pre-FFTed inputs

	cmp	ffttype, 4
	je	ymid4

;; Do FFT levels 1,2,3
;; On input the 64-byte cache lines hold these data values:
;;	0	+8	+8	+8	128	+8	+8	+8
;;	1
;;	...
;;	7
;;	32	...
;;	64	...
;;	96	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+8	+8	+8	32	+8	+8	+8
;;	1
;;	...
;;	7
;;	64	...
;;	128	...
;;	192	...
;; These are swizzled to:
;;	0	+64	+64	+64	32	+64	+64	+64
;;	1
;;	...
;;	31

;; Do 32 eight_reals_fft operations
;;	distance between fft data elements is 32
;;	do 8 macros each processing 32 data values

	start_timer 4
	mov	rdi, sincos1		;; Load sin/cos pointer
	bump	rdi, 8*YMM_SCD3
	yloop_init  8			;; 8 iterations
	yr4_fs4cl_eight_reals_fft_preload
b1b:	yr4_fs4cl_eight_reals_fft rsi, 64, 8*64, 16*64, rdi, YMM_SCD3, 8
	yloop	8, b1b, -8*64		;; Test loop counter, restore source pointer
	end_timer 4

;; Do FFT levels 4,5
;; On input the 64-byte cache lines hold these data values:
;;	0	+64	+64	+64	32	+64	+64	+64
;;	1
;;	...
;;	31
;; On output the 64-byte cache lines hold these data values:
;;	0	+64	+64	+64	8	+64	+64	+64
;;	1
;;	...
;;	7
;;	16	...
;;	32	...
;;	48	...

;; Do 8 eight_reals_fft operations
;; Do 24 four_complex_fft operations
;;	distance between fft data elements is 8
;;	do 8 macros each processing 32 data values

	start_timer 5
	bump	rdi, -8*YMM_SCD3-8*YMM_SCD3 ;; Load sin/cos pointer
	yloop_init  8			;; 8 iterations
	yr4_4cl_eight_reals_four_complex_djbfft_preload
b2b:	yr4_4cl_eight_reals_four_complex_djbfft rsi, 64, 8*64, 16*64, rdi, YMM_SCD3, 8
	yloop	8, b2b, -8*64		;; Test loop counter, restore source pointer
	end_timer 5

;; Do FFT levels 6,7
;; On input and output the 64-byte cache lines hold these data values:
;;	0	+64	+64	+64	8	+64	+64	+64
;;	1
;;	...
;;	7
;;	16	...
;;	32	...
;;	48	...

;; Do 1 sixteen_reals_fft operation
;; Do 15 eight_complex_fft operations
;;	distance between fft data elements is 1
;;	do 4 macros each processing 64 data values

	cmp	ffttype, 2
	je	ymid2
	ja	ymid3

	start_timer 6
;;	yr8_8cl_sixteen_reals_eight_complex_fft_final rsi, 8*64, 64, 2*64, 4*64
	yr8_dispatch call, y8real_fft_final, 64, 2*64, 4*64
	yloop_init  4, 1, 4		;; 3 more iterations
b3a:	yr8_8cl_eight_complex_fft_final rsi, 8*64, 64, 2*64, 4*64
	yloop	4, b3a			;; Test loop counter
	end_timer 6
	yfft_1_ret

ymid2:	start_timer 6
;;	yr8_8cl_sixteen_reals_eight_complex_with_square rsi, 8*64, 64, 2*64, 4*64
	yr8_dispatch call, y8real_square, 64, 2*64, 4*64
	yloop_init  4, 1, 4		;; 3 more iterations
	yr8_8cl_eight_complex_with_square_preload
b3b:	yr8_8cl_eight_complex_with_square rsi, 8*64, 64, 2*64, 4*64
	yloop	4, b3b			;; Test loop counter
	end_timer 6
	jmp	yfinish_unfft

ymid3:	start_timer 6
x64	mov	r8, DEST2ARG
	mov	rbp, DIST_TO_MULSRCARG
;;	yr8_8cl_sixteen_reals_eight_complex_with_mult rsi, 8*64, 64, 2*64, 4*64
	yr8_dispatch call, y8real_mult, 64, 2*64, 4*64
	yloop_init  4, 1, 4		;; 3 more iterations
	yr8_8cl_eight_complex_with_mult_preload
b3c:	yr8_8cl_eight_complex_with_mult rsi, 8*64, 64, 2*64, 4*64
	yloop	4, b3c			;; Test loop counter
x64	add	rsi, r8
	end_timer 6
	jmp	yfinish_unfft

ymid4:	start_timer 6
	mov	rbp, DIST_TO_MULSRCARG
;;	yr8_8cl_sixteen_reals_eight_complex_with_mulf rsi, 8*64, 64, 2*64, 4*64
	yr8_dispatch call, y8real_mulf, 64, 2*64, 4*64
	yloop_init  4, 1, 4		;; 3 more iterations
	yr8_8cl_eight_complex_with_mulf_preload
b3d:	yr8_8cl_eight_complex_with_mulf rsi, 8*64, 64, 2*64, 4*64
	yloop	4, b3d			;; Test loop counter
	end_timer 6
yfinish_unfft:
	bump	rsi, -4*8*64		;; Restore source pointer

;; Do inverse FFT levels 4,5
;; On input the 64-byte cache lines hold these data values:
;;	0	+64	+64	+64	8	+64	+64	+64
;;	1
;;	...
;;	7
;;	16	...
;;	32	...
;;	48	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+64	+64	+64	32	+64	+64	+64
;;	1
;;	...
;;	31

;; Do 8 eight_reals_unfft operations
;; Do 24 four_complex_unfft operations
;;	distance between fft data elements is 8
;;	do 8 macros each processing 32 data values

	start_timer 7
	mov	rdi, sincos1		;; Load sin/cos pointer
	yloop_init  8			;; 8 iterations
	yr4_4cl_eight_reals_four_complex_djbunfft_preload
c2b:	yr4_4cl_eight_reals_four_complex_djbunfft rsi, 64, 8*64, 16*64, rdi, YMM_SCD3, 8
	yloop	8, c2b, -8*64		;; Test loop counter, restore source pointer
	end_timer 7

;; Do inverse FFT levels 1,2,3
;; On input the 64-byte cache lines hold these data values:
;;	0	+64	+64	+64	32	+64	+64	+64
;;	1
;;	...
;;	31
;; These are swizzled to:
;;	0	+8	+8	+8	32	+8	+8	+8
;;	1	...
;;	...
;;	7
;;	64	...
;;	128	...
;;	192	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+8	+8	+8	128	+8	+8	+8
;;	1	...
;;	...
;;	7
;;	32	...
;;	64	...
;;	96	...

;; Do 32 eight_reals_unfft operations
;;	distance between fft data elements is 32
;;	do 8 macros each processing 32 data values

	start_timer 8
	yloop_init  8			;; 8 iterations
	yr4_s4cl_eight_reals_unfft_preload
c1b:	yr4_s4cl_eight_reals_unfft rsi, 64, 8*64, 16*64, rdi, YMM_SCD3, 8
	yloop	8, c1b			;; Test loop counter
	end_timer 8

	ytwo_dest_adjust
	yfft_3_ret
	ENDM

;; Perform a 256-element negacyclic FFT

yfft256p MACRO type
	LOCAL	ymid2p, ymid3p, ymid4p, yfinish_unfft
	LOCAL	b1b, b2b, b3a, b3b, b3c, b3d, c1b, c2b

;; Do a multiply with pre-FFTed inputs

	cmp	ffttype, 4
	je	ymid4p

;; Do FFT levels 1,2,3
;; On input the 64-byte cache lines hold these data values:
;;	0	+8	+8	+8	128	+8	+8	+8
;;	1
;;	...
;;	7
;;	32	...
;;	64	...
;;	96	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+8	+8	+8	32	+8	+8	+8
;;	1
;;	...
;;	7
;;	64	...
;;	128	...
;;	192	...
;; These are swizzled to:
;;	0	+64	+64	+64	32	+64	+64	+64
;;	1
;;	...
;;	31

;; Do 32 four_complex_first_fft operations
;;	distance between fft data elements is 32
;;	do 8 macros each processing 32 data values

	start_timer 4
	mov	rdi, sincos1		;; Load sin/cos pointer
	bump	rdi, 8*YMM_SCD2/4
	yloop_init  8			;; 8 iterations
	yr4_fs4cl_four_complex_first_djbfft_preload
b1b:	yr4_fs4cl_four_complex_first_djbfft rsi, 64, 8*64, 16*64, rdi, YMM_SCD4, 8
	yloop	8, b1b, -8*64		;; Test loop counter, restore source pointer
	end_timer 4

;; Do FFT levels 4,5
;; On input the 64-byte cache lines hold these data values:
;;	0	+64	+64	+64	32	+64	+64	+64
;;	1
;;	...
;;	31
;; On output the 64-byte cache lines hold these data values:
;;	0	+64	+64	+64	8	+64	+64	+64
;;	1
;;	...
;;	7
;;	16	...
;;	32	...
;;	48	...

;; Do 32 four_complex_fft operations
;;	distance between fft data elements is 8
;;	do 8 macros each processing 32 data values

	start_timer 5
	bump	rdi, -8*YMM_SCD4-8*YMM_SCD2/4 ;; Load sin/cos pointer
	yloop_init  8			;; 8 iterations
	yr4_b4cl_four_complex_djbfft_preload
b2b:	yr4_b4cl_four_complex_djbfft rsi, 64, 8*64, 16*64, rdi, YMM_SCD2/4, 8
	yloop	8, b2b, -8*64		;; Test loop counter, restore source pointer
	end_timer 5

;; Do FFT levels 6,7
;; On input and output the 64-byte cache lines hold these data values:
;;	0	+64	+64	+64	8	+64	+64	+64
;;	1
;;	...
;;	7
;;	16	...
;;	32	...
;;	48	...

;; Do 16 eight_complex_fft operations
;;	distance between fft data elements is 1
;;	do 4 macros each processing 64 data values

	cmp	ffttype, 2
	je	ymid2p
	ja	ymid3p

	start_timer 6
	yloop_init  4
b3a:	yr8_8cl_eight_complex_fft_final rsi, 8*64, 64, 2*64, 4*64
	yloop	4, b3a			;; Test loop counter
	end_timer 6
	yfft_1_ret

ymid2p:	start_timer 6
	yloop_init  4			;; 4 iterations
	yr8_8cl_eight_complex_with_square_preload
b3b:	yr8_8cl_eight_complex_with_square rsi, 8*64, 64, 2*64, 4*64
	yloop	4, b3b			;; Test loop counter
	end_timer 6
	jmp	yfinish_unfft

ymid3p:	start_timer 6
x64	mov	r8, DEST2ARG
	mov	rbp, DIST_TO_MULSRCARG
	yloop_init  4			;; 4 iterations
	yr8_8cl_eight_complex_with_mult_preload
b3c:	yr8_8cl_eight_complex_with_mult rsi, 8*64, 64, 2*64, 4*64
	yloop	4, b3c			;; Test loop counter
x64	add	rsi, r8
	end_timer 6
	jmp	yfinish_unfft

ymid4p:	start_timer 6
	mov	rbp, DIST_TO_MULSRCARG
	yloop_init  4			;; 4 iterations
	yr8_8cl_eight_complex_with_mulf_preload
b3d:	yr8_8cl_eight_complex_with_mulf rsi, 8*64, 64, 2*64, 4*64
	yloop	4, b3d			;; Test loop counter
	end_timer 6
yfinish_unfft:
	bump	rsi, -4*8*64		;; Restore source pointer

;; Do inverse FFT levels 4,5
;; On input the 64-byte cache lines hold these data values:
;;	0	+64	+64	+64	8	+64	+64	+64
;;	1
;;	...
;;	7
;;	16	...
;;	32	...
;;	48	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+64	+64	+64	32	+64	+64	+64
;;	1
;;	...
;;	31

;; Do 32 four_complex_unfft operations
;;	distance between fft data elements is 8
;;	do 8 macros each processing 32 data values

	start_timer 7
	mov	rdi, sincos1		;; Load sin/cos pointer
	yloop_init  8			;; 8 iterations
	yr4_b4cl_four_complex_djbunfft_preload
c2b:	yr4_b4cl_four_complex_djbunfft rsi, 64, 8*64, 16*64, rdi, YMM_SCD2/4, 8
	yloop	8, c2b, -8*64		;; Test loop counter, restore source pointer
	end_timer 7

;; Do inverse FFT levels 1,2,3
;; On input the 64-byte cache lines hold these data values:
;;	0	+64	+64	+64	32	+64	+64	+64
;;	1
;;	...
;;	31
;; These are swizzled to:
;;	0	+8	+8	+8	32	+8	+8	+8
;;	1	...
;;	...
;;	7
;;	64	...
;;	128	...
;;	192	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+8	+8	+8	128	+8	+8	+8
;;	1	...
;;	...
;;	7
;;	32	...
;;	64	...
;;	96	...

;; Do 32 four_complex_last_unfft operations
;;	distance between fft data elements is 32
;;	do 8 macros each processing 32 data values

	start_timer 8
	yloop_init  8			;; 8 iterations
	yr4_s4cl_four_complex_last_unfft_preload
c1b:	yr4_s4cl_four_complex_last_unfft rsi, 64, 8*64, 16*64, rdi, YMM_SCD4, 8
	yloop	8, c1b			;; Test loop counter
	end_timer 8

	ytwo_dest_adjust
	yfft_3_ret
	ENDM


;; Perform a 320-element FFT.

yfft320 MACRO type
	LOCAL	ymid2, ymid3, ymid4, yfinish_unfft
	LOCAL	b1b, b2b, b3a, b3b, b3c, b3d, c1b, c2b

;; Do a multiply with pre-FFTed inputs

	cmp	ffttype, 4
	je	ymid4

;; Do FFT levels 1,2,3
;; On input the 64-byte cache lines hold these data values:
;;	0	+10	+10	+10	160	+10	+10	+10
;;	1
;;	...
;;	9
;;	40	...
;;	80	...
;;	120	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+10	+10	+10	40	+10	+10	+10
;;	1
;;	...
;;	9
;;	80	...
;;	160	...
;;	240	...
;; These are swizzled to:
;;	0	+80	+80	+80	40	+80	+80	+80
;;	1
;;	...
;;	39

;; Do 40 eight_reals_fft operations
;;	distance between fft data elements is 40
;;	do 10 macros each processing 32 data values

	start_timer 4
	mov	rdi, sincos1		;; Load sin/cos pointer
	bump	rdi, 8*YMM_SCD4
	yloop_init  10			;; 10 iterations
	yr4_fs4cl_eight_reals_fft_preload
b1b:	yr4_fs4cl_eight_reals_fft rsi, 64, 10*64, 20*64, rdi, YMM_SCD3, 10
	yloop	10, b1b, -10*64		;; Test loop counter, restore source pointer
	end_timer 4

;; Do FFT levels 4,5
;; On input the 64-byte cache lines hold these data values:
;;	0	+80	+80	+80	40	+80	+80	+80
;;	1
;;	...
;;	39
;; On output the 64-byte cache lines hold these data values:
;;	0	+80	+80	+80	8	+80	+80	+80
;;	1
;;	...
;;	7
;;	16	...
;;	...
;;	64	...

;; Do 8 ten_reals_fft operations
;; Do 24 five_complex_fft operations
;;	distance between fft data elements is 8
;;	do 8 macros each processing 40 data values

	start_timer 5
	bump	rdi, -10*YMM_SCD3-8*YMM_SCD4 ;; Load sin/cos pointer
	yloop_init  8			;; 8 iterations
	yr5_5cl_ten_reals_five_complex_djbfft_preload
b2b:	yr5_5cl_ten_reals_five_complex_djbfft rsi, 64, 8*64, rdi, YMM_SCD4, 8
	yloop	8, b2b, -8*64		;; Test loop counter, restore source pointer
	end_timer 5

;; Do FFT levels 6,7
;; On input and output the 64-byte cache lines hold these data values:
;;	0	+80	+80	+80	8	+80	+80	+80
;;	1
;;	...
;;	7
;;	16	...
;;	...
;;	64	...

;; Do 1 sixteen_reals_fft operation
;; Do 19 eight_complex_fft operations
;;	distance between fft data elements is 1
;;	do 5 macros each processing 64 data values

	cmp	ffttype, 2
	je	ymid2
	ja	ymid3

	start_timer 6
;;	yr8_8cl_sixteen_reals_eight_complex_fft_final rsi, 8*64, 64, 2*64, 4*64
	yr8_dispatch call, y8real_fft_final, 64, 2*64, 4*64
	yloop_init  4			;; 4 more iterations
b3a:	yr8_8cl_eight_complex_fft_final rsi, 8*64, 64, 2*64, 4*64
	yloop	4, b3a			;; Test loop counter
	end_timer 6
	yfft_1_ret

ymid2:	start_timer 6
;;	yr8_8cl_sixteen_reals_eight_complex_with_square rsi, 8*64, 64, 2*64, 4*64
	yr8_dispatch call, y8real_square, 64, 2*64, 4*64
	yloop_init  4			;; 4 more iterations
	yr8_8cl_eight_complex_with_square_preload
b3b:	yr8_8cl_eight_complex_with_square rsi, 8*64, 64, 2*64, 4*64
	yloop	4, b3b			;; Test loop counter
	end_timer 6
	jmp	yfinish_unfft

ymid3:	start_timer 6
x64	mov	r8, DEST2ARG
	mov	rbp, DIST_TO_MULSRCARG
;;	yr8_8cl_sixteen_reals_eight_complex_with_mult rsi, 8*64, 64, 2*64, 4*64
	yr8_dispatch call, y8real_mult, 64, 2*64, 4*64
	yloop_init  4			;; 4 more iterations
	yr8_8cl_eight_complex_with_mult_preload
b3c:	yr8_8cl_eight_complex_with_mult rsi, 8*64, 64, 2*64, 4*64
	yloop	4, b3c			;; Test loop counter
x64	add	rsi, r8
	end_timer 6
	jmp	yfinish_unfft

ymid4:	start_timer 6
	mov	rbp, DIST_TO_MULSRCARG
;;	yr8_8cl_sixteen_reals_eight_complex_with_mulf rsi, 8*64, 64, 2*64, 4*64
	yr8_dispatch call, y8real_mulf, 64, 2*64, 4*64
	yloop_init  4			;; 4 more iterations
	yr8_8cl_eight_complex_with_mulf_preload
b3d:	yr8_8cl_eight_complex_with_mulf rsi, 8*64, 64, 2*64, 4*64
	yloop	4, b3d			;; Test loop counter
	end_timer 6
yfinish_unfft:
	bump	rsi, -5*8*64		;; Restore source pointer

;; Do inverse FFT levels 4,5
;; On input the 64-byte cache lines hold these data values:
;;	0	+80	+80	+80	8	+80	+80	+80
;;	1
;;	...
;;	7
;;	16	...
;;	...
;;	64	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+80	+80	+80	40	+80	+80	+80
;;	1
;;	...
;;	39

;; Do 8 five_reals_unfft operations
;; Do 24 five_complex_unfft operations
;;	distance between fft data elements is 8
;;	do 8 macros each processing 40 data values

	start_timer 7
	mov	rdi, sincos1		;; Load sin/cos pointer
	yloop_init  8			;; 8 iterations
	yr5_5cl_ten_reals_five_complex_djbunfft_preload
c2b:	yr5_5cl_ten_reals_five_complex_djbunfft rsi, 64, 8*64, rdi, YMM_SCD4, 8
	yloop	8, c2b, -8*64		;; Test loop counter, restore source pointer
	end_timer 7

;; Do inverse FFT levels 1,2,3
;; On input the 64-byte cache lines hold these data values:
;;	0	+80	+80	+80	40	+80	+80	+80
;;	1
;;	...
;;	39
;; These are swizzled to:
;;	0	+10	+10	+10	40	+10	+10	+10
;;	1	...
;;	...
;;	9
;;	80	...
;;	160	...
;;	240	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+10	+10	+10	160	+10	+10	+10
;;	1	...
;;	...
;;	9
;;	40	...
;;	80	...
;;	120	...

;; Do 40 eight_reals_unfft operations
;;	distance between fft data elements is 40
;;	do 10 macros each processing 32 data values

	start_timer 8
	yloop_init  10			;; 10 iterations
	yr4_s4cl_eight_reals_unfft_preload
c1b:	yr4_s4cl_eight_reals_unfft rsi, 64, 10*64, 20*64, rdi, YMM_SCD3, 10
	yloop	10, c1b			;; Test loop counter
	end_timer 8

	ytwo_dest_adjust
	yfft_3_ret
	ENDM

;; Perform a 320-element negacyclic FFT

yfft320p MACRO type
	LOCAL	ymid2p, ymid3p, ymid4p, yfinish_unfft
	LOCAL	b1b, b2b, b3a, b3b, b3c, b3d, c1b, c2b

;; Do a multiply with pre-FFTed inputs

	cmp	ffttype, 4
	je	ymid4p

;; Do FFT levels 1,2,3
;; On input the 64-byte cache lines hold these data values:
;;	0	+10	+10	+10	160	+10	+10	+10
;;	1
;;	...
;;	9
;;	40	...
;;	80	...
;;	120	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+10	+10	+10	40	+10	+10	+10
;;	1
;;	...
;;	9
;;	80	...
;;	160	...
;;	240	...
;; These are swizzled to:
;;	0	+80	+80	+80	40	+80	+80	+80
;;	1
;;	...
;;	39

;; Do 40 four_complex_first_fft operations
;;	distance between fft data elements is 40
;;	do 10 macros each processing 32 data values

	start_timer 4
	mov	rdi, sincos1		;; Load sin/cos pointer
	bump	rdi, 8*YMM_SCD2/4
	yloop_init  10			;; 10 iterations
	yr4_fs4cl_four_complex_first_djbfft_preload
b1b:	yr4_fs4cl_four_complex_first_djbfft rsi, 64, 10*64, 20*64, rdi, YMM_SCD4, 10
	yloop	10, b1b, -10*64		;; Test loop counter, restore source pointer
	end_timer 4

;; Do FFT levels 4,5
;; On input the 64-byte cache lines hold these data values:
;;	0	+80	+80	+80	40	+80	+80	+80
;;	1
;;	...
;;	39
;; On output the 64-byte cache lines hold these data values:
;;	0	+80	+80	+80	8	+80	+80	+80
;;	1
;;	...
;;	7
;;	16	...
;;	...
;;	64	...

;; Do 32 five_complex_fft operations
;;	distance between fft data elements is 4
;;	do 8 macros each processing 40 data values

	start_timer 5
	bump	rdi, -10*YMM_SCD4-8*YMM_SCD2/4 ;; Load sin/cos pointer
	yloop_init  8			;; 8 iterations
	yr5_b5cl_five_complex_djbfft_preload
b2b:	yr5_b5cl_five_complex_djbfft rsi, 64, 8*64, rdi, YMM_SCD2/4, 8
	yloop	8, b2b, -8*64		;; Test loop counter, restore source pointer
	end_timer 5

;; Do FFT levels 6,7
;; On input and output the 64-byte cache lines hold these data values:
;;	0	+80	+80	+80	8	+80	+80	+80
;;	1
;;	...
;;	7
;;	16	...
;;	...
;;	64	...

;; Do 20 eight_complex_fft operations
;;	distance between fft data elements is 1
;;	do 5 macros each processing 64 data values

	cmp	ffttype, 2
	je	ymid2p
	ja	ymid3p

	start_timer 6
	yloop_init  5			;; 5 iterations
b3a:	yr8_8cl_eight_complex_fft_final rsi, 8*64, 64, 2*64, 4*64
	yloop	5, b3a			;; Test loop counter
	end_timer 6
	yfft_1_ret

ymid2p:	start_timer 6
	yloop_init  5			;; 5 iterations
	yr8_8cl_eight_complex_with_square_preload
b3b:	yr8_8cl_eight_complex_with_square rsi, 8*64, 64, 2*64, 4*64
	yloop	5, b3b			;; Test loop counter
	end_timer 6
	jmp	yfinish_unfft

ymid3p:	start_timer 6
x64	mov	r8, DEST2ARG
	mov	rbp, DIST_TO_MULSRCARG
	yloop_init  5			;; 5 iterations
	yr8_8cl_eight_complex_with_mult_preload
b3c:	yr8_8cl_eight_complex_with_mult rsi, 8*64, 64, 2*64, 4*64
	yloop	5, b3c			;; Test loop counter
x64	add	rsi, r8
	end_timer 6
	jmp	yfinish_unfft

ymid4p:	start_timer 6
	mov	rbp, DIST_TO_MULSRCARG
	yloop_init  5			;; 5 iterations
	yr8_8cl_eight_complex_with_mulf_preload
b3d:	yr8_8cl_eight_complex_with_mulf rsi, 8*64, 64, 2*64, 4*64
	yloop	5, b3d			;; Test loop counter
	end_timer 6
yfinish_unfft:
	bump	rsi, -5*8*64		;; Restore source pointer

;; Do inverse FFT levels 4,5
;; On input the 64-byte cache lines hold these data values:
;;	0	+80	+80	+80	8	+80	+80	+80
;;	1
;;	...
;;	7
;;	16	...
;;	...
;;	64	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+80	+80	+80	40	+80	+80	+80
;;	1
;;	...
;;	39

;; Do 32 five_complex_unfft operations
;;	distance between fft data elements is 8
;;	do 8 macros each processing 40 data values

	start_timer 7
	mov	rdi, sincos1		;; Load sin/cos pointer
	yloop_init  8			;; 8 iterations
	yr5_b5cl_five_complex_djbunfft_preload
c2b:	yr5_b5cl_five_complex_djbunfft rsi, 64, 8*64, rdi, YMM_SCD2/4, 8
	yloop	8, c2b, -8*64		;; Test loop counter, restore source pointer
	end_timer 7

;; Do inverse FFT levels 1,2,3
;; On input the 64-byte cache lines hold these data values:
;;	0	+80	+80	+80	40	+80	+80	+80
;;	1
;;	...
;;	39
;; These are swizzled to:
;;	0	+10	+10	+10	40	+10	+10	+10
;;	1	...
;;	...
;;	9
;;	80	...
;;	160	...
;;	240	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+10	+10	+10	160	+10	+10	+10
;;	1	...
;;	...
;;	9
;;	40	...
;;	80	...
;;	120	...

;; Do 40 four_complex_last_unfft operations
;;	distance between fft data elements is 40
;;	do 10 macros each processing 32 data values

	start_timer 8
	yloop_init  10			;; 10 iterations
	yr4_s4cl_four_complex_last_unfft_preload
c1b:	yr4_s4cl_four_complex_last_unfft rsi, 64, 10*64, 20*64, rdi, YMM_SCD4, 10
	yloop	10, c1b			;; Test loop counter
	end_timer 8

	ytwo_dest_adjust
	yfft_3_ret
	ENDM


;; Perform a 384-element FFT.

yfft384 MACRO type
	LOCAL	ymid2, ymid3, ymid4, yfinish_unfft
	LOCAL	b1b, b2b, b3a, b3b, b4a, b4b, b4c, b4d, c1b, c2b, c3a, c3b

;; Do a multiply with pre-FFTed inputs

	cmp	ffttype, 4
	je	ymid4

;; Do FFT levels 1,2,3
;; On input the 64-byte cache lines hold these data values:
;;	0	12	24	36	192	+12	+12	+12
;;	1
;;	...
;;	11
;;	48	...
;;	96	...
;;	144	...
;; On output the 64-byte cache lines hold these data values:
;;	0	12	24	36	48	+12	+12	+12
;;	1
;;	...
;;	11
;;	96	...
;;	192	...
;;	288	...
;; These are swizzled to:
;;	0	96	192	288	48	+96	+96	+96
;;	1
;;	...
;;	47

;; Do 48 eight_reals_fft operations
;;	distance between fft data elements is 48
;;	do 12 macros each processing 32 data values

	start_timer 4
	mov	rdi, sincos1		;; Load sin/cos pointer
	bump	rdi, 16*YMM_SCD2+4*YMM_SCD3
	yloop_init  12			;; 12 iterations
	yr4_fs4cl_eight_reals_fft_preload
b1b:	yr4_fs4cl_eight_reals_fft rsi, 64, 12*64, 24*64, rdi, YMM_SCD3, 12
	yloop	12, b1b, -12*64		;; Test loop counter, restore source pointer
	end_timer 4

;; Do FFT levels 4,5
;; On input the 64-byte cache lines hold these data values:
;;	0	96	192	288	48	+96	+96	+96
;;	1
;;	...
;;	47
;; On output the 64-byte cache lines hold these data values:
;;	0	96	192	288	16	+96	+96	+96
;;	1
;;	...
;;	15
;;	32	...
;;	64	...

;; Do 16 six_reals_fft operations
;; Do 48 three_complex_fft operations
;;	distance between fft data elements is 16
;;	do 16 macros each processing 24 data values

	start_timer 5
	bump	rdi, -12*YMM_SCD3-16*YMM_SCD2 ;; Load sin/cos pointer
	yloop_init  16			;; 16 iterations
	yr3_3cl_six_reals_three_complex_djbfft_preload
b2b:	yr3_3cl_six_reals_three_complex_djbfft rsi, 64, 16*64, rdi, YMM_SCD2, 16
	yloop	16, b2b, -16*64		;; Test loop counter, restore source pointer
	end_timer 5

;; Do FFT levels 6,7
;; On input the 64-byte cache lines hold these data values:
;;	0	96	192	288	16	+96	+96	+96
;;	1
;;	...
;;	15
;;	32	...
;;	64	...
;; On output the 64-byte cache lines hold these data values:
;;	0	96	192	288	4	+96	+96	+96
;;	1
;;	...
;;	3
;;	8	...
;;	...
;;	88	...

;; Do 4 eight_reals_fft operations
;; Do 44 four_complex_fft operations
;;	distance between fft data elements is 4
;;	do 12 macros each processing 32 data values

	start_timer 6
	bump	rdi, -16*YMM_SCD2-4*YMM_SCD3 ;; Next sin/cos pointer
	yloop_init  12			;; 3 iterations of 4
	yr4_4cl_eight_reals_four_complex_djbfft_preload
b3a:	yr4_4cl_eight_reals_four_complex_djbfft rsi, 64, 4*64, 8*64, rdi, YMM_SCD3, 4
	yloop	4, b3a, -4*64+16*64, rdi, -4*YMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer

	loops_reset			;; 2 more iters of 4
	yr4_rb4cl_four_complex_djbfft_preload
b3b:	yr4_rb4cl_four_complex_djbfft rsi, 64, 4*64, 8*64, rdi, YMM_SCD3, 4
	yloop	4, b3b, -4*64+16*64, rdi, -4*YMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer
	yloop	3, b3b, -3*16*64	;; Test loop counter, restore source pointer
	end_timer 6

;; Do FFT levels 8,9
;; On input and output the 64-byte cache lines hold these data values:
;;	0	96	192	288	4	+96	+96	+96
;;	1
;;	...
;;	3
;;	8	...
;;	...
;;	88	...

;; Do 1 eight_reals_fft operation
;; Do 47 four_complex_fft operations
;;	distance between fft data elements is 1
;;	do 12 macros each processing 32 data values

	cmp	ffttype, 2
	je	ymid2
	ja	ymid3

	start_timer 7
;;	yr4_4cl_eight_reals_four_complex_fft_final rsi, 4*64, 64, 2*64
	yr4_dispatch call, yreal_fft_final, 64, 2*64
	yloop_init  11			;; 11 more iterations
b4a:	yr4_4cl_four_complex_fft_final rsi, 4*64, 64, 2*64
	yloop	11, b4a			;; Test loop counter
	end_timer 7
	yfft_1_ret

ymid2:	start_timer 7
;;	yr4_4cl_eight_reals_four_complex_with_square rsi, 4*64, 64, 2*64
	yr4_dispatch call, yreal_square, 64, 2*64
	yloop_init  11			;; 11 more iterations
	yr4_4cl_four_complex_with_square_preload
b4b:	yr4_4cl_four_complex_with_square rsi, 4*64, 64, 2*64, 11
	yloop	11, b4b			;; Test loop counter
	end_timer 7
	jmp	yfinish_unfft

ymid3:	start_timer 7
x64	mov	r8, DEST2ARG
	mov	rbp, DIST_TO_MULSRCARG
;;	yr4_4cl_eight_reals_four_complex_with_mult rsi, 4*64, 64, 2*64
	yr4_dispatch call, yreal_mult, 64, 2*64
	yloop_init  11			;; 11 more iterations
	yr4_4cl_four_complex_with_mult_preload
b4c:	yr4_4cl_four_complex_with_mult rsi, 4*64, 64, 2*64
	yloop	11, b4c			;; Test loop counter
x64	add	rsi, r8
	end_timer 7
	jmp	yfinish_unfft

ymid4:	start_timer 7
	mov	rbp, DIST_TO_MULSRCARG
;;	yr4_4cl_eight_reals_four_complex_with_mulf rsi, 4*64, 64, 2*64
	yr4_dispatch call, yreal_mulf, 64, 2*64
	yloop_init  11			;; 11 more iterations
	yr4_4cl_four_complex_with_mulf_preload
b4d:	yr4_4cl_four_complex_with_mulf rsi, 4*64, 64, 2*64
	yloop	11, b4d			;; Test loop counter
	end_timer 7
yfinish_unfft:
	bump	rsi, -12*4*64		;; Restore source pointer

;; Do inverse FFT levels 6,7
;; On input the 64-byte cache lines hold these data values:
;;	0	96	192	288	4	+96	+96	+96
;;	1
;;	...
;;	3
;;	8	...
;;	...
;;	88	...
;; On output the 64-byte cache lines hold these data values:
;;	0	96	192	288	16	+96	+96	+96
;;	1
;;	...
;;	15
;;	32	...
;;	64	...

;; Do 4 eight_reals_unfft operations
;; Do 44 four_complex_unfft operations
;;	distance between fft data elements is 4
;;	do 12 macros each processing 32 data values

	start_timer 8
	mov	rdi, sincos1		;; Load sin/cos pointer
	yloop_init  12			;; 3 iterations of 4
	yr4_4cl_eight_reals_four_complex_djbunfft_preload
c3a:	yr4_4cl_eight_reals_four_complex_djbunfft rsi, 64, 4*64, 8*64, rdi, YMM_SCD3, 4
	yloop	4, c3a, -4*64+16*64, rdi, -4*YMM_SCD3 ;; Test loop counter, next source pointer, restore sine/cosine pointer

	loops_reset			;; 2 more iters of 4
	yr4_rb4cl_four_complex_djbunfft_preload
c3b:	yr4_rb4cl_four_complex_djbunfft rsi, 64, 4*64, 8*64, rdi, YMM_SCD3, 4
	yloop	4, c3b, -4*64+16*64, rdi, -4*YMM_SCD3 ;; Test loop counter, next source pointer, restore sine/cosine pointer
	yloop	3, c3b, -3*16*64	;; Test loop counter, restore source pointer
	end_timer 8

;; Do inverse FFT levels 4,5
;; On input the 64-byte cache lines hold these data values:
;;	0	96	192	288	16	+96	+96	+96
;;	1
;;	...
;;	15
;;	32	...
;;	64	...
;; On output the 64-byte cache lines hold these data values:
;;	0	96	192	288	48	+96	+96	+96
;;	1
;;	...
;;	47

;; Do 16 six_reals_unfft operations
;; Do 48 three_complex_unfft operations
;;	distance between fft data elements is 16
;;	do 16 macros each processing 24 data values

	start_timer 9
	bump	rdi, 4*YMM_SCD3		;; Next sine/cosine pointer
	yloop_init  16			;; 16 iterations
	yr3_3cl_six_reals_three_complex_djbunfft_preload
c2b:	yr3_3cl_six_reals_three_complex_djbunfft rsi, 64, 16*64, rdi, YMM_SCD2, 16
	yloop	16, c2b, -16*64		;; Test loop counter, restore source pointer
	end_timer 9

;; Do inverse FFT levels 1,2,3
;; On input the 64-byte cache lines hold these data values:
;;	0	96	192	288	48	+96	+96	+96
;;	1
;;	...
;;	47
;; These are swizzled to:
;;	0	12	24	36	48	+12	+12	+12
;;	1	...
;;	...
;;	11
;;	96	...
;;	192	...
;;	288	...
;; On output the 64-byte cache lines hold these data values:
;;	0	12	24	36	192	+12	+12	+12
;;	1	...
;;	...
;;	11
;;	48	...
;;	96	...
;;	144	...

;; Do 48 eight_reals_unfft operations
;;	distance between fft data elements is 48
;;	do 12 macros each processing 32 data values

	start_timer 10
	yloop_init  12			;; 12 iterations
	yr4_s4cl_eight_reals_unfft_preload
c1b:	yr4_s4cl_eight_reals_unfft rsi, 64, 12*64, 24*64, rdi, YMM_SCD3, 12
	yloop	12, c1b			;; Test loop counter
	end_timer 10

	ytwo_dest_adjust
	yfft_3_ret
	ENDM

;; Perform a 384-element negacyclic FFT

yfft384p MACRO type
	LOCAL	ymid2p, ymid3p, ymid4p, yfinish_unfft
	LOCAL	b1b, b2b, b3b, b4a, b4b, b4c, b4d, c1b, c2b, c3b

;; Do a multiply with pre-FFTed inputs

	cmp	ffttype, 4
	je	ymid4p

;; Do FFT levels 1,2,3
;; On input the 64-byte cache lines hold these data values:
;;	0	12	24	36	192	+12	+12	+12
;;	1
;;	...
;;	11
;;	48	...
;;	96	...
;;	144	...
;; On output the 64-byte cache lines hold these data values:
;;	0	12	24	36	48	+12	+12	+12
;;	1
;;	...
;;	11
;;	96	...
;;	192	...
;;	288	...
;; These are swizzled to:
;;	0	96	192	288	48	+96	+96	+96
;;	1
;;	...
;;	47

;; Do 48 four_complex_first_fft operations
;;	distance between fft data elements is 48
;;	do 12 macros each processing 32 data values

	start_timer 4
	mov	rdi, sincos1		;; Load sin/cos pointer
	bump	rdi, 16*YMM_SCD1/4+4*YMM_SCD2/4
	yloop_init  12			;; 12 iterations
	yr4_fs4cl_four_complex_first_djbfft_preload
b1b:	yr4_fs4cl_four_complex_first_djbfft rsi, 64, 12*64, 24*64, rdi, YMM_SCD4, 12
	yloop	12, b1b, -12*64		;; Test loop counter, restore source pointer
	end_timer 4

;; Do FFT levels 4,5
;; On input the 64-byte cache lines hold these data values:
;;	0	96	192	288	48	+96	+96	+96
;;	1
;;	...
;;	47
;; On output the 64-byte cache lines hold these data values:
;;	0	96	192	288	16	+96	+96	+96
;;	1
;;	...
;;	15
;;	32	...
;;	64	...

;; Do 64 three_complex_fft operations
;;	distance between fft data elements is 16
;;	do 16 macros each processing 24 data values

	start_timer 5
	bump	rdi, -12*YMM_SCD4-16*YMM_SCD1/4 ;; Load sin/cos pointer
	yloop_init  16			;; 16 iterations
	yr3_b3cl_three_complex_djbfft_preload
b2b:	yr3_b3cl_three_complex_djbfft rsi, 64, 16*64, rdi, YMM_SCD1/4, 16
	yloop	16, b2b, -16*64		;; Test loop counter, restore source pointer
	end_timer 5

;; Do FFT levels 6,7
;; On input the 64-byte cache lines hold these data values:
;;	0	96	192	288	16	+96	+96	+96
;;	1
;;	...
;;	15
;;	32	...
;;	64	...
;; On output the 64-byte cache lines hold these data values:
;;	0	96	192	288	4	+96	+96	+96
;;	1
;;	...
;;	3
;;	8	...
;;	...
;;	88	...

;; Do 48 four_complex_fft operations
;;	distance between fft data elements is 4
;;	do 12 macros each processing 32 data values

	start_timer 6
	bump	rdi, -16*YMM_SCD1/4-4*YMM_SCD2/4 ;; Next sin/cos pointer
	yloop_init  12			;; 3 iterations of 4
	yr4_b4cl_four_complex_djbfft_preload
b3b:	yr4_b4cl_four_complex_djbfft rsi, 64, 4*64, 8*64, rdi, YMM_SCD2/4, 4
	yloop	4, b3b, -4*64+16*64, rdi, -4*YMM_SCD2/4 ;; Test loop counter, next source pointer, restore sin/cos pointer
	yloop	3, b3b, -3*16*64	;; Test loop counter, restore source pointer
	end_timer 6

;; Do FFT levels 8,9
;; On input and output the 64-byte cache lines hold these data values:
;;	0	96	192	288	4	+96	+96	+96
;;	1
;;	...
;;	3
;;	8	...
;;	...
;;	88	...

;; Do 48 four_complex_fft operations
;;	distance between fft data elements is 1
;;	do 12 macros each processing 32 data values

	cmp	ffttype, 2
	je	ymid2p
	ja	ymid3p

	start_timer 7
	yloop_init  12			;; 12 iterations
b4a:	yr4_4cl_four_complex_fft_final rsi, 4*64, 64, 2*64
	yloop	12, b4a			;; Test loop counter
	end_timer 7
	yfft_1_ret

ymid2p:	start_timer 7
	yloop_init  12			;; 12 iterations
	yr4_4cl_four_complex_with_square_preload
b4b:	yr4_4cl_four_complex_with_square rsi, 4*64, 64, 2*64, 12
	yloop	12, b4b			;; Test loop counter
	end_timer 7
	jmp	yfinish_unfft

ymid3p:	start_timer 7
x64	mov	r8, DEST2ARG
	mov	rbp, DIST_TO_MULSRCARG
	yloop_init  12			;; 12 iterations
	yr4_4cl_four_complex_with_mult_preload
b4c:	yr4_4cl_four_complex_with_mult rsi, 4*64, 64, 2*64
	yloop	12, b4c			;; Test loop counter
x64	add	rsi, r8
	end_timer 7
	jmp	yfinish_unfft

ymid4p:	start_timer 7
	mov	rbp, DIST_TO_MULSRCARG
	yloop_init  12			;; 12 iterations
	yr4_4cl_four_complex_with_mulf_preload
b4d:	yr4_4cl_four_complex_with_mulf rsi, 4*64, 64, 2*64
	yloop	12, b4d			;; Test loop counter
	end_timer 7
yfinish_unfft:
	bump	rsi, -12*4*64		;; Restore source pointer

;; Do inverse FFT levels 6,7
;; On input the 64-byte cache lines hold these data values:
;;	0	96	192	288	4	+96	+96	+96
;;	1
;;	...
;;	3
;;	8	...
;;	...
;;	88	...
;; On output the 64-byte cache lines hold these data values:
;;	0	96	192	288	16	+96	+96	+96
;;	1
;;	...
;;	15
;;	32	...
;;	64	...

;; Do 48 four_complex_unfft operations
;;	distance between fft data elements is 4
;;	do 12 macros each processing 32 data values

	start_timer 8
	mov	rdi, sincos1		;; Load sin/cos pointer
	yloop_init  12			;; 3 iterations of 4
	yr4_b4cl_four_complex_djbunfft_preload
c3b:	yr4_b4cl_four_complex_djbunfft rsi, 64, 4*64, 8*64, rdi, YMM_SCD2/4, 4
	yloop	4, c3b, -4*64+16*64, rdi, -4*YMM_SCD2/4 ;; Test loop counter, next source pointer, restore sine/cosine pointer
	yloop	3, c3b, -3*16*64	;; Test loop counter, restore source pointer
	end_timer 8

;; Do inverse FFT levels 4,5
;; On input the 64-byte cache lines hold these data values:
;;	0	96	192	288	16	+96	+96	+96
;;	1
;;	...
;;	15
;;	32	...
;;	64	...
;; On output the 64-byte cache lines hold these data values:
;;	0	96	192	288	48	+96	+96	+96
;;	1
;;	...
;;	47

;; Do 64 three_complex_unfft operations
;;	distance between fft data elements is 16
;;	do 16 macros each processing 24 data values

	start_timer 9
	bump	rdi, 4*YMM_SCD2/4	;; Next sine/cosine pointer
	yloop_init  16			;; 16 iterations
	yr3_b3cl_three_complex_djbunfft_preload
c2b:	yr3_b3cl_three_complex_djbunfft rsi, 64, 16*64, rdi, YMM_SCD1/4, 16
	yloop	16, c2b, -16*64		;; Test loop counter, restore source pointer
	end_timer 9

;; Do inverse FFT levels 1,2,3
;; On input the 64-byte cache lines hold these data values:
;;	0	96	192	288	48	+96	+96	+96
;;	1
;;	...
;;	47
;; These are swizzled to:
;;	0	12	24	36	48	+12	+12	+12
;;	1	...
;;	...
;;	11
;;	96	...
;;	192	...
;;	288	...
;; On output the 64-byte cache lines hold these data values:
;;	0	12	24	36	192	+12	+12	+12
;;	1	...
;;	...
;;	11
;;	48	...
;;	96	...
;;	144	...

;; Do 48 four_complex_last_unfft operations
;;	distance between fft data elements is 48
;;	do 12 macros each processing 32 data values

	start_timer 10
	yloop_init  12			;; 12 iterations
	yr4_s4cl_four_complex_last_unfft_preload
c1b:	yr4_s4cl_four_complex_last_unfft rsi, 64, 12*64, 24*64, rdi, YMM_SCD4, 12
	yloop	12, c1b			;; Test loop counter
	end_timer 10

	ytwo_dest_adjust
	yfft_3_ret
	ENDM


;; Perform a 512-element FFT.

yfft512 MACRO type
	LOCAL	ymid2, ymid3, ymid4, yfinish_unfft
	LOCAL	b1b, b2b, b3a, b3b, b4a, b4b, b4c, b4d, c1b, c2b, c3a, c3b

;; Do a multiply with pre-FFTed inputs

	cmp	ffttype, 4
	je	ymid4

;; Do FFT levels 1,2,3
;; On input the 64-byte cache lines hold these data values:
;;	0	16	32	48	256	+16	+16	+16
;;	1
;;	...
;;	15
;;	64	...
;;	128	...
;;	192	...
;; On output the 64-byte cache lines hold these data values:
;;	0	16	32	48	64	+16	+16	+16
;;	1
;;	...
;;	15
;;	128	...
;;	256	...
;;	384	...
;; These are swizzled to:
;;	0	128	256	384	64	+128	+128	+128
;;	1
;;	...
;;	63

;; Do 64 eight_reals_fft operations
;;	distance between fft data elements is 64
;;	do 16 macros each processing 32 data values

	start_timer 4
	mov	rdi, sincos1		;; Load sin/cos pointer
	bump	rdi, 16*YMM_SCD3
	yloop_init  16			;; 16 iterations
	yr4_fs4cl_eight_reals_fft_preload
b1b:	yr4_fs4cl_eight_reals_fft rsi, 64, 16*64, 32*64, rdi, YMM_SCD3, 16
	yloop	16, b1b, -16*64		;; Test loop counter, restore source pointer
	end_timer 4

;; Do FFT levels 4,5
;; On input the 64-byte cache lines hold these data values:
;;	0	128	256	384	64	+128	+128	+128
;;	1
;;	...
;;	63
;; On output the 64-byte cache lines hold these data values:
;;	0	128	256	384	16	+128	+128	+128
;;	1
;;	...
;;	15
;;	32	...
;;	64	...
;;	96	...

;; Do 16 eight_reals_fft operations
;; Do 48 four_complex_fft operations
;;	distance between fft data elements is 16
;;	do 16 macros each processing 32 data values

	start_timer 5
	bump	rdi, -16*YMM_SCD3-16*YMM_SCD3 ;; Load sin/cos pointer
	yloop_init  16			;; 16 iterations
	yr4_4cl_eight_reals_four_complex_djbfft_preload
b2b:	yr4_4cl_eight_reals_four_complex_djbfft rsi, 64, 16*64, 32*64, rdi, YMM_SCD3, 16
	yloop	16, b2b, -16*64, rdi, -16*YMM_SCD3 ;; Test loop counter, restore source pointer, restore sin/cos pointer
	end_timer 5

;; Do FFT levels 6,7
;; On input the 64-byte cache lines hold these data values:
;;	0	128	256	384	16	+128	+128	+128
;;	1
;;	...
;;	15
;;	32	...
;;	64	...
;;	96	...
;; On output the 64-byte cache lines hold these data values:
;;	0	128	256	384	4	+128	+128	+128
;;	1
;;	...
;;	3
;;	8	...
;;	...
;;	120	...

;; Do 4 eight_reals_fft operations
;; Do 60 four_complex_fft operations
;;	distance between fft data elements is 4
;;	do 16 macros each processing 32 data values

	start_timer 6
	yloop_init  16			;; 4 iterations of 4
	yr4_4cl_eight_reals_four_complex_djbfft_preload
b3a:	yr4_4cl_eight_reals_four_complex_djbfft rsi, 64, 4*64, 8*64, rdi, 4*YMM_SCD3, 4
	yloop	4, b3a, -4*64+16*64, rdi, -4*4*YMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer

	loops_reset			;; 3 more iters of 4
	yr4_rb4cl_four_complex_djbfft_preload
b3b:	yr4_rb4cl_four_complex_djbfft rsi, 64, 4*64, 8*64, rdi, 4*YMM_SCD3, 4
	yloop	4, b3b, -4*64+16*64, rdi, -4*4*YMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer
	yloop	4, b3b, -4*16*64	;; Test loop counter, restore source pointer
	end_timer 6

;; Do FFT levels 8,9
;; On input and output the 64-byte cache lines hold these data values:
;;	0	128	256	384	4	+128	+128	+128
;;	1
;;	...
;;	3
;;	8	...
;;	...
;;	120	...

;; Do 1 eight_reals_fft operation
;; Do 63 four_complex_fft operations
;;	distance between fft data elements is 1
;;	do 16 macros each processing 32 data values

	cmp	ffttype, 2
	je	ymid2
	ja	ymid3

	start_timer 7
;;	yr4_4cl_eight_reals_four_complex_fft_final rsi, 4*64, 64, 2*64
	yr4_dispatch call, yreal_fft_final, 64, 2*64
	yloop_init  16, 1, 16		;; 15 more iterations
b4a:	yr4_4cl_four_complex_fft_final rsi, 4*64, 64, 2*64
	yloop	16, b4a			;; Test loop counter
	end_timer 7
	yfft_1_ret

ymid2:	start_timer 7
;;	yr4_4cl_eight_reals_four_complex_with_square rsi, 4*64, 64, 2*64
	yr4_dispatch call, yreal_square, 64, 2*64
	yloop_init  16, 1, 16		;; 15 more iterations
	yr4_4cl_four_complex_with_square_preload
b4b:	yr4_4cl_four_complex_with_square rsi, 4*64, 64, 2*64, 1
	yloop	16, b4b			;; Test loop counter
	end_timer 7
	jmp	yfinish_unfft

ymid3:	start_timer 7
x64	mov	r8, DEST2ARG
	mov	rbp, DIST_TO_MULSRCARG
;;	yr4_4cl_eight_reals_four_complex_with_mult rsi, 4*64, 64, 2*64
	yr4_dispatch call, yreal_mult, 64, 2*64
	yloop_init  16, 1, 16		;; 15 more iterations
	yr4_4cl_four_complex_with_mult_preload
b4c:	yr4_4cl_four_complex_with_mult rsi, 4*64, 64, 2*64
	yloop	16, b4c			;; Test loop counter
x64	add	rsi, r8
	end_timer 7
	jmp	yfinish_unfft

ymid4:	start_timer 7
	mov	rbp, DIST_TO_MULSRCARG
;;	yr4_4cl_eight_reals_four_complex_with_mulf rsi, 4*64, 64, 2*64
	yr4_dispatch call, yreal_mulf, 64, 2*64
	yloop_init  16, 1, 16		;; 15 more iterations
	yr4_4cl_four_complex_with_mulf_preload
b4d:	yr4_4cl_four_complex_with_mulf rsi, 4*64, 64, 2*64
	yloop	16, b4d			;; Test loop counter
	end_timer 7
yfinish_unfft:
	bump	rsi, -16*4*64		;; Restore source pointer

;; Do inverse FFT levels 6,7
;; On input the 64-byte cache lines hold these data values:
;;	0	128	256	384	4	+128	+128	+128
;;	1
;;	...
;;	3
;;	8	...
;;	...
;;	120	...
;; On output the 64-byte cache lines hold these data values:
;;	0	128	256	384	16	+128	+128	+128
;;	1
;;	...
;;	15
;;	32	...
;;	64	...
;;	96	...

;; Do 4 eight_reals_unfft operations
;; Do 60 four_complex_unfft operations
;;	distance between fft data elements is 4
;;	do 16 macros each processing 32 data values

	start_timer 8
	mov	rdi, sincos1		;; Load sin/cos pointer
	yloop_init  16			;; 4 iterations of 4
	yr4_4cl_eight_reals_four_complex_djbunfft_preload
c3a:	yr4_4cl_eight_reals_four_complex_djbunfft rsi, 64, 4*64, 8*64, rdi, 4*YMM_SCD3, 4
	yloop	4, c3a, -4*64+16*64, rdi, -4*4*YMM_SCD3 ;; Test loop counter, next source pointer, restore sine/cosine pointer

	loops_reset			;; 3 more iters of 4
	yr4_rb4cl_four_complex_djbunfft_preload
c3b:	yr4_rb4cl_four_complex_djbunfft rsi, 64, 4*64, 8*64, rdi, 4*YMM_SCD3, 4
	yloop	4, c3b, -4*64+16*64, rdi, -4*4*YMM_SCD3 ;; Test loop counter, next source pointer, restore sine/cosine pointer
	yloop	4, c3b, -4*16*64	;; Test loop counter, restore source pointer
	end_timer 8

;; Do inverse FFT levels 4,5
;; On input the 64-byte cache lines hold these data values:
;;	0	128	256	384	16	+128	+128	+128
;;	1
;;	...
;;	15
;;	32	...
;;	64	...
;;	96	...
;; On output the 64-byte cache lines hold these data values:
;;	0	128	256	384	64	+128	+128	+128
;;	1
;;	...
;;	63

;; Do 16 eight_reals_unfft operations
;; Do 48 four_complex_unfft operations
;;	distance between fft data elements is 16
;;	do 16 macros each processing 32 data values

	start_timer 9
	yloop_init  16			;; 16 iterations
	yr4_4cl_eight_reals_four_complex_djbunfft_preload
c2b:	yr4_4cl_eight_reals_four_complex_djbunfft rsi, 64, 16*64, 32*64, rdi, YMM_SCD3, 16
	yloop	16, c2b, -16*64		;; Test loop counter, restore source pointer
	end_timer 9

;; Do inverse FFT levels 1,2,3
;; On input the 64-byte cache lines hold these data values:
;;	0	128	256	384	64	+128	+128	+128
;;	1
;;	...
;;	63
;; These are swizzled to:
;;	0	16	32	48	64	+16	+16	+16
;;	1	...
;;	...
;;	15
;;	128	...
;;	256	...
;;	384	...
;; On output the 64-byte cache lines hold these data values:
;;	0	16	32	48	256	+16	+16	+16
;;	1	...
;;	...
;;	15
;;	64	...
;;	128	...
;;	192	...

;; Do 64 eight_reals_unfft operations
;;	distance between fft data elements is 64
;;	do 16 macros each processing 32 data values

	start_timer 10
	yloop_init  16			;; 16 iterations
	yr4_s4cl_eight_reals_unfft_preload
c1b:	yr4_s4cl_eight_reals_unfft rsi, 64, 16*64, 32*64, rdi, YMM_SCD3, 16
	yloop	16, c1b			;; Test loop counter
	end_timer 10

	ytwo_dest_adjust
	yfft_3_ret
	ENDM

;; Perform a 512-element negacyclic FFT

yfft512p MACRO type
	LOCAL	ymid2p, ymid3p, ymid4p, yfinish_unfft
	LOCAL	b1b, b2b, b3b, b4a, b4b, b4c, b4d, c1b, c2b, c3b

;; Do a multiply with pre-FFTed inputs

	cmp	ffttype, 4
	je	ymid4p

;; Do FFT levels 1,2,3
;; On input the 64-byte cache lines hold these data values:
;;	0	16	32	48	256	+16	+16	+16
;;	1
;;	...
;;	15
;;	64	...
;;	128	...
;;	192	...
;; On output the 64-byte cache lines hold these data values:
;;	0	16	32	48	64	+16	+16	+16
;;	1
;;	...
;;	15
;;	128	...
;;	256	...
;;	384	...
;; These are swizzled to:
;;	0	128	256	384	64	+128	+128	+128
;;	1
;;	...
;;	63

;; Do 64 four_complex_first_fft operations
;;	distance between fft data elements is 64
;;	do 16 macros each processing 32 data values

	start_timer 4
	mov	rdi, sincos1		;; Load sin/cos pointer
	bump	rdi, 16*YMM_SCD2/4
	yloop_init  16			;; 16 iterations
	yr4_fs4cl_four_complex_first_djbfft_preload
b1b:	yr4_fs4cl_four_complex_first_djbfft rsi, 64, 16*64, 32*64, rdi, YMM_SCD4, 16
	yloop	16, b1b, -16*64		;; Test loop counter, restore source pointer
	end_timer 4

;; Do FFT levels 4,5
;; On input the 64-byte cache lines hold these data values:
;;	0	128	256	384	64	+128	+128	+128
;;	1
;;	...
;;	63
;; On output the 64-byte cache lines hold these data values:
;;	0	128	256	384	16	+128	+128	+128
;;	1
;;	...
;;	15
;;	32	...
;;	64	...
;;	96	...

;; Do 64 four_complex_fft operations
;;	distance between fft data elements is 16
;;	do 16 macros each processing 32 data values

	start_timer 5
	bump	rdi, -16*YMM_SCD4-16*YMM_SCD2/4 ;; Load sin/cos pointer
	yloop_init  16			;; 16 iterations
	yr4_b4cl_four_complex_djbfft_preload
b2b:	yr4_b4cl_four_complex_djbfft rsi, 64, 16*64, 32*64, rdi, YMM_SCD2/4, 16
	yloop	16, b2b, -16*64, rdi, -16*YMM_SCD2/4 ;; Test loop counter, restore source pointer, restore sin/cos pointer
	end_timer 5

;; Do FFT levels 6,7
;; On input the 64-byte cache lines hold these data values:
;;	0	128	256	384	16	+128	+128	+128
;;	1
;;	...
;;	15
;;	32	...
;;	64	...
;;	96	...
;; On output the 64-byte cache lines hold these data values:
;;	0	128	256	384	4	+128	+128	+128
;;	1
;;	...
;;	3
;;	8	...
;;	...
;;	120	...

;; Do 64 four_complex_fft operations
;;	distance between fft data elements is 4
;;	do 16 macros each processing 32 data values

	start_timer 6
	yloop_init  16			;; 4 iterations of 4
	yr4_b4cl_four_complex_djbfft_preload
b3b:	yr4_b4cl_four_complex_djbfft rsi, 64, 4*64, 8*64, rdi, 4*YMM_SCD2/4, 4
	yloop	4, b3b, -4*64+16*64, rdi, -4*4*YMM_SCD2/4 ;; Test loop counter, next source pointer, restore sin/cos pointer
	yloop	4, b3b, -4*16*64	;; Test loop counter, restore source pointer
	end_timer 6

;; Do FFT levels 8,9
;; On input and output the 64-byte cache lines hold these data values:
;;	0	128	256	384	4	+128	+128	+128
;;	1
;;	...
;;	3
;;	8	...
;;	...
;;	120	...

;; Do 64 four_complex_fft operations
;;	distance between fft data elements is 1
;;	do 16 macros each processing 32 data values

	cmp	ffttype, 2
	je	ymid2p
	ja	ymid3p

	start_timer 7
	yloop_init  16			;; 16 iterations
b4a:	yr4_4cl_four_complex_fft_final rsi, 4*64, 64, 2*64
	yloop	16, b4a			;; Test loop counter
	end_timer 7
	yfft_1_ret

ymid2p:	start_timer 7
	yloop_init  16			;; 16 iterations
	yr4_4cl_four_complex_with_square_preload
b4b:	yr4_4cl_four_complex_with_square rsi, 4*64, 64, 2*64, 16
	yloop	16, b4b			;; Test loop counter
	end_timer 7
	jmp	yfinish_unfft

ymid3p:	start_timer 7
x64	mov	r8, DEST2ARG
	mov	rbp, DIST_TO_MULSRCARG
	yloop_init  16			;; 16 iterations
	yr4_4cl_four_complex_with_mult_preload
b4c:	yr4_4cl_four_complex_with_mult rsi, 4*64, 64, 2*64
	yloop	16, b4c			;; Test loop counter
x64	add	rsi, r8
	end_timer 7
	jmp	yfinish_unfft

ymid4p:	start_timer 7
	mov	rbp, DIST_TO_MULSRCARG
	yloop_init  16			;; 16 iterations
	yr4_4cl_four_complex_with_mulf_preload
b4d:	yr4_4cl_four_complex_with_mulf rsi, 4*64, 64, 2*64
	yloop	16, b4d			;; Test loop counter
	end_timer 7
yfinish_unfft:
	bump	rsi, -16*4*64		;; Restore source pointer

;; Do inverse FFT levels 6,7
;; On input the 64-byte cache lines hold these data values:
;;	0	128	256	384	4	+128	+128	+128
;;	1
;;	...
;;	3
;;	8	...
;;	...
;;	120	...
;; On output the 64-byte cache lines hold these data values:
;;	0	128	256	384	16	+128	+128	+128
;;	1
;;	...
;;	15
;;	32	...
;;	64	...
;;	96	...

;; Do 64 four_complex_unfft operations
;;	distance between fft data elements is 4
;;	do 16 macros each processing 32 data values

	start_timer 8
	mov	rdi, sincos1		;; Load sin/cos pointer
	yloop_init  16			;; 4 iterations of 4
	yr4_b4cl_four_complex_djbunfft_preload
c3b:	yr4_b4cl_four_complex_djbunfft rsi, 64, 4*64, 8*64, rdi, 4*YMM_SCD2/4, 4
	yloop	4, c3b, -4*64+16*64, rdi, -4*4*YMM_SCD2/4 ;; Test loop counter, next source pointer, restore sine/cosine pointer
	yloop	4, c3b, -4*16*64	;; Test loop counter, restore source pointer
	end_timer 8

;; Do inverse FFT levels 4,5
;; On input the 64-byte cache lines hold these data values:
;;	0	128	256	384	16	+128	+128	+128
;;	1
;;	...
;;	15
;;	32	...
;;	64	...
;;	96	...
;; On output the 64-byte cache lines hold these data values:
;;	0	128	256	384	64	+128	+128	+128
;;	1
;;	...
;;	63

;; Do 64 four_complex_unfft operations
;;	distance between fft data elements is 16
;;	do 16 macros each processing 32 data values

	start_timer 9
	yloop_init  16			;; 16 iterations
	yr4_b4cl_four_complex_djbunfft_preload
c2b:	yr4_b4cl_four_complex_djbunfft rsi, 64, 16*64, 32*64, rdi, YMM_SCD2/4, 16
	yloop	16, c2b, -16*64		;; Test loop counter, restore source pointer
	end_timer 9

;; Do inverse FFT levels 1,2,3
;; On input the 64-byte cache lines hold these data values:
;;	0	128	256	384	64	+128	+128	+128
;;	1
;;	...
;;	63
;; These are swizzled to:
;;	0	16	32	48	64	+16	+16	+16
;;	1	...
;;	...
;;	15
;;	128	...
;;	256	...
;;	384	...
;; On output the 64-byte cache lines hold these data values:
;;	0	16	32	48	256	+16	+16	+16
;;	1	...
;;	...
;;	15
;;	64	...
;;	128	...
;;	192	...

;; Do 64 four_complex_last_unfft operations
;;	distance between fft data elements is 64
;;	do 16 macros each processing 32 data values

	start_timer 10
	yloop_init  16			;; 16 iterations
	yr4_s4cl_four_complex_last_unfft_preload
c1b:	yr4_s4cl_four_complex_last_unfft rsi, 64, 16*64, 32*64, rdi, YMM_SCD4, 16
	yloop	16, c1b			;; Test loop counter
	end_timer 10

	ytwo_dest_adjust
	yfft_3_ret
	ENDM


;; Perform a 640-element FFT.

yfft640 MACRO type
	LOCAL	ymid2, ymid3, ymid4, yfinish_unfft
	LOCAL	b1b, b2b, b3a, b3b, b4a, b4b, b4c, b4d, c1b, c2b, c3a, c3b

;; Do a multiply with pre-FFTed inputs

	cmp	ffttype, 4
	je	ymid4

;; Do FFT levels 1,2,3
;; On input the 64-byte cache lines hold these data values:
;;	0	+20	+20	+20	320	+20	+20	+20
;;	1
;;	...
;;	19
;;	80	...
;;	160	...
;;	240	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+20	+20	+20	80	+20	+20	+20
;;	1
;;	...
;;	19
;;	160	...
;;	320	...
;;	480	...
;; These are swizzled to:
;;	0	+160	+160	+160	80	+160	+160	+160
;;	1
;;	...
;;	79

;; Do 80 eight_reals_fft operations
;;	distance between fft data elements is 80
;;	do 20 macros each processing 32 data values

	start_timer 4
	mov	rdi, sincos1		;; Load sin/cos pointer
	bump	rdi, 16*YMM_SCD4+4*YMM_SCD3
	yloop_init  20			;; 20 iterations
	yr4_fs4cl_eight_reals_fft_preload
b1b:	yr4_fs4cl_eight_reals_fft rsi, 64, 20*64, 40*64, rdi, YMM_SCD3, 20
	yloop	20, b1b, -20*64		;; Test loop counter, restore source pointer
	end_timer 4

;; Do FFT levels 4,5
;; On input the 64-byte cache lines hold these data values:
;;	0	+160	+160	+160	80	+160	+160	+160
;;	1
;;	...
;;	79
;; On output the 64-byte cache lines hold these data values:
;;	0	+160	+160	+160	16	+160	+160	+160
;;	1
;;	...
;;	15
;;	32	...
;;	...
;;	128	...

;; Do 16 ten_reals_fft operations
;; Do 48 five_complex_fft operations
;;	distance between fft data elements is 16
;;	do 16 macros each processing 40 data values

	start_timer 5
	bump	rdi, -20*YMM_SCD3-16*YMM_SCD4 ;; Load sin/cos pointer
	yloop_init  16			;; 16 iterations
	yr5_5cl_ten_reals_five_complex_djbfft_preload
b2b:	yr5_5cl_ten_reals_five_complex_djbfft rsi, 64, 16*64, rdi, YMM_SCD4, 16
	yloop	16, b2b, -16*64		;; Test loop counter, restore source pointer
	end_timer 5

;; Do FFT levels 6,7
;; On input the 64-byte cache lines hold these data values:
;;	0	+160	+160	+160	16	+160	+160	+160
;;	1
;;	...
;;	15
;;	32	...
;;	...
;;	128	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+160	+160	+160	4	+160	+160	+160
;;	1
;;	...
;;	3
;;	8	...
;;	...
;;	152	...

;; Do 4 eight_reals_fft operations
;; Do 76 four_complex_fft operations
;;	distance between fft data elements is 4
;;	do 20 macros each processing 32 data values

	start_timer 6
	bump	rdi, -16*YMM_SCD4-4*YMM_SCD3 ;; Next sin/cos pointer
	yloop_init  20			;; 5 iterations of 4
	yr4_4cl_eight_reals_four_complex_djbfft_preload
b3a:	yr4_4cl_eight_reals_four_complex_djbfft rsi, 64, 4*64, 8*64, rdi, YMM_SCD3, 4
	yloop	4, b3a, -4*64+16*64, rdi, -4*YMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer

	loops_reset			;; 4 more iters of 4
	yr4_rb4cl_four_complex_djbfft_preload
b3b:	yr4_rb4cl_four_complex_djbfft rsi, 64, 4*64, 8*64, rdi, YMM_SCD3, 4
	yloop	4, b3b, -4*64+16*64, rdi, -4*YMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer
	yloop	5, b3b, -5*16*64	;; Test loop counter, restore source pointer
	end_timer 6

;; Do FFT levels 8,9
;; On input and output the 64-byte cache lines hold these data values:
;;	0	+160	+160	+160	4	+160	+160	+160
;;	1
;;	...
;;	3
;;	8	...
;;	...
;;	152	...

;; Do 1 eight_reals_fft operation
;; Do 79 four_complex_fft operations
;;	distance between fft data elements is 1
;;	do 20 macros each processing 32 data values

	cmp	ffttype, 2
	je	ymid2
	ja	ymid3

	start_timer 7
;;	yr4_4cl_eight_reals_four_complex_fft_final rsi, 4*64, 64, 2*64
	yr4_dispatch call, yreal_fft_final, 64, 2*64
	yloop_init  19			;; 19 more iterations
b4a:	yr4_4cl_four_complex_fft_final rsi, 4*64, 64, 2*64
	yloop	19, b4a			;; Test loop counter
	end_timer 7
	yfft_1_ret

ymid2:	start_timer 7
;;	yr4_4cl_eight_reals_four_complex_with_square rsi, 4*64, 64, 2*64
	yr4_dispatch call, yreal_square, 64, 2*64
	yloop_init  19			;; 19 more iterations
	yr4_4cl_four_complex_with_square_preload
b4b:	yr4_4cl_four_complex_with_square rsi, 4*64, 64, 2*64, 19
	yloop	19, b4b			;; Test loop counter
	end_timer 7
	jmp	yfinish_unfft

ymid3:	start_timer 7
x64	mov	r8, DEST2ARG
	mov	rbp, DIST_TO_MULSRCARG
;;	yr4_4cl_eight_reals_four_complex_with_mult rsi, 4*64, 64, 2*64
	yr4_dispatch call, yreal_mult, 64, 2*64
	yloop_init  19			;; 19 more iterations
	yr4_4cl_four_complex_with_mult_preload
b4c:	yr4_4cl_four_complex_with_mult rsi, 4*64, 64, 2*64
	yloop	19, b4c			;; Test loop counter
x64	add	rsi, r8
	end_timer 7
	jmp	yfinish_unfft

ymid4:	start_timer 7
	mov	rbp, DIST_TO_MULSRCARG
;;	yr4_4cl_eight_reals_four_complex_with_mulf rsi, 4*64, 64, 2*64
	yr4_dispatch call, yreal_mulf, 64, 2*64
	yloop_init  19			;; 19 more iterations
	yr4_4cl_four_complex_with_mulf_preload
b4d:	yr4_4cl_four_complex_with_mulf rsi, 4*64, 64, 2*64
	yloop	19, b4d			;; Test loop counter
	end_timer 7
yfinish_unfft:
	bump	rsi, -20*4*64		;; Restore source pointer

;; Do inverse FFT levels 6,7
;; On input the 64-byte cache lines hold these data values:
;;	0	+160	+160	+160	4	+160	+160	+160
;;	1
;;	...
;;	3
;;	8	...
;;	...
;;	152	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+160	+160	+160	16	+160	+160	+160
;;	1
;;	...
;;	15
;;	32	...
;;	...
;;	128	...

;; Do 4 eight_reals_unfft operations
;; Do 76 four_complex_unfft operations
;;	distance between fft data elements is 4
;;	do 20 macros each processing 32 data values

	start_timer 8
	mov	rdi, sincos1		;; Load sin/cos pointer
	yloop_init  20			;; 5 iterations of 4
	yr4_4cl_eight_reals_four_complex_djbunfft_preload
c3a:	yr4_4cl_eight_reals_four_complex_djbunfft rsi, 64, 4*64, 8*64, rdi, YMM_SCD3, 4
	yloop	4, c3a, -4*64+16*64, rdi, -4*YMM_SCD3 ;; Test loop counter, next source pointer, restore sine/cosine pointer

	loops_reset			;; 4 more iters of 4
	yr4_rb4cl_four_complex_djbunfft_preload
c3b:	yr4_rb4cl_four_complex_djbunfft rsi, 64, 4*64, 8*64, rdi, YMM_SCD3, 4
	yloop	4, c3b, -4*64+16*64, rdi, -4*YMM_SCD3 ;; Test loop counter, next source pointer, restore sine/cosine pointer
	yloop	5, c3b, -5*16*64	;; Test loop counter, restore source pointer
	end_timer 8

;; Do inverse FFT levels 4,5
;; On input the 64-byte cache lines hold these data values:
;;	0	+160	+160	+160	16	+160	+160	+160
;;	1
;;	...
;;	15
;;	32	...
;;	...
;;	128	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+160	+160	+160	80	+160	+160	+160
;;	1
;;	...
;;	79

;; Do 16 ten_reals_unfft operations
;; Do 48 five_complex_unfft operations
;;	distance between fft data elements is 16
;;	do 16 macros each processing 40 data values

	start_timer 9
	bump	rdi, 4*YMM_SCD3		;; Next sine/cosine pointer
	yloop_init  16			;; 16 iterations
	yr5_5cl_ten_reals_five_complex_djbunfft_preload
c2b:	yr5_5cl_ten_reals_five_complex_djbunfft rsi, 64, 16*64, rdi, YMM_SCD4, 16
	yloop	16, c2b, -16*64		;; Test loop counter, restore source pointer
	end_timer 9

;; Do inverse FFT levels 1,2,3
;; On input the 64-byte cache lines hold these data values:
;;	0	+160	+160	+160	80	+160	+160	+160
;;	1
;;	...
;;	79
;; These are swizzled to:
;;	0	+20	+20	+20	80	+20	+20	+20
;;	1	...
;;	...
;;	19
;;	160	...
;;	320	...
;;	480	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+20	+20	+20	320	+20	+20	+20
;;	1	...
;;	...
;;	19
;;	80	...
;;	160	...
;;	240	...

;; Do 80 eight_reals_unfft operations
;;	distance between fft data elements is 80
;;	do 20 macros each processing 32 data values

	start_timer 10
	yloop_init  20			;; 20 iterations
	yr4_s4cl_eight_reals_unfft_preload
c1b:	yr4_s4cl_eight_reals_unfft rsi, 64, 20*64, 40*64, rdi, YMM_SCD3, 20
	yloop	20, c1b			;; Test loop counter
	end_timer 10

	ytwo_dest_adjust
	yfft_3_ret
	ENDM

;; Perform a 640-element negacyclic FFT

yfft640p MACRO type
	LOCAL	ymid2p, ymid3p, ymid4p, yfinish_unfft
	LOCAL	b1b, b2b, b3b, b4a, b4b, b4c, b4d, c1b, c2b, c3b

;; Do a multiply with pre-FFTed inputs

	cmp	ffttype, 4
	je	ymid4p

;; Do FFT levels 1,2,3
;; On input the 64-byte cache lines hold these data values:
;;	0	+20	+20	+20	320	+20	+20	+20
;;	1
;;	...
;;	19
;;	80	...
;;	160	...
;;	240	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+20	+20	+20	80	+20	+20	+20
;;	1
;;	...
;;	19
;;	160	...
;;	320	...
;;	480	...
;; These are swizzled to:
;;	0	+160	+160	+160	80	+160	+160	+160
;;	1
;;	...
;;	79

;; Do 80 four_complex_first_fft operations
;;	distance between fft data elements is 80
;;	do 20 macros each processing 32 data values

	start_timer 4
	mov	rdi, sincos1		;; Load sin/cos pointer
	bump	rdi, 16*YMM_SCD2/4+4*YMM_SCD2/4
	yloop_init  20			;; 20 iterations
	yr4_fs4cl_four_complex_first_djbfft_preload
b1b:	yr4_fs4cl_four_complex_first_djbfft rsi, 64, 20*64, 40*64, rdi, YMM_SCD4, 20
	yloop	20, b1b, -20*64		;; Test loop counter, restore source pointer
	end_timer 4

;; Do FFT levels 4,5
;; On input the 64-byte cache lines hold these data values:
;;	0	+160	+160	+160	80	+160	+160	+160
;;	1
;;	...
;;	79
;; On output the 64-byte cache lines hold these data values:
;;	0	+160	+160	+160	16	+160	+160	+160
;;	1
;;	...
;;	15
;;	32	...
;;	...
;;	128	...

;; Do 64 five_complex_fft operations
;;	distance between fft data elements is 16
;;	do 16 macros each processing 40 data values

	start_timer 5
	bump	rdi, -20*YMM_SCD4-16*YMM_SCD2/4 ;; Load sin/cos pointer
	yloop_init  16			;; 16 iterations
	yr5_b5cl_five_complex_djbfft_preload
b2b:	yr5_b5cl_five_complex_djbfft rsi, 64, 16*64, rdi, YMM_SCD2/4, 16
	yloop	16, b2b, -16*64		;; Test loop counter, restore source pointer
	end_timer 5

;; Do FFT levels 6,7
;; On input the 64-byte cache lines hold these data values:
;;	0	+160	+160	+160	16	+160	+160	+160
;;	1
;;	...
;;	15
;;	32	...
;;	...
;;	128	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+160	+160	+160	4	+160	+160	+160
;;	1
;;	...
;;	3
;;	8	...
;;	...
;;	152	...

;; Do 80 four_complex_fft operations
;;	distance between fft data elements is 4
;;	do 20 macros each processing 32 data values

	start_timer 6
	bump	rdi, -16*YMM_SCD2/4-4*YMM_SCD2/4 ;; Next sin/cos pointer
	yloop_init  20			;; 5 iterations of 4
	yr4_b4cl_four_complex_djbfft_preload
b3b:	yr4_b4cl_four_complex_djbfft rsi, 64, 4*64, 8*64, rdi, YMM_SCD2/4, 4
	yloop	4, b3b, -4*64+16*64, rdi, -4*YMM_SCD2/4 ;; Test loop counter, next source pointer, restore sin/cos pointer
	yloop	5, b3b, -5*16*64	;; Test loop counter, restore source pointer
	end_timer 6

;; Do FFT levels 8,9
;; On input and output the 64-byte cache lines hold these data values:
;;	0	+160	+160	+160	4	+160	+160	+160
;;	1
;;	...
;;	3
;;	8	...
;;	...
;;	152	...

;; Do 80 four_complex_fft operations
;;	distance between fft data elements is 1
;;	do 20 macros each processing 32 data values

	cmp	ffttype, 2
	je	ymid2p
	ja	ymid3p

	start_timer 7
	yloop_init  20			;; 20 iterations
b4a:	yr4_4cl_four_complex_fft_final rsi, 4*64, 64, 2*64
	yloop	20, b4a			;; Test loop counter
	end_timer 7
	yfft_1_ret

ymid2p:	start_timer 7
	yloop_init  20			;; 20 iterations
	yr4_4cl_four_complex_with_square_preload
b4b:	yr4_4cl_four_complex_with_square rsi, 4*64, 64, 2*64, 20
	yloop	20, b4b			;; Test loop counter
	end_timer 7
	jmp	yfinish_unfft

ymid3p:	start_timer 7
x64	mov	r8, DEST2ARG
	mov	rbp, DIST_TO_MULSRCARG
	yloop_init  20			;; 20 iterations
	yr4_4cl_four_complex_with_mult_preload
b4c:	yr4_4cl_four_complex_with_mult rsi, 4*64, 64, 2*64
	yloop	20, b4c			;; Test loop counter
x64	add	rsi, r8
	end_timer 7
	jmp	yfinish_unfft

ymid4p:	start_timer 7
	mov	rbp, DIST_TO_MULSRCARG
	yloop_init  20			;; 20 iterations
	yr4_4cl_four_complex_with_mulf_preload
b4d:	yr4_4cl_four_complex_with_mulf rsi, 4*64, 64, 2*64
	yloop	20, b4d			;; Test loop counter
	end_timer 7
yfinish_unfft:
	bump	rsi, -20*4*64		;; Restore source pointer

;; Do inverse FFT levels 6,7
;; On input the 64-byte cache lines hold these data values:
;;	0	+160	+160	+160	4	+160	+160	+160
;;	1
;;	...
;;	3
;;	8	...
;;	...
;;	152	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+160	+160	+160	16	+160	+160	+160
;;	1
;;	...
;;	15
;;	32	...
;;	...
;;	128	...

;; Do 80 four_complex_unfft operations
;;	distance between fft data elements is 4
;;	do 20 macros each processing 32 data values

	start_timer 8
	mov	rdi, sincos1		;; Load sin/cos pointer
	yloop_init  20			;; 5 iterations of 4
	yr4_b4cl_four_complex_djbunfft_preload
c3b:	yr4_b4cl_four_complex_djbunfft rsi, 64, 4*64, 8*64, rdi, YMM_SCD2/4, 4
	yloop	4, c3b, -4*64+16*64, rdi, -4*YMM_SCD2/4 ;; Test loop counter, next source pointer, restore sine/cosine pointer
	yloop	5, c3b, -5*16*64	;; Test loop counter, restore source pointer
	end_timer 8

;; Do inverse FFT levels 4,5
;; On input the 64-byte cache lines hold these data values:
;;	0	+160	+160	+160	16	+160	+160	+160
;;	1
;;	...
;;	15
;;	32	...
;;	...
;;	128	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+160	+160	+160	80	+160	+160	+160
;;	1
;;	...
;;	79

;; Do 64 five_complex_unfft operations
;;	distance between fft data elements is 16
;;	do 16 macros each processing 40 data values

	start_timer 9
	bump	rdi, 4*YMM_SCD2/4	;; Next sine/cosine pointer
	yloop_init  16			;; 16 iterations
	yr5_b5cl_five_complex_djbunfft_preload
c2b:	yr5_b5cl_five_complex_djbunfft rsi, 64, 16*64, rdi, YMM_SCD2/4, 16
	yloop	16, c2b, -16*64		;; Test loop counter, restore source pointer
	end_timer 9

;; Do inverse FFT levels 1,2,3
;; On input the 64-byte cache lines hold these data values:
;;	0	+160	+160	+160	80	+160	+160	+160
;;	1
;;	...
;;	79
;; These are swizzled to:
;;	0	+20	+20	+20	80	+20	+20	+20
;;	1	...
;;	...
;;	19
;;	160	...
;;	320	...
;;	480	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+20	+20	+20	320	+20	+20	+20
;;	1	...
;;	...
;;	19
;;	80	...
;;	160	...
;;	240	...

;; Do 80 four_complex_last_unfft operations
;;	distance between fft data elements is 80
;;	do 20 macros each processing 32 data values

	start_timer 10
	yloop_init  20			;; 20 iterations
	yr4_s4cl_four_complex_last_unfft_preload
c1b:	yr4_s4cl_four_complex_last_unfft rsi, 64, 20*64, 40*64, rdi, YMM_SCD4, 20
	yloop	20, c1b			;; Test loop counter
	end_timer 10

	ytwo_dest_adjust
	yfft_3_ret
	ENDM


;; Perform a 768-element FFT.

yfft768 MACRO type
	LOCAL	ymid2, ymid3, ymid4, yfinish_unfft
	LOCAL	b1b, b2b, b3a, b3b, b4a, b4b, b4c, b4d, c1b, c2b, c3a, c3b

;; Do a multiply with pre-FFTed inputs

	cmp	ffttype, 4
	je	ymid4

;; Do FFT levels 1,2,3
;; On input the 64-byte cache lines hold these data values:
;;	0	+24	+24	+24	384	+24	+24	+24
;;	1
;;	...
;;	23
;;	96	...
;;	192	...
;;	288	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+24	+24	+24	96	+24	+24	+24
;;	1
;;	...
;;	23
;;	192	...
;;	384	...
;;	576	...
;; These are swizzled to:
;;	0	+192	+192	+192	96	+192	+192	+192
;;	1
;;	...
;;	95

;; Do 96 eight_reals_fft operations
;;	distance between fft data elements is 96
;;	do 24 macros each processing 32 data values

	start_timer 4
	mov	rdi, sincos1		;; Load sin/cos pointer
	bump	rdi, 32*YMM_SCD2+8*YMM_SCD3
	yloop_init  24			;; 24 iterations
	yr4_fs4cl_eight_reals_fft_preload
b1b:	yr4_fs4cl_eight_reals_fft rsi, 64, 24*64, 48*64, rdi, YMM_SCD3, 24
	yloop	24, b1b, -24*64		;; Test loop counter, restore source pointer
	end_timer 4

;; Do FFT levels 4,5
;; On input the 64-byte cache lines hold these data values:
;;	0	+192	+192	+192	96	+192	+192	+192
;;	1
;;	...
;;	95
;; On output the 64-byte cache lines hold these data values:
;;	0	+192	+192	+192	32	+192	+192	+192
;;	1
;;	...
;;	31
;;	64	...
;;	128	...

;; Do 32 six_reals_fft operations
;; Do 96 three_complex_fft operations
;;	distance between fft data elements is 32
;;	do 32 macros each processing 24 data values

	start_timer 5
	bump	rdi, -24*YMM_SCD3-32*YMM_SCD2 ;; Load sin/cos pointer
	yloop_init  32			;; 32 iterations
	yr3_3cl_six_reals_three_complex_djbfft_preload
b2b:	yr3_3cl_six_reals_three_complex_djbfft rsi, 64, 32*64, rdi, YMM_SCD2, 32
	yloop	32, b2b, -32*64		;; Test loop counter, restore source pointer
	end_timer 5

;; Do FFT levels 6,7
;; On input the 64-byte cache lines hold these data values:
;;	0	+192	+192	+192	32	+192	+192	+192
;;	1
;;	...
;;	31
;;	64	...
;;	128	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+192	+192	+192	8	+192	+192	+192
;;	1
;;	...
;;	7
;;	16	...
;;	...
;;	176	...

;; Do 8 eight_reals_fft operations
;; Do 88 four_complex_fft operations
;;	distance between fft data elements is 8
;;	do 24 macros each processing 32 data values

	start_timer 6
	bump	rdi, -32*YMM_SCD2-8*YMM_SCD3 ;; Next sin/cos pointer
	yloop_init  24			;; 3 iterations of 8
	yr4_4cl_eight_reals_four_complex_djbfft_preload
b3a:	yr4_4cl_eight_reals_four_complex_djbfft rsi, 64, 8*64, 16*64, rdi, YMM_SCD3, 8
	yloop	8, b3a, -8*64+32*64, rdi, -8*YMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer

	loops_reset			;; 2 more iters of 8
	yr4_rb4cl_four_complex_djbfft_preload
b3b:	yr4_rb4cl_four_complex_djbfft rsi, 64, 8*64, 16*64, rdi, YMM_SCD3, 8
	yloop	8, b3b, -8*64+32*64, rdi, -8*YMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer
	yloop	3, b3b, -3*32*64	;; Test loop counter, restore source pointer
	end_timer 6

;; Do FFT levels 8,9
;; On input and output the 64-byte cache lines hold these data values:
;;	0	+192	+192	+192	8	+192	+192	+192
;;	1
;;	...
;;	7
;;	16	...
;;	...
;;	176	...

;; Do 1 sixteen_reals_fft operation
;; Do 47 eight_complex_fft operations
;;	distance between fft data elements is 1
;;	do 12 macros each processing 64 data values

	cmp	ffttype, 2
	je	ymid2
	ja	ymid3

	start_timer 7
;;	yr8_8cl_sixteen_reals_eight_complex_fft_final rsi, 8*64, 64, 2*64, 4*64
	yr8_dispatch call, y8real_fft_final, 64, 2*64, 4*64
	yloop_init  11			;; 11 more iterations
b4a:	yr8_8cl_eight_complex_fft_final rsi, 8*64, 64, 2*64, 4*64
	yloop	11, b4a			;; Test loop counter
	end_timer 7
	yfft_1_ret

ymid2:	start_timer 7
;;	yr8_8cl_sixteen_reals_eight_complex_with_square rsi, 8*64, 64, 2*64, 4*64
	yr8_dispatch call, y8real_square, 64, 2*64, 4*64
	yloop_init  11			;; 11 more iterations
	yr8_8cl_eight_complex_with_square_preload
b4b:	yr8_8cl_eight_complex_with_square rsi, 8*64, 64, 2*64, 4*64
	yloop	11, b4b			;; Test loop counter
	end_timer 7
	jmp	yfinish_unfft

ymid3:	start_timer 7
x64	mov	r8, DEST2ARG
	mov	rbp, DIST_TO_MULSRCARG
;;	yr8_8cl_sixteen_reals_eight_complex_with_mult rsi, 8*64, 64, 2*64, 4*64
	yr8_dispatch call, y8real_mult, 64, 2*64, 4*64
	yloop_init  11			;; 11 more iterations
	yr8_8cl_eight_complex_with_mult_preload
b4c:	yr8_8cl_eight_complex_with_mult rsi, 8*64, 64, 2*64, 4*64
	yloop	11, b4c			;; Test loop counter
x64	add	rsi, r8
	end_timer 7
	jmp	yfinish_unfft

ymid4:	start_timer 7
	mov	rbp, DIST_TO_MULSRCARG
;;	yr8_8cl_sixteen_reals_eight_complex_with_mulf rsi, 8*64, 64, 2*64, 4*64
	yr8_dispatch call, y8real_mulf, 64, 2*64, 4*64
	yloop_init  11			;; 11 more iterations
	yr8_8cl_eight_complex_with_mulf_preload
b4d:	yr8_8cl_eight_complex_with_mulf rsi, 8*64, 64, 2*64, 4*64
	yloop	11, b4d			;; Test loop counter
	end_timer 7
yfinish_unfft:
	bump	rsi, -12*8*64		;; Restore source pointer

;; Do inverse FFT levels 6,7
;; On input the 64-byte cache lines hold these data values:
;;	0	+192	+192	+192	8	+192	+192	+192
;;	1
;;	...
;;	7
;;	16	...
;;	...
;;	176	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+192	+192	+192	32	+192	+192	+192
;;	1
;;	...
;;	31
;;	64	...
;;	128	...

;; Do 8 eight_reals_unfft operations
;; Do 88 four_complex_unfft operations
;;	distance between fft data elements is 8
;;	do 24 macros each processing 32 data values

	start_timer 8
	mov	rdi, sincos1		;; Load sin/cos pointer
	yloop_init  24			;; 3 iterations of 8
	yr4_4cl_eight_reals_four_complex_djbunfft_preload
c3a:	yr4_4cl_eight_reals_four_complex_djbunfft rsi, 64, 8*64, 16*64, rdi, YMM_SCD3, 8
	yloop	8, c3a, -8*64+32*64, rdi, -8*YMM_SCD3 ;; Test loop counter, next source pointer, restore sine/cosine pointer

	loops_reset			;; 2 more iters of 8
	yr4_rb4cl_four_complex_djbunfft_preload
c3b:	yr4_rb4cl_four_complex_djbunfft rsi, 64, 8*64, 16*64, rdi, YMM_SCD3, 8
	yloop	8, c3b, -8*64+32*64, rdi, -8*YMM_SCD3 ;; Test loop counter, next source pointer, restore sine/cosine pointer
	yloop	3, c3b, -3*32*64	;; Test loop counter, restore source pointer
	end_timer 8

;; Do inverse FFT levels 4,5
;; On input the 64-byte cache lines hold these data values:
;;	0	+192	+192	+192	32	+192	+192	+192
;;	1
;;	...
;;	31
;;	64	...
;;	128	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+192	+192	+192	96	+192	+192	+192
;;	1
;;	...
;;	95

;; Do 32 six_reals_unfft operations
;; Do 96 three_complex_unfft operations
;;	distance between fft data elements is 32
;;	do 32 macros each processing 24 data values

	start_timer 9
	bump	rdi, 8*YMM_SCD3		;; Next sine/cosine pointer
	yloop_init  32			;; 32 iterations
	yr3_3cl_six_reals_three_complex_djbunfft_preload
c2b:	yr3_3cl_six_reals_three_complex_djbunfft rsi, 64, 32*64, rdi, YMM_SCD2, 32
	yloop	32, c2b, -32*64		;; Test loop counter, restore source pointer
	end_timer 9

;; Do inverse FFT levels 1,2,3
;; On input the 64-byte cache lines hold these data values:
;;	0	+192	+192	+192	96	+192	+192	+192
;;	1
;;	...
;;	95
;; These are swizzled to:
;;	0	+24	+24	+24	96	+24	+24	+24
;;	1	...
;;	...
;;	23
;;	192	...
;;	384	...
;;	576	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+24	+24	+24	384	+24	+24	+24
;;	1	...
;;	...
;;	23
;;	96	...
;;	192	...
;;	288	...

;; Do 96 eight_reals_unfft operations
;;	distance between fft data elements is 96
;;	do 24 macros each processing 32 data values

	start_timer 10
	yloop_init  24			;; 24 iterations
	yr4_s4cl_eight_reals_unfft_preload
c1b:	yr4_s4cl_eight_reals_unfft rsi, 64, 24*64, 48*64, rdi, YMM_SCD3, 24
	yloop	24, c1b			;; Test loop counter
	end_timer 10

	ytwo_dest_adjust
	yfft_3_ret
	ENDM

;; Perform a 768-element negacyclic FFT

yfft768p MACRO type
	LOCAL	ymid2p, ymid3p, ymid4p, yfinish_unfft
	LOCAL	b1b, b2b, b3b, b4a, b4b, b4c, b4d, c1b, c2b, c3b

;; Do a multiply with pre-FFTed inputs

	cmp	ffttype, 4
	je	ymid4p

;; Do FFT levels 1,2,3
;; On input the 64-byte cache lines hold these data values:
;;	0	+24	+24	+24	384	+24	+24	+24
;;	1
;;	...
;;	23
;;	96	...
;;	192	...
;;	288	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+24	+24	+24	96	+24	+24	+24
;;	1
;;	...
;;	23
;;	192	...
;;	384	...
;;	576	...
;; These are swizzled to:
;;	0	+192	+192	+192	96	+192	+192	+192
;;	1
;;	...
;;	95

;; Do 96 four_complex_first_fft operations
;;	distance between fft data elements is 96
;;	do 24 macros each processing 32 data values

	start_timer 4
	mov	rdi, sincos1		;; Load sin/cos pointer
	bump	rdi, 32*YMM_SCD1/4+8*YMM_SCD2/4
	yloop_init  24			;; 24 iterations
	yr4_fs4cl_four_complex_first_djbfft_preload
b1b:	yr4_fs4cl_four_complex_first_djbfft rsi, 64, 24*64, 48*64, rdi, YMM_SCD4, 24
	yloop	24, b1b, -24*64		;; Test loop counter, restore source pointer
	end_timer 4

;; Do FFT levels 4,5
;; On input the 64-byte cache lines hold these data values:
;;	0	+192	+192	+192	96	+192	+192	+192
;;	1
;;	...
;;	95
;; On output the 64-byte cache lines hold these data values:
;;	0	+192	+192	+192	32	+192	+192	+192
;;	1
;;	...
;;	31
;;	64	...
;;	128	...

;; Do 128 three_complex_fft operations
;;	distance between fft data elements is 32
;;	do 32 macros each processing 24 data values

	start_timer 5
	bump	rdi, -24*YMM_SCD4-32*YMM_SCD1/4 ;; Load sin/cos pointer
	yloop_init  32			;; 32 iterations
	yr3_b3cl_three_complex_djbfft_preload
b2b:	yr3_b3cl_three_complex_djbfft rsi, 64, 32*64, rdi, YMM_SCD1/4, 32
	yloop	32, b2b, -32*64		;; Test loop counter, restore source pointer
	end_timer 5

;; Do FFT levels 6,7
;; On input the 64-byte cache lines hold these data values:
;;	0	+192	+192	+192	32	+192	+192	+192
;;	1
;;	...
;;	31
;;	64	...
;;	128	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+192	+192	+192	8	+192	+192	+192
;;	1
;;	...
;;	7
;;	16	...
;;	...
;;	176	...

;; Do 96 four_complex_fft operations
;;	distance between fft data elements is 8
;;	do 24 macros each processing 32 data values

	start_timer 6
	bump	rdi, -32*YMM_SCD1/4-8*YMM_SCD2/4 ;; Next sin/cos pointer
	yloop_init  24			;; 3 iterations of 8
	yr4_b4cl_four_complex_djbfft_preload
b3b:	yr4_b4cl_four_complex_djbfft rsi, 64, 8*64, 16*64, rdi, YMM_SCD2/4, 8
	yloop	8, b3b, -8*64+32*64, rdi, -8*YMM_SCD2/4 ;; Test loop counter, next source pointer, restore sin/cos pointer
	yloop	3, b3b, -3*32*64	;; Test loop counter, restore source pointer
	end_timer 6

;; Do FFT levels 8,9
;; On input and output the 64-byte cache lines hold these data values:
;;	0	+192	+192	+192	8	+192	+192	+192
;;	1
;;	...
;;	7
;;	16	...
;;	...
;;	176	...

;; Do 48 eight_complex_fft operations
;;	distance between fft data elements is 1
;;	do 12 macros each processing 64 data values

	cmp	ffttype, 2
	je	ymid2p
	ja	ymid3p

	start_timer 7
	yloop_init  12			;; 12 iterations
b4a:	yr8_8cl_eight_complex_fft_final rsi, 8*64, 64, 2*64, 4*64
	yloop	12, b4a			;; Test loop counter
	end_timer 7
	yfft_1_ret

ymid2p:	start_timer 7
	yloop_init  12			;; 12 iterations
	yr8_8cl_eight_complex_with_square_preload
b4b:	yr8_8cl_eight_complex_with_square rsi, 8*64, 64, 2*64, 4*64
	yloop	12, b4b			;; Test loop counter
	end_timer 7
	jmp	yfinish_unfft

ymid3p:	start_timer 7
x64	mov	r8, DEST2ARG
	mov	rbp, DIST_TO_MULSRCARG
	yloop_init  12			;; 12 iterations
	yr8_8cl_eight_complex_with_mult_preload
b4c:	yr8_8cl_eight_complex_with_mult rsi, 8*64, 64, 2*64, 4*64
	yloop	12, b4c			;; Test loop counter
x64	add	rsi, r8
	end_timer 7
	jmp	yfinish_unfft

ymid4p:	start_timer 7
	mov	rbp, DIST_TO_MULSRCARG
	yloop_init  12			;; 12 iterations
	yr8_8cl_eight_complex_with_mulf_preload
b4d:	yr8_8cl_eight_complex_with_mulf rsi, 8*64, 64, 2*64, 4*64
	yloop	12, b4d			;; Test loop counter
	end_timer 7
yfinish_unfft:
	bump	rsi, -12*8*64		;; Restore source pointer

;; Do inverse FFT levels 6,7
;; On input the 64-byte cache lines hold these data values:
;;	0	+192	+192	+192	8	+192	+192	+192
;;	1
;;	...
;;	7
;;	16	...
;;	...
;;	176	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+192	+192	+192	32	+192	+192	+192
;;	1
;;	...
;;	31
;;	64	...
;;	128	...

;; Do 96 four_complex_unfft operations
;;	distance between fft data elements is 8
;;	do 24 macros each processing 32 data values

	start_timer 8
	mov	rdi, sincos1		;; Load sin/cos pointer
	yloop_init  24			;; 3 iterations of 8
	yr4_b4cl_four_complex_djbunfft_preload
c3b:	yr4_b4cl_four_complex_djbunfft rsi, 64, 8*64, 16*64, rdi, YMM_SCD2/4, 8
	yloop	8, c3b, -8*64+32*64, rdi, -8*YMM_SCD2/4 ;; Test loop counter, next source pointer, restore sine/cosine pointer
	yloop	3, c3b, -3*32*64	;; Test loop counter, restore source pointer
	end_timer 8

;; Do inverse FFT levels 4,5
;; On input the 64-byte cache lines hold these data values:
;;	0	+192	+192	+192	32	+192	+192	+192
;;	1
;;	...
;;	31
;;	64	...
;;	128	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+192	+192	+192	96	+192	+192	+192
;;	1
;;	...
;;	95

;; Do 128 three_complex_unfft operations
;;	distance between fft data elements is 32
;;	do 32 macros each processing 24 data values

	start_timer 9
	bump	rdi, 8*YMM_SCD2/4	;; Next sine/cosine pointer
	yloop_init  32			;; 32 iterations
	yr3_b3cl_three_complex_djbunfft_preload
c2b:	yr3_b3cl_three_complex_djbunfft rsi, 64, 32*64, rdi, YMM_SCD1/4, 32
	yloop	32, c2b, -32*64		;; Test loop counter, restore source pointer
	end_timer 9

;; Do inverse FFT levels 1,2,3
;; On input the 64-byte cache lines hold these data values:
;;	0	+192	+192	+192	96	+192	+192	+192
;;	1
;;	...
;;	95
;; These are swizzled to:
;;	0	+24	+24	+24	96	+24	+24	+24
;;	1	...
;;	...
;;	23
;;	192	...
;;	384	...
;;	576	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+24	+24	+24	384	+24	+24	+24
;;	1	...
;;	...
;;	23
;;	96	...
;;	192	...
;;	288	...

;; Do 96 four_complex_last_unfft operations
;;	distance between fft data elements is 96
;;	do 24 macros each processing 32 data values

	start_timer 10
	yloop_init  24			;; 24 iterations
	yr4_s4cl_four_complex_last_unfft_preload
c1b:	yr4_s4cl_four_complex_last_unfft rsi, 64, 24*64, 48*64, rdi, YMM_SCD4, 24
	yloop	24, c1b			;; Test loop counter
	end_timer 10

	ytwo_dest_adjust
	yfft_3_ret
	ENDM


;; Perform a 1K-element FFT.

yfft1K MACRO type
	LOCAL	ymid2, ymid3, ymid4, yfinish_unfft
	LOCAL	b1b, b2b, b3a, b3b, b4a, b4b, b4c, b4d, c1b, c2b, c3a, c3b

;; Do a multiply with pre-FFTed inputs

	cmp	ffttype, 4
	je	ymid4

;; Do FFT levels 1,2,3
;; On input the 64-byte cache lines hold these data values:
;;	0	+32	+32	+32	512	+32	+32	+32
;;	1
;;	...
;;	31
;;	128	...
;;	256	...
;;	384	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+32	+32	+32	128	+32	+32	+32
;;	1
;;	...
;;	31
;;	256	...
;;	512	...
;;	768	...
;; These are swizzled to:
;;	0	+256	+256	+256	128	+256	+256	+256
;;	1
;;	...
;;	127

;; Do 128 eight_reals_fft operations
;;	distance between fft data elements is 128
;;	do 32 macros each processing 32 data values

	start_timer 4
	mov	rdi, sincos1		;; Load sin/cos pointer
	bump	rdi, 32*YMM_SCD3
	yloop_init  32			;; 32 iterations
	yr4_fs4cl_eight_reals_fft_preload
b1b:	yr4_fs4cl_eight_reals_fft rsi, 64, dist32, 2*dist32, rdi, YMM_SCD3, 32
	yloop	32, b1b, -32*64		;; Test loop counter, restore source pointer
	end_timer 4

;; Do FFT levels 4,5
;; On input the 64-byte cache lines hold these data values:
;;	0	+256	+256	+256	128	+256	+256	+256
;;	1
;;	...
;;	127
;; On output the 64-byte cache lines hold these data values:
;;	0	+256	+256	+256	32	+256	+256	+256
;;	1
;;	...
;;	31
;;	64	...
;;	128	...
;;	192	...

;; Do 32 eight_reals_fft operations
;; Do 96 four_complex_fft operations
;;	distance between fft data elements is 32
;;	do 32 macros each processing 32 data values

	start_timer 5
	bump	rdi, -32*YMM_SCD3-32*YMM_SCD3 ;; Load sin/cos pointer
	yloop_init  32			;; 32 iterations
	yr4_4cl_eight_reals_four_complex_djbfft_preload
b2b:	yr4_4cl_eight_reals_four_complex_djbfft rsi, 64, dist32, 2*dist32, rdi, YMM_SCD3, 32
	yloop	32, b2b, -32*64, rdi, -32*YMM_SCD3 ;; Test loop counter, restore source pointer, restore sin/cos pointer
	end_timer 5

;; Do FFT levels 6,7
;; On input the 64-byte cache lines hold these data values:
;;	0	+256	+256	+256	32	+256	+256	+256
;;	1
;;	...
;;	31
;;	64	...
;;	128	...
;;	192	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+256	+256	+256	8	+256	+256	+256
;;	1
;;	...
;;	7
;;	16	...
;;	...
;;	240	...

;; Do 8 eight_reals_fft operations
;; Do 120 four_complex_fft operations
;;	distance between fft data elements is 8
;;	do 32 macros each processing 32 data values

	start_timer 6
	yloop_init  32			;; 4 iterations of 8
	yr4_4cl_eight_reals_four_complex_djbfft_preload
b3a:	yr4_4cl_eight_reals_four_complex_djbfft rsi, 64, 8*64, 16*64, rdi, 4*YMM_SCD3, 8
	yloop	8, b3a, -8*64+dist32, rdi, -8*4*YMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer

	loops_reset			;; 3 more iters of 8
	yr4_rb4cl_four_complex_djbfft_preload
b3b:	yr4_rb4cl_four_complex_djbfft rsi, 64, 8*64, 16*64, rdi, 4*YMM_SCD3, 8
	yloop	8, b3b, -8*64+dist32, rdi, -8*4*YMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer
	yloop	4, b3b, -4*dist32	;; Test loop counter, restore source pointer
	end_timer 6

;; Do FFT levels 8,9,10
;; On input and output the 64-byte cache lines hold these data values:
;;	0	+256	+256	+256	8	+256	+256	+256
;;	1
;;	...
;;	7
;;	16	...
;;	...
;;	240	...

;; Do 1 sixteen_reals_fft operation
;; Do 63 eight_complex_fft operations
;;	distance between fft data elements is 1
;;	do 16 macros each processing 64 data values

	cmp	ffttype, 2
	je	ymid2
	ja	ymid3

	start_timer 7
;;	yr8_8cl_sixteen_reals_eight_complex_fft_final rsi, 8*64, 64, 2*64, 4*64
	yr8_dispatch call, y8real_fft_final, 64, 2*64, 4*64
	yloop_init  16, 1, 4		;; 15 more iterations
b4a:	yr8_8cl_eight_complex_fft_final rsi, 8*64, 64, 2*64, 4*64
	yloop	4, b4a, -4*8*64+dist32	;; Test loop counter, next source pointer
	yloop	4, b4a			;; Test loop counter
	end_timer 7
	yfft_1_ret

ymid2:	start_timer 7
;;	yr8_8cl_sixteen_reals_eight_complex_with_square rsi, 8*64, 64, 2*64, 4*64
	yr8_dispatch call, y8real_square, 64, 2*64, 4*64
	yloop_init  16, 1, 4		;; 15 more iterations
	yr8_8cl_eight_complex_with_square_preload
b4b:	yr8_8cl_eight_complex_with_square rsi, 8*64, 64, 2*64, 4*64
	yloop	4, b4b, -4*8*64+dist32	;; Test loop counter, next source pointer
	yloop	4, b4b			;; Test loop counter
	end_timer 7
	jmp	yfinish_unfft

ymid3:	start_timer 7
x64	mov	r8, DEST2ARG
	mov	rbp, DIST_TO_MULSRCARG
;;	yr8_8cl_sixteen_reals_eight_complex_with_mult rsi, 8*64, 64, 2*64, 4*64
	yr8_dispatch call, y8real_mult, 64, 2*64, 4*64
	yloop_init  16, 1, 4		;; 15 more iterations
	yr8_8cl_eight_complex_with_mult_preload
b4c:	yr8_8cl_eight_complex_with_mult rsi, 8*64, 64, 2*64, 4*64
	yloop	4, b4c, -4*8*64+dist32	;; Test loop counter, next source pointer
	yloop	4, b4c			;; Test loop counter
x64	add	rsi, r8
	end_timer 7
	jmp	yfinish_unfft

ymid4:	start_timer 7
	mov	rbp, DIST_TO_MULSRCARG
;;	yr8_8cl_sixteen_reals_eight_complex_with_mulf rsi, 8*64, 64, 2*64, 4*64
	yr8_dispatch call, y8real_mulf, 64, 2*64, 4*64
	yloop_init  16, 1, 4		;; 15 more iterations
	yr8_8cl_eight_complex_with_mulf_preload
b4d:	yr8_8cl_eight_complex_with_mulf rsi, 8*64, 64, 2*64, 4*64
	yloop	4, b4d, -4*8*64+dist32	;; Test loop counter, next source pointer
	yloop	4, b4d			;; Test loop counter
	end_timer 7
yfinish_unfft:
	bump	rsi, -4*dist32		;; Restore source pointer

;; Do inverse FFT levels 6,7
;; On input the 64-byte cache lines hold these data values:
;;	0	+256	+256	+256	8	+256	+256	+256
;;	1
;;	...
;;	7
;;	16	...
;;	...
;;	240	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+256	+256	+256	32	+256	+256	+256
;;	1
;;	...
;;	31
;;	64	...
;;	128	...
;;	192	...

;; Do 8 eight_reals_unfft operations
;; Do 120 four_complex_unfft operations
;;	distance between fft data elements is 8
;;	do 32 macros each processing 32 data values

	start_timer 8
	mov	rdi, sincos1		;; Load sin/cos pointer
	yloop_init  32			;; 4 iterations of 8
	yr4_4cl_eight_reals_four_complex_djbunfft_preload
c3a:	yr4_4cl_eight_reals_four_complex_djbunfft rsi, 64, 8*64, 16*64, rdi, 4*YMM_SCD3, 8
	yloop	8, c3a, -8*64+dist32, rdi, -8*4*YMM_SCD3 ;; Test loop counter, next source pointer, restore sine/cosine pointer

	loops_reset			;; 3 more iters of 8
	yr4_rb4cl_four_complex_djbunfft_preload
c3b:	yr4_rb4cl_four_complex_djbunfft rsi, 64, 8*64, 16*64, rdi, 4*YMM_SCD3, 8
	yloop	8, c3b, -8*64+dist32, rdi, -8*4*YMM_SCD3 ;; Test loop counter, next source pointer, restore sine/cosine pointer
	yloop	4, c3b, -4*dist32	;; Test loop counter, restore source pointer
	end_timer 8

;; Do inverse FFT levels 4,5
;; On input the 64-byte cache lines hold these data values:
;;	0	+256	+256	+256	32	+256	+256	+256
;;	1
;;	...
;;	31
;;	64	...
;;	128	...
;;	192	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+256	+256	+256	128	+256	+256	+256
;;	1
;;	...
;;	127

;; Do 32 eight_reals_unfft operations
;; Do 96 four_complex_unfft operations
;;	distance between fft data elements is 32
;;	do 32 macros each processing 32 data values

	start_timer 9
	yloop_init  32			;; 32 iterations
	yr4_4cl_eight_reals_four_complex_djbunfft_preload
c2b:	yr4_4cl_eight_reals_four_complex_djbunfft rsi, 64, dist32, 2*dist32, rdi, YMM_SCD3, 32
	yloop	32, c2b, -32*64		;; Test loop counter, restore source pointer
	end_timer 9

;; Do inverse FFT levels 1,2,3
;; On input the 64-byte cache lines hold these data values:
;;	0	+256	+256	+256	128	+256	+256	+256
;;	1
;;	...
;;	127
;; These are swizzled to:
;;	0	+32	+32	+32	128	+32	+32	+32
;;	1	...
;;	...
;;	31
;;	256	...
;;	512	...
;;	768	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+32	+32	+32	512	+32	+32	+32
;;	1	...
;;	...
;;	31
;;	128	...
;;	256	...
;;	384	...

;; Do 128 eight_reals_unfft operations
;;	distance between fft data elements is 128
;;	do 32 macros each processing 32 data values

	start_timer 10
	yloop_init  32			;; 32 iterations
	yr4_s4cl_eight_reals_unfft_preload
c1b:	yr4_s4cl_eight_reals_unfft rsi, 64, dist32, 2*dist32, rdi, YMM_SCD3, 32
	yloop	32, c1b			;; Test loop counter
	end_timer 10

	ytwo_dest_adjust
	yfft_3_ret
	ENDM

;; Perform a 1K-element negacyclic FFT

yfft1Kp MACRO type
	LOCAL	ymid2p, ymid3p, ymid4p, yfinish_unfft
	LOCAL	b1b, b2b, b3b, b4a, b4b, b4c, b4d, c1b, c2b, c3b

;; Do a multiply with pre-FFTed inputs

	cmp	ffttype, 4
	je	ymid4p

;; Do FFT levels 1,2,3
;; On input the 64-byte cache lines hold these data values:
;;	0	+32	+32	+32	512	+32	+32	+32
;;	1
;;	...
;;	31
;;	128	...
;;	256	...
;;	384	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+32	+32	+32	128	+32	+32	+32
;;	1
;;	...
;;	31
;;	256	...
;;	512	...
;;	768	...
;; These are swizzled to:
;;	0	+256	+256	+256	128	+256	+256	+256
;;	1
;;	...
;;	127

;; Do 128 four_complex_first_fft operations
;;	distance between fft data elements is 128
;;	do 32 macros each processing 32 data values

;; Do FFT levels 4,5
;; On input the 64-byte cache lines hold these data values:
;;	0	+256	+256	+256	128	+256	+256	+256
;;	1
;;	...
;;	127
;; On output the 64-byte cache lines hold these data values:
;;	0	+256	+256	+256	32	+256	+256	+256
;;	1
;;	...
;;	31
;;	64	...
;;	128	...
;;	192	...

;; Do 128 four_complex_fft operations
;;	distance between fft data elements is 32
;;	do 32 macros each processing 32 data values

IF OLDWAY EQ 1
	start_timer 4
	mov	rdi, sincos1		;; Load sin/cos pointer
	bump	rdi, 32*YMM_SCD2/4
	yloop_init  32			;; 32 iterations
	yr4_fs4cl_four_complex_first_djbfft_preload
b1b:	yr4_fs4cl_four_complex_first_djbfft rsi, 64, dist32, 2*dist32, rdi, YMM_SCD4, 32
	yloop	32, b1b, -32*64		;; Test loop counter, restore source pointer
	end_timer 4

	start_timer 5
	bump	rdi, -32*YMM_SCD4-32*YMM_SCD2/4 ;; Load sin/cos pointer
	yloop_init  32			;; 32 iterations
	yr4_b4cl_four_complex_djbfft_preload
b2b:	yr4_b4cl_four_complex_djbfft rsi, 64, dist32, 2*dist32, rdi, YMM_SCD2/4, 32
	yloop	32, b2b, -32*64, rdi, -32*YMM_SCD2/4 ;; Test loop counter, restore source pointer, restore sin/cos pointer
	end_timer 5
ELSE
	start_timer 4
	mov	rdi, sincos1		;; Load sin/cos pointers
	lea	rdx, [rdi+32*YMM_SCD2/4]
	yloop_init  32			;; 32 iterations
	yr4_fs4cl_sixteen_complex_first_djbfft_preload
b1b:	yr4_fs4cl_sixteen_complex_first_djbfft rsi, 64, dist32, 2*dist32, rdx, YMM_SCD4, rdi, YMM_SCD2/4, 32
	yloop	32, b1b, -32*64, rdi, -32*YMM_SCD2/4 ;; Test loop counter, restore source pointer, restore sin/cos pointer
	end_timer 4
ENDIF

;; Do FFT levels 6,7
;; On input the 64-byte cache lines hold these data values:
;;	0	+256	+256	+256	32	+256	+256	+256
;;	1
;;	...
;;	31
;;	64	...
;;	128	...
;;	192	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+256	+256	+256	8	+256	+256	+256
;;	1
;;	...
;;	7
;;	16	...
;;	...
;;	240	...

;; Do 128 four_complex_fft operations
;;	distance between fft data elements is 8
;;	do 32 macros each processing 32 data values

	start_timer 6
	yloop_init  32			;; 4 iterations of 8
	yr4_b4cl_four_complex_djbfft_preload
b3b:	yr4_b4cl_four_complex_djbfft rsi, 64, 8*64, 16*64, rdi, 4*YMM_SCD2/4, 8
	yloop	8, b3b, -8*64+dist32, rdi, -8*4*YMM_SCD2/4 ;; Test loop counter, next source pointer, restore sin/cos pointer
	yloop	4, b3b, -4*dist32	;; Test loop counter, restore source pointer
	end_timer 6

;; Do FFT levels 8,9
;; On input and output the 64-byte cache lines hold these data values:
;;	0	+256	+256	+256	8	+256	+256	+256
;;	1
;;	...
;;	7
;;	16	...
;;	...
;;	240	...

;; Do 64 eight_complex_fft operations
;;	distance between fft data elements is 1
;;	do 16 macros each processing 64 data values

	cmp	ffttype, 2
	je	ymid2p
	ja	ymid3p

	start_timer 7
	yloop_init  16			;; 16 iterations
b4a:	yr8_8cl_eight_complex_fft_final rsi, 8*64, 64, 2*64, 4*64
	yloop	4, b4a, -4*8*64+dist32	;; Test loop counter, next source pointer
	yloop	4, b4a			;; Test loop counter
	end_timer 7
	yfft_1_ret

ymid2p:	start_timer 7
	yloop_init  16			;; 16 iterations
	yr8_8cl_eight_complex_with_square_preload
b4b:	yr8_8cl_eight_complex_with_square rsi, 8*64, 64, 2*64, 4*64
	yloop	4, b4b, -4*8*64+dist32	;; Test loop counter, next source pointer
	yloop	4, b4b			;; Test loop counter
	end_timer 7
	jmp	yfinish_unfft

ymid3p:	start_timer 7
x64	mov	r8, DEST2ARG
	mov	rbp, DIST_TO_MULSRCARG
	yloop_init  16			;; 16 iterations
	yr8_8cl_eight_complex_with_mult_preload
b4c:	yr8_8cl_eight_complex_with_mult rsi, 8*64, 64, 2*64, 4*64
	yloop	4, b4c, -4*8*64+dist32	;; Test loop counter, next source pointer
	yloop	4, b4c			;; Test loop counter
x64	add	rsi, r8
	end_timer 7
	jmp	yfinish_unfft

ymid4p:	start_timer 7
	mov	rbp, DIST_TO_MULSRCARG
	yloop_init  16			;; 16 iterations
	yr8_8cl_eight_complex_with_mulf_preload
b4d:	yr8_8cl_eight_complex_with_mulf rsi, 8*64, 64, 2*64, 4*64
	yloop	4, b4d, -4*8*64+dist32	;; Test loop counter, next source pointer
	yloop	4, b4d			;; Test loop counter
	end_timer 7
yfinish_unfft:
	bump	rsi, -4*dist32		;; Restore source pointer

;; Do inverse FFT levels 6,7
;; On input the 64-byte cache lines hold these data values:
;;	0	+256	+256	+256	8	+256	+256	+256
;;	1
;;	...
;;	7
;;	16	...
;;	...
;;	240	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+256	+256	+256	32	+256	+256	+256
;;	1
;;	...
;;	31
;;	64	...
;;	128	...
;;	192	...

;; Do 128 four_complex_unfft operations
;;	distance between fft data elements is 8
;;	do 32 macros each processing 32 data values

	start_timer 8
	mov	rdi, sincos1		;; Load sin/cos pointer
	yloop_init  32			;; 4 iterations of 8
	yr4_b4cl_four_complex_djbunfft_preload
c3b:	yr4_b4cl_four_complex_djbunfft rsi, 64, 8*64, 16*64, rdi, 4*YMM_SCD2/4, 8
	yloop	8, c3b, -8*64+dist32, rdi, -8*4*YMM_SCD2/4 ;; Test loop counter, next source pointer, restore sine/cosine pointer
	yloop	4, c3b, -4*dist32	;; Test loop counter, restore source pointer
	end_timer 8

;; Do inverse FFT levels 4,5
;; On input the 64-byte cache lines hold these data values:
;;	0	+256	+256	+256	32	+256	+256	+256
;;	1
;;	...
;;	31
;;	64	...
;;	128	...
;;	192	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+256	+256	+256	128	+256	+256	+256
;;	1
;;	...
;;	127

;; Do 128 four_complex_unfft operations
;;	distance between fft data elements is 32
;;	do 32 macros each processing 32 data values

;; Do inverse FFT levels 1,2,3
;; On input the 64-byte cache lines hold these data values:
;;	0	+256	+256	+256	128	+256	+256	+256
;;	1
;;	...
;;	127
;; These are swizzled to:
;;	0	+32	+32	+32	128	+32	+32	+32
;;	1	...
;;	...
;;	31
;;	256	...
;;	512	...
;;	768	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+32	+32	+32	512	+32	+32	+32
;;	1	...
;;	...
;;	31
;;	128	...
;;	256	...
;;	384	...

;; Do 128 four_complex_last_unfft operations
;;	distance between fft data elements is 128
;;	do 32 macros each processing 32 data values

IF OLDWAY EQ 1
	start_timer 9
	yloop_init  32			;; 32 iterations
	yr4_b4cl_four_complex_djbunfft_preload
c2b:	yr4_b4cl_four_complex_djbunfft rsi, 64, dist32, 2*dist32, rdi, YMM_SCD2/4, 32
	yloop	32, c2b, -32*64		;; Test loop counter, restore source pointer
	end_timer 9

	start_timer 10
	yloop_init  32			;; 32 iterations
	yr4_s4cl_four_complex_last_unfft_preload
c1b:	yr4_s4cl_four_complex_last_unfft rsi, 64, dist32, 2*dist32, rdi, YMM_SCD4, 32
	yloop	32, c1b			;; Test loop counter
	end_timer 10
ELSE
	start_timer 10
	lea	rdx, [rdi+32*YMM_SCD2/4]
	yloop_init  32			;; 32 iterations
	yr4_s4cl_sixteen_complex_last_unfft_preload
c1b:	yr4_s4cl_sixteen_complex_last_unfft rsi, 64, dist32, 2*dist32, rdi, YMM_SCD2/4, rdx, YMM_SCD4, 32
	yloop	32, c1b			;; Test loop counter
	end_timer 10
ENDIF

	ytwo_dest_adjust
	yfft_3_ret
	ENDM


;; Perform a 1280-element FFT.

yfft1280 MACRO type
	LOCAL	ymid2, ymid3, ymid4, yfinish_unfft
	LOCAL	b1b, b2b, b3a, b3b, b4a, b4b, b4c, b4d, c1b, c2b, c3a, c3b

;; Do a multiply with pre-FFTed inputs

	cmp	ffttype, 4
	je	ymid4

;; Do FFT levels 1,2,3
;; On input the 64-byte cache lines hold these data values:
;;	0	+40	+40	+40	640	+40	+40	+40
;;	1
;;	...
;;	39
;;	160	...
;;	320	...
;;	480	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+40	+40	+40	160	+40	+40	+40
;;	1
;;	...
;;	39
;;	320	...
;;	640	...
;;	960	...
;; These are swizzled to:
;;	0	+320	+320	+320	160	+320	+320	+320
;;	1
;;	...
;;	159

;; Do 160 eight_reals_fft operations
;;	distance between fft data elements is 160
;;	do 40 macros each processing 32 data values

	start_timer 4
	mov	rdi, sincos1		;; Load sin/cos pointer
	bump	rdi, 32*YMM_SCD4+8*YMM_SCD3
	yloop_init  40			;; 40 iterations
	yr4_fs4cl_eight_reals_fft_preload
b1b:	yr4_fs4cl_eight_reals_fft rsi, 64, 40*64, 80*64, rdi, YMM_SCD3, 40
	yloop	40, b1b, -40*64		;; Test loop counter, restore source pointer
	end_timer 4

;; Do FFT levels 4,5
;; On input the 64-byte cache lines hold these data values:
;;	0	+320	+320	+320	160	+320	+320	+320
;;	1
;;	...
;;	159
;; On output the 64-byte cache lines hold these data values:
;;	0	+320	+320	+320	32	+320	+320	+320
;;	1
;;	...
;;	31
;;	64	...
;;	...
;;	256	...

;; Do 32 ten_reals_fft operations
;; Do 96 five_complex_fft operations
;;	distance between fft data elements is 32
;;	do 32 macros each processing 40 data values

	start_timer 5
	bump	rdi, -40*YMM_SCD3-32*YMM_SCD4 ;; Load sin/cos pointer
	yloop_init  32			;; 32 iterations
	yr5_5cl_ten_reals_five_complex_djbfft_preload
b2b:	yr5_5cl_ten_reals_five_complex_djbfft rsi, 64, 32*64, rdi, YMM_SCD4, 32
	yloop	32, b2b, -32*64		;; Test loop counter, restore source pointer
	end_timer 5

;; Do FFT levels 6,7
;; On input the 64-byte cache lines hold these data values:
;;	0	+320	+320	+320	32	+320	+320	+320
;;	1
;;	...
;;	31
;;	64	...
;;	...
;;	256	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+320	+320	+320	8	+320	+320	+320
;;	1
;;	...
;;	7
;;	16	...
;;	...
;;	304	...

;; Do 8 eight_reals_fft operations
;; Do 152 four_complex_fft operations
;;	distance between fft data elements is 8
;;	do 40 macros each processing 32 data values

	start_timer 6
	bump	rdi, -32*YMM_SCD4-8*YMM_SCD3 ;; Next sin/cos pointer
	yloop_init  40			;; 5 iterations of 8
	yr4_4cl_eight_reals_four_complex_djbfft_preload
b3a:	yr4_4cl_eight_reals_four_complex_djbfft rsi, 64, 8*64, 16*64, rdi, YMM_SCD3, 8
	yloop	8, b3a, -8*64+32*64, rdi, -8*YMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer

	loops_reset			;; 4 more iters of 8
	yr4_rb4cl_four_complex_djbfft_preload
b3b:	yr4_rb4cl_four_complex_djbfft rsi, 64, 8*64, 16*64, rdi, YMM_SCD3, 8
	yloop	8, b3b, -8*64+32*64, rdi, -8*YMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer
	yloop	5, b3b, -5*32*64	;; Test loop counter, restore source pointer
	end_timer 6

;; Do FFT levels 8,9
;; On input and output the 64-byte cache lines hold these data values:
;;	0	+320	+320	+320	8	+320	+320	+320
;;	1
;;	...
;;	7
;;	16	...
;;	...
;;	304	...

;; Do 1 sixteen_reals_fft operation
;; Do 79 eight_complex_fft operations
;;	distance between fft data elements is 1
;;	do 20 macros each processing 64 data values

	cmp	ffttype, 2
	je	ymid2
	ja	ymid3

	start_timer 7
;;	yr8_8cl_sixteen_reals_eight_complex_fft_final rsi, 8*64, 64, 2*64, 4*64
	yr8_dispatch call, y8real_fft_final, 64, 2*64, 4*64
	yloop_init  19			;; 19 more iterations
b4a:	yr8_8cl_eight_complex_fft_final rsi, 8*64, 64, 2*64, 4*64
	yloop	19, b4a			;; Test loop counter
	end_timer 7
	yfft_1_ret

ymid2:	start_timer 7
;;	yr8_8cl_sixteen_reals_eight_complex_with_square rsi, 8*64, 64, 2*64, 4*64
	yr8_dispatch call, y8real_square, 64, 2*64, 4*64
	yloop_init  19			;; 19 more iterations
	yr8_8cl_eight_complex_with_square_preload
b4b:	yr8_8cl_eight_complex_with_square rsi, 8*64, 64, 2*64, 4*64
	yloop	19, b4b			;; Test loop counter
	end_timer 7
	jmp	yfinish_unfft

ymid3:	start_timer 7
x64	mov	r8, DEST2ARG
	mov	rbp, DIST_TO_MULSRCARG
;;	yr8_8cl_sixteen_reals_eight_complex_with_mult rsi, 8*64, 64, 2*64, 4*64
	yr8_dispatch call, y8real_mult, 64, 2*64, 4*64
	yloop_init  19			;; 19 more iterations
	yr8_8cl_eight_complex_with_mult_preload
b4c:	yr8_8cl_eight_complex_with_mult rsi, 8*64, 64, 2*64, 4*64
	yloop	19, b4c			;; Test loop counter
x64	add	rsi, r8
	end_timer 7
	jmp	yfinish_unfft

ymid4:	start_timer 7
	mov	rbp, DIST_TO_MULSRCARG
;;	yr8_8cl_sixteen_reals_eight_complex_with_mulf rsi, 8*64, 64, 2*64, 4*64
	yr8_dispatch call, y8real_mulf, 64, 2*64, 4*64
	yloop_init  19			;; 19 more iterations
	yr8_8cl_eight_complex_with_mulf_preload
b4d:	yr8_8cl_eight_complex_with_mulf rsi, 8*64, 64, 2*64, 4*64
	yloop	19, b4d			;; Test loop counter
	end_timer 7
yfinish_unfft:
	bump	rsi, -20*8*64		;; Restore source pointer

;; Do inverse FFT levels 6,7
;; On input the 64-byte cache lines hold these data values:
;;	0	+320	+320	+320	8	+320	+320	+320
;;	1
;;	...
;;	7
;;	16	...
;;	...
;;	304	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+320	+320	+320	32	+320	+320	+320
;;	1
;;	...
;;	31
;;	64	...
;;	...
;;	256	...

;; Do 8 eight_reals_unfft operations
;; Do 152 four_complex_unfft operations
;;	distance between fft data elements is 8
;;	do 40 macros each processing 32 data values

	start_timer 8
	mov	rdi, sincos1		;; Load sin/cos pointer
	yloop_init  40			;; 5 iterations of 8
	yr4_4cl_eight_reals_four_complex_djbunfft_preload
c3a:	yr4_4cl_eight_reals_four_complex_djbunfft rsi, 64, 8*64, 16*64, rdi, YMM_SCD3, 8
	yloop	8, c3a, -8*64+32*64, rdi, -8*YMM_SCD3 ;; Test loop counter, next source pointer, restore sine/cosine pointer

	loops_reset			;; 4 more iters of 8
	yr4_rb4cl_four_complex_djbunfft_preload
c3b:	yr4_rb4cl_four_complex_djbunfft rsi, 64, 8*64, 16*64, rdi, YMM_SCD3, 8
	yloop	8, c3b, -8*64+32*64, rdi, -8*YMM_SCD3 ;; Test loop counter, next source pointer, restore sine/cosine pointer
	yloop	5, c3b, -5*32*64	;; Test loop counter, restore source pointer
	end_timer 8

;; Do inverse FFT levels 4,5
;; On input the 64-byte cache lines hold these data values:
;;	0	+320	+320	+320	32	+320	+320	+320
;;	1
;;	...
;;	31
;;	64	...
;;	...
;;	256	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+320	+320	+320	160	+320	+320	+320
;;	1
;;	...
;;	159

;; Do 32 ten_reals_unfft operations
;; Do 96 five_complex_unfft operations
;;	distance between fft data elements is 32
;;	do 32 macros each processing 40 data values

	start_timer 9
	bump	rdi, 8*YMM_SCD3		;; Next sine/cosine pointer
	yloop_init  32			;; 32 iterations
	yr5_5cl_ten_reals_five_complex_djbunfft_preload
c2b:	yr5_5cl_ten_reals_five_complex_djbunfft rsi, 64, 32*64, rdi, YMM_SCD4, 32
	yloop	32, c2b, -32*64		;; Test loop counter, restore source pointer
	end_timer 9

;; Do inverse FFT levels 1,2,3
;; On input the 64-byte cache lines hold these data values:
;;	0	+320	+320	+320	160	+320	+320	+320
;;	1
;;	...
;;	159
;; These are swizzled to:
;;	0	+40	+40	+40	160	+40	+40	+40
;;	1	...
;;	...
;;	39
;;	320	...
;;	640	...
;;	960	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+40	+40	+40	640	+40	+40	+40
;;	1	...
;;	...
;;	39
;;	160	...
;;	320	...
;;	480	...

;; Do 160 eight_reals_unfft operations
;;	distance between fft data elements is 160
;;	do 40 macros each processing 32 data values

	start_timer 10
	yloop_init  40			;; 40 iterations
	yr4_s4cl_eight_reals_unfft_preload
c1b:	yr4_s4cl_eight_reals_unfft rsi, 64, 40*64, 80*64, rdi, YMM_SCD3, 40
	yloop	40, c1b			;; Test loop counter
	end_timer 10

	ytwo_dest_adjust
	yfft_3_ret
	ENDM

;; Perform a 1280-element negacyclic FFT

yfft1280p MACRO type
	LOCAL	ymid2p, ymid3p, ymid4p, yfinish_unfft
	LOCAL	b1b, b2b, b3b, b4a, b4b, b4c, b4d, c1b, c2b, c3b

;; Do a multiply with pre-FFTed inputs

	cmp	ffttype, 4
	je	ymid4p

;; Do FFT levels 1,2,3
;; On input the 64-byte cache lines hold these data values:
;;	0	+40	+40	+40	640	+40	+40	+40
;;	1
;;	...
;;	39
;;	160	...
;;	320	...
;;	480	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+40	+40	+40	160	+40	+40	+40
;;	1
;;	...
;;	39
;;	320	...
;;	640	...
;;	960	...
;; These are swizzled to:
;;	0	+320	+320	+320	160	+320	+320	+320
;;	1
;;	...
;;	159

;; Do 160 four_complex_first_fft operations
;;	distance between fft data elements is 160
;;	do 40 macros each processing 32 data values

	start_timer 4
	mov	rdi, sincos1		;; Load sin/cos pointer
	bump	rdi, 32*YMM_SCD2/4+8*YMM_SCD2/4
	yloop_init  40			;; 40 iterations
	yr4_fs4cl_four_complex_first_djbfft_preload
b1b:	yr4_fs4cl_four_complex_first_djbfft rsi, 64, 40*64, 80*64, rdi, YMM_SCD4, 40
	yloop	40, b1b, -40*64		;; Test loop counter, restore source pointer
	end_timer 4

;; Do FFT levels 4,5
;; On input the 64-byte cache lines hold these data values:
;;	0	+320	+320	+320	160	+320	+320	+320
;;	1
;;	...
;;	159
;; On output the 64-byte cache lines hold these data values:
;;	0	+320	+320	+320	32	+320	+320	+320
;;	1
;;	...
;;	31
;;	64	...
;;	...
;;	256	...

;; Do 128 five_complex_fft operations
;;	distance between fft data elements is 32
;;	do 32 macros each processing 40 data values

	start_timer 5
	bump	rdi, -40*YMM_SCD4-32*YMM_SCD2/4 ;; Load sin/cos pointer
	yloop_init  32			;; 32 iterations
	yr5_b5cl_five_complex_djbfft_preload
b2b:	yr5_b5cl_five_complex_djbfft rsi, 64, 32*64, rdi, YMM_SCD2/4, 32
	yloop	32, b2b, -32*64		;; Test loop counter, restore source pointer
	end_timer 5

;; Do FFT levels 6,7
;; On input the 64-byte cache lines hold these data values:
;;	0	+320	+320	+320	32	+320	+320	+320
;;	1
;;	...
;;	31
;;	64	...
;;	...
;;	256	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+320	+320	+320	8	+320	+320	+320
;;	1
;;	...
;;	7
;;	16	...
;;	...
;;	304	...

;; Do 160 four_complex_fft operations
;;	distance between fft data elements is 8
;;	do 40 macros each processing 32 data values

	start_timer 6
	bump	rdi, -32*YMM_SCD2/4-8*YMM_SCD2/4 ;; Next sin/cos pointer
	yloop_init  40			;; 5 iterations of 8
	yr4_b4cl_four_complex_djbfft_preload
b3b:	yr4_b4cl_four_complex_djbfft rsi, 64, 8*64, 16*64, rdi, YMM_SCD2/4, 8
	yloop	8, b3b, -8*64+32*64, rdi, -8*YMM_SCD2/4 ;; Test loop counter, next source pointer, restore sin/cos pointer
	yloop	5, b3b, -5*32*64	;; Test loop counter, restore source pointer
	end_timer 6

;; Do FFT levels 8,9
;; On input and output the 64-byte cache lines hold these data values:
;;	0	+320	+320	+320	8	+320	+320	+320
;;	1
;;	...
;;	7
;;	16	...
;;	...
;;	304	...

;; Do 80 eight_complex_fft operations
;;	distance between fft data elements is 1
;;	do 20 macros each processing 64 data values

	cmp	ffttype, 2
	je	ymid2p
	ja	ymid3p

	start_timer 7
	yloop_init  20			;; 20 iterations
b4a:	yr8_8cl_eight_complex_fft_final rsi, 8*64, 64, 2*64, 4*64
	yloop	20, b4a			;; Test loop counter
	end_timer 7
	yfft_1_ret

ymid2p:	start_timer 7
	yloop_init  20			;; 20 iterations
	yr8_8cl_eight_complex_with_square_preload
b4b:	yr8_8cl_eight_complex_with_square rsi, 8*64, 64, 2*64, 4*64
	yloop	20, b4b			;; Test loop counter
	end_timer 7
	jmp	yfinish_unfft

ymid3p:	start_timer 7
x64	mov	r8, DEST2ARG
	mov	rbp, DIST_TO_MULSRCARG
	yloop_init  20			;; 20 iterations
	yr8_8cl_eight_complex_with_mult_preload
b4c:	yr8_8cl_eight_complex_with_mult rsi, 8*64, 64, 2*64, 4*64
	yloop	20, b4c			;; Test loop counter
x64	add	rsi, r8
	end_timer 7
	jmp	yfinish_unfft

ymid4p:	start_timer 7
	mov	rbp, DIST_TO_MULSRCARG
	yloop_init  20			;; 20 iterations
	yr8_8cl_eight_complex_with_mulf_preload
b4d:	yr8_8cl_eight_complex_with_mulf rsi, 8*64, 64, 2*64, 4*64
	yloop	20, b4d			;; Test loop counter
	end_timer 7
yfinish_unfft:
	bump	rsi, -20*8*64		;; Restore source pointer

;; Do inverse FFT levels 6,7
;; On input the 64-byte cache lines hold these data values:
;;	0	+320	+320	+320	8	+320	+320	+320
;;	1
;;	...
;;	7
;;	16	...
;;	...
;;	304	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+320	+320	+320	32	+320	+320	+320
;;	1
;;	...
;;	31
;;	64	...
;;	...
;;	256	...

;; Do 160 four_complex_unfft operations
;;	distance between fft data elements is 8
;;	do 40 macros each processing 32 data values

	start_timer 8
	mov	rdi, sincos1		;; Load sin/cos pointer
	yloop_init  40			;; 5 iterations of 8
	yr4_b4cl_four_complex_djbunfft_preload
c3b:	yr4_b4cl_four_complex_djbunfft rsi, 64, 8*64, 16*64, rdi, YMM_SCD2/4, 8
	yloop	8, c3b, -8*64+32*64, rdi, -8*YMM_SCD2/4 ;; Test loop counter, next source pointer, restore sine/cosine pointer
	yloop	5, c3b, -5*32*64	;; Test loop counter, restore source pointer
	end_timer 8

;; Do inverse FFT levels 4,5
;; On input the 64-byte cache lines hold these data values:
;;	0	+320	+320	+320	32	+320	+320	+320
;;	1
;;	...
;;	31
;;	64	...
;;	...
;;	256	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+320	+320	+320	160	+320	+320	+320
;;	1
;;	...
;;	159

;; Do 128 five_complex_unfft operations
;;	distance between fft data elements is 32
;;	do 32 macros each processing 40 data values

	start_timer 9
	bump	rdi, 8*YMM_SCD2/4	;; Next sine/cosine pointer
	yloop_init  32			;; 32 iterations
	yr5_b5cl_five_complex_djbunfft_preload
c2b:	yr5_b5cl_five_complex_djbunfft rsi, 64, 32*64, rdi, YMM_SCD2/4, 32
	yloop	32, c2b, -32*64		;; Test loop counter, restore source pointer
	end_timer 9

;; Do inverse FFT levels 1,2,3
;; On input the 64-byte cache lines hold these data values:
;;	0	+320	+320	+320	160	+320	+320	+320
;;	1
;;	...
;;	159
;; These are swizzled to:
;;	0	+40	+40	+40	160	+40	+40	+40
;;	1	...
;;	...
;;	39
;;	320	...
;;	640	...
;;	960	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+40	+40	+40	640	+40	+40	+40
;;	1	...
;;	...
;;	39
;;	160	...
;;	320	...
;;	480	...

;; Do 160 four_complex_last_unfft operations
;;	distance between fft data elements is 160
;;	do 40 macros each processing 32 data values

	start_timer 10
	yloop_init  40			;; 40 iterations
	yr4_s4cl_four_complex_last_unfft_preload
c1b:	yr4_s4cl_four_complex_last_unfft rsi, 64, 40*64, 80*64, rdi, YMM_SCD4, 40
	yloop	40, c1b			;; Test loop counter
	end_timer 10

	ytwo_dest_adjust
	yfft_3_ret
	ENDM


;; Perform a 1536-element FFT.

yfft1536 MACRO type
	LOCAL	ymid2, ymid3, ymid4, yfinish_unfft
	LOCAL	b1b, b2b, b3a, b3b, b4a, b4b, b5a, b5b, b5c, b5d, c1b, c2b, c3a, c3b, c4a, c4b

;; Do a multiply with pre-FFTed inputs

	cmp	ffttype, 4
	je	ymid4

;; Do FFT levels 1,2,3
;; On input the 64-byte cache lines hold these data values:
;;	0	48	96	144	768	+48	+48	+48
;;	1
;;	...
;;	47
;;	192	...
;;	384	...
;;	576	...
;; On output these are swizzled to:
;;	0	384	768	1152	192	+384	+384	+384
;;	1
;;	...
;;	191

;; Do 192 eight_reals_fft operations
;;	distance between fft data elements is 192
;;	do 48 macros each processing 32 data values

	start_timer 4
	mov	rdi, sincos1		;; Load sin/cos pointer
	bump	rdi, 64*YMM_SCD2+16*YMM_SCD3
	yloop_init  48			;; 48 iterations
	yr4_fs4cl_eight_reals_fft_preload
b1b:	yr4_fs4cl_eight_reals_fft rsi, 64, 3*dist16, 6*dist16, rdi, YMM_SCD3, 16
	yloop	16, b1b, -16*64+dist16	;; Test loop counter, next source pointer
	yloop	3, b1b, -3*dist16	;; Test loop counter, restore source pointer
	end_timer 4

;; Do FFT levels 4,5
;; On input the 64-byte cache lines hold these data values:
;;	0	384	768	1152	192	+384	+384	+384
;;	1
;;	...
;;	191
;; On output the 64-byte cache lines hold these data values:
;;	0	384	768	1152	64	+384	+384	+384
;;	1
;;	...
;;	63
;;	128	...
;;	256	...

;; Do 64 six_reals_fft operations
;; Do 192 three_complex_fft operations
;;	distance between fft data elements is 64
;;	do 64 macros each processing 24 data values

	start_timer 5
	bump	rdi, -48*YMM_SCD3-64*YMM_SCD2 ;; Load sin/cos pointer
	yloop_init  64			;; 64 iterations
	yr3_3cl_six_reals_three_complex_djbfft_preload
b2b:	yr3_3cl_six_reals_three_complex_djbfft rsi, 64, 4*dist16, rdi, YMM_SCD2, 16
	yloop	16, b2b, -16*64+dist16	;; Test loop counter, next source pointer
	yloop	4, b2b, -4*dist16	;; Test loop counter, restore source pointer
	end_timer 5

;; Do FFT levels 6,7
;; On input the 64-byte cache lines hold these data values:
;;	0	384	768	1152	64	+384	+384	+384
;;	1
;;	...
;;	63
;;	128	...
;;	256	...
;; On output the 64-byte cache lines hold these data values:
;;	0	384	768	1152	16	+384	+384	+384
;;	1
;;	...
;;	15
;;	32	...
;;	...
;;	352	...

;; Do 16 eight_reals_fft operations
;; Do 176 four_complex_fft operations
;;	distance between fft data elements is 16
;;	do 48 macros each processing 32 data values

	start_timer 6
	bump	rdi, -64*YMM_SCD2-16*YMM_SCD3 ;; Next sin/cos pointer
	yloop_init  48			;; 3 iterations of 16
	yr4_4cl_eight_reals_four_complex_djbfft_preload
b3a:	yr4_4cl_eight_reals_four_complex_djbfft rsi, 64, dist16, 2*dist16, rdi, YMM_SCD3, 16
	yloop	16, b3a, -16*64+4*dist16, rdi, -16*YMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer

	loops_reset			;; 2 more iters of 16
	yr4_rb4cl_four_complex_djbfft_preload
b3b:	yr4_rb4cl_four_complex_djbfft rsi, 64, dist16, 2*dist16, rdi, YMM_SCD3, 16
	yloop	16, b3b, -16*64+4*dist16, rdi, -16*YMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer
	yloop	3, b3b, -3*4*dist16	;; Test loop counter, restore source pointer
	end_timer 6

;; Do FFT levels 8,9
;; On input the 64-byte cache lines hold these data values:
;;	0	384	768	1152	16	+384	+384	+384
;;	1
;;	...
;;	15
;;	32	...
;;	...
;;	352	...
;; On output the 64-byte cache lines hold these data values:
;;	0	384	768	1152	4	+384	+384	+384
;;	1
;;	...
;;	3
;;	8	...
;;	...
;;	380	...

;; Do 4 eight_reals_fft operations
;; Do 188 four_complex_fft operations
;;	distance between fft data elements is 4
;;	do 48 macros each processing 32 data values

	start_timer 7
	yloop_init  48			;; 12 iterations of 4
	yr4_4cl_eight_reals_four_complex_djbfft_preload
b4a:	yr4_4cl_eight_reals_four_complex_djbfft rsi, 64, 4*64, 8*64, rdi, 4*YMM_SCD3, 4
	yloop	4, b4a, -4*64+dist16, rdi, -4*4*YMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer

	loops_reset			;; 11 more iters of 4
	yr4_rb4cl_four_complex_djbfft_preload
b4b:	yr4_rb4cl_four_complex_djbfft rsi, 64, 4*64, 8*64, rdi, 4*YMM_SCD3, 4
	yloop	4, b4b, -4*64+dist16, rdi, -4*4*YMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer
	yloop	12, b4b, -12*dist16	;; Test loop counter, restore source pointer
	end_timer 7

;; Do FFT levels 10,11
;; On input and output the 64-byte cache lines hold these data values:
;;	0	384	768	1152	4	+384	+384	+384
;;	1
;;	...
;;	3
;;	8	...
;;	...
;;	380	...

;; Do 1 eight_reals_fft operation
;; Do 191 four_complex_fft operations
;;	distance between fft data elements is 1
;;	do 48 macros each processing 32 data values

	cmp	ffttype, 2
	je	ymid2
	ja	ymid3

	start_timer 8
;;	yr4_4cl_eight_reals_four_complex_fft_final rsi, 4*64, 64, 2*64
	yr4_dispatch call, yreal_fft_final, 64, 2*64
	yloop_init  48, 1, 4		;; 47 more iterations
b5a:	yr4_4cl_four_complex_fft_final rsi, 4*64, 64, 2*64
	yloop	4, b5a, -4*4*64+dist16	;; Test loop counter, next source pointer
	yloop	12, b5a			;; Test loop counter
	end_timer 8
	yfft_1_ret

ymid2:	start_timer 8
;;	yr4_4cl_eight_reals_four_complex_with_square rsi, 4*64, 64, 2*64
	yr4_dispatch call, yreal_square, 64, 2*64
	yloop_init  48, 1, 4		;; 47 more iterations
	yr4_4cl_four_complex_with_square_preload
b5b:	yr4_4cl_four_complex_with_square rsi, 4*64, 64, 2*64, 1
	yloop	4, b5b, -4*4*64+dist16	;; Test loop counter, next source pointer
	yloop	12, b5b			;; Test loop counter
	end_timer 8
	jmp	yfinish_unfft

ymid3:	start_timer 8
x64	mov	r8, DEST2ARG
	mov	rbp, DIST_TO_MULSRCARG
;;	yr4_4cl_eight_reals_four_complex_with_mult rsi, 4*64, 64, 2*64
	yr4_dispatch call, yreal_mult, 64, 2*64
	yloop_init  48, 1, 4		;; 47 more iterations
	yr4_4cl_four_complex_with_mult_preload
b5c:	yr4_4cl_four_complex_with_mult rsi, 4*64, 64, 2*64
	yloop	4, b5c, -4*4*64+dist16	;; Test loop counter, next source pointer
	yloop	12, b5c			;; Test loop counter
x64	add	rsi, r8
	end_timer 8
	jmp	yfinish_unfft

ymid4:	start_timer 8
	mov	rbp, DIST_TO_MULSRCARG
;;	yr4_4cl_eight_reals_four_complex_with_mulf rsi, 4*64, 64, 2*64
	yr4_dispatch call, yreal_mulf, 64, 2*64
	yloop_init  48, 1, 4		;; 47 more iterations
	yr4_4cl_four_complex_with_mulf_preload
b5d:	yr4_4cl_four_complex_with_mulf rsi, 4*64, 64, 2*64
	yloop	4, b5d, -4*4*64+dist16	;; Test loop counter, next source pointer
	yloop	12, b5d			;; Test loop counter
	end_timer 8
yfinish_unfft:
	bump	rsi, -12*dist16		;; Restore source pointer

;; Do inverse FFT levels 8,9
;; On input the 64-byte cache lines hold these data values:
;;	0	384	768	1152	4	+384	+384	+384
;;	1
;;	...
;;	3
;;	8	...
;;	...
;;	380	...
;; On output the 64-byte cache lines hold these data values:
;;	0	384	768	1152	16	+384	+384	+384
;;	1
;;	...
;;	15
;;	32	...
;;	...
;;	352	...

;; Do 4 eight_reals_unfft operations
;; Do 188 four_complex_unfft operations
;;	distance between fft data elements is 4
;;	do 48 macros each processing 32 data values

	start_timer 9
	mov	rdi, sincos1		;; Load sin/cos pointer
	yloop_init  48			;; 12 iterations of 4
	yr4_4cl_eight_reals_four_complex_djbunfft_preload
c4a:	yr4_4cl_eight_reals_four_complex_djbunfft rsi, 64, 4*64, 8*64, rdi, 4*YMM_SCD3, 4
	yloop	4, c4a, -4*64+dist16, rdi, -4*4*YMM_SCD3 ;; Test loop counter, next source pointer, restore sine/cosine pointer

	loops_reset			;; 11 more iters of 4
	yr4_rb4cl_four_complex_djbunfft_preload
c4b:	yr4_rb4cl_four_complex_djbunfft rsi, 64, 4*64, 8*64, rdi, 4*YMM_SCD3, 4
	yloop	4, c4b, -4*64+dist16, rdi, -4*4*YMM_SCD3 ;; Test loop counter, next source pointer, restore sine/cosine pointer
	yloop	12, c4b, -12*dist16	;; Test loop counter, restore source pointer
	end_timer 9

;; Do inverse FFT levels 6,7
;; On input the 64-byte cache lines hold these data values:
;;	0	384	768	1152	16	+384	+384	+384
;;	1
;;	...
;;	15
;;	32	...
;;	...
;;	352	...
;; On output the 64-byte cache lines hold these data values:
;;	0	384	768	1152	64	+384	+384	+384
;;	1
;;	...
;;	63
;;	128	...
;;	256	...

;; Do 16 eight_reals_unfft operations
;; Do 176 four_complex_unfft operations
;;	distance between fft data elements is 16
;;	do 48 macros each processing 32 data values

	start_timer 10
	yloop_init  48			;; 3 iterations of 16
	yr4_4cl_eight_reals_four_complex_djbunfft_preload
c3a:	yr4_4cl_eight_reals_four_complex_djbunfft rsi, 64, dist16, 2*dist16, rdi, YMM_SCD3, 16
	yloop	16, c3a, -16*64+4*dist16, rdi, -16*YMM_SCD3 ;; Test loop counter, next source pointer, restore sine/cosine pointer

	loops_reset			;; 2 more iters of 16
	yr4_rb4cl_four_complex_djbunfft_preload
c3b:	yr4_rb4cl_four_complex_djbunfft rsi, 64, dist16, 2*dist16, rdi, YMM_SCD3, 16
	yloop	16, c3b, -16*64+4*dist16, rdi, -16*YMM_SCD3 ;; Test loop counter, next source pointer, restore sine/cosine pointer
	yloop	3, c3b, -3*4*dist16	;; Test loop counter, restore source pointer
	end_timer 10

;; Do inverse FFT levels 4,5
;; On input the 64-byte cache lines hold these data values:
;;	0	384	768	1152	64	+384	+384	+384
;;	1
;;	...
;;	63
;;	128	...
;;	256	...
;; On output the 64-byte cache lines hold these data values:
;;	0	384	768	1152	192	+384	+384	+384
;;	1
;;	...
;;	191

;; Do 64 six_reals_unfft operations
;; Do 192 three_complex_unfft operations
;;	distance between fft data elements is 64
;;	do 64 macros each processing 24 data values

	start_timer 11
	bump	rdi, 16*YMM_SCD3	;; Next sine/cosine pointer
	yloop_init  64			;; 64 iterations
	yr3_3cl_six_reals_three_complex_djbunfft_preload
c2b:	yr3_3cl_six_reals_three_complex_djbunfft rsi, 64, 4*dist16, rdi, YMM_SCD2, 16
	yloop	16, c2b, -16*64+dist16	;; Test loop counter, next source pointer
	yloop	4, c2b, -4*dist16	;; Test loop counter, restore source pointer
	end_timer 11

;; Do inverse FFT levels 1,2,3
;; On input the 64-byte cache lines hold these data values:
;;	0	384	768	1152	192	+384	+384	+384
;;	1
;;	...
;;	191
;; These are swizzled to:
;;	0	48	96	144	192	+48	+48	+48
;;	1	...
;;	...
;;	47
;;	384	...
;;	768	...
;;	1152	...
;; On output the 64-byte cache lines hold these data values:
;;	0	48	96	144	768	+48	+48	+48
;;	1	...
;;	...
;;	47
;;	192	...
;;	384	...
;;	576	...

;; Do 192 eight_reals_unfft operations
;;	distance between fft data elements is 192
;;	do 48 macros each processing 32 data values

	start_timer 12
	yloop_init  48			;; 48 iterations
	yr4_s4cl_eight_reals_unfft_preload
c1b:	yr4_s4cl_eight_reals_unfft rsi, 64, 3*dist16, 6*dist16, rdi, YMM_SCD3, 16
	yloop	16, c1b, -16*64+dist16	;; Test loop counter, next source pointer
	yloop	3, c1b			;; Test loop counter
	end_timer 12

	ytwo_dest_adjust
	yfft_3_ret
	ENDM

;; Perform a 1536-element negacyclic FFT

yfft1536p MACRO type
	LOCAL	ymid2p, ymid3p, ymid4p, yfinish_unfft
	LOCAL	b1b, b2b, b3b, b4b, b5a, b5b, b5c, b5d, c1b, c2b, c3b, c4b

;; Do a multiply with pre-FFTed inputs

	cmp	ffttype, 4
	je	ymid4p

;; Do FFT levels 1,2,3
;; On input the 64-byte cache lines hold these data values:
;;	0	48	96	144	768	+48	+48	+48
;;	1
;;	...
;;	47
;;	192	...
;;	384	...
;;	576	...
;; On output these are swizzled to:
;;	0	384	768	1152	192	+384	+384	+384
;;	1
;;	...
;;	191

;; Do 192 four_complex_first_fft operations
;;	distance between fft data elements is 192
;;	do 48 macros each processing 32 data values

	start_timer 4
	mov	rdi, sincos1		;; Load sin/cos pointer
	bump	rdi, 64*YMM_SCD1/4+16*YMM_SCD2/4
	yloop_init  48			;; 48 iterations
	yr4_fs4cl_four_complex_first_djbfft_preload
b1b:	yr4_fs4cl_four_complex_first_djbfft rsi, 64, 3*dist16, 6*dist16, rdi, YMM_SCD4, 16
	yloop	16, b1b, -16*64+dist16	;; Test loop counter, next source pointer
	yloop	3, b1b, -3*dist16	;; Test loop counter, restore source pointer
	end_timer 4

;; Do FFT levels 4,5
;; On input the 64-byte cache lines hold these data values:
;;	0	384	768	1152	192	+384	+384	+384
;;	1
;;	...
;;	191
;; On output the 64-byte cache lines hold these data values:
;;	0	384	768	1152	64	+384	+384	+384
;;	1
;;	...
;;	63
;;	128	...
;;	256	...

;; Do 256 three_complex_fft operations
;;	distance between fft data elements is 64
;;	do 64 macros each processing 24 data values

	start_timer 5
	bump	rdi, -48*YMM_SCD4-64*YMM_SCD1/4 ;; Load sin/cos pointer
	yloop_init  64			;; 64 iterations
	yr3_b3cl_three_complex_djbfft_preload
b2b:	yr3_b3cl_three_complex_djbfft rsi, 64, 4*dist16, rdi, YMM_SCD1/4, 16
	yloop	16, b2b, -16*64+dist16	;; Test loop counter, next source pointer
	yloop	4, b2b, -4*dist16	;; Test loop counter, restore source pointer
	end_timer 5

;; Do FFT levels 6,7
;; On input the 64-byte cache lines hold these data values:
;;	0	384	768	1152	64	+384	+384	+384
;;	1
;;	...
;;	63
;;	128	...
;;	256	...
;; On output the 64-byte cache lines hold these data values:
;;	0	384	768	1152	16	+384	+384	+384
;;	1
;;	...
;;	15
;;	32	...
;;	...
;;	352	...

;; Do 192 four_complex_fft operations
;;	distance between fft data elements is 16
;;	do 48 macros each processing 32 data values

	start_timer 6
	bump	rdi, -64*YMM_SCD1/4-16*YMM_SCD2/4 ;; Next sin/cos pointer
	yloop_init  48			;; 3 iterations of 16
	yr4_b4cl_four_complex_djbfft_preload
b3b:	yr4_b4cl_four_complex_djbfft rsi, 64, dist16, 2*dist16, rdi, YMM_SCD2/4, 16
	yloop	16, b3b, -16*64+4*dist16, rdi, -16*YMM_SCD2/4 ;; Test loop counter, next source pointer, restore sin/cos pointer
	yloop	3, b3b, -3*4*dist16	;; Test loop counter, restore source pointer
	end_timer 6

;; Do FFT levels 8,9
;; On input the 64-byte cache lines hold these data values:
;;	0	384	768	1152	16	+384	+384	+384
;;	1
;;	...
;;	15
;;	32	...
;;	...
;;	352	...
;; On output the 64-byte cache lines hold these data values:
;;	0	384	768	1152	4	+384	+384	+384
;;	1
;;	...
;;	3
;;	8	...
;;	...
;;	380	...

;; Do 192 four_complex_fft operations
;;	distance between fft data elements is 4
;;	do 48 macros each processing 32 data values

	start_timer 7
	yloop_init  48			;; 12 iterations of 4
	yr4_b4cl_four_complex_djbfft_preload
b4b:	yr4_b4cl_four_complex_djbfft rsi, 64, 4*64, 8*64, rdi, 4*YMM_SCD2/4, 4
	yloop	4, b4b, -4*64+dist16, rdi, -4*4*YMM_SCD2/4 ;; Test loop counter, next source pointer, restore sin/cos pointer
	yloop	12, b4b, -12*dist16	;; Test loop counter, restore source pointer
	end_timer 7

;; Do FFT levels 10,11
;; On input and output the 64-byte cache lines hold these data values:
;;	0	384	768	1152	4	+384	+384	+384
;;	1
;;	...
;;	3
;;	8	...
;;	...
;;	380	...

;; Do 192 four_complex_fft operations
;;	distance between fft data elements is 1
;;	do 48 macros each processing 32 data values

	cmp	ffttype, 2
	je	ymid2p
	ja	ymid3p

	start_timer 8
	yloop_init  48			;; 48 iterations
b5a:	yr4_4cl_four_complex_fft_final rsi, 4*64, 64, 2*64
	yloop	4, b5a, -4*4*64+dist16	;; Test loop counter, next source pointer
	yloop	12, b5a			;; Test loop counter
	end_timer 8
	yfft_1_ret

ymid2p:	start_timer 8
	yloop_init  48			;; 48 iterations
	yr4_4cl_four_complex_with_square_preload
b5b:	yr4_4cl_four_complex_with_square rsi, 4*64, 64, 2*64, 4
	yloop	4, b5b, -4*4*64+dist16	;; Test loop counter, next source pointer
	yloop	12, b5b			;; Test loop counter
	end_timer 8
	jmp	yfinish_unfft

ymid3p:	start_timer 8
x64	mov	r8, DEST2ARG
	mov	rbp, DIST_TO_MULSRCARG
	yloop_init  48			;; 48 iterations
	yr4_4cl_four_complex_with_mult_preload
b5c:	yr4_4cl_four_complex_with_mult rsi, 4*64, 64, 2*64
	yloop	4, b5c, -4*4*64+dist16	;; Test loop counter, next source pointer
	yloop	12, b5c			;; Test loop counter
x64	add	rsi, r8
	end_timer 8
	jmp	yfinish_unfft

ymid4p:	start_timer 8
	mov	rbp, DIST_TO_MULSRCARG
	yloop_init  48			;; 48 iterations
	yr4_4cl_four_complex_with_mulf_preload
b5d:	yr4_4cl_four_complex_with_mulf rsi, 4*64, 64, 2*64
	yloop	4, b5d, -4*4*64+dist16	;; Test loop counter, next source pointer
	yloop	12, b5d			;; Test loop counter
	end_timer 8
yfinish_unfft:
	bump	rsi, -12*dist16		;; Restore source pointer

;; Do inverse FFT levels 8,9
;; On input the 64-byte cache lines hold these data values:
;;	0	384	768	1152	4	+384	+384	+384
;;	1
;;	...
;;	3
;;	8	...
;;	...
;;	380	...
;; On output the 64-byte cache lines hold these data values:
;;	0	384	768	1152	16	+384	+384	+384
;;	1
;;	...
;;	15
;;	32	...
;;	...
;;	352	...

;; Do 192 four_complex_unfft operations
;;	distance between fft data elements is 4
;;	do 48 macros each processing 32 data values

	start_timer 9
	mov	rdi, sincos1		;; Load sin/cos pointer
	yloop_init  48			;; 12 iterations of 4
	yr4_b4cl_four_complex_djbunfft_preload
c4b:	yr4_b4cl_four_complex_djbunfft rsi, 64, 4*64, 8*64, rdi, 4*YMM_SCD2/4, 4
	yloop	4, c4b, -4*64+dist16, rdi, -4*4*YMM_SCD2/4 ;; Test loop counter, next source pointer, restore sine/cosine pointer
	yloop	12, c4b, -12*dist16	;; Test loop counter, restore source pointer
	end_timer 9

;; Do inverse FFT levels 6,7
;; On input the 64-byte cache lines hold these data values:
;;	0	384	768	1152	16	+384	+384	+384
;;	1
;;	...
;;	15
;;	32	...
;;	...
;;	352	...
;; On output the 64-byte cache lines hold these data values:
;;	0	384	768	1152	64	+384	+384	+384
;;	1
;;	...
;;	63
;;	128	...
;;	256	...

;; Do 192 four_complex_unfft operations
;;	distance between fft data elements is 16
;;	do 48 macros each processing 32 data values

	start_timer 10
	yloop_init  48			;; 3 iterations of 16
	yr4_b4cl_four_complex_djbunfft_preload
c3b:	yr4_b4cl_four_complex_djbunfft rsi, 64, dist16, 2*dist16, rdi, YMM_SCD2/4, 16
	yloop	16, c3b, -16*64+4*dist16, rdi, -16*YMM_SCD2/4 ;; Test loop counter, next source pointer, restore sine/cosine pointer
	yloop	3, c3b, -3*4*dist16	;; Test loop counter, restore source pointer
	end_timer 10

;; Do inverse FFT levels 4,5
;; On input the 64-byte cache lines hold these data values:
;;	0	384	768	1152	64	+384	+384	+384
;;	1
;;	...
;;	63
;;	128	...
;;	256	...
;; On output the 64-byte cache lines hold these data values:
;;	0	384	768	1152	192	+384	+384	+384
;;	1
;;	...
;;	191

;; Do 256 three_complex_unfft operations
;;	distance between fft data elements is 64
;;	do 64 macros each processing 24 data values

	start_timer 11
	bump	rdi, 16*YMM_SCD2/4	;; Next sine/cosine pointer
	yloop_init  64			;; 64 iterations
	yr3_b3cl_three_complex_djbunfft_preload
c2b:	yr3_b3cl_three_complex_djbunfft rsi, 64, 4*dist16, rdi, YMM_SCD1/4, 16
	yloop	16, c2b, -16*64+dist16	;; Test loop counter, next source pointer
	yloop	4, c2b, -4*dist16	;; Test loop counter, restore source pointer
	end_timer 11

;; Do inverse FFT levels 1,2,3
;; On input the 64-byte cache lines hold these data values:
;;	0	384	768	1152	192	+384	+384	+384
;;	1
;;	...
;;	191
;; These are swizzled to:
;;	0	48	96	144	192	+48	+48	+48
;;	1	...
;;	...
;;	47
;;	384	...
;;	768	...
;;	1152	...
;; On output the 64-byte cache lines hold these data values:
;;	0	48	96	144	768	+48	+48	+48
;;	1	...
;;	...
;;	47
;;	192	...
;;	384	...
;;	576	...

;; Do 192 four_complex_last_unfft operations
;;	distance between fft data elements is 192
;;	do 48 macros each processing 32 data values

	start_timer 12
	yloop_init  48			;; 48 iterations
	yr4_s4cl_four_complex_last_unfft_preload
c1b:	yr4_s4cl_four_complex_last_unfft rsi, 64, 3*dist16, 6*dist16, rdi, YMM_SCD4, 16
	yloop	16, c1b, -16*64+dist16	;; Test loop counter, next source pointer
	yloop	3, c1b			;; Test loop counter
	end_timer 12

	ytwo_dest_adjust
	yfft_3_ret
	ENDM


;; Perform a 2K-element FFT.

yfft2K MACRO type
	LOCAL	ymid2, ymid3, ymid4, yfinish_unfft
	LOCAL	b1b, b2b, b3a, b3b, b4a, b4b, b5a, b5b, b5c, b5d, c1b, c2b, c3a, c3b, c4a, c4b

;; Do a multiply with pre-FFTed inputs

	cmp	ffttype, 4
	je	ymid4

;; Do FFT levels 1,2,3
;; On input the 64-byte cache lines hold these data values:
;;	0	64	128	192	1024	+64	+64	+64
;;	1
;;	...
;;	63
;;	256	...
;;	512	...
;;	768	...
;; On output these are swizzled to:
;;	0	512	1024	1536	256	+512	+512	+512
;;	1
;;	...
;;	255

;; Do 256 eight_reals_fft operations
;;	distance between fft data elements is 256
;;	do 64 macros each processing 32 data values

	start_timer 4
	mov	rdi, sincos1		;; Load sin/cos pointer
	bump	rdi, 64*YMM_SCD3
	yloop_init  64			;; 64 iterations
	yr4_fs4cl_eight_reals_fft_preload
;; Loop below reworked to avoid 4KB distances
;;b1b:	yr4_fs4cl_eight_reals_fft rsi, 64, dist64, 2*dist64, rdi, YMM_SCD3, 64
;;	yloop	64, b1b, -64*64	;; Test loop counter, bump	rsi		;; Restore source pointer
b1b:	yr4_fs4cl_eight_reals_fft rsi, 4*64, dist64, 2*dist64, rdi, 4*YMM_SCD3, 16
	yloop	16, b1b, -16*4*64+64, rdi, -16*4*YMM_SCD3+YMM_SCD3 ;; Test loop counter, next source pointer, next sin/cos pointer
	yloop	4, b1b, -4*64		;; Test loop counter, restore source pointer
	end_timer 4

;; Do FFT levels 4,5
;; On input the 64-byte cache lines hold these data values:
;;	0	512	1024	1536	256	+512	+512	+512
;;	1
;;	...
;;	255
;; On output the 64-byte cache lines hold these data values:
;;	0	512	1024	1536	64	+512	+512	+512
;;	1
;;	...
;;	63
;;	128	...
;;	256	...
;;	384	...

;; Do 64 eight_reals_fft operations
;; Do 192 four_complex_fft operations
;;	distance between fft data elements is 64
;;	do 64 macros each processing 32 data values

	start_timer 5
	bump	rdi, -4*YMM_SCD3-64*YMM_SCD3 ;; Load sin/cos pointer
	yloop_init  64			;; 64 iterations
	yr4_4cl_eight_reals_four_complex_djbfft_preload
;; Loop below reworked to avoid 4KB distances
;;b2b:	yr4_4cl_eight_reals_four_complex_djbfft rsi, 64, dist64, 2*dist64, rdi, YMM_SCD3, 64
;;	yloop	64, b2b, -64*64, rdi, -64*YMM_SCD3 ;; Test loop counter, restore source pointer, restore sin/cos pointer
b2b:	yr4_4cl_eight_reals_four_complex_djbfft rsi, 4*64, dist64, 2*dist64, rdi, 4*YMM_SCD3, 16
	yloop	16, b2b, -16*4*64+64, rdi, -16*4*YMM_SCD3+YMM_SCD3 ;; Test loop counter, next source pointer, next sin/cos pointer
	yloop	4, b2b, -4*64, rdi, -4*YMM_SCD3 ;; Test loop counter, restore source pointer, restore sin/cos pointer
	end_timer 5

;; Do FFT levels 6,7
;; On input the 64-byte cache lines hold these data values:
;;	0	512	1024	1536	64	+512	+512	+512
;;	1
;;	...
;;	63
;;	128	...
;;	256	...
;;	384	...
;; On output the 64-byte cache lines hold these data values:
;;	0	512	1024	1536	16	+512	+512	+512
;;	1
;;	...
;;	15
;;	32	...
;;	...
;;	480	...

;; Do 16 eight_reals_fft operations
;; Do 240 four_complex_fft operations
;;	distance between fft data elements is 16
;;	do 64 macros each processing 32 data values

	start_timer 6
	yloop_init  64			;; 4 iterations of 16
	yr4_4cl_eight_reals_four_complex_djbfft_preload
b3a:	yr4_4cl_eight_reals_four_complex_djbfft rsi, 64, 16*64, 32*64, rdi, 4*YMM_SCD3, 16
	yloop	16, b3a, -16*64+dist64, rdi, -16*4*YMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer

	loops_reset			;; 3 more iters of 16
	yr4_rb4cl_four_complex_djbfft_preload
b3b:	yr4_rb4cl_four_complex_djbfft rsi, 64, 16*64, 32*64, rdi, 4*YMM_SCD3, 16
	yloop	16, b3b, -16*64+dist64, rdi, -16*4*YMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer
	yloop	4, b3b, -4*dist64	;; Test loop counter, restore source pointer
	end_timer 6

;; Do FFT levels 8,9
;; On input the 64-byte cache lines hold these data values:
;;	0	512	1024	1536	16	+512	+512	+512
;;	1
;;	...
;;	15
;;	32	...
;;	...
;;	480	...
;; On output the 64-byte cache lines hold these data values:
;;	0	512	1024	1536	4	+512	+512	+512
;;	1
;;	...
;;	3
;;	8	...
;;	...
;;	504	...

;; Do 4 eight_reals_fft operations
;; Do 252 four_complex_fft operations
;;	distance between fft data elements is 4
;;	do 64 macros each processing 32 data values

	start_timer 7
	yloop_init  64			;; 16 iterations of 4
	yr4_4cl_eight_reals_four_complex_djbfft_preload
b4a:	yr4_4cl_eight_reals_four_complex_djbfft rsi, 64, 4*64, 8*64, rdi, 16*YMM_SCD3, 4
	yloop	4, b4a, -4*64+16*64, rdi, -4*16*YMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer

	loops_reset			;; 15 more iters of 4
	yr4_rb4cl_four_complex_djbfft_preload
b4b:	yr4_rb4cl_four_complex_djbfft rsi, 64, 4*64, 8*64, rdi, 16*YMM_SCD3, 4
	yloop	4, b4b, -4*64+16*64, rdi, -4*16*YMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer
	yloop	4, b4b, -4*16*64+dist64	;; Test loop counter, next source pointer
	yloop	4, b4b, -4*dist64	;; Test loop counter, restore source pointer
	end_timer 7

;; Do FFT levels 10,11
;; On input and output the 64-byte cache lines hold these data values:
;;	0	512	1024	1536	4	+512	+512	+512
;;	1
;;	...
;;	3
;;	8	...
;;	...
;;	504	...

;; Do 1 eight_reals_fft operation
;; Do 255 four_complex_fft operations
;;	distance between fft data elements is 1
;;	do 64 macros each processing 32 data values

	cmp	ffttype, 2
	je	ymid2
	ja	ymid3

	start_timer 8
;;	yr4_4cl_eight_reals_four_complex_fft_final rsi, 4*64, 64, 2*64
	yr4_dispatch call, yreal_fft_final, 64, 2*64
	yloop_init  64, 1, 16		;; 63 more iterations
b5a:	yr4_4cl_four_complex_fft_final rsi, 4*64, 64, 2*64
	yloop	16, b5a, -16*4*64+dist64 ;; Test loop counter, next source pointer
	yloop	4, b5a			;; Test loop counter
	end_timer 8
	yfft_1_ret

ymid2:	start_timer 8
;;	yr4_4cl_eight_reals_four_complex_with_square rsi, 4*64, 64, 2*64
	yr4_dispatch call, yreal_square, 64, 2*64
	yloop_init  64, 1, 16		;; 63 more iterations
	yr4_4cl_four_complex_with_square_preload
b5b:	yr4_4cl_four_complex_with_square rsi, 4*64, 64, 2*64, 1
	yloop	16, b5b, -16*4*64+dist64 ;; Test loop counter, next source pointer
	yloop	4, b5b			;; Test loop counter
	end_timer 8
	jmp	yfinish_unfft

ymid3:	start_timer 8
x64	mov	r8, DEST2ARG
	mov	rbp, DIST_TO_MULSRCARG
;;	yr4_4cl_eight_reals_four_complex_with_mult rsi, 4*64, 64, 2*64
	yr4_dispatch call, yreal_mult, 64, 2*64
	yloop_init  64, 1, 16		;; 63 more iterations
	yr4_4cl_four_complex_with_mult_preload
b5c:	yr4_4cl_four_complex_with_mult rsi, 4*64, 64, 2*64
	yloop	16, b5c, -16*4*64+dist64 ;; Test loop counter, next source pointer
	yloop	4, b5c			;; Test loop counter
x64	add	rsi, r8
	end_timer 8
	jmp	yfinish_unfft

ymid4:	start_timer 8
	mov	rbp, DIST_TO_MULSRCARG
;;	yr4_4cl_eight_reals_four_complex_with_mulf rsi, 4*64, 64, 2*64
	yr4_dispatch call, yreal_mulf, 64, 2*64
	yloop_init  64, 1, 16		;; 63 more iterations
	yr4_4cl_four_complex_with_mulf_preload
b5d:	yr4_4cl_four_complex_with_mulf rsi, 4*64, 64, 2*64
	yloop	16, b5d, -16*4*64+dist64 ;; Test loop counter, next source pointer
	yloop	4, b5d			;; Test loop counter
	end_timer 8
yfinish_unfft:
	bump	rsi, -4*dist64		;; Restore source pointer

;; Do inverse FFT levels 8,9
;; On input the 64-byte cache lines hold these data values:
;;	0	512	1024	1536	4	+512	+512	+512
;;	1
;;	...
;;	3
;;	8	...
;;	...
;;	504	...
;; On output the 64-byte cache lines hold these data values:
;;	0	512	1024	1536	16	+512	+512	+512
;;	1
;;	...
;;	15
;;	32	...
;;	...
;;	480	...

;; Do 4 eight_reals_unfft operations
;; Do 252 four_complex_unfft operations
;;	distance between fft data elements is 4
;;	do 64 macros each processing 32 data values

	start_timer 9
	mov	rdi, sincos1		;; Load sin/cos pointer
	yloop_init  64			;; 16 iterations of 4
	yr4_4cl_eight_reals_four_complex_djbunfft_preload
c4a:	yr4_4cl_eight_reals_four_complex_djbunfft rsi, 64, 4*64, 8*64, rdi, 16*YMM_SCD3, 4
	yloop	4, c4a, -4*64+16*64, rdi, -4*16*YMM_SCD3 ;; Test loop counter, next source pointer, restore sine/cosine pointer

	loops_reset			;; 15 more iters of 4
	yr4_rb4cl_four_complex_djbunfft_preload
c4b:	yr4_rb4cl_four_complex_djbunfft rsi, 64, 4*64, 8*64, rdi, 16*YMM_SCD3, 4
	yloop	4, c4b, -4*64+16*64, rdi, -4*16*YMM_SCD3 ;; Test loop counter, next source pointer, restore sine/cosine pointer
	yloop	4, c4b, -4*16*64+dist64	;; Test loop counter, next source pointer
	yloop	4, c4b, -4*dist64	;; Test loop counter, restore source pointer
	end_timer 9

;; Do inverse FFT levels 6,7
;; On input the 64-byte cache lines hold these data values:
;;	0	512	1024	1536	16	+512	+512	+512
;;	1
;;	...
;;	63
;;	128	...
;;	256	...
;;	384	...
;; On output the 64-byte cache lines hold these data values:
;;	0	512	1024	1536	64	+512	+512	+512
;;	1
;;	...
;;	63
;;	128	...
;;	256	...
;;	384	...

;; Do 16 eight_reals_unfft operations
;; Do 240 four_complex_unfft operations
;;	distance between fft data elements is 16
;;	do 64 macros each processing 32 data values

	start_timer 10
	yloop_init  64			;; 4 iterations of 16
	yr4_4cl_eight_reals_four_complex_djbunfft_preload
c3a:	yr4_4cl_eight_reals_four_complex_djbunfft rsi, 64, 16*64, 32*64, rdi, 4*YMM_SCD3, 16
	yloop	16, c3a, -16*64+dist64, rdi, -16*4*YMM_SCD3 ;; Test loop counter, next source pointer, restore sine/cosine pointer

	loops_reset			;; 3 more iters of 16
	yr4_rb4cl_four_complex_djbunfft_preload
c3b:	yr4_rb4cl_four_complex_djbunfft rsi, 64, 16*64, 32*64, rdi, 4*YMM_SCD3, 16
	yloop	16, c3b, -16*64+dist64, rdi, -16*4*YMM_SCD3 ;; Test loop counter, next source pointer, restore sine/cosine pointer
	yloop	4, c3b, -4*dist64	;; Test loop counter, restore source pointer
	end_timer 10

;; Do inverse FFT levels 4,5
;; On input the 64-byte cache lines hold these data values:
;;	0	512	1024	1536	64	+512	+512	+512
;;	1
;;	...
;;	63
;;	128	...
;;	256	...
;;	384	...
;; On output the 64-byte cache lines hold these data values:
;;	0	512	1024	1536	256	+512	+512	+512
;;	1
;;	...
;;	255

;; Do 64 eight_reals_unfft operations
;; Do 192 four_complex_unfft operations
;;	distance between fft data elements is 64
;;	do 64 macros each processing 32 data values

	start_timer 11
	yloop_init  64			;; 64 iterations
	yr4_4cl_eight_reals_four_complex_djbunfft_preload
;; Loop below reworked to avoid 4KB distances
;;c2b:	yr4_4cl_eight_reals_four_complex_djbunfft rsi, 64, dist64, 2*dist64, rdi, YMM_SCD3, 64
;;	yloop	64, c2b, -64*64	;; Test loop counter, bump	rsi		;; Restore source pointer
c2b:	yr4_4cl_eight_reals_four_complex_djbunfft rsi, 4*64, dist64, 2*dist64, rdi, 4*YMM_SCD3, 16
	yloop	16, c2b, -16*4*64+64, rdi, -16*4*YMM_SCD3+YMM_SCD3 ;; Test loop counter, next source pointer, next sine/cosine pointer
	yloop	4, c2b, -4*64, rdi, -4*YMM_SCD3+64*YMM_SCD3 ;; Test loop counter, restore source pointer, next sine/cosine pointer
	end_timer 11

;; Do inverse FFT levels 1,2,3
;; On input the 64-byte cache lines hold these data values:
;;	0	512	1024	1536	256	+512	+512	+512
;;	1
;;	...
;;	255
;; These are swizzled to:
;;	0	64	128	192	256	+64	+64	+64
;;	1	...
;;	...
;;	63
;;	512	...
;;	1024	...
;;	1536	...
;; On output the 64-byte cache lines hold these data values:
;;	0	64	128	192	1024	+64	+64	+64
;;	1	...
;;	...
;;	63
;;	256	...
;;	512	...
;;	768	...

;; Do 256 eight_reals_unfft operations
;;	distance between fft data elements is 256
;;	do 64 macros each processing 32 data values

	start_timer 12
	yloop_init  64			;; 64 iterations
	yr4_s4cl_eight_reals_unfft_preload
;; Loop below reworked to avoid 4KB distances
;;c1b:	yr4_s4cl_eight_reals_unfft rsi, 64, dist64, 2*dist64, rdi, YMM_SCD3, 64
;;	yloop	64, c1b			;; Test loop counter
c1b:	yr4_s4cl_eight_reals_unfft rsi, 4*64, dist64, 2*dist64, rdi, 4*YMM_SCD3, 16
	yloop	16, c1b, -16*4*64+64, rdi, -16*4*YMM_SCD3+YMM_SCD3 ;; Test loop counter, next source pointer, next sine/cosine pointer
	yloop	4, c1b			;; Test loop counter
	end_timer 12

	ytwo_dest_adjust
	yfft_3_ret
	ENDM

;; Perform a 2048-element negacyclic FFT

yfft2Kp MACRO type
	LOCAL	ymid2p, ymid3p, ymid4p, yfinish_unfft
	LOCAL	b1b, b2b, b3b, b4b, b5a, b5b, b5c, b5d, c1b, c2b, c3b, c4b

;; Do a multiply with pre-FFTed inputs

	cmp	ffttype, 4
	je	ymid4p

;; Do FFT levels 1,2,3
;; On input the 64-byte cache lines hold these data values:
;;	0	64	128	192	1024	+64	+64	+64
;;	1
;;	...
;;	63
;;	256	...
;;	512	...
;;	768	...
;; On output these are swizzled to:
;;	0	512	1024	1536	256	+512	+512	+512
;;	1
;;	...
;;	255

;; Do 256 four_complex_first_fft operations
;;	distance between fft data elements is 256
;;	do 64 macros each processing 32 data values

	start_timer 4
	mov	rdi, sincos1		;; Load sin/cos pointer
	bump	rdi, 64*YMM_SCD2/4
	yloop_init  64			;; 64 iterations
	yr4_fs4cl_four_complex_first_djbfft_preload
;; Loop below reworked to avoid 4KB distances
;;b1b:	yr4_fs4cl_four_complex_first_djbfft rsi, 64, dist64, 2*dist64, rdi, YMM_SCD4, 64
;;	yloop	64, b1b, -64*64	;; Test loop counter, bump	rsi		;; Restore source pointer
b1b:	yr4_fs4cl_four_complex_first_djbfft rsi, 4*64, dist64, 2*dist64, rdi, 4*YMM_SCD4, 16
	yloop	16, b1b, -16*4*64+64, rdi, -16*4*YMM_SCD4+YMM_SCD4 ;; Test loop counter, next source pointer, next premultiplier pointer
	yloop	4, b1b, -4*64		;; Test loop counter, restore source pointer
	end_timer 4

;; Do FFT levels 4,5
;; On input the 64-byte cache lines hold these data values:
;;	0	512	1024	1536	256	+512	+512	+512
;;	1
;;	...
;;	255
;; On output the 64-byte cache lines hold these data values:
;;	0	512	1024	1536	64	+512	+512	+512
;;	1
;;	...
;;	63
;;	128	...
;;	256	...
;;	384	...

;; Do 256 four_complex_fft operations
;;	distance between fft data elements is 64
;;	do 64 macros each processing 32 data values

	start_timer 5
	bump	rdi, -4*YMM_SCD4-64*YMM_SCD2/4 ;; Load sin/cos pointer
	yloop_init  64			;; 64 iterations
	yr4_b4cl_four_complex_djbfft_preload
;; Loop below reworked to avoid 4KB distances
;;b2b:	yr4_b4cl_four_complex_djbfft rsi, 64, dist64, 2*dist64, rdi, YMM_SCD2/4, 64
;;	yloop	64, b2b, -64*64, rdi, -64*YMM_SCD2/4 ;; Test loop counter, restore source pointer, restore sin/cos pointer
b2b:	yr4_b4cl_four_complex_djbfft rsi, 4*64, dist64, 2*dist64, rdi, 4*YMM_SCD2/4, 16
	yloop	16, b2b, -16*4*64+64, rdi, -16*4*YMM_SCD2/4+YMM_SCD2/4 ;; Test loop counter, next source pointer, next sin/cos pointer
	yloop	4, b2b, -4*64, rdi, -4*YMM_SCD2/4 ;; Test loop counter, restore source pointer, restore sin/cos pointer
	end_timer 5

;; Do FFT levels 6,7
;; On input the 64-byte cache lines hold these data values:
;;	0	512	1024	1536	64	+512	+512	+512
;;	1
;;	...
;;	63
;;	128	...
;;	256	...
;;	384	...
;; On output the 64-byte cache lines hold these data values:
;;	0	512	1024	1536	16	+512	+512	+512
;;	1
;;	...
;;	15
;;	32	...
;;	...
;;	480	...

;; Do 256 four_complex_fft operations
;;	distance between fft data elements is 16
;;	do 64 macros each processing 32 data values

	start_timer 6
	yloop_init  64			;; 4 iterations of 16
	yr4_b4cl_four_complex_djbfft_preload
b3b:	yr4_b4cl_four_complex_djbfft rsi, 64, 16*64, 32*64, rdi, 4*YMM_SCD2/4, 16
	yloop	16, b3b, -16*64+dist64, rdi, -16*4*YMM_SCD2/4 ;; Test loop counter, next source pointer, restore sin/cos pointer
	yloop	4, b3b, -4*dist64	;; Test loop counter, restore source pointer
	end_timer 6

;; Do FFT levels 8,9
;; On input the 64-byte cache lines hold these data values:
;;	0	512	1024	1536	16	+512	+512	+512
;;	1
;;	...
;;	15
;;	32	...
;;	...
;;	480	...
;; On output the 64-byte cache lines hold these data values:
;;	0	512	1024	1536	4	+512	+512	+512
;;	1
;;	...
;;	3
;;	8	...
;;	...
;;	504	...

;; Do 256 four_complex_fft operations
;;	distance between fft data elements is 4
;;	do 64 macros each processing 32 data values

	start_timer 7
	yloop_init  64			;; 16 iterations of 4
	yr4_b4cl_four_complex_djbfft_preload
b4b:	yr4_b4cl_four_complex_djbfft rsi, 64, 4*64, 8*64, rdi, 16*YMM_SCD2/4, 4
	yloop	4, b4b, -4*64+16*64, rdi, -4*16*YMM_SCD2/4 ;; Test loop counter, next source pointer, restore sin/cos pointer
	yloop	4, b4b, -4*16*64+dist64	;; Test loop counter, next source pointer
	yloop	4, b4b, -4*dist64	;; Test loop counter, restore source pointer
	end_timer 7

;; Do FFT levels 10,11
;; On input and output the 64-byte cache lines hold these data values:
;;	0	512	1024	1536	4	+512	+512	+512
;;	1
;;	...
;;	3
;;	8	...
;;	...
;;	504	...

;; Do 256 four_complex_fft operations
;;	distance between fft data elements is 1
;;	do 64 macros each processing 32 data values

	cmp	ffttype, 2
	je	ymid2p
	ja	ymid3p

	start_timer 8
	yloop_init  64			;; 64 iterations
b5a:	yr4_4cl_four_complex_fft_final rsi, 4*64, 64, 2*64
	yloop	16, b5a, -16*4*64+dist64 ;; Test loop counter, next source pointer
	yloop	4, b5a			;; Test loop counter
	end_timer 8
	yfft_1_ret

ymid2p:	start_timer 8
	yloop_init  64			;; 64 iterations
	yr4_4cl_four_complex_with_square_preload
b5b:	yr4_4cl_four_complex_with_square rsi, 4*64, 64, 2*64, 16
	yloop	16, b5b, -16*4*64+dist64 ;; Test loop counter, next source pointer
	yloop	4, b5b			;; Test loop counter
	end_timer 8
	jmp	yfinish_unfft

ymid3p:	start_timer 8
x64	mov	r8, DEST2ARG
	mov	rbp, DIST_TO_MULSRCARG
	yloop_init  64			;; 64 iterations
	yr4_4cl_four_complex_with_mult_preload
b5c:	yr4_4cl_four_complex_with_mult rsi, 4*64, 64, 2*64
	yloop	16, b5c, -16*4*64+dist64 ;; Test loop counter, next source pointer
	yloop	4, b5c			;; Test loop counter
x64	add	rsi, r8
	end_timer 8
	jmp	yfinish_unfft

ymid4p:	start_timer 8
	mov	rbp, DIST_TO_MULSRCARG
	yloop_init  64			;; 64 iterations
	yr4_4cl_four_complex_with_mulf_preload
b5d:	yr4_4cl_four_complex_with_mulf rsi, 4*64, 64, 2*64
	yloop	16, b5d, -16*4*64+dist64 ;; Test loop counter, next source pointer
	yloop	4, b5d			;; Test loop counter
	end_timer 8
yfinish_unfft:
	bump	rsi, -4*dist64		;; Restore source pointer

;; Do inverse FFT levels 8,9
;; On input the 64-byte cache lines hold these data values:
;;	0	512	1024	1536	4	+512	+512	+512
;;	1
;;	...
;;	3
;;	8	...
;;	...
;;	504	...
;; On output the 64-byte cache lines hold these data values:
;;	0	512	1024	1536	16	+512	+512	+512
;;	1
;;	...
;;	15
;;	32	...
;;	...
;;	480	...

;; Do 256 four_complex_unfft operations
;;	distance between fft data elements is 4
;;	do 64 macros each processing 32 data values

	start_timer 9
	mov	rdi, sincos1		;; Load sin/cos pointer
	yloop_init  64			;; 16 iterations of 4
	yr4_b4cl_four_complex_djbunfft_preload
c4b:	yr4_b4cl_four_complex_djbunfft rsi, 64, 4*64, 8*64, rdi, 16*YMM_SCD2/4, 4
	yloop	4, c4b, -4*64+16*64, rdi, -4*16*YMM_SCD2/4 ;; Test loop counter, next source pointer, restore sine/cosine pointer
	yloop	4, c4b, -4*16*64+dist64	;; Test loop counter, next source pointer
	yloop	4, c4b, -4*dist64	;; Test loop counter, restore source pointer
	end_timer 9

;; Do inverse FFT levels 6,7
;; On input the 64-byte cache lines hold these data values:
;;	0	512	1024	1536	16	+512	+512	+512
;;	1
;;	...
;;	15
;;	32	...
;;	...
;;	480	...
;; On output the 64-byte cache lines hold these data values:
;;	0	512	1024	1536	64	+512	+512	+512
;;	1
;;	...
;;	63
;;	128	...
;;	256	...
;;	384	...

;; Do 256 four_complex_unfft operations
;;	distance between fft data elements is 16
;;	do 64 macros each processing 32 data values

	start_timer 10
	yloop_init  64			;; 4 iterations of 16
	yr4_b4cl_four_complex_djbunfft_preload
c3b:	yr4_b4cl_four_complex_djbunfft rsi, 64, 16*64, 32*64, rdi, 4*YMM_SCD2/4, 16
	yloop	16, c3b, -16*64+dist64, rdi, -16*4*YMM_SCD2/4 ;; Test loop counter, next source pointer, restore sine/cosine pointer
	yloop	4, c3b, -4*dist64	;; Test loop counter, restore source pointer
	end_timer 10

;; Do inverse FFT levels 4,5
;; On input the 64-byte cache lines hold these data values:
;;	0	512	1024	1536	64	+512	+512	+512
;;	1
;;	...
;;	63
;;	128	...
;;	256	...
;;	384	...
;; On output the 64-byte cache lines hold these data values:
;;	0	512	1024	1536	256	+512	+512	+512
;;	1
;;	...
;;	255

;; Do 256 four_complex_unfft operations
;;	distance between fft data elements is 64
;;	do 64 macros each processing 32 data values

	start_timer 11
	yloop_init  64			;; 64 iterations
	yr4_b4cl_four_complex_djbunfft_preload
;; Loop below reworked to avoid 4KB distances
;;c2b:	yr4_b4cl_four_complex_djbunfft rsi, 64, dist64, 2*dist64, rdi, YMM_SCD2/4, 64
;;	yloop	64, c2b, -64*64		;; Test loop counter, restore source pointer
c2b:	yr4_b4cl_four_complex_djbunfft rsi, 4*64, dist64, 2*dist64, rdi, 4*YMM_SCD2/4, 16
	yloop	16, c2b, -16*4*64+64, rdi, -16*4*YMM_SCD2/4+YMM_SCD2/4 ;; Test loop counter, next source pointer, next sine/cosine pointer
	yloop	4, c2b, -4*64, rdi, -4*YMM_SCD2/4+64*YMM_SCD2/4 ;; Test loop counter, restore source pointer, next sine/cosine pointer
	end_timer 11

;; Do inverse FFT levels 1,2,3
;; On input the 64-byte cache lines hold these data values:
;;	0	512	1024	1536	256	+512	+512	+512
;;	1
;;	...
;;	255
;; These are swizzled to:
;;	0	64	128	192	256	+64	+64	+64
;;	1	...
;;	...
;;	63
;;	512	...
;;	1024	...
;;	1536	...
;; On output the 64-byte cache lines hold these data values:
;;	0	64	128	192	1024	+64	+64	+64
;;	1	...
;;	...
;;	63
;;	256	...
;;	512	...
;;	768	...

;; Do 256 four_complex_last_unfft operations
;;	distance between fft data elements is 256
;;	do 64 macros each processing 32 data values

	start_timer 12
	yloop_init  64			;; 64 iterations
	yr4_s4cl_four_complex_last_unfft_preload
;; Loop below reworked to avoid 4KB distances
;;c1b:	yr4_s4cl_four_complex_last_unfft rsi, 64, dist64, 2*dist64, rdi, YMM_SCD4, 64
;;	yloop	64, c1b			;; Test loop counter
c1b:	yr4_s4cl_four_complex_last_unfft rsi, 4*64, dist64, 2*dist64, rdi, 4*YMM_SCD4, 16
	yloop	16, c1b, -16*4*64+64, rdi, -16*4*YMM_SCD4+YMM_SCD4 ;; Test loop counter, next source pointer, next sine/cosine pointer
	yloop	4, c1b			;; Test loop counter
	end_timer 12

	ytwo_dest_adjust
	yfft_3_ret
	ENDM


;; Perform a 2560-element FFT.

yfft2560 MACRO type
	LOCAL	ymid2, ymid3, ymid4, yfinish_unfft
	LOCAL	b1b, b2b, b3a, b3b, b4a, b4b, b5a, b5b, b5c, b5d, c1b, c2b, c3a, c3b, c4a, c4b

;; Do a multiply with pre-FFTed inputs

	cmp	ffttype, 4
	je	ymid4

;; Do FFT levels 1,2,3
;; On input the 64-byte cache lines hold these data values:
;;	0	+80	+80	+80	1280	+80	+80	+80
;;	1
;;	...
;;	79
;;	320	...
;;	640	...
;;	960	...
;; On output these are swizzled to:
;;	0	+640	+640	+640	320	+640	+640	+640
;;	1
;;	...
;;	319

;; Do 320 eight_reals_fft operations
;;	distance between fft data elements is 320
;;	do 80 macros each processing 32 data values

	start_timer 4
	mov	rdi, sincos1		;; Load sin/cos pointer
	bump	rdi, 64*YMM_SCD4+16*YMM_SCD3
	yloop_init  80			;; 80 iterations
	yr4_fs4cl_eight_reals_fft_preload
b1b:	yr4_fs4cl_eight_reals_fft rsi, 64, 5*dist16, 10*dist16, rdi, YMM_SCD3, 16
	yloop	16, b1b, -16*64+dist16	;; Test loop counter, next source pointer
	yloop	5, b1b, -5*dist16	;; Test loop counter, restore source pointer
	end_timer 4

;; Do FFT levels 4,5
;; On input the 64-byte cache lines hold these data values:
;;	0	+640	+640	+640	320	+640	+640	+640
;;	1
;;	...
;;	319
;; On output the 64-byte cache lines hold these data values:
;;	0	+640	+640	+640	64	+640	+640	+640
;;	1
;;	...
;;	63
;;	128	...
;;	...
;;	512	...

;; Do 64 ten_reals_fft operations
;; Do 192 five_complex_fft operations
;;	distance between fft data elements is 64
;;	do 64 macros each processing 40 data values

	start_timer 5
	bump	rdi, -80*YMM_SCD3-64*YMM_SCD4 ;; Load sin/cos pointer
	yloop_init  64			;; 64 iterations
	yr5_5cl_ten_reals_five_complex_djbfft_preload
b2b:	yr5_5cl_ten_reals_five_complex_djbfft rsi, 64, 4*dist16, rdi, YMM_SCD4, 16
	yloop	16, b2b, -16*64+dist16	;; Test loop counter, next source pointer
	yloop	4, b2b, -4*dist16	;; Test loop counter, restore source pointer
	end_timer 5

;; Do FFT levels 6,7
;; On input the 64-byte cache lines hold these data values:
;;	0	+640	+640	+640	64	+640	+640	+640
;;	1
;;	...
;;	63
;;	128	...
;;	...
;;	512	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+640	+640	+640	16	+640	+640	+640
;;	1
;;	...
;;	15
;;	32	...
;;	...
;;	608	...

;; Do 16 eight_reals_fft operations
;; Do 304 four_complex_fft operations
;;	distance between fft data elements is 16
;;	do 80 macros each processing 32 data values

	start_timer 6
	bump	rdi, -64*YMM_SCD4-16*YMM_SCD3 ;; Next sin/cos pointer
	yloop_init  80			;; 5 iterations of 16
	yr4_4cl_eight_reals_four_complex_djbfft_preload
b3a:	yr4_4cl_eight_reals_four_complex_djbfft rsi, 64, dist16, 2*dist16, rdi, YMM_SCD3, 16
	yloop	16, b3a, -16*64+4*dist16, rdi, -16*YMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer

	loops_reset			;; 4 more iters of 16
	yr4_rb4cl_four_complex_djbfft_preload
b3b:	yr4_rb4cl_four_complex_djbfft rsi, 64, dist16, 2*dist16, rdi, YMM_SCD3, 16
	yloop	16, b3b, -16*64+4*dist16, rdi, -16*YMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer
	yloop	5, b3b, -5*4*dist16	;; Test loop counter, restore source pointer
	end_timer 6

;; Do FFT levels 8,9
;; On input the 64-byte cache lines hold these data values:
;;	0	+640	+640	+640	16	+640	+640	+640
;;	1
;;	...
;;	15
;;	32	...
;;	...
;;	608	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+640	+640	+640	4	+640	+640	+640
;;	1
;;	...
;;	3
;;	8	...
;;	...
;;	632	...

;; Do 4 eight_reals_fft operations
;; Do 316 four_complex_fft operations
;;	distance between fft data elements is 4
;;	do 80 macros each processing 32 data values

	start_timer 7
	yloop_init  80			;; 20 iterations of 4
	yr4_4cl_eight_reals_four_complex_djbfft_preload
b4a:	yr4_4cl_eight_reals_four_complex_djbfft rsi, 64, 4*64, 8*64, rdi, 4*YMM_SCD3, 4
	yloop	4, b4a, -4*64+dist16, rdi, -4*4*YMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer

	loops_reset			;; 19 more iters of 4
	yr4_rb4cl_four_complex_djbfft_preload
b4b:	yr4_rb4cl_four_complex_djbfft rsi, 64, 4*64, 8*64, rdi, 4*YMM_SCD3, 4
	yloop	4, b4b, -4*64+dist16, rdi, -4*4*YMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer
	yloop	20, b4b, -20*dist16	;; Test loop counter, restore source pointer
	end_timer 7

;; Do FFT levels 10,11
;; On input and output the 64-byte cache lines hold these data values:
;;	0	+640	+640	+640	4	+640	+640	+640
;;	1
;;	...
;;	3
;;	8	...
;;	...
;;	632	...

;; Do 1 eight_reals_fft operation
;; Do 319 four_complex_fft operations
;;	distance between fft data elements is 1
;;	do 80 macros each processing 32 data values

	cmp	ffttype, 2
	je	ymid2
	ja	ymid3

	start_timer 8
;;	yr4_4cl_eight_reals_four_complex_fft_final rsi, 4*64, 64, 2*64
	yr4_dispatch call, yreal_fft_final, 64, 2*64
	yloop_init  80, 1, 4		;; 79 more iterations
b5a:	yr4_4cl_four_complex_fft_final rsi, 4*64, 64, 2*64
	yloop	4, b5a, -4*4*64+dist16	;; Test loop counter, next source pointer
	yloop	20, b5a			;; Test loop counter
	end_timer 8
	yfft_1_ret

ymid2:	start_timer 8
;;	yr4_4cl_eight_reals_four_complex_with_square rsi, 4*64, 64, 2*64
	yr4_dispatch call, yreal_square, 64, 2*64
	yloop_init  80, 1, 4		;; 79 more iterations
	yr4_4cl_four_complex_with_square_preload
b5b:	yr4_4cl_four_complex_with_square rsi, 4*64, 64, 2*64, 1
	yloop	4, b5b, -4*4*64+dist16	;; Test loop counter, next source pointer
	yloop	20, b5b			;; Test loop counter
	end_timer 8
	jmp	yfinish_unfft

ymid3:	start_timer 8
x64	mov	r8, DEST2ARG
	mov	rbp, DIST_TO_MULSRCARG
;;	yr4_4cl_eight_reals_four_complex_with_mult rsi, 4*64, 64, 2*64
	yr4_dispatch call, yreal_mult, 64, 2*64
	yloop_init  80, 1, 4		;; 79 more iterations
	yr4_4cl_four_complex_with_mult_preload
b5c:	yr4_4cl_four_complex_with_mult rsi, 4*64, 64, 2*64
	yloop	4, b5c, -4*4*64+dist16	;; Test loop counter, next source pointer
	yloop	20, b5c			;; Test loop counter
x64	add	rsi, r8
	end_timer 8
	jmp	yfinish_unfft

ymid4:	start_timer 8
	mov	rbp, DIST_TO_MULSRCARG
;;	yr4_4cl_eight_reals_four_complex_with_mulf rsi, 4*64, 64, 2*64
	yr4_dispatch call, yreal_mulf, 64, 2*64
	yloop_init  80, 1, 4		;; 79 more iterations
	yr4_4cl_four_complex_with_mulf_preload
b5d:	yr4_4cl_four_complex_with_mulf rsi, 4*64, 64, 2*64
	yloop	4, b5d, -4*4*64+dist16	;; Test loop counter, next source pointer
	yloop	20, b5d			;; Test loop counter
	end_timer 8
yfinish_unfft:
	bump	rsi, -20*dist16		;; Restore source pointer

;; Do inverse FFT levels 8,9
;; On input the 64-byte cache lines hold these data values:
;;	0	+640	+640	+640	4	+640	+640	+640
;;	1
;;	...
;;	3
;;	8	...
;;	...
;;	632	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+640	+640	+640	16	+640	+640	+640
;;	1
;;	...
;;	15
;;	32	...
;;	...
;;	608	...

;; Do 4 eight_reals_unfft operations
;; Do 316 four_complex_unfft operations
;;	distance between fft data elements is 4
;;	do 80 macros each processing 32 data values

	start_timer 9
	mov	rdi, sincos1		;; Load sin/cos pointer
	yloop_init  80			;; 20 iterations of 4
	yr4_4cl_eight_reals_four_complex_djbunfft_preload
c4a:	yr4_4cl_eight_reals_four_complex_djbunfft rsi, 64, 4*64, 8*64, rdi, 4*YMM_SCD3, 4
	yloop	4, c4a, -4*64+dist16, rdi, -4*4*YMM_SCD3 ;; Test loop counter, next source pointer, restore sine/cosine pointer

	loops_reset			;; 19 more iters of 4
	yr4_rb4cl_four_complex_djbunfft_preload
c4b:	yr4_rb4cl_four_complex_djbunfft rsi, 64, 4*64, 8*64, rdi, 4*YMM_SCD3, 4
	yloop	4, c4b, -4*64+dist16, rdi, -4*4*YMM_SCD3 ;; Test loop counter, next source pointer, restore sine/cosine pointer
	yloop	20, c4b, -20*dist16	;; Test loop counter, restore source pointer
	end_timer 9

;; Do inverse FFT levels 6,7
;; On input the 64-byte cache lines hold these data values:
;;	0	+640	+640	+640	16	+640	+640	+640
;;	1
;;	...
;;	15
;;	32	...
;;	...
;;	608	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+640	+640	+640	64	+640	+640	+640
;;	1
;;	...
;;	63
;;	128	...
;;	...
;;	512	...

;; Do 16 eight_reals_unfft operations
;; Do 304 four_complex_unfft operations
;;	distance between fft data elements is 16
;;	do 80 macros each processing 32 data values

	start_timer 10
	yloop_init  80			;; 5 iterations of 16
	yr4_4cl_eight_reals_four_complex_djbunfft_preload
c3a:	yr4_4cl_eight_reals_four_complex_djbunfft rsi, 64, dist16, 2*dist16, rdi, YMM_SCD3, 16
	yloop	16, c3a, -16*64+4*dist16, rdi, -16*YMM_SCD3 ;; Test loop counter, next source pointer, restore sine/cosine pointer

	loops_reset			;; 4 more iters of 16
	yr4_rb4cl_four_complex_djbunfft_preload
c3b:	yr4_rb4cl_four_complex_djbunfft rsi, 64, dist16, 2*dist16, rdi, YMM_SCD3, 16
	yloop	16, c3b, -16*64+4*dist16, rdi, -16*YMM_SCD3 ;; Test loop counter, next source pointer, restore sine/cosine pointer
	yloop	5, c3b, -5*4*dist16	;; Test loop counter, restore source pointer
	end_timer 10

;; Do inverse FFT levels 4,5
;; On input the 64-byte cache lines hold these data values:
;;	0	+640	+640	+640	64	+640	+640	+640
;;	1
;;	...
;;	63
;;	128	...
;;	...
;;	512	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+640	+640	+640	320	+640	+640	+640
;;	1
;;	...
;;	319

;; Do 64 ten_reals_unfft operations
;; Do 192 five_complex_unfft operations
;;	distance between fft data elements is 64
;;	do 64 macros each processing 40 data values

	start_timer 11
	bump	rdi, 16*YMM_SCD3	;; Next sine/cosine pointer
	yloop_init  64			;; 64 iterations
	yr5_5cl_ten_reals_five_complex_djbunfft_preload
c2b:	yr5_5cl_ten_reals_five_complex_djbunfft rsi, 64, 4*dist16, rdi, YMM_SCD4, 16
	yloop	16, c2b, -16*64+dist16	;; Test loop counter, next source pointer
	yloop	4, c2b, -4*dist16	;; Test loop counter, restore source pointer
	end_timer 11

;; Do inverse FFT levels 1,2,3
;; On input the 64-byte cache lines hold these data values:
;;	0	+640	+640	+640	320	+640	+640	+640
;;	1
;;	...
;;	319
;; These are swizzled to:
;;	0	+80	+80	+80	320	+80	+80	+80
;;	1	...
;;	...
;;	79
;;	640	...
;;	1280	...
;;	1920	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+80	+80	+80	1280	+80	+80	+80
;;	1	...
;;	...
;;	79
;;	320	...
;;	640	...
;;	960	...

;; Do 320 eight_reals_unfft operations
;;	distance between fft data elements is 320
;;	do 80 macros each processing 32 data values

	start_timer 12
	yloop_init  80			;; 80 iterations
	yr4_s4cl_eight_reals_unfft_preload
c1b:	yr4_s4cl_eight_reals_unfft rsi, 64, 5*dist16, 10*dist16, rdi, YMM_SCD3, 16
	yloop	16, c1b, -16*64+dist16	;; Test loop counter, next source pointer
	yloop	5, c1b			;; Test loop counter
	end_timer 12

	ytwo_dest_adjust
	yfft_3_ret
	ENDM

;; Perform a 2560-element negacyclic FFT

yfft2560p MACRO type
	LOCAL	ymid2p, ymid3p, ymid4p, yfinish_unfft
	LOCAL	b1b, b2b, b3b, b4b, b5a, b5b, b5c, b5d, c1b, c2b, c3b, c4b

;; Do a multiply with pre-FFTed inputs

	cmp	ffttype, 4
	je	ymid4p

;; Do FFT levels 1,2,3
;; On input the 64-byte cache lines hold these data values:
;;	0	+80	+80	+80	1280	+80	+80	+80
;;	1
;;	...
;;	79
;;	320	...
;;	640	...
;;	960	...
;; On output these are swizzled to:
;;	0	+640	+640	+640	320	+640	+640	+640
;;	1
;;	...
;;	319

;; Do 320 four_complex_first_fft operations
;;	distance between fft data elements is 320
;;	do 80 macros each processing 32 data values

	start_timer 4
	mov	rdi, sincos1		;; Load sin/cos pointer
	bump	rdi, 64*YMM_SCD2/4+16*YMM_SCD2/4
	yloop_init  80			;; 80 iterations
	yr4_fs4cl_four_complex_first_djbfft_preload
b1b:	yr4_fs4cl_four_complex_first_djbfft rsi, 64, 5*dist16, 10*dist16, rdi, YMM_SCD4, 16
	yloop	16, b1b, -16*64+dist16	;; Test loop counter, next source pointer
	yloop	5, b1b, -5*dist16	;; Test loop counter, restore source pointer
	end_timer 4

;; Do FFT levels 4,5
;; On input the 64-byte cache lines hold these data values:
;;	0	+640	+640	+640	320	+640	+640	+640
;;	1
;;	...
;;	319
;; On output the 64-byte cache lines hold these data values:
;;	0	+640	+640	+640	64	+640	+640	+640
;;	1
;;	...
;;	63
;;	128	...
;;	...
;;	512	...

;; Do 256 three_complex_fft operations
;;	distance between fft data elements is 64
;;	do 64 macros each processing 40 data values

	start_timer 5
	bump	rdi, -80*YMM_SCD4-64*YMM_SCD2/4 ;; Load sin/cos pointer
	yloop_init  64			;; 64 iterations
	yr5_b5cl_five_complex_djbfft_preload
b2b:	yr5_b5cl_five_complex_djbfft rsi, 64, 4*dist16, rdi, YMM_SCD2/4, 16
	yloop	16, b2b, -16*64+dist16	;; Test loop counter, next source pointer
	yloop	4, b2b, -4*dist16	;; Test loop counter, restore source pointer
	end_timer 5

;; Do FFT levels 6,7
;; On input the 64-byte cache lines hold these data values:
;;	0	+640	+640	+640	64	+640	+640	+640
;;	1
;;	...
;;	63
;;	128	...
;;	...
;;	512	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+640	+640	+640	16	+640	+640	+640
;;	1
;;	...
;;	15
;;	32	...
;;	...
;;	608	...

;; Do 320 four_complex_fft operations
;;	distance between fft data elements is 16
;;	do 80 macros each processing 32 data values

	start_timer 6
	bump	rdi, -64*YMM_SCD2/4-16*YMM_SCD2/4 ;; Next sin/cos pointer
	yloop_init  80			;; 5 iterations of 16
	yr4_b4cl_four_complex_djbfft_preload
b3b:	yr4_b4cl_four_complex_djbfft rsi, 64, dist16, 2*dist16, rdi, YMM_SCD2/4, 16
	yloop	16, b3b, -16*64+4*dist16, rdi, -16*YMM_SCD2/4 ;; Test loop counter, next source pointer, restore sin/cos pointer
	yloop	5, b3b, -5*4*dist16	;; Test loop counter, restore source pointer
	end_timer 6

;; Do FFT levels 8,9
;; On input the 64-byte cache lines hold these data values:
;;	0	+640	+640	+640	16	+640	+640	+640
;;	1
;;	...
;;	15
;;	32	...
;;	...
;;	608	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+640	+640	+640	4	+640	+640	+640
;;	1
;;	...
;;	3
;;	8	...
;;	...
;;	632	...

;; Do 320 four_complex_fft operations
;;	distance between fft data elements is 4
;;	do 80 macros each processing 32 data values

	start_timer 7
	yloop_init  80			;; 20 iterations of 4
	yr4_b4cl_four_complex_djbfft_preload
b4b:	yr4_b4cl_four_complex_djbfft rsi, 64, 4*64, 8*64, rdi, 4*YMM_SCD2/4, 4
	yloop	4, b4b, -4*64+dist16, rdi, -4*4*YMM_SCD2/4 ;; Test loop counter, next source pointer, restore sin/cos pointer
	yloop	20, b4b, -20*dist16	;; Test loop counter, restore source pointer
	end_timer 7

;; Do FFT levels 10,11
;; On input and output the 64-byte cache lines hold these data values:
;;	0	+640	+640	+640	4	+640	+640	+640
;;	1
;;	...
;;	3
;;	8	...
;;	...
;;	632	...

;; Do 320 four_complex_fft operations
;;	distance between fft data elements is 1
;;	do 80 macros each processing 32 data values

	cmp	ffttype, 2
	je	ymid2p
	ja	ymid3p

	start_timer 8
	yloop_init  80			;; 80 iterations
b5a:	yr4_4cl_four_complex_fft_final rsi, 4*64, 64, 2*64
	yloop	4, b5a, -4*4*64+dist16	;; Test loop counter, next source pointer
	yloop	20, b5a			;; Test loop counter
	end_timer 8
	yfft_1_ret

ymid2p:	start_timer 8
	yloop_init  80			;; 80 iterations
	yr4_4cl_four_complex_with_square_preload
b5b:	yr4_4cl_four_complex_with_square rsi, 4*64, 64, 2*64, 4
	yloop	4, b5b, -4*4*64+dist16	;; Test loop counter, next source pointer
	yloop	20, b5b			;; Test loop counter
	end_timer 8
	jmp	yfinish_unfft

ymid3p:	start_timer 8
x64	mov	r8, DEST2ARG
	mov	rbp, DIST_TO_MULSRCARG
	yloop_init  80			;; 80 iterations
	yr4_4cl_four_complex_with_mult_preload
b5c:	yr4_4cl_four_complex_with_mult rsi, 4*64, 64, 2*64
	yloop	4, b5c, -4*4*64+dist16	;; Test loop counter, next source pointer
	yloop	20, b5c			;; Test loop counter
x64	add	rsi, r8
	end_timer 8
	jmp	yfinish_unfft

ymid4p:	start_timer 8
	mov	rbp, DIST_TO_MULSRCARG
	yloop_init  80			;; 80 iterations
	yr4_4cl_four_complex_with_mulf_preload
b5d:	yr4_4cl_four_complex_with_mulf rsi, 4*64, 64, 2*64
	yloop	4, b5d, -4*4*64+dist16	;; Test loop counter, next source pointer
	yloop	20, b5d			;; Test loop counter
	end_timer 8
yfinish_unfft:
	bump	rsi, -20*dist16		;; Restore source pointer

;; Do inverse FFT levels 8,9
;; On input the 64-byte cache lines hold these data values:
;;	0	+640	+640	+640	4	+640	+640	+640
;;	1
;;	...
;;	3
;;	8	...
;;	...
;;	632	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+640	+640	+640	16	+640	+640	+640
;;	1
;;	...
;;	15
;;	32	...
;;	...
;;	608	...

;; Do 320 four_complex_unfft operations
;;	distance between fft data elements is 4
;;	do 80 macros each processing 32 data values

	start_timer 9
	mov	rdi, sincos1		;; Load sin/cos pointer
	yloop_init  80			;; 20 iterations of 4
	yr4_b4cl_four_complex_djbunfft_preload
c4b:	yr4_b4cl_four_complex_djbunfft rsi, 64, 4*64, 8*64, rdi, 4*YMM_SCD2/4, 4
	yloop	4, c4b, -4*64+dist16, rdi, -4*4*YMM_SCD2/4 ;; Test loop counter, next source pointer, restore sine/cosine pointer
	yloop	20, c4b, -20*dist16	;; Test loop counter, restore source pointer
	end_timer 9

;; Do inverse FFT levels 6,7
;; On input the 64-byte cache lines hold these data values:
;;	0	+640	+640	+640	16	+640	+640	+640
;;	1
;;	...
;;	15
;;	32	...
;;	...
;;	608	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+640	+640	+640	64	+640	+640	+640
;;	1
;;	...
;;	63
;;	128	...
;;	...
;;	512	...

;; Do 320 four_complex_unfft operations
;;	distance between fft data elements is 16
;;	do 80 macros each processing 32 data values

	start_timer 10
	yloop_init  80			;; 5 iterations of 16
	yr4_b4cl_four_complex_djbunfft_preload
c3b:	yr4_b4cl_four_complex_djbunfft rsi, 64, dist16, 2*dist16, rdi, YMM_SCD2/4, 16
	yloop	16, c3b, -16*64+4*dist16, rdi, -16*YMM_SCD2/4 ;; Test loop counter, next source pointer, restore sine/cosine pointer
	yloop	5, c3b, -5*4*dist16	;; Test loop counter, restore source pointer
	end_timer 10

;; Do inverse FFT levels 4,5
;; On input the 64-byte cache lines hold these data values:
;;	0	+640	+640	+640	64	+640	+640	+640
;;	1
;;	...
;;	63
;;	128	...
;;	...
;;	512	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+640	+640	+640	320	+640	+640	+640
;;	1
;;	...
;;	319

;; Do 256 five_complex_unfft operations
;;	distance between fft data elements is 64
;;	do 64 macros each processing 40 data values

	start_timer 11
	bump	rdi, 16*YMM_SCD2/4	;; Next sine/cosine pointer
	yloop_init  64			;; 64 iterations
	yr5_b5cl_five_complex_djbunfft_preload
c2b:	yr5_b5cl_five_complex_djbunfft rsi, 64, 4*dist16, rdi, YMM_SCD2/4, 16
	yloop	16, c2b, -16*64+dist16	;; Test loop counter, next source pointer
	yloop	4, c2b, -4*dist16	;; Test loop counter, restore source pointer
	end_timer 11

;; Do inverse FFT levels 1,2,3
;; On input the 64-byte cache lines hold these data values:
;;	0	+640	+640	+640	320	+640	+640	+640
;;	1
;;	...
;;	319
;; These are swizzled to:
;;	0	+80	+80	+80	320	+80	+80	+80
;;	1	...
;;	...
;;	79
;;	640	...
;;	1280	...
;;	1920	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+80	+80	+80	1280	+80	+80	+80
;;	1	...
;;	...
;;	79
;;	320	...
;;	640	...
;;	960	...

;; Do 320 four_complex_last_unfft operations
;;	distance between fft data elements is 320
;;	do 80 macros each processing 32 data values

	start_timer 12
	yloop_init  80			;; 80 iterations
	yr4_s4cl_four_complex_last_unfft_preload
c1b:	yr4_s4cl_four_complex_last_unfft rsi, 64, 5*dist16, 10*dist16, rdi, YMM_SCD4, 16
	yloop	16, c1b, -16*64+dist16	;; Test loop counter, next source pointer
	yloop	5, c1b			;; Test loop counter
	end_timer 12

	ytwo_dest_adjust
	yfft_3_ret
	ENDM


;; Perform a 3K-element FFT.

yfft3K MACRO type
	LOCAL	ymid2, ymid3, ymid4, yfinish_unfft
	LOCAL	b1b, b2b, b3a, b3b, b4a, b4b, b5a, b5b, b5c, b5d, c1b, c2b, c3a, c3b, c4a, c4b

;; Do a multiply with pre-FFTed inputs

	cmp	ffttype, 4
	je	ymid4

;; Do FFT levels 1,2,3
;; On input the 64-byte cache lines hold these data values:
;;	0	+96	+96	+96	1536	+96	+96	+96
;;	1
;;	...
;;	95
;;	384	...
;;	768	...
;;	1152	...
;; On output these are swizzled to:
;;	0	+768	+768	+768	384	+768	+768	+768
;;	1
;;	...
;;	383

;; Do 384 eight_reals_fft operations
;;	distance between fft data elements is 384
;;	do 96 macros each processing 32 data values

	start_timer 4
	mov	rdi, sincos1		;; Load sin/cos pointer
	bump	rdi, 128*YMM_SCD2+32*YMM_SCD3
	yloop_init  96			;; 96 iterations
	yr4_fs4cl_eight_reals_fft_preload
b1b:	yr4_fs4cl_eight_reals_fft rsi, 64, 3*dist32, 6*dist32, rdi, YMM_SCD3, 32
	yloop	32, b1b, -32*64+dist32	;; Test loop counter, next source pointer
	yloop	3, b1b, -3*dist32	;; Test loop counter, restore source pointer
	end_timer 4

;; Do FFT levels 4,5
;; On input the 64-byte cache lines hold these data values:
;;	0	+768	+768	+768	384	+768	+768	+768
;;	1
;;	...
;;	383
;; On output the 64-byte cache lines hold these data values:
;;	0	+768	+768	+768	128	+768	+768	+768
;;	1
;;	...
;;	127
;;	256	...
;;	512	...

;; Do 128 six_reals_fft operations
;; Do 384 three_complex_fft operations
;;	distance between fft data elements is 128
;;	do 128 macros each processing 24 data values

	start_timer 5
	bump	rdi, -96*YMM_SCD3-128*YMM_SCD2 ;; Load sin/cos pointer
	yloop_init  128			;; 128 iterations
	yr3_3cl_six_reals_three_complex_djbfft_preload
b2b:	yr3_3cl_six_reals_three_complex_djbfft rsi, 64, 4*dist32, rdi, YMM_SCD2, 32
	yloop	32, b2b, -32*64+dist32	;; Test loop counter, next source pointer
	yloop	4, b2b, -4*dist32	;; Test loop counter, restore source pointer
	end_timer 5

;; Do FFT levels 6,7
;; On input the 64-byte cache lines hold these data values:
;;	0	+768	+768	+768	128	+768	+768	+768
;;	1
;;	...
;;	127
;;	256	...
;;	512	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+768	+768	+768	32	+768	+768	+768
;;	1
;;	...
;;	31
;;	64	...
;;	...
;;	704	...

;; Do 32 eight_reals_fft operations
;; Do 352 four_complex_fft operations
;;	distance between fft data elements is 32
;;	do 96 macros each processing 32 data values

	start_timer 6
	bump	rdi, -128*YMM_SCD2-32*YMM_SCD3 ;; Next sin/cos pointer
	yloop_init  96			;; 3 iterations of 32
	yr4_4cl_eight_reals_four_complex_djbfft_preload
b3a:	yr4_4cl_eight_reals_four_complex_djbfft rsi, 64, dist32, 2*dist32, rdi, YMM_SCD3, 32
	yloop	32, b3a, -32*64+4*dist32, rdi, -32*YMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer

	loops_reset			;; 2 more iters of 32
	yr4_rb4cl_four_complex_djbfft_preload
b3b:	yr4_rb4cl_four_complex_djbfft rsi, 64, dist32, 2*dist32, rdi, YMM_SCD3, 32
	yloop	32, b3b, -32*64+4*dist32, rdi, -32*YMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer
	yloop	3, b3b, -3*4*dist32	;; Test loop counter, restore source pointer
	end_timer 6

;; Do FFT levels 8,9
;; On input the 64-byte cache lines hold these data values:
;;	0	+768	+768	+768	32	+768	+768	+768
;;	1
;;	...
;;	31
;;	64	...
;;	...
;;	704	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+768	+768	+768	8	+768	+768	+768
;;	1
;;	...
;;	7
;;	16	...
;;	...
;;	752	...

;; Do 8 eight_reals_fft operations
;; Do 376 four_complex_fft operations
;;	distance between fft data elements is 8
;;	do 96 macros each processing 32 data values

	start_timer 7
	yloop_init  96			;; 12 iterations of 8
	yr4_4cl_eight_reals_four_complex_djbfft_preload
b4a:	yr4_4cl_eight_reals_four_complex_djbfft rsi, 64, 8*64, 16*64, rdi, 4*YMM_SCD3, 8
	yloop	8, b4a, -8*64+dist32, rdi, -8*4*YMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer

	loops_reset			;; 11 more iters of 8
	yr4_rb4cl_four_complex_djbfft_preload
b4b:	yr4_rb4cl_four_complex_djbfft rsi, 64, 8*64, 16*64, rdi, 4*YMM_SCD3, 8
	yloop	8, b4b, -8*64+dist32, rdi, -8*4*YMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer
	yloop	12, b4b, -12*dist32	;; Test loop counter, restore source pointer
	end_timer 7

;; Do FFT levels 10,11
;; On input and output the 64-byte cache lines hold these data values:
;;	0	+768	+768	+768	8	+768	+768	+768
;;	1
;;	...
;;	7
;;	16	...
;;	...
;;	752	...

;; Do 1 sixteen_reals_fft operation
;; Do 191 eight_complex_fft operations
;;	distance between fft data elements is 1
;;	do 48 macros each processing 64 data values

	cmp	ffttype, 2
	je	ymid2
	ja	ymid3

	start_timer 8
;;	yr8_8cl_sixteen_reals_eight_complex_fft_final rsi, 8*64, 64, 2*64, 4*64
	yr8_dispatch call, y8real_fft_final, 64, 2*64, 4*64
	yloop_init  48, 1, 4		;; 47 more iterations
b5a:	yr8_8cl_eight_complex_fft_final rsi, 8*64, 64, 2*64, 4*64
	yloop	4, b5a, -4*8*64+dist32	;; Test loop counter, next source pointer
	yloop	12, b5a			;; Test loop counter
	end_timer 8
	yfft_1_ret

ymid2:	start_timer 8
;;	yr8_8cl_sixteen_reals_eight_complex_with_square rsi, 8*64, 64, 2*64, 4*64
	yr8_dispatch call, y8real_square, 64, 2*64, 4*64
	yloop_init  48, 1, 4		;; 47 more iterations
	yr8_8cl_eight_complex_with_square_preload
b5b:	yr8_8cl_eight_complex_with_square rsi, 8*64, 64, 2*64, 4*64
	yloop	4, b5b, -4*8*64+dist32	;; Test loop counter, next source pointer
	yloop	12, b5b			;; Test loop counter
	end_timer 8
	jmp	yfinish_unfft

ymid3:	start_timer 8
x64	mov	r8, DEST2ARG
	mov	rbp, DIST_TO_MULSRCARG
;;	yr8_8cl_sixteen_reals_eight_complex_with_mult rsi, 8*64, 64, 2*64, 4*64
	yr8_dispatch call, y8real_mult, 64, 2*64, 4*64
	yloop_init  48, 1, 4		;; 47 more iterations
	yr8_8cl_eight_complex_with_mult_preload
b5c:	yr8_8cl_eight_complex_with_mult rsi, 8*64, 64, 2*64, 4*64
	yloop	4, b5c, -4*8*64+dist32	;; Test loop counter, next source pointer
	yloop	12, b5c			;; Test loop counter
x64	add	rsi, r8
	end_timer 8
	jmp	yfinish_unfft

ymid4:	start_timer 8
	mov	rbp, DIST_TO_MULSRCARG
;;	yr8_8cl_sixteen_reals_eight_complex_with_mulf rsi, 8*64, 64, 2*64, 4*64
	yr8_dispatch call, y8real_mulf, 64, 2*64, 4*64
	yloop_init  48, 1, 4		;; 47 more iterations
	yr8_8cl_eight_complex_with_mulf_preload
b5d:	yr8_8cl_eight_complex_with_mulf rsi, 8*64, 64, 2*64, 4*64
	yloop	4, b5d, -4*8*64+dist32	;; Test loop counter, next source pointer
	yloop	12, b5d			;; Test loop counter
	end_timer 8
yfinish_unfft:
	bump	rsi, -12*dist32		;; Restore source pointer

;; Do inverse FFT levels 8,9
;; On input the 64-byte cache lines hold these data values:
;;	0	+768	+768	+768	8	+768	+768	+768
;;	1
;;	...
;;	7
;;	16	...
;;	...
;;	752	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+768	+768	+768	32	+768	+768	+768
;;	1
;;	...
;;	31
;;	64	...
;;	...
;;	704	...

;; Do 8 eight_reals_unfft operations
;; Do 376 four_complex_unfft operations
;;	distance between fft data elements is 8
;;	do 96 macros each processing 32 data values

	start_timer 9
	mov	rdi, sincos1		;; Load sin/cos pointer
	yloop_init  96			;; 12 iterations of 8
	yr4_4cl_eight_reals_four_complex_djbunfft_preload
c4a:	yr4_4cl_eight_reals_four_complex_djbunfft rsi, 64, 8*64, 16*64, rdi, 4*YMM_SCD3, 8
	yloop	8, c4a, -8*64+dist32, rdi, -8*4*YMM_SCD3 ;; Test loop counter, next source pointer, restore sine/cosine pointer

	loops_reset			;; 11 more iters of 8
	yr4_rb4cl_four_complex_djbunfft_preload
c4b:	yr4_rb4cl_four_complex_djbunfft rsi, 64, 8*64, 16*64, rdi, 4*YMM_SCD3, 8
	yloop	8, c4b, -8*64+dist32, rdi, -8*4*YMM_SCD3 ;; Test loop counter, next source pointer, restore sine/cosine pointer
	yloop	12, c4b, -12*dist32	;; Test loop counter, restore source pointer
	end_timer 9

;; Do inverse FFT levels 6,7
;; On input the 64-byte cache lines hold these data values:
;;	0	+768	+768	+768	32	+768	+768	+768
;;	1
;;	...
;;	31
;;	64	...
;;	...
;;	704	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+768	+768	+768	128	+768	+768	+768
;;	1
;;	...
;;	127
;;	256	...
;;	512	...

;; Do 32 eight_reals_unfft operations
;; Do 352 four_complex_unfft operations
;;	distance between fft data elements is 32
;;	do 96 macros each processing 32 data values

	start_timer 10
	yloop_init  96			;; 3 iterations of 32
	yr4_4cl_eight_reals_four_complex_djbunfft_preload
c3a:	yr4_4cl_eight_reals_four_complex_djbunfft rsi, 64, dist32, 2*dist32, rdi, YMM_SCD3, 32
	yloop	32, c3a, -32*64+4*dist32, rdi, -32*YMM_SCD3 ;; Test loop counter, next source pointer, restore sine/cosine pointer

	loops_reset			;; 2 more iters of 32
	yr4_rb4cl_four_complex_djbunfft_preload
c3b:	yr4_rb4cl_four_complex_djbunfft rsi, 64, dist32, 2*dist32, rdi, YMM_SCD3, 32
	yloop	32, c3b, -32*64+4*dist32, rdi, -32*YMM_SCD3 ;; Test loop counter, next source pointer, restore sine/cosine pointer
	yloop	3, c3b, -3*4*dist32	;; Test loop counter, restore source pointer
	end_timer 10

;; Do inverse FFT levels 4,5
;; On input the 64-byte cache lines hold these data values:
;;	0	+768	+768	+768	128	+768	+768	+768
;;	1
;;	...
;;	127
;;	256	...
;;	512	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+768	+768	+768	384	+768	+768	+768
;;	1
;;	...
;;	383

;; Do 128 six_reals_unfft operations
;; Do 384 three_complex_unfft operations
;;	distance between fft data elements is 128
;;	do 128 macros each processing 24 data values

	start_timer 11
	bump	rdi, 32*YMM_SCD3	;; Next sine/cosine pointer
	yloop_init  128			;; 128 iterations
	yr3_3cl_six_reals_three_complex_djbunfft_preload
c2b:	yr3_3cl_six_reals_three_complex_djbunfft rsi, 64, 4*dist32, rdi, YMM_SCD2, 32
	yloop	32, c2b, -32*64+dist32	;; Test loop counter, next source pointer
	yloop	4, c2b, -4*dist32	;; Test loop counter, restore source pointer
	end_timer 11

;; Do inverse FFT levels 1,2,3
;; On input the 64-byte cache lines hold these data values:
;;	0	+768	+768	+768	384	+768	+768	+768
;;	1
;;	...
;;	383
;; These are swizzled to:
;;	0	+96	+96	+96	384	+96	+96	+96
;;	1	...
;;	...
;;	95
;;	768	...
;;	1536	...
;;	2304	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+96	+96	+96	1536	+96	+96	+96
;;	1	...
;;	...
;;	95
;;	384	...
;;	768	...
;;	1152	...

;; Do 384 eight_reals_unfft operations
;;	distance between fft data elements is 384
;;	do 96 macros each processing 32 data values

	start_timer 12
	yloop_init  96			;; 96 iterations
	yr4_s4cl_eight_reals_unfft_preload
c1b:	yr4_s4cl_eight_reals_unfft rsi, 64, 3*dist32, 6*dist32, rdi, YMM_SCD3, 32
	yloop	32, c1b, -32*64+dist32	;; Test loop counter, next source pointer
	yloop	3, c1b			;; Test loop counter
	end_timer 12

	ytwo_dest_adjust
	yfft_3_ret
	ENDM

;; Perform a 3K-element negacyclic FFT

yfft3Kp MACRO type
	LOCAL	ymid2p, ymid3p, ymid4p, yfinish_unfft
	LOCAL	b1b, b2b, b3b, b4b, b5a, b5b, b5c, b5d, c1b, c2b, c3b, c4b

;; Do a multiply with pre-FFTed inputs

	cmp	ffttype, 4
	je	ymid4p

;; Do FFT levels 1,2,3
;; On input the 64-byte cache lines hold these data values:
;;	0	+96	+96	+96	1536	+96	+96	+96
;;	1
;;	...
;;	95
;;	384	...
;;	768	...
;;	1152	...
;; On output these are swizzled to:
;;	0	+768	+768	+768	384	+768	+768	+768
;;	1
;;	...
;;	383

;; Do 384 four_complex_first_fft operations
;;	distance between fft data elements is 384
;;	do 96 macros each processing 32 data values

	start_timer 4
	mov	rdi, sincos1		;; Load sin/cos pointer
	bump	rdi, 128*YMM_SCD1/4+32*YMM_SCD2/4
	yloop_init  96			;; 96 iterations
	yr4_fs4cl_four_complex_first_djbfft_preload
b1b:	yr4_fs4cl_four_complex_first_djbfft rsi, 64, 3*dist32, 6*dist32, rdi, YMM_SCD4, 32
	yloop	32, b1b, -32*64+dist32	;; Test loop counter, next source pointer
	yloop	3, b1b, -3*dist32	;; Test loop counter, restore source pointer
	end_timer 4

;; Do FFT levels 4,5
;; On input the 64-byte cache lines hold these data values:
;;	0	+768	+768	+768	384	+768	+768	+768
;;	1
;;	...
;;	383
;; On output the 64-byte cache lines hold these data values:
;;	0	+768	+768	+768	128	+768	+768	+768
;;	1
;;	...
;;	127
;;	256	...
;;	512	...

;; Do 512 three_complex_fft operations
;;	distance between fft data elements is 128
;;	do 128 macros each processing 24 data values

	start_timer 5
	bump	rdi, -96*YMM_SCD4-128*YMM_SCD1/4 ;; Load sin/cos pointer
	yloop_init  128			;; 128 iterations
	yr3_b3cl_three_complex_djbfft_preload
b2b:	yr3_b3cl_three_complex_djbfft rsi, 64, 4*dist32, rdi, YMM_SCD1/4, 32
	yloop	32, b2b, -32*64+dist32	;; Test loop counter, next source pointer
	yloop	4, b2b, -4*dist32	;; Test loop counter, restore source pointer
	end_timer 5

;; Do FFT levels 6,7
;; On input the 64-byte cache lines hold these data values:
;;	0	+768	+768	+768	128	+768	+768	+768
;;	1
;;	...
;;	127
;;	256	...
;;	512	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+768	+768	+768	32	+768	+768	+768
;;	1
;;	...
;;	31
;;	64	...
;;	...
;;	704	...

;; Do 384 four_complex_fft operations
;;	distance between fft data elements is 32
;;	do 96 macros each processing 32 data values

	start_timer 6
	bump	rdi, -128*YMM_SCD1/4-32*YMM_SCD2/4 ;; Next sin/cos pointer
	yloop_init  96			;; 3 iterations of 32
	yr4_b4cl_four_complex_djbfft_preload
b3b:	yr4_b4cl_four_complex_djbfft rsi, 64, dist32, 2*dist32, rdi, YMM_SCD2/4, 32
	yloop	32, b3b, -32*64+4*dist32, rdi, -32*YMM_SCD2/4 ;; Test loop counter, next source pointer, restore sin/cos pointer
	yloop	3, b3b, -3*4*dist32	;; Test loop counter, restore source pointer
	end_timer 6

;; Do FFT levels 8,9
;; On input the 64-byte cache lines hold these data values:
;;	0	+768	+768	+768	32	+768	+768	+768
;;	1
;;	...
;;	31
;;	64	...
;;	...
;;	704	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+768	+768	+768	8	+768	+768	+768
;;	1
;;	...
;;	7
;;	16	...
;;	...
;;	752	...

;; Do 384 four_complex_fft operations
;;	distance between fft data elements is 8
;;	do 96 macros each processing 32 data values

	start_timer 7
	yloop_init  96			;; 12 iterations of 8
	yr4_b4cl_four_complex_djbfft_preload
b4b:	yr4_b4cl_four_complex_djbfft rsi, 64, 8*64, 16*64, rdi, 4*YMM_SCD2/4, 8
	yloop	8, b4b, -8*64+dist32, rdi, -8*4*YMM_SCD2/4 ;; Test loop counter, next source pointer, restore sin/cos pointer
	yloop	12, b4b, -12*dist32	;; Test loop counter, restore source pointer
	end_timer 7

;; Do FFT levels 10,11
;; On input and output the 64-byte cache lines hold these data values:
;;	0	+768	+768	+768	8	+768	+768	+768
;;	1
;;	...
;;	7
;;	16	...
;;	...
;;	752	...

;; Do 192 eight_complex_fft operations
;;	distance between fft data elements is 1
;;	do 48 macros each processing 64 data values

	cmp	ffttype, 2
	je	ymid2p
	ja	ymid3p

	start_timer 8
	yloop_init  48			;; 48 iterations
b5a:	yr8_8cl_eight_complex_fft_final rsi, 8*64, 64, 2*64, 4*64
	yloop	4, b5a, -4*8*64+dist32	;; Test loop counter, next source pointer
	yloop	12, b5a			;; Test loop counter
	end_timer 8
	yfft_1_ret

ymid2p:	start_timer 8
	yloop_init  48			;; 48 iterations
	yr8_8cl_eight_complex_with_square_preload
b5b:	yr8_8cl_eight_complex_with_square rsi, 8*64, 64, 2*64, 4*64
	yloop	4, b5b, -4*8*64+dist32	;; Test loop counter, next source pointer
	yloop	12, b5b			;; Test loop counter
	end_timer 8
	jmp	yfinish_unfft

ymid3p:	start_timer 8
x64	mov	r8, DEST2ARG
	mov	rbp, DIST_TO_MULSRCARG
	yloop_init  48			;; 48 iterations
	yr8_8cl_eight_complex_with_mult_preload
b5c:	yr8_8cl_eight_complex_with_mult rsi, 8*64, 64, 2*64, 4*64
	yloop	4, b5c, -4*8*64+dist32	;; Test loop counter, next source pointer
	yloop	12, b5c			;; Test loop counter
x64	add	rsi, r8
	end_timer 8
	jmp	yfinish_unfft

ymid4p:	start_timer 8
	mov	rbp, DIST_TO_MULSRCARG
	yloop_init  48			;; 48 iterations
	yr8_8cl_eight_complex_with_mulf_preload
b5d:	yr8_8cl_eight_complex_with_mulf rsi, 8*64, 64, 2*64, 4*64
	yloop	4, b5d, -4*8*64+dist32	;; Test loop counter, next source pointer
	yloop	12, b5d			;; Test loop counter
	end_timer 8
yfinish_unfft:
	bump	rsi, -12*dist32		;; Restore source pointer

;; Do inverse FFT levels 8,9
;; On input the 64-byte cache lines hold these data values:
;;	0	+768	+768	+768	8	+768	+768	+768
;;	1
;;	...
;;	7
;;	16	...
;;	...
;;	752	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+768	+768	+768	32	+768	+768	+768
;;	1
;;	...
;;	31
;;	64	...
;;	...
;;	704	...

;; Do 384 four_complex_unfft operations
;;	distance between fft data elements is 8
;;	do 96 macros each processing 32 data values

	start_timer 9
	mov	rdi, sincos1		;; Load sin/cos pointer
	yloop_init  96			;; 12 iterations of 8
	yr4_b4cl_four_complex_djbunfft_preload
c4b:	yr4_b4cl_four_complex_djbunfft rsi, 64, 8*64, 16*64, rdi, 4*YMM_SCD2/4, 8
	yloop	8, c4b, -8*64+dist32, rdi, -8*4*YMM_SCD2/4 ;; Test loop counter, next source pointer, restore sine/cosine pointer
	yloop	12, c4b, -12*dist32	;; Test loop counter, restore source pointer
	end_timer 9

;; Do inverse FFT levels 6,7
;; On input the 64-byte cache lines hold these data values:
;;	0	+768	+768	+768	32	+768	+768	+768
;;	1
;;	...
;;	31
;;	64	...
;;	...
;;	704	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+768	+768	+768	128	+768	+768	+768
;;	1
;;	...
;;	127
;;	256	...
;;	512	...

;; Do 384 four_complex_unfft operations
;;	distance between fft data elements is 32
;;	do 96 macros each processing 32 data values

	start_timer 10
	yloop_init  96			;; 3 iterations of 32
	yr4_b4cl_four_complex_djbunfft_preload
c3b:	yr4_b4cl_four_complex_djbunfft rsi, 64, dist32, 2*dist32, rdi, YMM_SCD2/4, 32
	yloop	32, c3b, -32*64+4*dist32, rdi, -32*YMM_SCD2/4 ;; Test loop counter, next source pointer, restore sine/cosine pointer
	yloop	3, c3b, -3*4*dist32	;; Test loop counter, restore source pointer
	end_timer 10

;; Do inverse FFT levels 4,5
;; On input the 64-byte cache lines hold these data values:
;;	0	+768	+768	+768	128	+768	+768	+768
;;	1
;;	...
;;	127
;;	256	...
;;	512	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+768	+768	+768	384	+768	+768	+768
;;	1
;;	...
;;	383

;; Do 512 three_complex_unfft operations
;;	distance between fft data elements is 128
;;	do 128 macros each processing 24 data values

	start_timer 11
	bump	rdi, 32*YMM_SCD2/4	;; Next sine/cosine pointer
	yloop_init  128			;; 128 iterations
	yr3_b3cl_three_complex_djbunfft_preload
c2b:	yr3_b3cl_three_complex_djbunfft rsi, 64, 4*dist32, rdi, YMM_SCD1/4, 32
	yloop	32, c2b, -32*64+dist32	;; Test loop counter, next source pointer
	yloop	4, c2b, -4*dist32	;; Test loop counter, restore source pointer
	end_timer 11

;; Do inverse FFT levels 1,2,3
;; On input the 64-byte cache lines hold these data values:
;;	0	+768	+768	+768	384	+768	+768	+768
;;	1
;;	...
;;	383
;; These are swizzled to:
;;	0	+96	+96	+96	384	+96	+96	+96
;;	1	...
;;	...
;;	95
;;	768	...
;;	1536	...
;;	2304	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+96	+96	+96	1536	+96	+96	+96
;;	1	...
;;	...
;;	95
;;	384	...
;;	768	...
;;	1152	...

;; Do 384 four_complex_last_unfft operations
;;	distance between fft data elements is 384
;;	do 96 macros each processing 32 data values

	start_timer 12
	yloop_init  96			;; 96 iterations
	yr4_s4cl_four_complex_last_unfft_preload
c1b:	yr4_s4cl_four_complex_last_unfft rsi, 64, 3*dist32, 6*dist32, rdi, YMM_SCD4, 32
	yloop	32, c1b, -32*64+dist32	;; Test loop counter, next source pointer
	yloop	3, c1b			;; Test loop counter
	end_timer 12

	ytwo_dest_adjust
	yfft_3_ret
	ENDM


;; Perform a 4K-element FFT.

yfft4K MACRO type
	LOCAL	ymid2, ymid3, ymid4, yfinish_unfft
	LOCAL	b1b, b2b, b3a, b3b, b4a, b4b, b5a, b5b, b5c, b5d, c1b, c2b, c3a, c3b, c4a, c4b

;; Do a multiply with pre-FFTed inputs

	cmp	ffttype, 4
	je	ymid4

;; Do FFT levels 1,2,3
;; On input the 64-byte cache lines hold these data values:
;;	0	+128	+128	+128	2K	+128	+128	+128
;;	1
;;	...
;;	127
;;	512	...
;;	1024	...
;;	1536	...
;; On output these are swizzled to:
;;	0	+1K	+1K	+1K	512	+1K	+1K	+1K
;;	1
;;	...
;;	511

;; Do 512 eight_reals_fft operations
;;	distance between fft data elements is 512
;;	do 128 macros each processing 32 data values

	start_timer 4
	mov	rdi, sincos1		;; Load sin/cos pointer
	bump	rdi, 128*YMM_SCD3
	yloop_init  128			;; 128 iterations
	yr4_fs4cl_eight_reals_fft_preload
b1b:	yr4_fs4cl_eight_reals_fft rsi, 64, 2*dist64, 4*dist64, rdi, YMM_SCD3, 64
	yloop	64, b1b, -64*64+dist64	;; Test loop counter, next source pointer
	yloop	2, b1b, -2*dist64	;; Test loop counter, restore source pointer
	end_timer 4

;; Do FFT levels 4,5
;; On input the 64-byte cache lines hold these data values:
;;	0	+1K	+1K	+1K	512	+1K	+1K	+1K
;;	1
;;	...
;;	511
;; On output the 64-byte cache lines hold these data values:
;;	0	+1K	+1K	+1K	128	+1K	+1K	+1K
;;	1
;;	...
;;	127
;;	256	...
;;	512	...
;;	768	...

;; Do 128 eight_reals_fft operations
;; Do 384 four_complex_fft operations
;;	distance between fft data elements is 128
;;	do 128 macros each processing 32 data values

	start_timer 5
	bump	rdi, -128*YMM_SCD3-128*YMM_SCD3 ;; Load sin/cos pointer
	yloop_init  128			;; 128 iterations
	yr4_4cl_eight_reals_four_complex_djbfft_preload
b2b:	yr4_4cl_eight_reals_four_complex_djbfft rsi, 64, 2*dist64, 4*dist64, rdi, YMM_SCD3, 64
	yloop	64, b2b, -64*64+dist64	;; Test loop counter, next source pointer
	yloop	2, b2b, -2*dist64, rdi, -128*YMM_SCD3 ;; Test loop counter, restore source pointer, restore sin/cos pointer
	end_timer 5

;; Do FFT levels 6,7
;; On input the 64-byte cache lines hold these data values:
;;	0	+1K	+1K	+1K	128	+1K	+1K	+1K
;;	1
;;	...
;;	127
;;	256	...
;;	512	...
;;	768	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+1K	+1K	+1K	32	+1K	+1K	+1K
;;	1
;;	...
;;	31
;;	64	...
;;	...
;;	960	...

;; Do 32 eight_reals_fft operations
;; Do 480 four_complex_fft operations
;;	distance between fft data elements is 32
;;	do 128 macros each processing 32 data values

	start_timer 6
	yloop_init  128			;; 4 iterations of 32
	yr4_4cl_eight_reals_four_complex_djbfft_preload
;; Loop below reworked to avoid 4KB distances
;;b3a:	yr4_4cl_eight_reals_four_complex_djbfft rsi, 64, 32*64, dist64, rdi, 4*YMM_SCD3, 32
;;	yloop	32, b3a, -32*64+2*dist64, rdi, -32*4*YMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer
b3a:	yr4_4cl_eight_reals_four_complex_djbfft rsi, 2*64, 32*64, dist64, rdi, 2*4*YMM_SCD3, 16
	yloop	16, b3a, -16*2*64+64, rdi, -16*2*4*YMM_SCD3+4*YMM_SCD3 ;; Test loop counter, next source pointer, next sin/cos pointer
	yloop	2, b3a, -2*64+2*dist64, rdi, -2*4*YMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer

	loops_reset			;; 3 more iters of 32
	yr4_rb4cl_four_complex_djbfft_preload
;; Loop below reworked to avoid 4KB distances
;;b3b:	yr4_rb4cl_four_complex_djbfft rsi, 64, 32*64, dist64, rdi, 4*YMM_SCD3, 32
;;	yloop	32, b3b, -32*64+2*dist64, rdi, -32*4*YMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer
b3b:	yr4_rb4cl_four_complex_djbfft rsi, 2*64, 32*64, dist64, rdi, 2*4*YMM_SCD3, 16
	yloop	16, b3b, -16*2*64+64, rdi, -16*2*4*YMM_SCD3+4*YMM_SCD3 ;; Test loop counter, next source pointer, next sin/cos pointer
	yloop	2, b3b, -2*64+2*dist64, rdi, -2*4*YMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer
	yloop	4, b3b, -4*2*dist64	;; Test loop counter, restore source pointer
	end_timer 6

;; Do FFT levels 8,9
;; On input the 64-byte cache lines hold these data values:
;;	0	+1K	+1K	+1K	32	+1K	+1K	+1K
;;	1
;;	...
;;	31
;;	64	...
;;	...
;;	960	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+1K	+1K	+1K	8	+1K	+1K	+1K
;;	1
;;	...
;;	7
;;	16	...
;;	...
;;	1008	...

;; Do 8 eight_reals_fft operations
;; Do 504 four_complex_fft operations
;;	distance between fft data elements is 8
;;	do 128 macros each processing 32 data values

	start_timer 7
	yloop_init  128			;; 16 iterations of 8
	yr4_4cl_eight_reals_four_complex_djbfft_preload
b4a:	yr4_4cl_eight_reals_four_complex_djbfft rsi, 64, 8*64, 16*64, rdi, 16*YMM_SCD3, 8
	yloop	8, b4a, -8*64+32*64, rdi, -8*16*YMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer

	loops_reset			;; 15 more iters of 8
	yr4_rb4cl_four_complex_djbfft_preload
b4b:	yr4_rb4cl_four_complex_djbfft rsi, 64, 8*64, 16*64, rdi, 16*YMM_SCD3, 8
	yloop	8, b4b, -8*64+32*64, rdi, -8*16*YMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer
	yloop	2, b4b, -2*32*64+dist64	;; Test loop counter, next source pointer
	yloop	8, b4b, -8*dist64	;; Test loop counter, restore source pointer
	end_timer 7

;; Do FFT levels 10,11
;; On input and output the 64-byte cache lines hold these data values:
;;	0	+1K	+1K	+1K	8	+1K	+1K	+1K
;;	1
;;	...
;;	7
;;	16	...
;;	...
;;	1008	...

;; Do 1 sixteen_reals_fft operation
;; Do 255 eight_complex_fft operations
;;	distance between fft data elements is 1
;;	do 128 macros each processing 64 data values

	cmp	ffttype, 2
	je	ymid2
	ja	ymid3

	start_timer 8
;;	yr8_8cl_sixteen_reals_eight_complex_fft_final rsi, 8*64, 64, 2*64, 4*64
	yr8_dispatch call, y8real_fft_final, 64, 2*64, 4*64
	yloop_init  64, 1, 8		;; 63 more iterations
b5a:	yr8_8cl_eight_complex_fft_final rsi, 8*64, 64, 2*64, 4*64
	yloop	8, b5a, -8*8*64+dist64	;; Test loop counter, next source pointer
	yloop	8, b5a			;; Test loop counter
	end_timer 8
	yfft_1_ret

ymid2:	start_timer 8
;;	yr8_8cl_sixteen_reals_eight_complex_with_square rsi, 8*64, 64, 2*64, 4*64
	yr8_dispatch call, y8real_square, 64, 2*64, 4*64
	yloop_init  64, 1, 8		;; 63 more iterations
	yr8_8cl_eight_complex_with_square_preload
b5b:	yr8_8cl_eight_complex_with_square rsi, 8*64, 64, 2*64, 4*64
	yloop	8, b5b, -8*8*64+dist64	;; Test loop counter, next source pointer
	yloop	8, b5b			;; Test loop counter
	end_timer 8
	jmp	yfinish_unfft

ymid3:	start_timer 8
x64	mov	r8, DEST2ARG
	mov	rbp, DIST_TO_MULSRCARG
;;	yr8_8cl_sixteen_reals_eight_complex_with_mult rsi, 8*64, 64, 2*64, 4*64
	yr8_dispatch call, y8real_mult, 64, 2*64, 4*64
	yloop_init  64, 1, 8		;; 63 more iterations
	yr8_8cl_eight_complex_with_mult_preload
b5c:	yr8_8cl_eight_complex_with_mult rsi, 8*64, 64, 2*64, 4*64
	yloop	8, b5c, -8*8*64+dist64	;; Test loop counter, next source pointer
	yloop	8, b5c			;; Test loop counter
x64	add	rsi, r8
	end_timer 8
	jmp	yfinish_unfft

ymid4:	start_timer 8
	mov	rbp, DIST_TO_MULSRCARG
;;	yr8_8cl_sixteen_reals_eight_complex_with_mulf rsi, 8*64, 64, 2*64, 4*64
	yr8_dispatch call, y8real_mulf, 64, 2*64, 4*64
	yloop_init  64, 1, 8		;; 63 more iterations
	yr8_8cl_eight_complex_with_mulf_preload
b5d:	yr8_8cl_eight_complex_with_mulf rsi, 8*64, 64, 2*64, 4*64
	yloop	8, b5d, -8*8*64+dist64	;; Test loop counter, next source pointer
	yloop	8, b5d			;; Test loop counter
	end_timer 8
yfinish_unfft:
	bump	rsi, -8*dist64		;; Restore source pointer

;; Do inverse FFT levels 8,9
;; On input the 64-byte cache lines hold these data values:
;;	0	+1K	+1K	+1K	8	+1K	+1K	+1K
;;	1
;;	...
;;	7
;;	16	...
;;	...
;;	1008	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+1K	+1K	+1K	32	+1K	+1K	+1K
;;	1
;;	...
;;	31
;;	64	...
;;	...
;;	960	...

;; Do 8 eight_reals_unfft operations
;; Do 504 four_complex_unfft operations
;;	distance between fft data elements is 8
;;	do 128 macros each processing 32 data values

	start_timer 9
	mov	rdi, sincos1		;; Load sin/cos pointer
	yloop_init  128			;; 16 iterations of 8
	yr4_4cl_eight_reals_four_complex_djbunfft_preload
c4a:	yr4_4cl_eight_reals_four_complex_djbunfft rsi, 64, 8*64, 16*64, rdi, 16*YMM_SCD3, 8
	yloop	8, c4a, -8*64+32*64, rdi, -8*16*YMM_SCD3 ;; Test loop counter, next source pointer, restore sine/cosine pointer

	loops_reset			;; 15 more iters of 8
	yr4_rb4cl_four_complex_djbunfft_preload
c4b:	yr4_rb4cl_four_complex_djbunfft rsi, 64, 8*64, 16*64, rdi, 16*YMM_SCD3, 8
	yloop	8, c4b, -8*64+32*64, rdi, -8*16*YMM_SCD3 ;; Test loop counter, next source pointer, restore sine/cosine pointer
	yloop	2, c4b, -2*32*64+dist64	;; Test loop counter, next source pointer
	yloop	8, c4b, -8*dist64	;; Test loop counter, restore source pointer
	end_timer 9

;; Do inverse FFT levels 6,7
;; On input the 64-byte cache lines hold these data values:
;;	0	+1K	+1K	+1K	32	+1K	+1K	+1K
;;	1
;;	...
;;	31
;;	64	...
;;	...
;;	960	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+1K	+1K	+1K	128	+1K	+1K	+1K
;;	1
;;	...
;;	127
;;	256	...
;;	512	...
;;	768	...

;; Do 32 eight_reals_unfft operations
;; Do 480 four_complex_unfft operations
;;	distance between fft data elements is 32
;;	do 128 macros each processing 32 data values

	start_timer 10
	yloop_init  128			;; 4 iterations of 32
	yr4_4cl_eight_reals_four_complex_djbunfft_preload
;; Loop below reworked to avoid 4KB distances
;;c3a:	yr4_4cl_eight_reals_four_complex_djbunfft rsi, 64, 32*64, dist64, rdi, 4*YMM_SCD3, 32
;;	yloop	32, c3a, -32*64+2*dist64, rdi, -32*4*YMM_SCD3 ;; Test loop counter, next source pointer, restore sine/cosine pointer
c3a:	yr4_4cl_eight_reals_four_complex_djbunfft rsi, 2*64, 32*64, dist64, rdi, 2*4*YMM_SCD3, 16
	yloop	16, c3a, -16*2*64+64, rdi, -16*2*4*YMM_SCD3+4*YMM_SCD3 ;; Test loop counter, next source pointer, next sine/cosine pointer
	yloop	2, c3a, -2*64+2*dist64, rdi, -2*4*YMM_SCD3 ;; Test loop counter, next source pointer, restore sine/cosine pointer

	loops_reset			;; 3 more iters of 32
	yr4_rb4cl_four_complex_djbunfft_preload
;; Loop below reworked to avoid 4KB distances
;;c3b:	yr4_rb4cl_four_complex_djbunfft rsi, 64, 32*64, dist64, rdi, 4*YMM_SCD3, 32
;;	yloop	32, c3b, -32*64+2*dist64, rdi, -32*4*YMM_SCD3 ;; Test loop counter, next source pointer, restore sine/cosine pointer
c3b:	yr4_rb4cl_four_complex_djbunfft rsi, 2*64, 32*64, dist64, rdi, 2*4*YMM_SCD3, 16
	yloop	16, c3b, -16*2*64+64, rdi, -16*2*4*YMM_SCD3+4*YMM_SCD3 ;; Test loop counter, next source pointer, next sine/cosine pointer
	yloop	2, c3b, -2*64+2*dist64, rdi, -2*4*YMM_SCD3 ;; Test loop counter, next source pointer, restore sine/cosine pointer
	yloop	4, c3b, -4*2*dist64	;; Test loop counter, restore source pointer
	end_timer 10

;; Do inverse FFT levels 4,5
;; On input the 64-byte cache lines hold these data values:
;;	0	+1K	+1K	+1K	128	+1K	+1K	+1K
;;	1
;;	...
;;	127
;;	256	...
;;	512	...
;;	768	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+1K	+1K	+1K	512	+1K	+1K	+1K
;;	1
;;	...
;;	511

;; Do 128 eight_reals_unfft operations
;; Do 384 four_complex_unfft operations
;;	distance between fft data elements is 128
;;	do 128 macros each processing 32 data values

	start_timer 11
	yloop_init  128			;; 128 iterations
	yr4_4cl_eight_reals_four_complex_djbunfft_preload
c2b:	yr4_4cl_eight_reals_four_complex_djbunfft rsi, 64, 2*dist64, 4*dist64, rdi, YMM_SCD3, 64
	yloop	64, c2b, -64*64+dist64	;; Test loop counter, next source pointer
	yloop	2, c2b, -2*dist64	;; Test loop counter, restore source pointer
	end_timer 11

;; Do inverse FFT levels 1,2,3
;; On input the 64-byte cache lines hold these data values:
;;	0	+1K	+1K	+1K	512	+1K	+1K	+1K
;;	1
;;	...
;;	511
;; These are swizzled to:
;;	0	+128	+128	+128	512	+128	+128	+128
;;	1	...
;;	...
;;	127
;;	1K	...
;;	2K	...
;;	3K	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+128	+128	+128	2K	+128	+128	+128
;;	1	...
;;	...
;;	127
;;	512	...
;;	1024	...
;;	1536	...

;; Do 512 eight_reals_unfft operations
;;	distance between fft data elements is 512
;;	do 128 macros each processing 32 data values

	start_timer 12
	yloop_init  128			;; 128 iterations
	yr4_s4cl_eight_reals_unfft_preload
c1b:	yr4_s4cl_eight_reals_unfft rsi, 64, 2*dist64, 4*dist64, rdi, YMM_SCD3, 64
	yloop	64, c1b, -64*64+dist64	;; Test loop counter, next source pointer
	yloop	2, c1b			;; Test loop counter
	end_timer 12

	ytwo_dest_adjust
	yfft_3_ret
	ENDM

;; Perform a 4K-element negacyclic FFT

yfft4Kp MACRO type
	LOCAL	ymid2p, ymid3p, ymid4p, yfinish_unfft
	LOCAL	b1b, b2b, b3b, b4b, b5a, b5b, b5c, b5d, c1b, c2b, c3b, c4b

;; Do a multiply with pre-FFTed inputs

	cmp	ffttype, 4
	je	ymid4p

;; Do FFT levels 1,2,3
;; On input the 64-byte cache lines hold these data values:
;;	0	+128	+128	+128	2K	+128	+128	+128
;;	1
;;	...
;;	127
;;	512	...
;;	1024	...
;;	1536	...
;; On output these are swizzled to:
;;	0	+1K	+1K	+1K	512	+1K	+1K	+1K
;;	1
;;	...
;;	511

;; Do 512 four_complex_first_fft operations
;;	distance between fft data elements is 512
;;	do 128 macros each processing 32 data values

	start_timer 4
	mov	rdi, sincos1		;; Load sin/cos pointer
	bump	rdi, 128*YMM_SCD2/4
	yloop_init  128			;; 128 iterations
	yr4_fs4cl_four_complex_first_djbfft_preload
b1b:	yr4_fs4cl_four_complex_first_djbfft rsi, 64, 2*dist64, 4*dist64, rdi, YMM_SCD4, 64
	yloop	64, b1b, -64*64+dist64	;; Test loop counter, next source pointer
	yloop	2, b1b, -2*dist64	;; Test loop counter, restore source pointer
	end_timer 4

;; Do FFT levels 4,5
;; On input the 64-byte cache lines hold these data values:
;;	0	+1K	+1K	+1K	512	+1K	+1K	+1K
;;	1
;;	...
;;	511
;; On output the 64-byte cache lines hold these data values:
;;	0	+1K	+1K	+1K	128	+1K	+1K	+1K
;;	1
;;	...
;;	127
;;	256	...
;;	512	...
;;	768	...

;; Do 512 four_complex_fft operations
;;	distance between fft data elements is 128
;;	do 128 macros each processing 32 data values

	start_timer 5
	bump	rdi, -128*YMM_SCD4-128*YMM_SCD2/4 ;; Load sin/cos pointer
	yloop_init  128			;; 128 iterations
	yr4_b4cl_four_complex_djbfft_preload
b2b:	yr4_b4cl_four_complex_djbfft rsi, 64, 2*dist64, 4*dist64, rdi, YMM_SCD2/4, 64
	yloop	64, b2b, -64*64+dist64	;; Test loop counter, next source pointer
	yloop	2, b2b, -2*dist64, rdi, -128*YMM_SCD2/4 ;; Test loop counter, restore source pointer, restore sin/cos pointer
	end_timer 5

;; Do FFT levels 6,7
;; On input the 64-byte cache lines hold these data values:
;;	0	+1K	+1K	+1K	128	+1K	+1K	+1K
;;	1
;;	...
;;	127
;;	256	...
;;	512	...
;;	768	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+1K	+1K	+1K	32	+1K	+1K	+1K
;;	1
;;	...
;;	31
;;	64	...
;;	...
;;	960	...

;; Do 512 four_complex_fft operations
;;	distance between fft data elements is 32
;;	do 128 macros each processing 32 data values

	start_timer 6
	yloop_init  128			;; 4 iterations of 32
	yr4_b4cl_four_complex_djbfft_preload
;; Loop below reworked to avoid 4KB distances
;;b3b:	yr4_b4cl_four_complex_djbfft rsi, 64, 32*64, dist64, rdi, 4*YMM_SCD2/4, 32
;;	yloop	32, b3b, -32*64+2*dist64, rdi, -32*4*YMM_SCD2/4 ;; Test loop counter, next source pointer, restore sin/cos pointer
b3b:	yr4_b4cl_four_complex_djbfft rsi, 2*64, 32*64, dist64, rdi, 2*4*YMM_SCD2/4, 16
	yloop	16, b3b, -16*2*64+64, rdi, -16*2*4*YMM_SCD2/4+4*YMM_SCD2/4 ;; Test loop counter, next source pointer, next sin/cos pointer
	yloop	2, b3b, -2*64+2*dist64, rdi, -2*4*YMM_SCD2/4 ;; Test loop counter, next source pointer, restore sin/cos pointer
	yloop	4, b3b, -4*2*dist64	;; Test loop counter, restore source pointer
	end_timer 6

;; Do FFT levels 8,9
;; On input the 64-byte cache lines hold these data values:
;;	0	+1K	+1K	+1K	32	+1K	+1K	+1K
;;	1
;;	...
;;	31
;;	64	...
;;	...
;;	960	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+1K	+1K	+1K	8	+1K	+1K	+1K
;;	1
;;	...
;;	7
;;	16	...
;;	...
;;	1008	...

;; Do 512 four_complex_fft operations
;;	distance between fft data elements is 8
;;	do 128 macros each processing 32 data values

	start_timer 7
	yloop_init  128			;; 16 iterations of 8
	yr4_b4cl_four_complex_djbfft_preload
b4b:	yr4_b4cl_four_complex_djbfft rsi, 64, 8*64, 16*64, rdi, 16*YMM_SCD2/4, 8
	yloop	8, b4b, -8*64+32*64, rdi, -8*16*YMM_SCD2/4 ;; Test loop counter, next source pointer, restore sin/cos pointer
	yloop	2, b4b, -2*32*64+dist64	;; Test loop counter, next source pointer
	yloop	8, b4b, -8*dist64	;; Test loop counter, restore source pointer
	end_timer 7

;; Do FFT levels 10,11
;; On input and output the 64-byte cache lines hold these data values:
;;	0	+1K	+1K	+1K	8	+1K	+1K	+1K
;;	1
;;	...
;;	7
;;	16	...
;;	...
;;	1008	...

;; Do 256 eight_complex_fft operations
;;	distance between fft data elements is 1
;;	do 64 macros each processing 64 data values

	cmp	ffttype, 2
	je	ymid2p
	ja	ymid3p

	start_timer 8
	yloop_init  64			;; 64 iterations
b5a:	yr8_8cl_eight_complex_fft_final rsi, 8*64, 64, 2*64, 4*64
	yloop	8, b5a, -8*8*64+dist64	;; Test loop counter, next source pointer
	yloop	8, b5a			;; Test loop counter
	end_timer 8
	yfft_1_ret

ymid2p:	start_timer 8
	yloop_init  64			;; 64 iterations
	yr8_8cl_eight_complex_with_square_preload
b5b:	yr8_8cl_eight_complex_with_square rsi, 8*64, 64, 2*64, 4*64
	yloop	8, b5b, -8*8*64+dist64	;; Test loop counter, next source pointer
	yloop	8, b5b			;; Test loop counter
	end_timer 8
	jmp	yfinish_unfft

ymid3p:	start_timer 8
x64	mov	r8, DEST2ARG
	mov	rbp, DIST_TO_MULSRCARG
	yloop_init  64			;; 64 iterations
	yr8_8cl_eight_complex_with_mult_preload
b5c:	yr8_8cl_eight_complex_with_mult rsi, 8*64, 64, 2*64, 4*64
	yloop	8, b5c, -8*8*64+dist64	;; Test loop counter, next source pointer
	yloop	8, b5c			;; Test loop counter
x64	add	rsi, r8
	end_timer 8
	jmp	yfinish_unfft

ymid4p:	start_timer 8
	mov	rbp, DIST_TO_MULSRCARG
	yloop_init  64			;; 64 iterations
	yr8_8cl_eight_complex_with_mulf_preload
b5d:	yr8_8cl_eight_complex_with_mulf rsi, 8*64, 64, 2*64, 4*64
	yloop	8, b5d, -8*8*64+dist64	;; Test loop counter, next source pointer
	yloop	8, b5d			;; Test loop counter
	end_timer 8
yfinish_unfft:
	bump	rsi, -8*dist64		;; Restore source pointer

;; Do inverse FFT levels 8,9
;; On input the 64-byte cache lines hold these data values:
;;	0	+1K	+1K	+1K	8	+1K	+1K	+1K
;;	1
;;	...
;;	7
;;	16	...
;;	...
;;	1008	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+1K	+1K	+1K	32	+1K	+1K	+1K
;;	1
;;	...
;;	31
;;	64	...
;;	...
;;	960	...

;; Do 512 four_complex_unfft operations
;;	distance between fft data elements is 8
;;	do 128 macros each processing 32 data values

	start_timer 9
	mov	rdi, sincos1		;; Load sin/cos pointer
	yloop_init  128			;; 16 iterations of 8
	yr4_b4cl_four_complex_djbunfft_preload
c4b:	yr4_b4cl_four_complex_djbunfft rsi, 64, 8*64, 16*64, rdi, 16*YMM_SCD2/4, 8
	yloop	8, c4b, -8*64+32*64, rdi, -8*16*YMM_SCD2/4 ;; Test loop counter, next source pointer, restore sine/cosine pointer
	yloop	2, c4b, -2*32*64+dist64	;; Test loop counter, nextsource pointer
	yloop	8, c4b, -8*dist64	;; Test loop counter, restore source pointer
	end_timer 9

;; Do inverse FFT levels 6,7
;; On input the 64-byte cache lines hold these data values:
;;	0	+1K	+1K	+1K	32	+1K	+1K	+1K
;;	1
;;	...
;;	31
;;	64	...
;;	...
;;	960	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+1K	+1K	+1K	128	+1K	+1K	+1K
;;	1
;;	...
;;	127
;;	256	...
;;	512	...
;;	768	...

;; Do 512 four_complex_unfft operations
;;	distance between fft data elements is 32
;;	do 128 macros each processing 32 data values

	start_timer 10
	yloop_init  128			;; 4 iterations of 32
	yr4_b4cl_four_complex_djbunfft_preload
;; Loop below reworked to avoid 4KB distances
;;c3b:	yr4_b4cl_four_complex_djbunfft rsi, 64, 32*64, dist64, rdi, 4*YMM_SCD2/4, 32
;;	yloop	32, c3b, -32*64+2*dist64, rdi, -32*4*YMM_SCD2/4 ;; Test loop counter, next source pointer, restore sine/cosine pointer
c3b:	yr4_b4cl_four_complex_djbunfft rsi, 2*64, 32*64, dist64, rdi, 2*4*YMM_SCD2/4, 16
	yloop	16, c3b, -16*2*64+64, rdi, -16*2*4*YMM_SCD2/4+4*YMM_SCD2/4 ;; Test loop counter, next source pointer, next sine/cosine pointer
	yloop	2, c3b, -2*64+2*dist64, rdi, -2*4*YMM_SCD2/4 ;; Test loop counter, next source pointer, restore sine/cosine pointer
	yloop	4, c3b, -4*2*dist64	;; Test loop counter, restore source pointer
	end_timer 10

;; Do inverse FFT levels 4,5
;; On input the 64-byte cache lines hold these data values:
;;	0	+1K	+1K	+1K	128	+1K	+1K	+1K
;;	1
;;	...
;;	127
;;	256	...
;;	512	...
;;	768	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+1K	+1K	+1K	512	+1K	+1K	+1K
;;	1
;;	...
;;	511

;; Do 512 four_complex_unfft operations
;;	distance between fft data elements is 128
;;	do 128 macros each processing 32 data values

	start_timer 11
	yloop_init  128			;; 128 iterations
	yr4_b4cl_four_complex_djbunfft_preload
c2b:	yr4_b4cl_four_complex_djbunfft rsi, 64, 2*dist64, 4*dist64, rdi, YMM_SCD2/4, 64
	yloop	64, c2b, -64*64+dist64	;; Test loop counter, next source pointer
	yloop	2, c2b, -2*dist64	;; Test loop counter, restore source pointer
	end_timer 11

;; Do inverse FFT levels 1,2,3
;; On input the 64-byte cache lines hold these data values:
;;	0	+1K	+1K	+1K	512	+1K	+1K	+1K
;;	1
;;	...
;;	511
;; These are swizzled to:
;;	0	+128	+128	+128	512	+128	+128	+128
;;	1	...
;;	...
;;	127
;;	1K	...
;;	2K	...
;;	3K	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+128	+128	+128	2K	+128	+128	+128
;;	1	...
;;	...
;;	127
;;	512	...
;;	1024	...
;;	1536	...

;; Do 512 four_complex_last_unfft operations
;;	distance between fft data elements is 512
;;	do 128 macros each processing 32 data values

	start_timer 12
	yloop_init  128			;; 128 iterations
	yr4_s4cl_four_complex_last_unfft_preload
c1b:	yr4_s4cl_four_complex_last_unfft rsi, 64, 2*dist64, 4*dist64, rdi, YMM_SCD4, 64
	yloop	64, c1b, -64*64+dist64	;; Test loop counter, next source pointer
	yloop	2, c1b			;; Test loop counter
	end_timer 12

	ytwo_dest_adjust
	yfft_3_ret
	ENDM


;; Perform a 5K-element FFT.

yfft5K MACRO type
	LOCAL	ymid2, ymid3, ymid4, yfinish_unfft
	LOCAL	b1b, b2b, b3a, b3b, b4a, b4b, b5a, b5b, b5c, b5d, c1b, c2b, c3a, c3b, c4a, c4b

;; Do a multiply with pre-FFTed inputs

	cmp	ffttype, 4
	je	ymid4

;; Do FFT levels 1,2,3
;; On input the 64-byte cache lines hold these data values:
;;	0	+160	+160	+160	2560	+160	+160	+160
;;	1
;;	...
;;	159
;;	640	...
;;	1280	...
;;	1920	...
;; On output these are swizzled to:
;;	0	+1280	+1280	+1280	640	+1280	+1280	+1280
;;	1
;;	...
;;	639

;; Do 640 eight_reals_fft operations
;;	distance between fft data elements is 640
;;	do 160 macros each processing 32 data values

	start_timer 4
	mov	rdi, sincos1		;; Load sin/cos pointer
	bump	rdi, 128*YMM_SCD4+32*YMM_SCD3
	yloop_init  160			;; 160 iterations
	yr4_fs4cl_eight_reals_fft_preload
b1b:	yr4_fs4cl_eight_reals_fft rsi, 64, 5*dist32, 10*dist32, rdi, YMM_SCD3, 32
	yloop	32, b1b, -32*64+dist32	;; Test loop counter, next source pointer
	yloop	5, b1b, -5*dist32	;; Test loop counter, restore source pointer
	end_timer 4

;; Do FFT levels 4,5
;; On input the 64-byte cache lines hold these data values:
;;	0	+1280	+1280	+1280	640	+1280	+1280	+1280
;;	1
;;	...
;;	639
;; On output the 64-byte cache lines hold these data values:
;;	0	+1280	+1280	+1280	128	+1280	+1280	+1280
;;	1
;;	...
;;	127
;;	256	...
;;	...
;;	1024	...

;; Do 128 ten_reals_fft operations
;; Do 384 five_complex_fft operations
;;	distance between fft data elements is 128
;;	do 128 macros each processing 40 data values

	start_timer 5
	bump	rdi, -160*YMM_SCD3-128*YMM_SCD4 ;; Load sin/cos pointer
	yloop_init  128			;; 128 iterations
	yr5_5cl_ten_reals_five_complex_djbfft_preload
b2b:	yr5_5cl_ten_reals_five_complex_djbfft rsi, 64, 4*dist32, rdi, YMM_SCD4, 32
	yloop	32, b2b, -32*64+dist32	;; Test loop counter, next source pointer
	yloop	4, b2b, -4*dist32	;; Test loop counter, restore source pointer
	end_timer 5

;; Do FFT levels 6,7
;; On input the 64-byte cache lines hold these data values:
;;	0	+1280	+1280	+1280	128	+1280	+1280	+1280
;;	1
;;	...
;;	127
;;	256	...
;;	...
;;	1024	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+1280	+1280	+1280	32	+1280	+1280	+1280
;;	1
;;	...
;;	31
;;	64	...
;;	...
;;	1216	...

;; Do 32 eight_reals_fft operations
;; Do 608 four_complex_fft operations
;;	distance between fft data elements is 32
;;	do 160 macros each processing 32 data values

	start_timer 6
	bump	rdi, -128*YMM_SCD4-32*YMM_SCD3 ;; Next sin/cos pointer
	yloop_init  160			;; 5 iterations of 32
	yr4_4cl_eight_reals_four_complex_djbfft_preload
b3a:	yr4_4cl_eight_reals_four_complex_djbfft rsi, 64, dist32, 2*dist32, rdi, YMM_SCD3, 32
	yloop	32, b3a, -32*64+4*dist32, rdi, -32*YMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer

	loops_reset			;; 4 more iters of 32
	yr4_rb4cl_four_complex_djbfft_preload
b3b:	yr4_rb4cl_four_complex_djbfft rsi, 64, dist32, 2*dist32, rdi, YMM_SCD3, 32
	yloop	32, b3b, -32*64+4*dist32, rdi, -32*YMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer
	yloop	5, b3b, -5*4*dist32	;; Test loop counter, restore source pointer
	end_timer 6

;; Do FFT levels 8,9
;; On input the 64-byte cache lines hold these data values:
;;	0	+1280	+1280	+1280	32	+1280	+1280	+1280
;;	1
;;	...
;;	31
;;	64	...
;;	...
;;	1216	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+1280	+1280	+1280	8	+1280	+1280	+1280
;;	1
;;	...
;;	7
;;	16	...
;;	...
;;	1264	...

;; Do 8 eight_reals_fft operations
;; Do 632 four_complex_fft operations
;;	distance between fft data elements is 8
;;	do 160 macros each processing 32 data values

	start_timer 7
	yloop_init  160			;; 20 iterations of 8
	yr4_4cl_eight_reals_four_complex_djbfft_preload
b4a:	yr4_4cl_eight_reals_four_complex_djbfft rsi, 64, 8*64, 16*64, rdi, 4*YMM_SCD3, 8
	yloop	8, b4a, -8*64+dist32, rdi, -8*4*YMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer

	loops_reset			;; 19 more iters of 8
	yr4_rb4cl_four_complex_djbfft_preload
b4b:	yr4_rb4cl_four_complex_djbfft rsi, 64, 8*64, 16*64, rdi, 4*YMM_SCD3, 8
	yloop	8, b4b, -8*64+dist32, rdi, -8*4*YMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer
	yloop	20, b4b, -20*dist32	;; Test loop counter, restore source pointer
	end_timer 7

;; Do FFT levels 10,11
;; On input and output the 64-byte cache lines hold these data values:
;;	0	+1280	+1280	+1280	8	+1280	+1280	+1280
;;	1
;;	...
;;	7
;;	16	...
;;	...
;;	1264	...

;; Do 1 sixteen_reals_fft operation
;; Do 319 eight_complex_fft operations
;;	distance between fft data elements is 1
;;	do 80 macros each processing 64 data values

	cmp	ffttype, 2
	je	ymid2
	ja	ymid3

	start_timer 8
;;	yr8_8cl_sixteen_reals_eight_complex_fft_final rsi, 8*64, 64, 2*64, 4*64
	yr8_dispatch call, y8real_fft_final, 64, 2*64, 4*64
	yloop_init  80, 1, 4		;; 79 more iterations
b5a:	yr8_8cl_eight_complex_fft_final rsi, 8*64, 64, 2*64, 4*64
	yloop	4, b5a, -4*8*64+dist32	;; Test loop counter, next source pointer
	yloop	20, b5a			;; Test loop counter
	end_timer 8
	yfft_1_ret

ymid2:	start_timer 8
;;	yr8_8cl_sixteen_reals_eight_complex_with_square rsi, 8*64, 64, 2*64, 4*64
	yr8_dispatch call, y8real_square, 64, 2*64, 4*64
	yloop_init  80, 1, 4		;; 79 more iterations
	yr8_8cl_eight_complex_with_square_preload
b5b:	yr8_8cl_eight_complex_with_square rsi, 8*64, 64, 2*64, 4*64
	yloop	4, b5b, -4*8*64+dist32	;; Test loop counter, next source pointer
	yloop	20, b5b			;; Test loop counter
	end_timer 8
	jmp	yfinish_unfft

ymid3:	start_timer 8
x64	mov	r8, DEST2ARG
	mov	rbp, DIST_TO_MULSRCARG
;;	yr8_8cl_sixteen_reals_eight_complex_with_mult rsi, 8*64, 64, 2*64, 4*64
	yr8_dispatch call, y8real_mult, 64, 2*64, 4*64
	yloop_init  80, 1, 4		;; 79 more iterations
	yr8_8cl_eight_complex_with_mult_preload
b5c:	yr8_8cl_eight_complex_with_mult rsi, 8*64, 64, 2*64, 4*64
	yloop	4, b5c, -4*8*64+dist32	;; Test loop counter, next source pointer
	yloop	20, b5c			;; Test loop counter
x64	add	rsi, r8
	end_timer 8
	jmp	yfinish_unfft

ymid4:	start_timer 8
	mov	rbp, DIST_TO_MULSRCARG
;;	yr8_8cl_sixteen_reals_eight_complex_with_mulf rsi, 8*64, 64, 2*64, 4*64
	yr8_dispatch call, y8real_mulf, 64, 2*64, 4*64
	yloop_init  80, 1, 4		;; 79 more iterations
	yr8_8cl_eight_complex_with_mulf_preload
b5d:	yr8_8cl_eight_complex_with_mulf rsi, 8*64, 64, 2*64, 4*64
	yloop	4, b5d, -4*8*64+dist32	;; Test loop counter, next source pointer
	yloop	20, b5d			;; Test loop counter
	end_timer 8
yfinish_unfft:
	bump	rsi, -20*dist32		;; Restore source pointer

;; Do inverse FFT levels 8,9
;; On input the 64-byte cache lines hold these data values:
;;	0	+1280	+1280	+1280	8	+1280	+1280	+1280
;;	1
;;	...
;;	7
;;	16	...
;;	...
;;	1264	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+1280	+1280	+1280	32	+1280	+1280	+1280
;;	1
;;	...
;;	31
;;	64	...
;;	...
;;	1216	...

;; Do 8 eight_reals_unfft operations
;; Do 632 four_complex_unfft operations
;;	distance between fft data elements is 8
;;	do 160 macros each processing 32 data values

	start_timer 9
	mov	rdi, sincos1		;; Load sin/cos pointer
	yloop_init  160			;; 20 iterations of 8
	yr4_4cl_eight_reals_four_complex_djbunfft_preload
c4a:	yr4_4cl_eight_reals_four_complex_djbunfft rsi, 64, 8*64, 16*64, rdi, 4*YMM_SCD3, 8
	yloop	8, c4a, -8*64+dist32, rdi, -8*4*YMM_SCD3 ;; Test loop counter, next source pointer, restore sine/cosine pointer

	loops_reset			;; 19 more iters of 8
	yr4_rb4cl_four_complex_djbunfft_preload
c4b:	yr4_rb4cl_four_complex_djbunfft rsi, 64, 8*64, 16*64, rdi, 4*YMM_SCD3, 8
	yloop	8, c4b, -8*64+dist32, rdi, -8*4*YMM_SCD3 ;; Test loop counter, next source pointer, restore sine/cosine pointer
	yloop	20, c4b, -20*dist32	;; Test loop counter, restore source pointer
	end_timer 9

;; Do inverse FFT levels 6,7
;; On input the 64-byte cache lines hold these data values:
;;	0	+1280	+1280	+1280	32	+1280	+1280	+1280
;;	1
;;	...
;;	31
;;	64	...
;;	...
;;	1216	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+1280	+1280	+1280	128	+1280	+1280	+1280
;;	1
;;	...
;;	127
;;	256	...
;;	...
;;	1024	...

;; Do 32 eight_reals_unfft operations
;; Do 608 four_complex_unfft operations
;;	distance between fft data elements is 32
;;	do 160 macros each processing 32 data values

	start_timer 10
	yloop_init  160			;; 5 iterations of 32
	yr4_4cl_eight_reals_four_complex_djbunfft_preload
c3a:	yr4_4cl_eight_reals_four_complex_djbunfft rsi, 64, dist32, 2*dist32, rdi, YMM_SCD3, 32
	yloop	32, c3a, -32*64+4*dist32, rdi, -32*YMM_SCD3 ;; Test loop counter, next source pointer, restore sine/cosine pointer

	loops_reset			;; 4 more iters of 32
	yr4_rb4cl_four_complex_djbunfft_preload
c3b:	yr4_rb4cl_four_complex_djbunfft rsi, 64, dist32, 2*dist32, rdi, YMM_SCD3, 32
	yloop	32, c3b, -32*64+4*dist32, rdi, -32*YMM_SCD3 ;; Test loop counter, next source pointer, restore sine/cosine pointer
	yloop	5, c3b, -5*4*dist32	;; Test loop counter, restore source pointer
	end_timer 10

;; Do inverse FFT levels 4,5
;; On input the 64-byte cache lines hold these data values:
;;	0	+1280	+1280	+1280	128	+1280	+1280	+1280
;;	1
;;	...
;;	127
;;	256	...
;;	...
;;	1024	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+1280	+1280	+1280	640	+1280	+1280	+1280
;;	1
;;	...
;;	639

;; Do 128 ten_reals_unfft operations
;; Do 384 five_complex_unfft operations
;;	distance between fft data elements is 128
;;	do 128 macros each processing 40 data values

	start_timer 11
	bump	rdi, 32*YMM_SCD3	;; Next sine/cosine pointer
	yloop_init  128			;; 128 iterations
	yr5_5cl_ten_reals_five_complex_djbunfft_preload
c2b:	yr5_5cl_ten_reals_five_complex_djbunfft rsi, 64, 4*dist32, rdi, YMM_SCD4, 32
	yloop	32, c2b, -32*64+dist32	;; Test loop counter, next source pointer
	yloop	4, c2b, -4*dist32	;; Test loop counter, restore source pointer
	end_timer 11

;; Do inverse FFT levels 1,2,3
;; On input the 64-byte cache lines hold these data values:
;;	0	+1280	+1280	+1280	640	+1280	+1280	+1280
;;	1
;;	...
;;	639
;; These are swizzled to:
;;	0	+160	+160	+160	640	+160	+160	+160
;;	1	...
;;	...
;;	159
;;	1280	...
;;	2560	...
;;	3840	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+160	+160	+160	2560	+160	+160	+160
;;	1	...
;;	...
;;	159
;;	640	...
;;	1280	...
;;	1920	...

;; Do 640 eight_reals_unfft operations
;;	distance between fft data elements is 640
;;	do 160 macros each processing 32 data values

	start_timer 12
	yloop_init  160			;; 160 iterations
	yr4_s4cl_eight_reals_unfft_preload
c1b:	yr4_s4cl_eight_reals_unfft rsi, 64, 5*dist32, 10*dist32, rdi, YMM_SCD3, 32
	yloop	32, c1b, -32*64+dist32	;; Test loop counter, next source pointer
	yloop	5, c1b			;; Test loop counter
	end_timer 12

	ytwo_dest_adjust
	yfft_3_ret
	ENDM

;; Perform a 5K-element negacyclic FFT

yfft5Kp MACRO type
	LOCAL	ymid2p, ymid3p, ymid4p, yfinish_unfft
	LOCAL	b1b, b2b, b3b, b4b, b5a, b5b, b5c, b5d, c1b, c2b, c3b, c4b

;; Do a multiply with pre-FFTed inputs

	cmp	ffttype, 4
	je	ymid4p

;; Do FFT levels 1,2,3
;; On input the 64-byte cache lines hold these data values:
;;	0	+160	+160	+160	2560	+160	+160	+160
;;	1
;;	...
;;	159
;;	640	...
;;	1280	...
;;	1920	...
;; On output these are swizzled to:
;;	0	+1280	+1280	+1280	640	+1280	+1280	+1280
;;	1
;;	...
;;	639

;; Do 640 four_complex_first_fft operations
;;	distance between fft data elements is 640
;;	do 160 macros each processing 32 data values

	start_timer 4
	mov	rdi, sincos1		;; Load sin/cos pointer
	bump	rdi, 128*YMM_SCD2/4+32*YMM_SCD2/4
	yloop_init  160			;; 160 iterations
	yr4_fs4cl_four_complex_first_djbfft_preload
b1b:	yr4_fs4cl_four_complex_first_djbfft rsi, 64, 5*dist32, 10*dist32, rdi, YMM_SCD4, 32
	yloop	32, b1b, -32*64+dist32	;; Test loop counter, next source pointer
	yloop	5, b1b, -5*dist32	;; Test loop counter, restore source pointer
	end_timer 4

;; Do FFT levels 4,5
;; On input the 64-byte cache lines hold these data values:
;;	0	+1280	+1280	+1280	640	+1280	+1280	+1280
;;	1
;;	...
;;	639
;; On output the 64-byte cache lines hold these data values:
;;	0	+1280	+1280	+1280	128	+1280	+1280	+1280
;;	1
;;	...
;;	127
;;	256	...
;;	...
;;	1024	...

;; Do 512 three_complex_fft operations
;;	distance between fft data elements is 128
;;	do 128 macros each processing 40 data values

	start_timer 5
	bump	rdi, -160*YMM_SCD4-128*YMM_SCD2/4 ;; Load sin/cos pointer
	yloop_init  128			;; 128 iterations
	yr5_b5cl_five_complex_djbfft_preload
b2b:	yr5_b5cl_five_complex_djbfft rsi, 64, 4*dist32, rdi, YMM_SCD2/4, 32
	yloop	32, b2b, -32*64+dist32	;; Test loop counter, next source pointer
	yloop	4, b2b, -4*dist32	;; Test loop counter, restore source pointer
	end_timer 5

;; Do FFT levels 6,7
;; On input the 64-byte cache lines hold these data values:
;;	0	+1280	+1280	+1280	128	+1280	+1280	+1280
;;	1
;;	...
;;	127
;;	256	...
;;	...
;;	1024	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+1280	+1280	+1280	32	+1280	+1280	+1280
;;	1
;;	...
;;	31
;;	64	...
;;	...
;;	1216	...

;; Do 640 four_complex_fft operations
;;	distance between fft data elements is 32
;;	do 160 macros each processing 32 data values

	start_timer 6
	bump	rdi, -128*YMM_SCD2/4-32*YMM_SCD2/4 ;; Next sin/cos pointer
	yloop_init  160			;; 5 iterations of 32
	yr4_b4cl_four_complex_djbfft_preload
b3b:	yr4_b4cl_four_complex_djbfft rsi, 64, dist32, 2*dist32, rdi, YMM_SCD2/4, 32
	yloop	32, b3b, -32*64+4*dist32, rdi, -32*YMM_SCD2/4 ;; Test loop counter, next source pointer, restore sin/cos pointer
	yloop	5, b3b, -5*4*dist32	;; Test loop counter, restore source pointer
	end_timer 6

;; Do FFT levels 8,9
;; On input the 64-byte cache lines hold these data values:
;;	0	+1280	+1280	+1280	32	+1280	+1280	+1280
;;	1
;;	...
;;	31
;;	64	...
;;	...
;;	1216	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+1280	+1280	+1280	8	+1280	+1280	+1280
;;	1
;;	...
;;	7
;;	16	...
;;	...
;;	1264	...

;; Do 640 four_complex_fft operations
;;	distance between fft data elements is 8
;;	do 160 macros each processing 32 data values

	start_timer 7
	yloop_init  160			;; 20 iterations of 8
	yr4_b4cl_four_complex_djbfft_preload
b4b:	yr4_b4cl_four_complex_djbfft rsi, 64, 8*64, 16*64, rdi, 4*YMM_SCD2/4, 8
	yloop	8, b4b, -8*64+dist32, rdi, -8*4*YMM_SCD2/4 ;; Test loop counter, next source pointer, restore sin/cos pointer
	yloop	20, b4b, -20*dist32	;; Test loop counter, restore source pointer
	end_timer 7

;; Do FFT levels 10,11
;; On input and output the 64-byte cache lines hold these data values:
;;	0	+1280	+1280	+1280	8	+1280	+1280	+1280
;;	1
;;	...
;;	7
;;	16	...
;;	...
;;	1264	...

;; Do 320 eight_complex_fft operations
;;	distance between fft data elements is 1
;;	do 80 macros each processing 64 data values

	cmp	ffttype, 2
	je	ymid2p
	ja	ymid3p

	start_timer 8
	yloop_init  80			;; 80 iterations
b5a:	yr8_8cl_eight_complex_fft_final rsi, 8*64, 64, 2*64, 4*64
	yloop	4, b5a, -4*8*64+dist32	;; Test loop counter, next source pointer
	yloop	20, b5a			;; Test loop counter
	end_timer 8
	yfft_1_ret

ymid2p:	start_timer 8
	yloop_init  80			;; 80 iterations
	yr8_8cl_eight_complex_with_square_preload
b5b:	yr8_8cl_eight_complex_with_square rsi, 8*64, 64, 2*64, 4*64
	yloop	4, b5b, -4*8*64+dist32	;; Test loop counter, next source pointer
	yloop	20, b5b			;; Test loop counter
	end_timer 8
	jmp	yfinish_unfft

ymid3p:	start_timer 8
x64	mov	r8, DEST2ARG
	mov	rbp, DIST_TO_MULSRCARG
	yloop_init  80			;; 80 iterations
	yr8_8cl_eight_complex_with_mult_preload
b5c:	yr8_8cl_eight_complex_with_mult rsi, 8*64, 64, 2*64, 4*64
	yloop	4, b5c, -4*8*64+dist32	;; Test loop counter, next source pointer
	yloop	20, b5c			;; Test loop counter
x64	add	rsi, r8
	end_timer 8
	jmp	yfinish_unfft

ymid4p:	start_timer 8
	mov	rbp, DIST_TO_MULSRCARG
	yloop_init  80			;; 80 iterations
	yr8_8cl_eight_complex_with_mulf_preload
b5d:	yr8_8cl_eight_complex_with_mulf rsi, 8*64, 64, 2*64, 4*64
	yloop	4, b5d, -4*8*64+dist32	;; Test loop counter, next source pointer
	yloop	20, b5d			;; Test loop counter
	end_timer 8
yfinish_unfft:
	bump	rsi, -20*dist32		;; Restore source pointer

;; Do inverse FFT levels 8,9
;; On input the 64-byte cache lines hold these data values:
;;	0	+1280	+1280	+1280	8	+1280	+1280	+1280
;;	1
;;	...
;;	7
;;	16	...
;;	...
;;	1264	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+1280	+1280	+1280	32	+1280	+1280	+1280
;;	1
;;	...
;;	31
;;	64	...
;;	...
;;	1216	...

;; Do 640 four_complex_unfft operations
;;	distance between fft data elements is 8
;;	do 160 macros each processing 32 data values

	start_timer 9
	mov	rdi, sincos1		;; Load sin/cos pointer
	yloop_init  160			;; 20 iterations of 8
	yr4_b4cl_four_complex_djbunfft_preload
c4b:	yr4_b4cl_four_complex_djbunfft rsi, 64, 8*64, 16*64, rdi, 4*YMM_SCD2/4, 8
	yloop	8, c4b, -8*64+dist32, rdi, -8*4*YMM_SCD2/4 ;; Test loop counter, next source pointer, restore sine/cosine pointer
	yloop	20, c4b, -20*dist32	;; Test loop counter, restore source pointer
	end_timer 9

;; Do inverse FFT levels 6,7
;; On input the 64-byte cache lines hold these data values:
;;	0	+1280	+1280	+1280	32	+1280	+1280	+1280
;;	1
;;	...
;;	31
;;	64	...
;;	...
;;	1216	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+1280	+1280	+1280	128	+1280	+1280	+1280
;;	1
;;	...
;;	127
;;	256	...
;;	...
;;	1024	...

;; Do 640 four_complex_unfft operations
;;	distance between fft data elements is 32
;;	do 160 macros each processing 32 data values

	start_timer 10
	yloop_init  160			;; 5 iterations of 32
	yr4_b4cl_four_complex_djbunfft_preload
c3b:	yr4_b4cl_four_complex_djbunfft rsi, 64, dist32, 2*dist32, rdi, YMM_SCD2/4, 32
	yloop	32, c3b, -32*64+4*dist32, rdi, -32*YMM_SCD2/4 ;; Test loop counter, next source pointer, restore sine/cosine pointer
	yloop	5, c3b, -5*4*dist32	;; Test loop counter, restore source pointer
	end_timer 10

;; Do inverse FFT levels 4,5
;; On input the 64-byte cache lines hold these data values:
;;	0	+1280	+1280	+1280	128	+1280	+1280	+1280
;;	1
;;	...
;;	127
;;	256	...
;;	...
;;	1024	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+1280	+1280	+1280	640	+1280	+1280	+1280
;;	1
;;	...
;;	639

;; Do 512 five_complex_unfft operations
;;	distance between fft data elements is 128
;;	do 128 macros each processing 40 data values

	start_timer 11
	bump	rdi, 32*YMM_SCD2/4	;; Next sine/cosine pointer
	yloop_init  128			;; 128 iterations
	yr5_b5cl_five_complex_djbunfft_preload
c2b:	yr5_b5cl_five_complex_djbunfft rsi, 64, 4*dist32, rdi, YMM_SCD2/4, 32
	yloop	32, c2b, -32*64+dist32	;; Test loop counter, next source pointer
	yloop	4, c2b, -4*dist32	;; Test loop counter, restore source pointer
	end_timer 11

;; Do inverse FFT levels 1,2,3
;; On input the 64-byte cache lines hold these data values:
;;	0	+1280	+1280	+1280	640	+1280	+1280	+1280
;;	1
;;	...
;;	639
;; These are swizzled to:
;;	0	+160	+160	+160	640	+160	+160	+160
;;	1	...
;;	...
;;	159
;;	1280	...
;;	2560	...
;;	3840	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+160	+160	+160	2560	+160	+160	+160
;;	1	...
;;	...
;;	159
;;	640	...
;;	1280	...
;;	1920	...

;; Do 640 four_complex_last_unfft operations
;;	distance between fft data elements is 640
;;	do 160 macros each processing 32 data values

	start_timer 12
	yloop_init  160			;; 160 iterations
	yr4_s4cl_four_complex_last_unfft_preload
c1b:	yr4_s4cl_four_complex_last_unfft rsi, 64, 5*dist32, 10*dist32, rdi, YMM_SCD4, 32
	yloop	32, c1b, -32*64+dist32	;; Test loop counter, next source pointer
	yloop	5, c1b			;; Test loop counter
	end_timer 12

	ytwo_dest_adjust
	yfft_3_ret
	ENDM


;; Perform a 6K-element FFT.

yfft6K MACRO type
	LOCAL	ymid2, ymid3, ymid4, yfinish_unfft
	LOCAL	b1b, b2b, b3a, b3b, b4a, b4b, b5a, b5b, b6a, b6b, b6c, b6d
	LOCAL	c1b, c2b, c3a, c3b, c4a, c4b, c5a, c5b

;; Do a multiply with pre-FFTed inputs

	cmp	ffttype, 4
	je	ymid4

;; Do FFT levels 1,2,3
;; On input the 64-byte cache lines hold these data values:
;;	0	192	384	576	3K	+192	+192	+192
;;	1
;;	...
;;	191
;;	768	...
;;	1536	...
;;	2304	...
;; On output these are swizzled to:
;;	0	+1536	+1536	+1536	768	+1536	+1536	+1536
;;	1
;;	...
;;	767

;; Do 768 eight_reals_fft operations
;;	distance between fft data elements is 768
;;	do 192 macros each processing 32 data values

	start_timer 4
	mov	rdi, sincos1		;; Load sin/cos pointer
	bump	rdi, 256*YMM_SCD2+64*YMM_SCD3
	yloop_init  192			;; 192 iterations
	yr4_fs4cl_eight_reals_fft_preload
b1b:	yr4_fs4cl_eight_reals_fft rsi, 64, 3*dist64, 6*dist64, rdi, YMM_SCD3, 64
	yloop	64, b1b, -64*64+dist64	;; Test loop counter, next source pointer
	yloop	3, b1b, -3*dist64	;; Test loop counter, restore source pointer
	end_timer 4

;; Do FFT levels 4,5
;; On input the 64-byte cache lines hold these data values:
;;	0	+1536	+1536	+1536	768	+1536	+1536	+1536
;;	1
;;	...
;;	767
;; On output the 64-byte cache lines hold these data values:
;;	0	+1536	+1536	+1536	256	+1536K	+1536	+1536
;;	1
;;	...
;;	255
;;	512	...
;;	1024	...

;; Do 256 six_reals_fft operations
;; Do 768 three_complex_fft operations
;;	distance between fft data elements is 256
;;	do 256 macros each processing 24 data values

	start_timer 5
	bump	rdi, -192*YMM_SCD3-256*YMM_SCD2 ;; Load sin/cos pointer
	yloop_init  256			;; 256 iterations
	yr3_3cl_six_reals_three_complex_djbfft_preload
b2b:	yr3_3cl_six_reals_three_complex_djbfft rsi, 64, 4*dist64, rdi, YMM_SCD2, 64
	yloop	64, b2b, -64*64+dist64	;; Test loop counter, next source pointer
	yloop	4, b2b, -4*dist64	;; Test loop counter, restore source pointer
	end_timer 5

;; Do FFT levels 6,7
;; On input the 64-byte cache lines hold these data values:
;;	0	+1536	+1536	+1536	256	+1536K	+1536	+1536
;;	1
;;	...
;;	255
;;	512	...
;;	1024	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+1536	+1536	+1536	64	+1536	+1536	+1536
;;	1
;;	...
;;	63
;;	128	...
;;	...
;;	1408	...

;; Do 64 eight_reals_fft operations
;; Do 704 four_complex_fft operations
;;	distance between fft data elements is 64
;;	do 192 macros each processing 32 data values

	start_timer 6
	bump	rdi, -256*YMM_SCD2-64*YMM_SCD3 ;; Restore sin/cos pointer
	yloop_init  192			;; 3 iterations of 64
	yr4_4cl_eight_reals_four_complex_djbfft_preload
;; Loop below reworked to avoid 4KB distances
;;b3a:	yr4_4cl_eight_reals_four_complex_djbfft rsi, 64, dist64, 2*dist64, rdi, YMM_SCD3, 64
;;	yloop	64, b3a, -64*64+4*dist64, rdi, -64*YMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer
b3a:	yr4_4cl_eight_reals_four_complex_djbfft rsi, 4*64, dist64, 2*dist64, rdi, 4*YMM_SCD3, 16
	yloop	16, b3a, -16*4*64+64, rdi, -16*4*YMM_SCD3+YMM_SCD3 ;; Test loop counter, next source pointer, next sin/cos pointer
	yloop	4, b3a, -4*64+4*dist64, rdi, -4*YMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer

	loops_reset			;; 2 more iters of 64
	yr4_rb4cl_four_complex_djbfft_preload
;; Loop below reworked to avoid 4KB distances
;;b3b:	yr4_rb4cl_four_complex_djbfft rsi, 64, dist64, 2*dist64, rdi, YMM_SCD3, 64
;;	yloop	64, b3b, -64*64+4*dist64, rdi, -64*YMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer
b3b:	yr4_rb4cl_four_complex_djbfft rsi, 4*64, dist64, 2*dist64, rdi, 4*YMM_SCD3, 16
	yloop	16, b3b, -16*4*64+64, rdi, -16*4*YMM_SCD3+YMM_SCD3 ;; Test loop counter, next source pointer, next sin/cos pointer
	yloop	4, b3b, -4*64+4*dist64, rdi, -4*YMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer
	yloop	3, b3b, -3*4*dist64	;; Test loop counter, restore source pointer
	end_timer 6

;; Do FFT levels 8,9
;; On input the 64-byte cache lines hold these data values:
;;	0	+1536	+1536	+1536	64	+1536	+1536	+1536
;;	1
;;	...
;;	63
;;	128	...
;;	...
;;	1408	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+1536	+1536	+1536	16	+1536	+1536	+1536
;;	1
;;	...
;;	15
;;	32	...
;;	...
;;	1504	...

;; Do 16 eight_reals_fft operations
;; Do 752 four_complex_fft operations
;;	distance between fft data elements is 16
;;	do 192 macros each processing 32 data values

	start_timer 7
	yloop_init  192			;; 12 iterations of 16
	yr4_4cl_eight_reals_four_complex_djbfft_preload
b4a:	yr4_4cl_eight_reals_four_complex_djbfft rsi, 64, 16*64, 32*64, rdi, 4*YMM_SCD3, 16
	yloop	16, b4a, -16*64+dist64, rdi, -16*4*YMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer

	loops_reset			;; 11 more iters of 16
	yr4_rb4cl_four_complex_djbfft_preload
b4b:	yr4_rb4cl_four_complex_djbfft rsi, 64, 16*64, 32*64, rdi, 4*YMM_SCD3, 16
	yloop	16, b4b, -16*64+dist64, rdi, -16*4*YMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer
	yloop	12, b4b, -12*dist64	;; Test loop counter, restore source pointer
	end_timer 7

;; Do FFT levels 10,11
;; On input the 64-byte cache lines hold these data values:
;;	0	+1536	+1536	+1536	16	+1536	+1536	+1536
;;	1
;;	...
;;	15
;;	32	...
;;	...
;;	1504	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+1536	+1536	+1536	4	+1536	+1536	+1536
;;	1
;;	...
;;	3
;;	8	...
;;	...
;;	1528	...

;; Do 4 eight_reals_fft operations
;; Do 764 four_complex_fft operations
;;	distance between fft data elements is 4
;;	do 192 macros each processing 32 data values

	start_timer 8
	yloop_init  192			;; 48 iterations of 4
	yr4_4cl_eight_reals_four_complex_djbfft_preload
b5a:	yr4_4cl_eight_reals_four_complex_djbfft rsi, 64, 4*64, 8*64, rdi, 16*YMM_SCD3, 4
	yloop	4, b5a, -4*64+16*64, rdi, -4*16*YMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer

	loops_reset			;; 47 more iters of 4
	yr4_rb4cl_four_complex_djbfft_preload
b5b:	yr4_rb4cl_four_complex_djbfft rsi, 64, 4*64, 8*64, rdi, 16*YMM_SCD3, 4
	yloop	4, b5b, -4*64+16*64, rdi, -4*16*YMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer
	yloop	4, b5b, -4*16*64+dist64	;; Test loop counter, next source pointer
	yloop	12, b5b, -12*dist64	;; Test loop counter, restore source pointer
	end_timer 8

;; Do FFT levels 12,13
;; On input and output the 64-byte cache lines hold these data values:
;;	0	+1536	+1536	+1536	4	+1536	+1536	+1536
;;	1
;;	...
;;	3
;;	8	...
;;	...
;;	1528	...

;; Do 1 eight_reals_fft operation
;; Do 767 four_complex_fft operations
;;	distance between fft data elements is 1
;;	do 192 macros each processing 32 data values

	cmp	ffttype, 2
	je	ymid2
	ja	ymid3

	start_timer 9
;;	yr4_4cl_eight_reals_four_complex_fft_final rsi, 4*64, 64, 2*64
	yr4_dispatch call, yreal_fft_final, 64, 2*64
	yloop_init  192, 1, 16		;; 191 more iterations
b6a:	yr4_4cl_four_complex_fft_final rsi, 4*64, 64, 2*64
	yloop	16, b6a, -16*4*64+dist64 ;; Test loop counter, next source pointer
	yloop	12, b6a			;; Test loop counter
	end_timer 9
	yfft_1_ret

ymid2:	start_timer 9
;;	yr4_4cl_eight_reals_four_complex_with_square rsi, 4*64, 64, 2*64
	yr4_dispatch call, yreal_square, 64, 2*64
	yloop_init  192, 1, 16		;; 191 more iterations
	yr4_4cl_four_complex_with_square_preload
b6b:	yr4_4cl_four_complex_with_square rsi, 4*64, 64, 2*64, 1
	yloop	16, b6b, -16*4*64+dist64 ;; Test loop counter, next source pointer
	yloop	12, b6b			;; Test loop counter
	end_timer 9
	jmp	yfinish_unfft

ymid3:	start_timer 9
x64	mov	r8, DEST2ARG
	mov	rbp, DIST_TO_MULSRCARG
;;	yr4_4cl_eight_reals_four_complex_with_mult rsi, 4*64, 64, 2*64
	yr4_dispatch call, yreal_mult, 64, 2*64
	yloop_init  192, 1, 16		;; 191 more iterations
	yr4_4cl_four_complex_with_mult_preload
b6c:	yr4_4cl_four_complex_with_mult rsi, 4*64, 64, 2*64
	yloop	16, b6c, -16*4*64+dist64 ;; Test loop counter, next source pointer
	yloop	12, b6c			;; Test loop counter
x64	add	rsi, r8
	end_timer 9
	jmp	yfinish_unfft

ymid4:	start_timer 9
	mov	rbp, DIST_TO_MULSRCARG
;;	yr4_4cl_eight_reals_four_complex_with_mulf rsi, 4*64, 64, 2*64
	yr4_dispatch call, yreal_mulf, 64, 2*64
	yloop_init  192, 1, 16		;; 191 more iterations
	yr4_4cl_four_complex_with_mulf_preload
b6d:	yr4_4cl_four_complex_with_mulf rsi, 4*64, 64, 2*64
	yloop	16, b6d, -16*4*64+dist64 ;; Test loop counter, next source pointer
	yloop	12, b6d			;; Test loop counter
	end_timer 9
yfinish_unfft:
	bump	rsi, -12*dist64		;; Restore source pointer

;; Do inverse FFT levels 10,11
;; On input the 64-byte cache lines hold these data values:
;;	0	+1536	+1536	+1536	4	+1536	+1536	+1536
;;	1
;;	...
;;	3
;;	8	...
;;	...
;;	1528	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+1536	+1536	+1536	16	+1536	+1536	+1536
;;	1
;;	...
;;	15
;;	32	...
;;	...
;;	1504	...

;; Do 4 eight_reals_unfft operations
;; Do 764 four_complex_unfft operations
;;	distance between fft data elements is 4
;;	do 192 macros each processing 32 data values

	start_timer 10
	mov	rdi, sincos1		;; Load sin/cos pointer
	yloop_init  192			;; 48 iterations of 4
	yr4_4cl_eight_reals_four_complex_djbunfft_preload
c5a:	yr4_4cl_eight_reals_four_complex_djbunfft rsi, 64, 4*64, 8*64, rdi, 16*YMM_SCD3, 4
	yloop	4, c5a, -4*64+16*64, rdi, -4*16*YMM_SCD3 ;; Test loop counter, next source pointer, restore sine/cosine pointer

	loops_reset			;; 47 more iters of 4
	yr4_rb4cl_four_complex_djbunfft_preload
c5b:	yr4_rb4cl_four_complex_djbunfft rsi, 64, 4*64, 8*64, rdi, 16*YMM_SCD3, 4
	yloop	4, c5b, -4*64+16*64, rdi, -4*16*YMM_SCD3 ;; Test loop counter, next source pointer, restore sine/cosine pointer
	yloop	4, c5b, -4*16*64+dist64	;; Test loop counter, next source pointer
	yloop	12, c5b, -12*dist64	;; Test loop counter, restore source pointer
	end_timer 10

;; Do inverse FFT levels 8,9
;; On input the 64-byte cache lines hold these data values:
;;	0	+1536	+1536	+1536	16	+1536	+1536	+1536
;;	1
;;	...
;;	15
;;	32	...
;;	...
;;	1504	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+1536	+1536	+1536	64	+1536	+1536	+1536
;;	1
;;	...
;;	63
;;	128	...
;;	...
;;	1408	...

;; Do 16 eight_reals_unfft operations
;; Do 748 four_complex_unfft operations
;;	distance between fft data elements is 16
;;	do 192 macros each processing 32 data values

	start_timer 11
	yloop_init  192			;; 12 iterations of 16
	yr4_4cl_eight_reals_four_complex_djbunfft_preload
c4a:	yr4_4cl_eight_reals_four_complex_djbunfft rsi, 64, 16*64, 32*64, rdi, 4*YMM_SCD3, 16
	yloop	16, c4a, -16*64+dist64, rdi, -16*4*YMM_SCD3 ;; Test loop counter, next source pointer, restore sine/cosine pointer

	loops_reset			;; 11 more iters of 16
	yr4_rb4cl_four_complex_djbunfft_preload
c4b:	yr4_rb4cl_four_complex_djbunfft rsi, 64, 16*64, 32*64, rdi, 4*YMM_SCD3, 16
	yloop	16, c4b, -16*64+dist64, rdi, -16*4*YMM_SCD3 ;; Test loop counter, next source pointer, restore sine/cosine pointer
	yloop	12, c4b, -12*dist64	;; Test loop counter, restore source pointer
	end_timer 11

;; Do inverse FFT levels 6,7
;; On input the 64-byte cache lines hold these data values:
;;	0	+1536	+1536	+1536	64	+1536	+1536	+1536
;;	1
;;	...
;;	63
;;	128	...
;;	...
;;	1408	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+1536	+1536	+1536	256	+1536	+1536	+1536
;;	1
;;	...
;;	255
;;	512	...
;;	1024	...

;; Do 64 eight_reals_unfft operations
;; Do 708 four_complex_unfft operations
;;	distance between fft data elements is 64
;;	do 192 macros each processing 32 data values

	start_timer 12
	yloop_init  192			;; 3 iterations of 64
	yr4_4cl_eight_reals_four_complex_djbunfft_preload
;; Loop below reworked to avoid 4KB distances
;;c3a:	yr4_4cl_eight_reals_four_complex_djbunfft rsi, 64, dist64, 2*dist64, rdi, YMM_SCD3, 64
;;	yloop	64, c3a, -64*64+4*dist64, rdi, -64*YMM_SCD3 ;; Test loop counter, next source pointer, restore sine/cosine pointer
c3a:	yr4_4cl_eight_reals_four_complex_djbunfft rsi, 4*64, dist64, 2*dist64, rdi, 4*YMM_SCD3, 16
	yloop	16, c3a, -16*4*64+64, rdi, -16*4*YMM_SCD3+YMM_SCD3 ;; Test loop counter, next source pointer, next sine/cosine pointer
	yloop	4, c3a, -4*64+4*dist64, rdi, -4*YMM_SCD3 ;; Test loop counter, next source pointer, restore sine/cosine pointer

	loops_reset			;; 2 more iters of 64
	yr4_rb4cl_four_complex_djbunfft_preload
;; Loop below reworked to avoid 4KB distances
;;c3b:	yr4_rb4cl_four_complex_djbunfft rsi, 64, dist64, 2*dist64, rdi, YMM_SCD3, 64
;;	yloop	64, c3b, -64*64+4*dist64, rdi, -64*YMM_SCD3 ;; Test loop counter, next source pointer, restore sine/cosine pointer
c3b:	yr4_rb4cl_four_complex_djbunfft rsi, 4*64, dist64, 2*dist64, rdi, 4*YMM_SCD3, 16
	yloop	16, c3b, -16*4*64+64, rdi, -16*4*YMM_SCD3+YMM_SCD3 ;; Test loop counter, next source pointer, next sine/cosine pointer
	yloop	4, c3b, -4*64+4*dist64, rdi, -4*YMM_SCD3 ;; Test loop counter, next source pointer, restore sine/cosine pointer
	yloop	3, c3b, -3*4*dist64	;; Test loop counter, restore source pointer
	end_timer 12

;; Do inverse FFT levels 4,5
;; On input the 64-byte cache lines hold these data values:
;;	0	+1536	+1536	+1536	256	+1536	+1536	+1536
;;	1
;;	...
;;	255
;;	512	...
;;	1024	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+1536	+1536	+1536	768	+1536	+1536	+1536
;;	1
;;	...
;;	767

;; Do 256 six_reals_unfft operations
;; Do 768 three_complex_unfft operations
;;	distance between fft data elements is 256
;;	do 256 macros each processing 24 data values

	start_timer 13
	bump	rdi, 64*YMM_SCD3	;; Next sine/cosine pointer
	yloop_init  256			;; 256 iterations
	yr3_3cl_six_reals_three_complex_djbunfft_preload
c2b:	yr3_3cl_six_reals_three_complex_djbunfft rsi, 64, 4*dist64, rdi, YMM_SCD2, 64
	yloop	64, c2b, -64*64+dist64	;; Test loop counter, next source pointer
	yloop	4, c2b, -4*dist64	;; Test loop counter, restore source pointer
	end_timer 13

;; Do inverse FFT levels 1,2,3
;; On input the 64-byte cache lines hold these data values:
;;	0	+1536	+1536	+1536	768	+1536	+1536	+1536
;;	1
;;	...
;;	767
;; These are swizzled to:
;;	0	+192	+192	+192	768	+192	+192	+192
;;	1	...
;;	...
;;	191
;;	1536	...
;;	3072	...
;;	4608	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+192	+192	+192	3K	+192	+192	+192
;;	1	...
;;	...
;;	191
;;	768	...
;;	1536	...
;;	2304	...

;; Do 768 eight_reals_unfft operations
;;	distance between fft data elements is 768
;;	do 192 macros each processing 32 data values

	start_timer 14
	yloop_init  192			;; 192 iterations
	yr4_s4cl_eight_reals_unfft_preload
c1b:	yr4_s4cl_eight_reals_unfft rsi, 64, 3*dist64, 6*dist64, rdi, YMM_SCD3, 64
	yloop	64, c1b, -64*64+dist64	;; Test loop counter, next source pointer
	yloop	3, c1b			;; Test loop counter
	end_timer 14

	ytwo_dest_adjust
	yfft_3_ret
	ENDM

;; Perform a 6K-element negacyclic FFT

yfft6Kp MACRO type
	LOCAL	ymid2p, ymid3p, ymid4p, yfinish_unfft
	LOCAL	b1b, b2b, b3b, b4b, b5b, b6a, b6b, b6c, b6d, c1b, c2b, c3b, c4b, c5b

;; Do a multiply with pre-FFTed inputs

	cmp	ffttype, 4
	je	ymid4p

;; Do FFT levels 1,2,3
;; On input the 64-byte cache lines hold these data values:
;;	0	192	384	576	3K	+192	+192	+192
;;	1
;;	...
;;	191
;;	768	...
;;	1536	...
;;	2304	...
;; On output these are swizzled to:
;;	0	+1536	+1536	+1536	768	+1536	+1536	+1536
;;	1
;;	...
;;	767

;; Do 768 four_complex_first_fft operations
;;	distance between fft data elements is 768
;;	do 192 macros each processing 32 data values

	start_timer 4
	mov	rdi, sincos1		;; Load sin/cos pointer
	bump	rdi, 256*YMM_SCD1/4+64*YMM_SCD2/4
	yloop_init  192			;; 192 iterations
	yr4_fs4cl_four_complex_first_djbfft_preload
b1b:	yr4_fs4cl_four_complex_first_djbfft rsi, 64, 3*dist64, 6*dist64, rdi, YMM_SCD4, 64
	yloop	64, b1b, -64*64+dist64	;; Test loop counter, next source pointer
	yloop	3, b1b, -3*dist64	;; Test loop counter, restore source pointer
	end_timer 4

;; Do FFT levels 4,5
;; On input the 64-byte cache lines hold these data values:
;;	0	+1536	+1536	+1536	768	+1536	+1536	+1536
;;	1
;;	...
;;	767
;; On output the 64-byte cache lines hold these data values:
;;	0	+1536	+1536	+1536	256	+1536K	+1536	+1536
;;	1
;;	...
;;	255
;;	512	...
;;	1024	...

;; Do 1024 three_complex_fft operations
;;	distance between fft data elements is 256
;;	do 256 macros each processing 24 data values

	start_timer 5
	bump	rdi, -192*YMM_SCD4-256*YMM_SCD1/4 ;; Load sin/cos pointer
	yloop_init  256			;; 256 iterations
	yr3_b3cl_three_complex_djbfft_preload
b2b:	yr3_b3cl_three_complex_djbfft rsi, 64, 4*dist64, rdi, YMM_SCD1/4, 64
	yloop	64, b2b, -64*64+dist64	;; Test loop counter, next source pointer
	yloop	4, b2b, -4*dist64	;; Test loop counter, restore source pointer
	end_timer 5

;; Do FFT levels 6,7
;; On input the 64-byte cache lines hold these data values:
;;	0	+1536	+1536	+1536	256	+1536K	+1536	+1536
;;	1
;;	...
;;	255
;;	512	...
;;	1024	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+1536	+1536	+1536	64	+1536	+1536	+1536
;;	1
;;	...
;;	63
;;	128	...
;;	...
;;	1408	...

;; Do 768 four_complex_fft operations
;;	distance between fft data elements is 64
;;	do 192 macros each processing 32 data values

	start_timer 6
	bump	rdi, -256*YMM_SCD1/4-64*YMM_SCD2/4 ;; Next sin/cos pointer
	yloop_init  192			;; 3 iterations of 64
	yr4_b4cl_four_complex_djbfft_preload
;; Loop below reworked to avoid 4KB distances
;;b3b:	yr4_b4cl_four_complex_djbfft rsi, 64, dist64, 2*dist64, rdi, YMM_SCD2/4, 64
;;	yloop	64, b3b, -64*64+4*dist64, rdi, -64*YMM_SCD2/4 ;; Test loop counter, next source pointer, restore sin/cos pointer
b3b:	yr4_b4cl_four_complex_djbfft rsi, 4*64, dist64, 2*dist64, rdi, 4*YMM_SCD2/4, 16
	yloop	16, b3b, -16*4*64+64, rdi, -16*4*YMM_SCD2/4+YMM_SCD2/4 ;; Test loop counter, next source pointer, next sin/cos pointer
	yloop	4, b3b, -4*64+4*dist64, rdi, -4*YMM_SCD2/4 ;; Test loop counter, next source pointer, restore sin/cos pointer
	yloop	3, b3b, -3*4*dist64	;; Test loop counter, restore source pointer
	end_timer 6

;; Do FFT levels 8,9
;; On input the 64-byte cache lines hold these data values:
;;	0	+1536	+1536	+1536	64	+1536	+1536	+1536
;;	1
;;	...
;;	63
;;	128	...
;;	...
;;	1408	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+1536	+1536	+1536	16	+1536	+1536	+1536
;;	1
;;	...
;;	15
;;	32	...
;;	...
;;	1504	...

;; Do 768 four_complex_fft operations
;;	distance between fft data elements is 16
;;	do 192 macros each processing 32 data values

	start_timer 7
	yloop_init  192			;; 12 iterations of 16
	yr4_b4cl_four_complex_djbfft_preload
b4b:	yr4_b4cl_four_complex_djbfft rsi, 64, 16*64, 32*64, rdi, 4*YMM_SCD2/4, 16
	yloop	16, b4b, -16*64+dist64, rdi, -16*4*YMM_SCD2/4 ;; Test loop counter, next source pointer, restore sin/cos pointer
	yloop	12, b4b, -12*dist64	;; Test loop counter, restore source pointer
	end_timer 7

;; Do FFT levels 10,11
;; On input the 64-byte cache lines hold these data values:
;;	0	+1536	+1536	+1536	16	+1536	+1536	+1536
;;	1
;;	...
;;	15
;;	32	...
;;	...
;;	1504	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+1536	+1536	+1536	4	+1536	+1536	+1536
;;	1
;;	...
;;	3
;;	8	...
;;	...
;;	1528	...

;; Do 768 four_complex_fft operations
;;	distance between fft data elements is 4
;;	do 192 macros each processing 32 data values

	start_timer 8
	yloop_init  192			;; 48 iterations of 4
	yr4_b4cl_four_complex_djbfft_preload
b5b:	yr4_b4cl_four_complex_djbfft rsi, 64, 4*64, 8*64, rdi, 16*YMM_SCD2/4, 4
	yloop	4, b5b, -4*64+16*64, rdi, -4*16*YMM_SCD2/4 ;; Test loop counter, next source pointer, restore sin/cos pointer
	yloop	4, b5b, -4*16*64+dist64	;; Test loop counter, next source pointer
	yloop	12, b5b, -12*dist64	;; Test loop counter, restore source pointer
	end_timer 8

;; Do FFT levels 12,13
;; On input and output the 64-byte cache lines hold these data values:
;;	0	+1536	+1536	+1536	4	+1536	+1536	+1536
;;	1
;;	...
;;	3
;;	8	...
;;	...
;;	1528	...

;; Do 768 four_complex_fft operations
;;	distance between fft data elements is 1
;;	do 192 macros each processing 32 data values

	cmp	ffttype, 2
	je	ymid2p
	ja	ymid3p

	start_timer 9
	yloop_init  192			;; 192 iterations
b6a:	yr4_4cl_four_complex_fft_final rsi, 4*64, 64, 2*64
	yloop	16, b6a, -16*4*64+dist64 ;; Test loop counter, next source pointer
	yloop	12, b6a			;; Test loop counter
	end_timer 9
	yfft_1_ret

ymid2p:	start_timer 9
	yloop_init  192			;; 192 iterations
	yr4_4cl_four_complex_with_square_preload
b6b:	yr4_4cl_four_complex_with_square rsi, 4*64, 64, 2*64, 16
	yloop	16, b6b, -16*4*64+dist64 ;; Test loop counter, next source pointer
	yloop	12, b6b			;; Test loop counter
	end_timer 9
	jmp	yfinish_unfft

ymid3p:	start_timer 9
x64	mov	r8, DEST2ARG
	mov	rbp, DIST_TO_MULSRCARG
	yloop_init  192			;; 192 iterations
	yr4_4cl_four_complex_with_mult_preload
b6c:	yr4_4cl_four_complex_with_mult rsi, 4*64, 64, 2*64
	yloop	16, b6c, -16*4*64+dist64 ;; Test loop counter, next source pointer
	yloop	12, b6c			;; Test loop counter
x64	add	rsi, r8
	end_timer 9
	jmp	yfinish_unfft

ymid4p:	start_timer 9
	mov	rbp, DIST_TO_MULSRCARG
	yloop_init  192			;; 192 iterations
	yr4_4cl_four_complex_with_mulf_preload
b6d:	yr4_4cl_four_complex_with_mulf rsi, 4*64, 64, 2*64
	yloop	16, b6d, -16*4*64+dist64 ;; Test loop counter, next source pointer
	yloop	12, b6d			;; Test loop counter
	end_timer 9
yfinish_unfft:
	bump	rsi, -12*dist64		;; Restore source pointer

;; Do inverse FFT levels 10,11
;; On input the 64-byte cache lines hold these data values:
;;	0	+1536	+1536	+1536	4	+1536	+1536	+1536
;;	1
;;	...
;;	3
;;	8	...
;;	...
;;	1528	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+1536	+1536	+1536	16	+1536	+1536	+1536
;;	1
;;	...
;;	15
;;	32	...
;;	...
;;	1504	...

;; Do 768 four_complex_unfft operations
;;	distance between fft data elements is 4
;;	do 192 macros each processing 32 data values

	start_timer 10
	mov	rdi, sincos1		;; Load sin/cos pointer
	yloop_init  192			;; 48 iterations of 4
	yr4_b4cl_four_complex_djbunfft_preload
c5b:	yr4_b4cl_four_complex_djbunfft rsi, 64, 4*64, 8*64, rdi, 16*YMM_SCD2/4, 4
	yloop	4, c5b, -4*64+16*64, rdi, -4*16*YMM_SCD2/4 ;; Test loop counter, next source pointer, restore sine/cosine pointer
	yloop	4, c5b, -4*16*64+dist64	;; Test loop counter, next source pointer
	yloop	12, c5b, -12*dist64	;; Test loop counter, restore source pointer
	end_timer 10

;; Do inverse FFT levels 8,9
;; On input the 64-byte cache lines hold these data values:
;;	0	+1536	+1536	+1536	16	+1536	+1536	+1536
;;	1
;;	...
;;	15
;;	32	...
;;	...
;;	1504	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+1536	+1536	+1536	64	+1536	+1536	+1536
;;	1
;;	...
;;	63
;;	128	...
;;	...
;;	1408	...

;; Do 768 four_complex_unfft operations
;;	distance between fft data elements is 16
;;	do 192 macros each processing 32 data values

	start_timer 11
	yloop_init  192			;; 12 iterations of 16
	yr4_b4cl_four_complex_djbunfft_preload
c4b:	yr4_b4cl_four_complex_djbunfft rsi, 64, 16*64, 32*64, rdi, 4*YMM_SCD2/4, 16
	yloop	16, c4b, -16*64+dist64, rdi, -16*4*YMM_SCD2/4 ;; Test loop counter, next source pointer, restore sine/cosine pointer
	yloop	12, c4b, -12*dist64	;; Test loop counter, restore source pointer
	end_timer 11

;; Do inverse FFT levels 6,7
;; On input the 64-byte cache lines hold these data values:
;;	0	+1536	+1536	+1536	64	+1536	+1536	+1536
;;	1
;;	...
;;	63
;;	128	...
;;	...
;;	1408	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+1536	+1536	+1536	256	+1536	+1536	+1536
;;	1
;;	...
;;	255
;;	512	...
;;	1024	...

;; Do 768 four_complex_unfft operations
;;	distance between fft data elements is 64
;;	do 192 macros each processing 32 data values

	start_timer 12
	yloop_init  192			;; 3 iterations of 64
	yr4_b4cl_four_complex_djbunfft_preload
;; Loop below reworked to avoid 4KB distances
;;c3b:	yr4_b4cl_four_complex_djbunfft rsi, 64, dist64, 2*dist64, rdi, YMM_SCD2/4, 64
;;	yloop	64, c3b, -64*64+4*dist64, rdi, -64*YMM_SCD2/4 ;; Test loop counter, next source pointer, restore sine/cosine pointer
c3b:	yr4_b4cl_four_complex_djbunfft rsi, 4*64, dist64, 2*dist64, rdi, 4*YMM_SCD2/4, 16
	yloop	16, c3b, -16*4*64+64, rdi, -16*4*YMM_SCD2/4+YMM_SCD2/4 ;; Test loop counter, next source pointer, next sine/cosine pointer
	yloop	4, c3b, -4*64+4*dist64, rdi, -4*YMM_SCD2/4 ;; Test loop counter, next source pointer, restore sine/cosine pointer
	yloop	3, c3b, -3*4*dist64	;; Test loop counter, restore source pointer
	end_timer 12

;; Do inverse FFT levels 4,5
;; On input the 64-byte cache lines hold these data values:
;;	0	+1536	+1536	+1536	256	+1536	+1536	+1536
;;	1
;;	...
;;	255
;;	512	...
;;	1024	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+1536	+1536	+1536	768	+1536	+1536	+1536
;;	1
;;	...
;;	767

;; Do 1024 three_complex_unfft operations
;;	distance between fft data elements is 256
;;	do 256 macros each processing 24 data values

	start_timer 13
	bump	rdi, 64*YMM_SCD2/4	;; Restore sine/cosine pointer
	yloop_init  256			;; 256 iterations
	yr3_b3cl_three_complex_djbunfft_preload
c2b:	yr3_b3cl_three_complex_djbunfft rsi, 64, 4*dist64, rdi, YMM_SCD1/4, 64
	yloop	64, c2b, -64*64+dist64	;; Test loop counter, next source pointer
	yloop	4, c2b, -4*dist64	;; Test loop counter, restore source pointer
	end_timer 13

;; Do inverse FFT levels 1,2,3
;; On input the 64-byte cache lines hold these data values:
;;	0	+1536	+1536	+1536	768	+1536	+1536	+1536
;;	1
;;	...
;;	767
;; These are swizzled to:
;;	0	+192	+192	+192	768	+192	+192	+192
;;	1	...
;;	...
;;	191
;;	1536	...
;;	3072	...
;;	4608	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+192	+192	+192	3K	+192	+192	+192
;;	1	...
;;	...
;;	191
;;	768	...
;;	1536	...
;;	2304	...

;; Do 768 four_complex_last_unfft operations
;;	distance between fft data elements is 768
;;	do 192 macros each processing 32 data values

	start_timer 14
	yloop_init  192			;; 192 iterations
	yr4_s4cl_four_complex_last_unfft_preload
c1b:	yr4_s4cl_four_complex_last_unfft rsi, 64, 3*dist64, 6*dist64, rdi, YMM_SCD4, 64
	yloop	64, c1b, -64*64+dist64	;; Test loop counter, next source pointer
	yloop	3, c1b			;; Test loop counter
	end_timer 14

	ytwo_dest_adjust
	yfft_3_ret
	ENDM


;; Perform a 8K-element FFT.

yfft8K MACRO type
	LOCAL	ymid2, ymid3, ymid4, yfinish_unfft
	LOCAL	b1b, b2b, b3a, b3b, b4a, b4b, b5a, b5b, b6a, b6b, b6c, b6d
	LOCAL	c1b, c2b, c3a, c3b, c4a, c4b, c5a, c5b

;; Do a multiply with pre-FFTed inputs

	cmp	ffttype, 4
	je	ymid4

;; Do FFT levels 1,2,3
;; On input the 64-byte cache lines hold these data values:
;;	0	256	512	768	4K	+256	+256	+256
;;	1
;;	...
;;	255
;;	1K	...
;;	2K	...
;;	3K	...
;; On output these are swizzled to:
;;	0	2K	4K	6K	1K	+2K	+2K	+2K
;;	1
;;	...
;;	1023

;; Do 1024 eight_reals_fft operations
;;	distance between fft data elements is 1024
;;	do 256 macros each processing 32 data values

	start_timer 4
	mov	rdi, sincos1		;; Load sin/cos pointer
	bump	rdi, 256*YMM_SCD3
	yloop_init  256			;; 256 iterations
	yr4_fs4cl_eight_reals_fft_preload
b1b:	yr4_fs4cl_eight_reals_fft rsi, 64, 4*dist64, 8*dist64, rdi, YMM_SCD3, 64
	yloop	64, b1b, -64*64+dist64	;; Test loop counter, next source pointer
	yloop	4, b1b, -4*dist64	;; Test loop counter, restore source pointer
	end_timer 4

;; Do FFT levels 4,5
;; On input the 64-byte cache lines hold these data values:
;;	0	2K	4K	6K	1K	+2K	+2K	+2K
;;	1
;;	...
;;	1023
;; On output the 64-byte cache lines hold these data values:
;;	0	2K	4K	6K	256	+2K	+2K	+2K
;;	1
;;	...
;;	255
;;	512	...
;;	1024	...
;;	1536	...

;; Do 256 eight_reals_fft operations
;; Do 768 four_complex_fft operations
;;	distance between fft data elements is 256
;;	do 256 macros each processing 32 data values

	start_timer 5
	bump	rdi, -256*YMM_SCD3-256*YMM_SCD3 ;; Load sin/cos pointer
	yloop_init  256			;; 256 iterations
	yr4_4cl_eight_reals_four_complex_djbfft_preload
b2b:	yr4_4cl_eight_reals_four_complex_djbfft rsi, 64, 4*dist64, 8*dist64, rdi, YMM_SCD3, 64
	yloop	64, b2b, -64*64+dist64	;; Test loop counter, next source pointer
	yloop	4, b2b, -4*dist64, rdi, -256*YMM_SCD3 ;; Test loop counter, restore source pointer, restore sin/cos pointer
	end_timer 5

;; Do FFT levels 6,7
;; On input the 64-byte cache lines hold these data values:
;;	0	2K	4K	6K	256	+2K	+2K	+2K
;;	1
;;	...
;;	255
;;	512	...
;;	1024	...
;;	1536	...
;; On output the 64-byte cache lines hold these data values:
;;	0	2K	4K	6K	64	+2K	+2K	+2K
;;	1
;;	...
;;	63
;;	128	...
;;	...
;;	1920	...

;; Do 64 eight_reals_fft operations
;; Do 960 four_complex_fft operations
;;	distance between fft data elements is 64
;;	do 256 macros each processing 32 data values

	start_timer 6
	yloop_init  256			;; 4 iterations of 64
	yr4_4cl_eight_reals_four_complex_djbfft_preload
;; Loop below reworked to avoid 4KB distances
;;b3a:	yr4_4cl_eight_reals_four_complex_djbfft rsi, 64, dist64, 2*dist64, rdi, 4*YMM_SCD3, 64
;;	yloop	64, b3a, -64*64+4*dist64, rdi, -64*4*YMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer
b3a:	yr4_4cl_eight_reals_four_complex_djbfft rsi, 4*64, dist64, 2*dist64, rdi, 4*4*YMM_SCD3, 16
	yloop	16, b3a, -16*4*64+64, rdi, -16*4*4*YMM_SCD3+4*YMM_SCD3 ;; Test loop counter, next source pointer, next sin/cos pointer
	yloop	4, b3a, -4*64+4*dist64, rdi, -4*4*YMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer

	loops_reset			;; 3 more iters of 64
	yr4_rb4cl_four_complex_djbfft_preload
;; Loop below reworked to avoid 4KB distances
;;b3b:	yr4_rb4cl_four_complex_djbfft rsi, 64, dist64, 2*dist64, rdi, 4*YMM_SCD3, 64
;;	yloop	64, b3b, -64*64+4*dist64, rdi, -64*4*YMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer
b3b:	yr4_rb4cl_four_complex_djbfft rsi, 4*64, dist64, 2*dist64, rdi, 4*4*YMM_SCD3, 16
	yloop	16, b3b, -16*4*64+64, rdi, -16*4*4*YMM_SCD3+4*YMM_SCD3 ;; Test loop counter, next source pointer, next sin/cos pointer
	yloop	4, b3b, -4*64+4*dist64, rdi, -4*4*YMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer
	yloop	4, b3b, -4*4*dist64	;; Test loop counter, restore source pointer
	end_timer 6

;; Do FFT levels 8,9
;; On input the 64-byte cache lines hold these data values:
;;	0	2K	4K	6K	64	+2K	+2K	+2K
;;	1
;;	...
;;	63
;;	128	...
;;	...
;;	1920	...
;; On output the 64-byte cache lines hold these data values:
;;	0	2K	4K	6K	16	+2K	+2K	+2K
;;	1
;;	...
;;	15
;;	32	...
;;	...
;;	2016	...

;; Do 16 eight_reals_fft operations
;; Do 1008 four_complex_fft operations
;;	distance between fft data elements is 16
;;	do 256 macros each processing 32 data values

	start_timer 7
	yloop_init  256			;; 16 iterations of 16
	yr4_4cl_eight_reals_four_complex_djbfft_preload
b4a:	yr4_4cl_eight_reals_four_complex_djbfft rsi, 64, 16*64, 32*64, rdi, 16*YMM_SCD3, 16
	yloop	16, b4a, -16*64+dist64, rdi, -16*16*YMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer

	loops_reset			;; 15 more iters of 16
	yr4_rb4cl_four_complex_djbfft_preload
b4b:	yr4_rb4cl_four_complex_djbfft rsi, 64, 16*64, 32*64, rdi, 16*YMM_SCD3, 16
	yloop	16, b4b, -16*64+dist64, rdi, -16*16*YMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer
	yloop	16, b4b, -16*dist64	;; Test loop counter, restore source pointer
	end_timer 7

;; Do FFT levels 10,11
;; On input the 64-byte cache lines hold these data values:
;;	0	2K	4K	6K	16	+2K	+2K	+2K
;;	1
;;	...
;;	15
;;	32	...
;;	...
;;	2016	...
;; On output the 64-byte cache lines hold these data values:
;;	0	2K	4K	6K	4	+2K	+2K	+2K
;;	1
;;	...
;;	3
;;	8	...
;;	...
;;	2040	...

;; Do 4 eight_reals_fft operations
;; Do 1020 four_complex_fft operations
;;	distance between fft data elements is 4
;;	do 256 macros each processing 32 data values

	start_timer 8
	yloop_init  256			;; 64 iterations of 4
	yr4_4cl_eight_reals_four_complex_djbfft_preload
b5a:	yr4_4cl_eight_reals_four_complex_djbfft rsi, 64, 4*64, 8*64, rdi, 64*YMM_SCD3, 4
	yloop	4, b5a, -4*64+16*64, rdi, -4*64*YMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer

	loops_reset			;; 63 more iters of 4
	yr4_rb4cl_four_complex_djbfft_preload
b5b:	yr4_rb4cl_four_complex_djbfft rsi, 64, 4*64, 8*64, rdi, 64*YMM_SCD3, 4
	yloop	4, b5b, -4*64+16*64, rdi, -4*64*YMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer
	yloop	4, b5b, -4*16*64+dist64	;; Test loop counter, next source pointer
	yloop	16, b5b, -16*dist64	;; Test loop counter, restore source pointer
	end_timer 8

;; Do FFT levels 12,13
;; On input and output the 64-byte cache lines hold these data values:
;;	0	2K	4K	6K	4	+2K	+2K	+2K
;;	1
;;	...
;;	3
;;	8	...
;;	...
;;	2040	...

;; Do 1 eight_reals_fft operation
;; Do 1023 four_complex_fft operations
;;	distance between fft data elements is 1
;;	do 256 macros each processing 32 data values

	cmp	ffttype, 2
	je	ymid2
	ja	ymid3

	start_timer 9
;;	yr4_4cl_eight_reals_four_complex_fft_final rsi, 4*64, 64, 2*64
	yr4_dispatch call, yreal_fft_final, 64, 2*64
	yloop_init  256, 1, 16		;; 255 more iterations
b6a:	yr4_4cl_four_complex_fft_final rsi, 4*64, 64, 2*64
	yloop	16, b6a, -16*4*64+dist64 ;; Test loop counter, next source pointer
	yloop	16, b6a			;; Test loop counter
	end_timer 9
	yfft_1_ret

ymid2:	start_timer 9
;;	yr4_4cl_eight_reals_four_complex_with_square rsi, 4*64, 64, 2*64
	yr4_dispatch call, yreal_square, 64, 2*64
	yloop_init  256, 1, 16		;; 255 more iterations
	yr4_4cl_four_complex_with_square_preload
b6b:	yr4_4cl_four_complex_with_square rsi, 4*64, 64, 2*64, 1
	yloop	16, b6b, -16*4*64+dist64 ;; Test loop counter, next source pointer
	yloop	16, b6b			;; Test loop counter
	end_timer 9
	jmp	yfinish_unfft

ymid3:	start_timer 9
x64	mov	r8, DEST2ARG
	mov	rbp, DIST_TO_MULSRCARG
;;	yr4_4cl_eight_reals_four_complex_with_mult rsi, 4*64, 64, 2*64
	yr4_dispatch call, yreal_mult, 64, 2*64
	yloop_init  256, 1, 16		;; 255 more iterations
	yr4_4cl_four_complex_with_mult_preload
b6c:	yr4_4cl_four_complex_with_mult rsi, 4*64, 64, 2*64
	yloop	16, b6c, -16*4*64+dist64 ;; Test loop counter, next source pointer
	yloop	16, b6c			;; Test loop counter
x64	add	rsi, r8
	end_timer 9
	jmp	yfinish_unfft

ymid4:	start_timer 9
	mov	rbp, DIST_TO_MULSRCARG
;;	yr4_4cl_eight_reals_four_complex_with_mulf rsi, 4*64, 64, 2*64
	yr4_dispatch call, yreal_mulf, 64, 2*64
	yloop_init  256, 1, 16		;; 255 more iterations
	yr4_4cl_four_complex_with_mulf_preload
b6d:	yr4_4cl_four_complex_with_mulf rsi, 4*64, 64, 2*64
	yloop	16, b6d, -16*4*64+dist64 ;; Test loop counter, next source pointer
	yloop	16, b6d			;; Test loop counter
	end_timer 9
yfinish_unfft:
	bump	rsi, -16*dist64		;; Restore source pointer

;; Do inverse FFT levels 10,11
;; On input the 64-byte cache lines hold these data values:
;;	0	2K	4K	6K	4	+2K	+2K	+2K
;;	1
;;	...
;;	3
;;	8	...
;;	...
;;	2040	...
;; On output the 64-byte cache lines hold these data values:
;;	0	2K	4K	6K	16	+2K	+2K	+2K
;;	1
;;	...
;;	15
;;	32	...
;;	...
;;	2016	...

;; Do 4 eight_reals_unfft operations
;; Do 1020 four_complex_unfft operations
;;	distance between fft data elements is 4
;;	do 256 macros each processing 32 data values

	start_timer 10
	mov	rdi, sincos1		;; Load sin/cos pointer
	yloop_init  256			;; 64 iterations of 4
	yr4_4cl_eight_reals_four_complex_djbunfft_preload
c5a:	yr4_4cl_eight_reals_four_complex_djbunfft rsi, 64, 4*64, 8*64, rdi, 64*YMM_SCD3, 4
	yloop	4, c5a, -4*64+16*64, rdi, -4*64*YMM_SCD3 ;; Test loop counter, next source pointer, restore sine/cosine pointer

	loops_reset			;; 63 more iters of 4
	yr4_rb4cl_four_complex_djbunfft_preload
c5b:	yr4_rb4cl_four_complex_djbunfft rsi, 64, 4*64, 8*64, rdi, 64*YMM_SCD3, 4
	yloop	4, c5b, -4*64+16*64, rdi, -4*64*YMM_SCD3 ;; Test loop counter, next source pointer, restore sine/cosine pointer
	yloop	4, c5b, -4*16*64+dist64	;; Test loop counter, next source pointer
	yloop	16, c5b, -16*dist64	;; Test loop counter, restore source pointer
	end_timer 10

;; Do inverse FFT levels 8,9
;; On input the 64-byte cache lines hold these data values:
;;	0	2K	4K	6K	16	+2K	+2K	+2K
;;	1
;;	...
;;	15
;;	32	...
;;	...
;;	2016	...
;; On output the 64-byte cache lines hold these data values:
;;	0	2K	4K	6K	64	+2K	+2K	+2K
;;	1
;;	...
;;	63
;;	128	...
;;	...
;;	1920	...

;; Do 16 eight_reals_unfft operations
;; Do 1008 four_complex_unfft operations
;;	distance between fft data elements is 16
;;	do 256 macros each processing 32 data values

	start_timer 11
	yloop_init  256			;; 16 iterations of 16
	yr4_4cl_eight_reals_four_complex_djbunfft_preload
c4a:	yr4_4cl_eight_reals_four_complex_djbunfft rsi, 64, 16*64, 32*64, rdi, 16*YMM_SCD3, 16
	yloop	16, c4a, -16*64+dist64, rdi, -16*16*YMM_SCD3 ;; Test loop counter, next source pointer, restore sine/cosine pointer

	loops_reset			;; 15 more iters of 16
	yr4_rb4cl_four_complex_djbunfft_preload
c4b:	yr4_rb4cl_four_complex_djbunfft rsi, 64, 16*64, 32*64, rdi, 16*YMM_SCD3, 16
	yloop	16, c4b, -16*64+dist64, rdi, -16*16*YMM_SCD3 ;; Test loop counter, next source pointer, restore sine/cosine pointer
	yloop	16, c4b, -16*dist64	;; Test loop counter, restore source pointer
	end_timer 11

;; Do inverse FFT levels 6,7
;; On input the 64-byte cache lines hold these data values:
;;	0	2K	4K	6K	64	+2K	+2K	+2K
;;	1
;;	...
;;	63
;;	128	...
;;	...
;;	1920	...
;; On output the 64-byte cache lines hold these data values:
;;	0	2K	4K	6K	256	+2K	+2K	+2K
;;	1
;;	...
;;	255
;;	512	...
;;	1024	...
;;	1536	...

;; Do 64 eight_reals_unfft operations
;; Do 1960 four_complex_unfft operations
;;	distance between fft data elements is 64
;;	do 256 macros each processing 32 data values

	start_timer 12
	yloop_init  256			;; 4 iterations of 64
	yr4_4cl_eight_reals_four_complex_djbunfft_preload
;; Loop below reworked to avoid 4KB distances
;;c3a:	yr4_4cl_eight_reals_four_complex_djbunfft rsi, 64, dist64, 2*dist64, rdi, 4*YMM_SCD3, 64
;;	yloop	64, c3a, -64*64+4*dist64, rdi, -64*4*YMM_SCD3 ;; Test loop counter, next source pointer, restore sine/cosine pointer
c3a:	yr4_4cl_eight_reals_four_complex_djbunfft rsi, 4*64, dist64, 2*dist64, rdi, 4*4*YMM_SCD3, 16
	yloop	16, c3a, -16*4*64+64, rdi, -16*4*4*YMM_SCD3+4*YMM_SCD3 ;; Test loop counter, next source pointer, next sine/cosine pointer
	yloop	4, c3a, -4*64+4*dist64, rdi, -4*4*YMM_SCD3 ;; Test loop counter, next source pointer, restore sine/cosine pointer

	loops_reset			;; 3 more iters of 64
	yr4_rb4cl_four_complex_djbunfft_preload
;; Loop below reworked to avoid 4KB distances
;;c3b:	yr4_rb4cl_four_complex_djbunfft rsi, 64, dist64, 2*dist64, rdi, 4*YMM_SCD3, 64
;;	yloop	64, c3b, -64*64+4*dist64, rdi, -64*4*YMM_SCD3 ;; Test loop counter, next source pointer, restore sine/cosine pointer
c3b:	yr4_rb4cl_four_complex_djbunfft rsi, 4*64, dist64, 2*dist64, rdi, 4*4*YMM_SCD3, 16
	yloop	16, c3b, -16*4*64+64, rdi, -16*4*4*YMM_SCD3+4*YMM_SCD3 ;; Test loop counter, next source pointer, next sine/cosine pointer
	yloop	4, c3b, -4*64+4*dist64, rdi, -4*4*YMM_SCD3 ;; Test loop counter, next source pointer, restore sine/cosine pointer
	yloop	4, c3b, -4*4*dist64	;; Test loop counter, restore source pointer
	end_timer 12

;; Do inverse FFT levels 4,5
;; On input the 64-byte cache lines hold these data values:
;;	0	2K	4K	6K	256	+2K	+2K	+2K
;;	1
;;	...
;;	255
;;	512	...
;;	1024	...
;;	1536	...
;; On output the 64-byte cache lines hold these data values:
;;	0	2K	4K	6K	1024	+2K	+2K	+2K
;;	1
;;	...
;;	1023

;; Do 256 eight_reals_unfft operations
;; Do 768 four_complex_unfft operations
;;	distance between fft data elements is 256
;;	do 256 macros each processing 32 data values

	start_timer 13
	yloop_init  256			;; 256 iterations
	yr4_4cl_eight_reals_four_complex_djbunfft_preload
c2b:	yr4_4cl_eight_reals_four_complex_djbunfft rsi, 64, 4*dist64, 8*dist64, rdi, YMM_SCD3, 64
	yloop	64, c2b, -64*64+dist64	;; Test loop counter, next source pointer
	yloop	4, c2b, -4*dist64	;; Test loop counter, restore source pointer
	end_timer 13

;; Do inverse FFT levels 1,2,3
;; On input the 64-byte cache lines hold these data values:
;;	0	2K	4K	6K	1024	+2K	+2K	+2K
;;	1
;;	...
;;	1023
;; These are swizzled to:
;;	0	256	512	768	1024	+256	+256	+256
;;	1	...
;;	...
;;	255
;;	2K	...
;;	4K	...
;;	6K	...
;; On output the 64-byte cache lines hold these data values:
;;	0	256	512	768	4K	+256	+256	+256
;;	1	...
;;	...
;;	255
;;	1K	...
;;	2K	...
;;	3K	...

;; Do 1024 eight_reals_unfft operations
;;	distance between fft data elements is 1024
;;	do 256 macros each processing 32 data values

	start_timer 14
	yloop_init  256			;; 256 iterations
	yr4_s4cl_eight_reals_unfft_preload
c1b:	yr4_s4cl_eight_reals_unfft rsi, 64, 4*dist64, 8*dist64, rdi, YMM_SCD3, 64
	yloop	64, c1b, -64*64+dist64	;; Test loop counter, next source pointer
	yloop	4, c1b			;; Test loop counter
	end_timer 14

	ytwo_dest_adjust
	yfft_3_ret
	ENDM

;; Perform a 8K-element negacyclic FFT

yfft8Kp MACRO type
	LOCAL	ymid2p, ymid3p, ymid4p, yfinish_unfft
	LOCAL	b1b, b2b, b3b, b4b, b5b, b6a, b6b, b6c, b6d, c1b, c2b, c3b, c4b, c5b

;; Do a multiply with pre-FFTed inputs

	cmp	ffttype, 4
	je	ymid4p

;; Do FFT levels 1,2,3
;; On input the 64-byte cache lines hold these data values:
;;	0	256	512	768	4K	+256	+256	+256
;;	1
;;	...
;;	255
;;	1K	...
;;	2K	...
;;	3K	...
;; On output these are swizzled to:
;;	0	2K	4K	6K	1K	+2K	+2K	+2K
;;	1
;;	...
;;	1023

;; Do 1024 four_complex_first_fft operations
;;	distance between fft data elements is 1024
;;	do 256 macros each processing 32 data values

;; Do FFT levels 4,5
;; On input the 64-byte cache lines hold these data values:
;;	0	2K	4K	6K	1K	+2K	+2K	+2K
;;	1
;;	...
;;	1023
;; On output the 64-byte cache lines hold these data values:
;;	0	2K	4K	6K	256	+2K	+2K	+2K
;;	1
;;	...
;;	255
;;	512	...
;;	1024	...
;;	1536	...

;; Do 1024 four_complex_fft operations
;;	distance between fft data elements is 256
;;	do 256 macros each processing 32 data values

IF OLDWAY EQ 1
	start_timer 4
	mov	rdi, sincos1		;; Load sin/cos pointer
	bump	rdi, 256*YMM_SCD2/4
	yloop_init  256			;; 256 iterations
	yr4_fs4cl_four_complex_first_djbfft_preload
b1b:	yr4_fs4cl_four_complex_first_djbfft rsi, 64, 4*dist64, 8*dist64, rdi, YMM_SCD4, 64
	yloop	64, b1b, -64*64+dist64	;; Test loop counter, next source pointer
	yloop	4, b1b, -4*dist64	;; Test loop counter, restore source pointer
	end_timer 4

	start_timer 5
	bump	rdi, -256*YMM_SCD4-256*YMM_SCD2/4 ;; Load sin/cos pointer
	yloop_init  256			;; 256 iterations
	yr4_b4cl_four_complex_djbfft_preload
b2b:	yr4_b4cl_four_complex_djbfft rsi, 64, 4*dist64, 8*dist64, rdi, YMM_SCD2/4, 64
	yloop	64, b2b, -64*64+dist64	;; Test loop counter, next source pointer
	yloop	4, b2b, -4*dist64, rdi, -256*YMM_SCD2/4 ;; Test loop counter, restore source pointer, restore sin/cos pointer
	end_timer 5
ELSE
	start_timer 4
	mov	rdi, sincos1		;; Load sin/cos pointers
	lea	rdx, [rdi+256*YMM_SCD2/4]
	yloop_init  256			;; 256 iterations
	yr4_fs4cl_sixteen_complex_first_djbfft_preload
b1b:	yr4_fs4cl_sixteen_complex_first_djbfft rsi, 64, 4*dist64, 8*dist64, rdx, YMM_SCD4, rdi, YMM_SCD2/4, 64
	yloop	64, b1b, -64*64+dist64	;; Test loop counter, next source pointer
	yloop	4, b1b, -4*dist64, rdi, -256*YMM_SCD2/4 ;; Test loop counter, restore source pointer, restore sin/cos pointer
	end_timer 4
ENDIF

;; Do FFT levels 6,7
;; On input the 64-byte cache lines hold these data values:
;;	0	2K	4K	6K	256	+2K	+2K	+2K
;;	1
;;	...
;;	255
;;	512	...
;;	1024	...
;;	1536	...
;; On output the 64-byte cache lines hold these data values:
;;	0	2K	4K	6K	64	+2K	+2K	+2K
;;	1
;;	...
;;	63
;;	128	...
;;	...
;;	1920	...

;; Do 1024 four_complex_fft operations
;;	distance between fft data elements is 64
;;	do 256 macros each processing 32 data values

	start_timer 6
	yloop_init  256			;; 4 iterations of 64
	yr4_b4cl_four_complex_djbfft_preload
;; Loop below reworked to avoid 4KB distances
;;b3b:	yr4_b4cl_four_complex_djbfft rsi, 64, dist64, 2*dist64, rdi, 4*YMM_SCD2/4, 64
;;	yloop	64, b3b, -64*64+4*dist64, rdi, -64*4*YMM_SCD2/4 ;; Test loop counter, next source pointer, restore sin/cos pointer
b3b:	yr4_b4cl_four_complex_djbfft rsi, 4*64, dist64, 2*dist64, rdi, 4*4*YMM_SCD2/4, 16
	yloop	16, b3b, -16*4*64+64, rdi, -16*4*4*YMM_SCD2/4+4*YMM_SCD2/4 ;; Test loop counter, next source pointer, next sin/cos pointer
	yloop	4, b3b, -4*64+4*dist64, rdi, -4*4*YMM_SCD2/4 ;; Test loop counter, next source pointer, restore sin/cos pointer
	yloop	4, b3b, -4*4*dist64	;; Test loop counter, restore source pointer
	end_timer 6

;; Do FFT levels 8,9
;; On input the 64-byte cache lines hold these data values:
;;	0	2K	4K	6K	64	+2K	+2K	+2K
;;	1
;;	...
;;	63
;;	128	...
;;	...
;;	1920	...
;; On output the 64-byte cache lines hold these data values:
;;	0	2K	4K	6K	16	+2K	+2K	+2K
;;	1
;;	...
;;	15
;;	32	...
;;	...
;;	2016	...

;; Do 1024 four_complex_fft operations
;;	distance between fft data elements is 16
;;	do 256 macros each processing 32 data values

	start_timer 7
	yloop_init  256			;; 16 iterations of 16
	yr4_b4cl_four_complex_djbfft_preload
b4b:	yr4_b4cl_four_complex_djbfft rsi, 64, 16*64, 32*64, rdi, 16*YMM_SCD2/4, 16
	yloop	16, b4b, -16*64+dist64, rdi, -16*16*YMM_SCD2/4 ;; Test loop counter, next source pointer, restore sin/cos pointer
	yloop	16, b4b, -16*dist64	;; Test loop counter, restore source pointer
	end_timer 7

;; Do FFT levels 10,11
;; On input the 64-byte cache lines hold these data values:
;;	0	2K	4K	6K	16	+2K	+2K	+2K
;;	1
;;	...
;;	15
;;	32	...
;;	...
;;	2016	...
;; On output the 64-byte cache lines hold these data values:
;;	0	2K	4K	6K	4	+2K	+2K	+2K
;;	1
;;	...
;;	3
;;	8	...
;;	...
;;	2040	...

;; Do 1024 four_complex_fft operations
;;	distance between fft data elements is 4
;;	do 256 macros each processing 32 data values

	start_timer 8
	yloop_init  256			;; 64 iterations of 4
	yr4_b4cl_four_complex_djbfft_preload
b5b:	yr4_b4cl_four_complex_djbfft rsi, 64, 4*64, 8*64, rdi, 64*YMM_SCD2/4, 4
	yloop	4, b5b, -4*64+16*64, rdi, -4*64*YMM_SCD2/4 ;; Test loop counter, next source pointer, restore sin/cos pointer
	yloop	4, b5b, -4*16*64+dist64	;; Test loop counter, next source pointer
	yloop	16, b5b, -16*dist64	;; Test loop counter, restore source pointer
	end_timer 8

;; Do FFT levels 12,13
;; On input and output the 64-byte cache lines hold these data values:
;;	0	2K	4K	6K	4	+2K	+2K	+2K
;;	1
;;	...
;;	3
;;	8	...
;;	...
;;	2040	...

;; Do 1024 four_complex_fft operations
;;	distance between fft data elements is 1
;;	do 256 macros each processing 32 data values

	cmp	ffttype, 2
	je	ymid2p
	ja	ymid3p

	start_timer 9
	yloop_init  256			;; 256 iterations
b6a:	yr4_4cl_four_complex_fft_final rsi, 4*64, 64, 2*64
	yloop	16, b6a, -16*4*64+dist64 ;; Test loop counter, next source pointer
	yloop	16, b6a			;; Test loop counter
	end_timer 9
	yfft_1_ret

ymid2p:	start_timer 9
	yloop_init  256			;; 256 iterations
	yr4_4cl_four_complex_with_square_preload
b6b:	yr4_4cl_four_complex_with_square rsi, 4*64, 64, 2*64, 16
	yloop	16, b6b, -16*4*64+dist64 ;; Test loop counter, next source pointer
	yloop	16, b6b			;; Test loop counter
	end_timer 9
	jmp	yfinish_unfft

ymid3p:	start_timer 9
x64	mov	r8, DEST2ARG
	mov	rbp, DIST_TO_MULSRCARG
	yloop_init  256			;; 256 iterations
	yr4_4cl_four_complex_with_mult_preload
b6c:	yr4_4cl_four_complex_with_mult rsi, 4*64, 64, 2*64
	yloop	16, b6c, -16*4*64+dist64 ;; Test loop counter, next source pointer
	yloop	16, b6c			;; Test loop counter
x64	add	rsi, r8
	end_timer 9
	jmp	yfinish_unfft

ymid4p:	start_timer 9
	mov	rbp, DIST_TO_MULSRCARG
	yloop_init  256			;; 256 iterations
	yr4_4cl_four_complex_with_mulf_preload
b6d:	yr4_4cl_four_complex_with_mulf rsi, 4*64, 64, 2*64
	yloop	16, b6d, -16*4*64+dist64 ;; Test loop counter, next source pointer
	yloop	16, b6d			;; Test loop counter
	end_timer 9
yfinish_unfft:
	bump	rsi, -16*dist64		;; Restore source pointer

;; Do inverse FFT levels 10,11
;; On input the 64-byte cache lines hold these data values:
;;	0	2K	4K	6K	4	+2K	+2K	+2K
;;	1
;;	...
;;	3
;;	8	...
;;	...
;;	2040	...
;; On output the 64-byte cache lines hold these data values:
;;	0	2K	4K	6K	16	+2K	+2K	+2K
;;	1
;;	...
;;	15
;;	32	...
;;	...
;;	2016	...

;; Do 1024 four_complex_unfft operations
;;	distance between fft data elements is 4
;;	do 256 macros each processing 32 data values

	start_timer 10
	mov	rdi, sincos1		;; Load sin/cos pointer
	yloop_init  256			;; 64 iterations of 4
	yr4_b4cl_four_complex_djbunfft_preload
c5b:	yr4_b4cl_four_complex_djbunfft rsi, 64, 4*64, 8*64, rdi, 64*YMM_SCD2/4, 4
	yloop	4, c5b, -4*64+16*64, rdi, -4*64*YMM_SCD2/4 ;; Test loop counter, next source pointer, restore sine/cosine pointer
	yloop	4, c5b, -4*16*64+dist64	;; Test loop counter, next source pointer
	yloop	16, c5b, -16*dist64	;; Test loop counter, restore source pointer
	end_timer 10

;; Do inverse FFT levels 8,9
;; On input the 64-byte cache lines hold these data values:
;;	0	2K	4K	6K	16	+2K	+2K	+2K
;;	1
;;	...
;;	15
;;	32	...
;;	...
;;	2016	...
;; On output the 64-byte cache lines hold these data values:
;;	0	2K	4K	6K	64	+2K	+2K	+2K
;;	1
;;	...
;;	63
;;	128	...
;;	...
;;	1920	...

;; Do 1024 four_complex_unfft operations
;;	distance between fft data elements is 16
;;	do 256 macros each processing 32 data values

	start_timer 11
	yloop_init  256			;; 16 iterations of 16
	yr4_b4cl_four_complex_djbunfft_preload
c4b:	yr4_b4cl_four_complex_djbunfft rsi, 64, 16*64, 32*64, rdi, 16*YMM_SCD2/4, 16
	yloop	16, c4b, -16*64+dist64, rdi, -16*16*YMM_SCD2/4 ;; Test loop counter, next source pointer, restore sine/cosine pointer
	yloop	16, c4b, -16*dist64	;; Test loop counter, restore source pointer
	end_timer 11

;; Do inverse FFT levels 6,7
;; On input the 64-byte cache lines hold these data values:
;;	0	2K	4K	6K	64	+2K	+2K	+2K
;;	1
;;	...
;;	63
;;	128	...
;;	...
;;	1920	...
;; On output the 64-byte cache lines hold these data values:
;;	0	2K	4K	6K	256	+2K	+2K	+2K
;;	1
;;	...
;;	255
;;	512	...
;;	1024	...
;;	1536	...

;; Do 1024 four_complex_unfft operations
;;	distance between fft data elements is 64
;;	do 256 macros each processing 32 data values

	start_timer 12
	yloop_init  256			;; 4 iterations of 64
	yr4_b4cl_four_complex_djbunfft_preload
;; Loop below reworked to avoid 4KB distances
;;c3b:	yr4_b4cl_four_complex_djbunfft rsi, 64, dist64, 2*dist64, rdi, 4*YMM_SCD2/4, 64
;;	yloop	64, c3b, -64*64+4*dist64, rdi, -64*4*YMM_SCD2/4 ;; Test loop counter, next source pointer, restore sine/cosine pointer
c3b:	yr4_b4cl_four_complex_djbunfft rsi, 4*64, dist64, 2*dist64, rdi, 4*4*YMM_SCD2/4, 16
	yloop	16, c3b, -16*4*64+64, rdi, -16*4*4*YMM_SCD2/4+4*YMM_SCD2/4 ;; Test loop counter, next source pointer, next sine/cosine pointer
	yloop	4, c3b, -4*64+4*dist64, rdi, -4*4*YMM_SCD2/4 ;; Test loop counter, next source pointer, restore sine/cosine pointer
	yloop	4, c3b, -4*4*dist64	;; Test loop counter, restore source pointer
	end_timer 12

;; Do inverse FFT levels 4,5
;; On input the 64-byte cache lines hold these data values:
;;	0	2K	4K	6K	256	+2K	+2K	+2K
;;	1
;;	...
;;	255
;;	512	...
;;	1024	...
;;	1536	...
;; On output the 64-byte cache lines hold these data values:
;;	0	2K	4K	6K	1024	+2K	+2K	+2K
;;	1
;;	...
;;	1023

;; Do 1024 four_complex_unfft operations
;;	distance between fft data elements is 256
;;	do 256 macros each processing 32 data values

;; Do inverse FFT levels 1,2,3
;; On input the 64-byte cache lines hold these data values:
;;	0	2K	4K	6K	1024	+2K	+2K	+2K
;;	1
;;	...
;;	1023
;; These are swizzled to:
;;	0	256	512	768	1024	+256	+256	+256
;;	1	...
;;	...
;;	255
;;	2K	...
;;	4K	...
;;	6K	...
;; On output the 64-byte cache lines hold these data values:
;;	0	256	512	768	4K	+256	+256	+256
;;	1	...
;;	...
;;	255
;;	1K	...
;;	2K	...
;;	3K	...

;; Do 1024 four_complex_last_unfft operations
;;	distance between fft data elements is 1024
;;	do 256 macros each processing 32 data values

IF OLDWAY EQ 1
	start_timer 13
	yloop_init  256			;; 256 iterations
	yr4_b4cl_four_complex_djbunfft_preload
c2b:	yr4_b4cl_four_complex_djbunfft rsi, 64, 4*dist64, 8*dist64, rdi, YMM_SCD2/4, 64
	yloop	64, c2b, -64*64+dist64	;; Test loop counter, next source pointer
	yloop	4, c2b, -4*dist64	;; Test loop counter, restore source pointer
	end_timer 13

	start_timer 14
	yloop_init  256			;; 256 iterations
	yr4_s4cl_four_complex_last_unfft_preload
c1b:	yr4_s4cl_four_complex_last_unfft rsi, 64, 4*dist64, 8*dist64, rdi, YMM_SCD4, 64
	yloop	64, c1b, -64*64+dist64	;; Test loop counter, next source pointer
	yloop	4, c1b			;; Test loop counter
	end_timer 14
ELSE
	start_timer 14
	lea	rdx, [rdi+256*YMM_SCD2/4]
	yloop_init  256			;; 256 iterations
	yr4_s4cl_sixteen_complex_last_unfft_preload
c1b:	yr4_s4cl_sixteen_complex_last_unfft rsi, 64, 4*dist64, 8*dist64, rdi, YMM_SCD2/4, rdx, YMM_SCD4, 64
	yloop	64, c1b, -64*64+dist64	;; Test loop counter, next source pointer
	yloop	4, c1b			;; Test loop counter
	end_timer 14
ENDIF

	ytwo_dest_adjust
	yfft_3_ret
	ENDM


;; Perform a 10K-element FFT.

yfft10K MACRO type
	LOCAL	ymid2, ymid3, ymid4, yfinish_unfft
	LOCAL	b1b, b2b, b3a, b3b, b4a, b4b, b5a, b5b, b6a, b6b, b6c, b6d
	LOCAL	c1b, c2b, c3a, c3b, c4a, c4b, c5a, c5b

;; Do a multiply with pre-FFTed inputs

	cmp	ffttype, 4
	je	ymid4

;; Do FFT levels 1,2,3
;; On input the 64-byte cache lines hold these data values:
;;	0	+320	+320	+320	5K	+320	+320	+320
;;	1
;;	...
;;	319
;;	1280	...
;;	2560	...
;;	3840	...
;; On output these are swizzled to:
;;	0	+2560	+2560	+2560	1280	+2560	+2560	+2560
;;	1
;;	...
;;	1279

;; Do 1280 eight_reals_fft operations
;;	distance between fft data elements is 1280
;;	do 320 macros each processing 32 data values

	start_timer 4
	mov	rdi, sincos1		;; Load sin/cos pointer
	bump	rdi, 256*YMM_SCD4+64*YMM_SCD3
	yloop_init  320			;; 320 iterations
	yr4_fs4cl_eight_reals_fft_preload
b1b:	yr4_fs4cl_eight_reals_fft rsi, 64, 5*dist64, 10*dist64, rdi, YMM_SCD3, 64
	yloop	64, b1b, -64*64+dist64	;; Test loop counter, next source pointer
	yloop	5, b1b, -5*dist64	;; Test loop counter, restore source pointer
	end_timer 4

;; Do FFT levels 4,5
;; On input the 64-byte cache lines hold these data values:
;;	0	+2560	+2560	+2560	1280	+2560	+2560	+2560
;;	1
;;	...
;;	1279
;; On output the 64-byte cache lines hold these data values:
;;	0	+2560	+2560	+2560	256	+2560	+2560	+2560
;;	1
;;	...
;;	255
;;	512	...
;;	....
;;	2048	...

;; Do 256 ten_reals_fft operations
;; Do 768 five_complex_fft operations
;;	distance between fft data elements is 256
;;	do 256 macros each processing 40 data values

	start_timer 5
	bump	rdi, -320*YMM_SCD3-256*YMM_SCD4 ;; Load sin/cos pointer
	yloop_init  256			;; 256 iterations
	yr5_5cl_ten_reals_five_complex_djbfft_preload
b2b:	yr5_5cl_ten_reals_five_complex_djbfft rsi, 64, 4*dist64, rdi, YMM_SCD4, 64
	yloop	64, b2b, -64*64+dist64	;; Test loop counter, next source pointer
	yloop	4, b2b, -4*dist64	;; Test loop counter, restore source pointer
	end_timer 5

;; Do FFT levels 6,7
;; On input the 64-byte cache lines hold these data values:
;;	0	+2560	+2560	+2560	256	+2560	+2560	+2560
;;	1
;;	...
;;	255
;;	512	...
;;	....
;;	2048	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+2560	+2560	+2560	64	+2560	+2560	+2560
;;	1
;;	...
;;	63
;;	128	...
;;	...
;;	2432	...

;; Do 64 eight_reals_fft operations
;; Do 1216 four_complex_fft operations
;;	distance between fft data elements is 64
;;	do 320 macros each processing 32 data values

	start_timer 6
	bump	rdi, -256*YMM_SCD4-64*YMM_SCD3 ;; Restore sin/cos pointer
	yloop_init  320			;; 5 iterations of 64
	yr4_4cl_eight_reals_four_complex_djbfft_preload
;; Loop below reworked to avoid 4KB distances
;;b3a:	yr4_4cl_eight_reals_four_complex_djbfft rsi, 64, dist64, 2*dist64, rdi, YMM_SCD3, 64
;;	yloop	64, b3a, -64*64+4*dist64, rdi, -64*YMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer
b3a:	yr4_4cl_eight_reals_four_complex_djbfft rsi, 4*64, dist64, 2*dist64, rdi, 4*YMM_SCD3, 16
	yloop	16, b3a, -16*4*64+64, rdi, -16*4*YMM_SCD3+YMM_SCD3 ;; Test loop counter, next source pointer, next sin/cos pointer
	yloop	4, b3a, -4*64+4*dist64, rdi, -4*YMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer

	loops_reset			;; 4 more iters of 64
	yr4_rb4cl_four_complex_djbfft_preload
;; Loop below reworked to avoid 4KB distances
;;b3b:	yr4_rb4cl_four_complex_djbfft rsi, 64, dist64, 2*dist64, rdi, YMM_SCD3, 64
;;	yloop	64, b3b, -64*64+4*dist64, rdi, -64*YMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer
b3b:	yr4_rb4cl_four_complex_djbfft rsi, 4*64, dist64, 2*dist64, rdi, 4*YMM_SCD3, 16
	yloop	16, b3b, -16*4*64+64, rdi, -16*4*YMM_SCD3+YMM_SCD3 ;; Test loop counter, next source pointer, next sin/cos pointer
	yloop	4, b3b, -4*64+4*dist64, rdi, -4*YMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer
	yloop	5, b3b, -5*4*dist64	;; Test loop counter, restore source pointer
	end_timer 6

;; Do FFT levels 8,9
;; On input the 64-byte cache lines hold these data values:
;;	0	+2560	+2560	+2560	64	+2560	+2560	+2560
;;	1
;;	...
;;	63
;;	128	...
;;	...
;;	2432	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+2560	+2560	+2560	16	+2560	+2560	+2560
;;	1
;;	...
;;	15
;;	32	...
;;	...
;;	2528	...

;; Do 16 eight_reals_fft operations
;; Do 1264 four_complex_fft operations
;;	distance between fft data elements is 16
;;	do 320 macros each processing 32 data values

	start_timer 7
	yloop_init  320			;; 20 iterations of 16
	yr4_4cl_eight_reals_four_complex_djbfft_preload
b4a:	yr4_4cl_eight_reals_four_complex_djbfft rsi, 64, 16*64, 32*64, rdi, 4*YMM_SCD3, 16
	yloop	16, b4a, -16*64+dist64, rdi, -16*4*YMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer

	loops_reset			;; 19 more iters of 16
	yr4_rb4cl_four_complex_djbfft_preload
b4b:	yr4_rb4cl_four_complex_djbfft rsi, 64, 16*64, 32*64, rdi, 4*YMM_SCD3, 16
	yloop	16, b4b, -16*64+dist64, rdi, -16*4*YMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer
	yloop	20, b4b, -20*dist64	;; Test loop counter, restore source pointer
	end_timer 7

;; Do FFT levels 10,11
;; On input the 64-byte cache lines hold these data values:
;;	0	+2560	+2560	+2560	16	+2560	+2560	+2560
;;	1
;;	...
;;	15
;;	32	...
;;	...
;;	2528	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+2560	+2560	+2560	4	+2560	+2560	+2560
;;	1
;;	...
;;	3
;;	8	...
;;	...
;;	2552	...

;; Do 4 eight_reals_fft operations
;; Do 1276 four_complex_fft operations
;;	distance between fft data elements is 4
;;	do 320 macros each processing 32 data values

	start_timer 8
	yloop_init  320			;; 80 iterations of 4
	yr4_4cl_eight_reals_four_complex_djbfft_preload
b5a:	yr4_4cl_eight_reals_four_complex_djbfft rsi, 64, 4*64, 8*64, rdi, 16*YMM_SCD3, 4
	yloop	4, b5a, -4*64+16*64, rdi, -4*16*YMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer

	loops_reset			;; 79 more iters of 4
	yr4_rb4cl_four_complex_djbfft_preload
b5b:	yr4_rb4cl_four_complex_djbfft rsi, 64, 4*64, 8*64, rdi, 16*YMM_SCD3, 4
	yloop	4, b5b, -4*64+16*64, rdi, -4*16*YMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer
	yloop	4, b5b, -4*16*64+dist64	;; Test loop counter, next source pointer
	yloop	20, b5b, -20*dist64	;; Test loop counter, restore source pointer
	end_timer 8

;; Do FFT levels 12,13
;; On input and output the 64-byte cache lines hold these data values:
;;	0	+2560	+2560	+2560	4	+2560	+2560	+2560
;;	1
;;	...
;;	3
;;	8	...
;;	...
;;	2552	...

;; Do 1 eight_reals_fft operation
;; Do 1279 four_complex_fft operations
;;	distance between fft data elements is 1
;;	do 320 macros each processing 32 data values

	cmp	ffttype, 2
	je	ymid2
	ja	ymid3

	start_timer 9
;;	yr4_4cl_eight_reals_four_complex_fft_final rsi, 4*64, 64, 2*64
	yr4_dispatch call, yreal_fft_final, 64, 2*64
	yloop_init  320, 1, 16		;; 319 more iterations
b6a:	yr4_4cl_four_complex_fft_final rsi, 4*64, 64, 2*64
	yloop	16, b6a, -16*4*64+dist64 ;; Test loop counter, next source pointer
	yloop	20, b6a			;; Test loop counter
	end_timer 9
	yfft_1_ret

ymid2:	start_timer 9
;;	yr4_4cl_eight_reals_four_complex_with_square rsi, 4*64, 64, 2*64
	yr4_dispatch call, yreal_square, 64, 2*64
	yloop_init  320, 1, 16		;; 319 more iterations
	yr4_4cl_four_complex_with_square_preload
b6b:	yr4_4cl_four_complex_with_square rsi, 4*64, 64, 2*64, 1
	yloop	16, b6b, -16*4*64+dist64 ;; Test loop counter, next source pointer
	yloop	20, b6b			;; Test loop counter
	end_timer 9
	jmp	yfinish_unfft

ymid3:	start_timer 9
x64	mov	r8, DEST2ARG
	mov	rbp, DIST_TO_MULSRCARG
;;	yr4_4cl_eight_reals_four_complex_with_mult rsi, 4*64, 64, 2*64
	yr4_dispatch call, yreal_mult, 64, 2*64
	yloop_init  320, 1, 16		;; 319 more iterations
	yr4_4cl_four_complex_with_mult_preload
b6c:	yr4_4cl_four_complex_with_mult rsi, 4*64, 64, 2*64
	yloop	16, b6c, -16*4*64+dist64 ;; Test loop counter, next source pointer
	yloop	20, b6c			;; Test loop counter
x64	add	rsi, r8
	end_timer 9
	jmp	yfinish_unfft

ymid4:	start_timer 9
	mov	rbp, DIST_TO_MULSRCARG
;;	yr4_4cl_eight_reals_four_complex_with_mulf rsi, 4*64, 64, 2*64
	yr4_dispatch call, yreal_mulf, 64, 2*64
	yloop_init  320, 1, 16		;; 319 more iterations
	yr4_4cl_four_complex_with_mulf_preload
b6d:	yr4_4cl_four_complex_with_mulf rsi, 4*64, 64, 2*64
	yloop	16, b6d, -16*4*64+dist64 ;; Test loop counter, next source pointer
	yloop	20, b6d			;; Test loop counter
	end_timer 9
yfinish_unfft:
	bump	rsi, -20*dist64		;; Restore source pointer

;; Do inverse FFT levels 10,11
;; On input the 64-byte cache lines hold these data values:
;;	0	+2560	+2560	+2560	4	+2560	+2560	+2560
;;	1
;;	...
;;	3
;;	8	...
;;	...
;;	2552	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+2560	+2560	+2560	16	+2560	+2560	+2560
;;	1
;;	...
;;	15
;;	32	...
;;	...
;;	2528	...

;; Do 4 eight_reals_unfft operations
;; Do 1276 four_complex_unfft operations
;;	distance between fft data elements is 4
;;	do 320 macros each processing 32 data values

	start_timer 10
	mov	rdi, sincos1		;; Load sin/cos pointer
	yloop_init  320			;; 80 iterations of 4
	yr4_4cl_eight_reals_four_complex_djbunfft_preload
c5a:	yr4_4cl_eight_reals_four_complex_djbunfft rsi, 64, 4*64, 8*64, rdi, 16*YMM_SCD3, 4
	yloop	4, c5a, -4*64+16*64, rdi, -4*16*YMM_SCD3 ;; Test loop counter, next source pointer, restore sine/cosine pointer

	loops_reset			;; 79 more iters of 4
	yr4_rb4cl_four_complex_djbunfft_preload
c5b:	yr4_rb4cl_four_complex_djbunfft rsi, 64, 4*64, 8*64, rdi, 16*YMM_SCD3, 4
	yloop	4, c5b, -4*64+16*64, rdi, -4*16*YMM_SCD3 ;; Test loop counter, next source pointer, restore sine/cosine pointer
	yloop	4, c5b, -4*16*64+dist64	;; Test loop counter, next source pointer
	yloop	20, c5b, -20*dist64	;; Test loop counter, restore source pointer
	end_timer 10

;; Do inverse FFT levels 8,9
;; On input the 64-byte cache lines hold these data values:
;;	0	+2560	+2560	+2560	16	+2560	+2560	+2560
;;	1
;;	...
;;	15
;;	32	...
;;	...
;;	2528	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+2560	+2560	+2560	64	+2560	+2560	+2560
;;	1
;;	...
;;	63
;;	128	...
;;	...
;;	2432	...

;; Do 16 eight_reals_unfft operations
;; Do 1264 four_complex_unfft operations
;;	distance between fft data elements is 16
;;	do 320 macros each processing 32 data values

	start_timer 11
	yloop_init  320			;; 20 iterations of 16
	yr4_4cl_eight_reals_four_complex_djbunfft_preload
c4a:	yr4_4cl_eight_reals_four_complex_djbunfft rsi, 64, 16*64, 32*64, rdi, 4*YMM_SCD3, 16
	yloop	16, c4a, -16*64+dist64, rdi, -16*4*YMM_SCD3 ;; Test loop counter, next source pointer, restore sine/cosine pointer

	loops_reset			;; 19 more iters of 16
	yr4_rb4cl_four_complex_djbunfft_preload
c4b:	yr4_rb4cl_four_complex_djbunfft rsi, 64, 16*64, 32*64, rdi, 4*YMM_SCD3, 16
	yloop	16, c4b, -16*64+dist64, rdi, -16*4*YMM_SCD3 ;; Test loop counter, next source pointer, restore sine/cosine pointer
	yloop	20, c4b, -20*dist64	;; Test loop counter, restore source pointer
	end_timer 11

;; Do inverse FFT levels 6,7
;; On input the 64-byte cache lines hold these data values:
;;	0	+2560	+2560	+2560	64	+2560	+2560	+2560
;;	1
;;	...
;;	63
;;	128	...
;;	...
;;	2432	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+2560	+2560	+2560	256	+2560	+2560	+2560
;;	1
;;	...
;;	255
;;	512	...
;;	...
;;	2048	...

;; Do 64 eight_reals_unfft operations
;; Do 1216 four_complex_unfft operations
;;	distance between fft data elements is 64
;;	do 320 macros each processing 32 data values

	start_timer 12
	yloop_init  320			;; 5 iterations of 64
	yr4_4cl_eight_reals_four_complex_djbunfft_preload
;; Loop below reworked to avoid 4KB distances
;;c3a:	yr4_4cl_eight_reals_four_complex_djbunfft rsi, 64, dist64, 2*dist64, rdi, YMM_SCD3, 64
;;	yloop	64, c3a, -64*64+4*dist64, rdi, -64*YMM_SCD3 ;; Test loop counter, next source pointer, restore sine/cosine pointer
c3a:	yr4_4cl_eight_reals_four_complex_djbunfft rsi, 4*64, dist64, 2*dist64, rdi, 4*YMM_SCD3, 16
	yloop	16, c3a, -16*4*64+64, rdi, -16*4*YMM_SCD3+YMM_SCD3 ;; Test loop counter, next source pointer, next sine/cosine pointer
	yloop	4, c3a, -4*64+4*dist64, rdi, -4*YMM_SCD3 ;; Test loop counter, next source pointer, restore sine/cosine pointer

	loops_reset			;; 4 more iters of 64
	yr4_rb4cl_four_complex_djbunfft_preload
;; Loop below reworked to avoid 4KB distances
;;c3b:	yr4_rb4cl_four_complex_djbunfft rsi, 64, dist64, 2*dist64, rdi, YMM_SCD3, 64
;;	yloop	64, c3b, -64*64+4*dist64, rdi, -64*YMM_SCD3 ;; Test loop counter, next source pointer, restore sine/cosine pointer
c3b:	yr4_rb4cl_four_complex_djbunfft rsi, 4*64, dist64, 2*dist64, rdi, 4*YMM_SCD3, 16
	yloop	16, c3b, -16*4*64+64, rdi, -16*4*YMM_SCD3+YMM_SCD3 ;; Test loop counter, next source pointer, next sine/cosine pointer
	yloop	4, c3b, -4*64+4*dist64, rdi, -4*YMM_SCD3 ;; Test loop counter, next source pointer, restore sine/cosine pointer
	yloop	5, c3b, -5*4*dist64	;; Test loop counter, restore source pointer
	end_timer 12

;; Do inverse FFT levels 4,5
;; On input the 64-byte cache lines hold these data values:
;;	0	+2560	+2560	+2560	256	+2560	+2560	+2560
;;	1
;;	...
;;	255
;;	512	...
;;	...
;;	2048	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+2560	+2560	+2560	1280	+2560	+2560	+2560
;;	1
;;	...
;;	1279

;; Do 256 ten_reals_unfft operations
;; Do 768 five_complex_unfft operations
;;	distance between fft data elements is 256
;;	do 256 macros each processing 40 data values

	start_timer 13
	bump	rdi, 64*YMM_SCD3	;; Next sine/cosine pointer
	yloop_init  256			;; 256 iterations
	yr5_5cl_ten_reals_five_complex_djbunfft_preload
c2b:	yr5_5cl_ten_reals_five_complex_djbunfft rsi, 64, 4*dist64, rdi, YMM_SCD4, 64
	yloop	64, c2b, -64*64+dist64	;; Test loop counter, next source pointer
	yloop	4, c2b, -4*dist64	;; Test loop counter, restore source pointer
	end_timer 13

;; Do inverse FFT levels 1,2,3
;; On input the 64-byte cache lines hold these data values:
;;	0	+2560	+2560	+2560	1280	+2560	+2560	+2560
;;	1
;;	...
;;	1279
;; These are swizzled to:
;;	0	+320	+320	+320	1280	+320	+320	+320
;;	1	...
;;	...
;;	319
;;	2560	...
;;	5120	...
;;	7680	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+320	+320	+320	5K	+320	+320	+320
;;	1	...
;;	...
;;	319
;;	1280	...
;;	2560	...
;;	3840	...

;; Do 1280 eight_reals_unfft operations
;;	distance between fft data elements is 1280
;;	do 320 macros each processing 32 data values

	start_timer 14
	yloop_init  320			;; 320 iterations
	yr4_s4cl_eight_reals_unfft_preload
c1b:	yr4_s4cl_eight_reals_unfft rsi, 64, 5*dist64, 10*dist64, rdi, YMM_SCD3, 64
	yloop	64, c1b, -64*64+dist64	;; Test loop counter, next source pointer
	yloop	5, c1b			;; Test loop counter
	end_timer 14

	ytwo_dest_adjust
	yfft_3_ret
	ENDM

;; Perform a 10K-element negacyclic FFT

yfft10Kp MACRO type
	LOCAL	ymid2p, ymid3p, ymid4p, yfinish_unfft
	LOCAL	b1b, b2b, b3b, b4b, b5b, b6a, b6b, b6c, b6d, c1b, c2b, c3b, c4b, c5b

;; Do a multiply with pre-FFTed inputs

	cmp	ffttype, 4
	je	ymid4p

;; Do FFT levels 1,2,3
;; On input the 64-byte cache lines hold these data values:
;;	0	+320	+320	+320	5K	+320	+320	+320
;;	1
;;	...
;;	319
;;	1280	...
;;	2560	...
;;	3840	...
;; On output these are swizzled to:
;;	0	+2560	+2560	+2560	1280	+2560	+2560	+2560
;;	1
;;	...
;;	1279

;; Do 1280 four_complex_first_fft operations
;;	distance between fft data elements is 1280
;;	do 320 macros each processing 32 data values

	start_timer 4
	mov	rdi, sincos1		;; Load sin/cos pointer
	bump	rdi, 256*YMM_SCD2/4+64*YMM_SCD2/4
	yloop_init  320			;; 320 iterations
	yr4_fs4cl_four_complex_first_djbfft_preload
b1b:	yr4_fs4cl_four_complex_first_djbfft rsi, 64, 5*dist64, 10*dist64, rdi, YMM_SCD4, 64
	yloop	64, b1b, -64*64+dist64	;; Test loop counter, next source pointer
	yloop	5, b1b, -5*dist64	;; Test loop counter, restore source pointer
	end_timer 4

;; Do FFT levels 4,5
;; On input the 64-byte cache lines hold these data values:
;;	0	+2560	+2560	+2560	1280	+2560	+2560	+2560
;;	1
;;	...
;;	1279
;; On output the 64-byte cache lines hold these data values:
;;	0	+2560	+2560	+2560	256	+2560	+2560	+2560
;;	1
;;	...
;;	255
;;	512	...
;;	....
;;	2048	...

;; Do 1024 five_complex_fft operations
;;	distance between fft data elements is 256
;;	do 256 macros each processing 40 data values

	start_timer 5
	bump	rdi, -320*YMM_SCD4-256*YMM_SCD2/4 ;; Load sin/cos pointer
	yloop_init  256			;; 256 iterations
	yr5_b5cl_five_complex_djbfft_preload
b2b:	yr5_b5cl_five_complex_djbfft rsi, 64, 4*dist64, rdi, YMM_SCD2/4, 64
	yloop	64, b2b, -64*64+dist64	;; Test loop counter, next source pointer
	yloop	4, b2b, -4*dist64	;; Test loop counter, restore source pointer
	end_timer 5

;; Do FFT levels 6,7
;; On input the 64-byte cache lines hold these data values:
;;	0	+2560	+2560	+2560	256	+2560	+2560	+2560
;;	1
;;	...
;;	255
;;	512	...
;;	....
;;	2048	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+2560	+2560	+2560	64	+2560	+2560	+2560
;;	1
;;	...
;;	63
;;	128	...
;;	...
;;	2432	...

;; Do 1280 four_complex_fft operations
;;	distance between fft data elements is 64
;;	do 320 macros each processing 32 data values

	start_timer 6
	bump	rdi, -256*YMM_SCD2/4-64*YMM_SCD2/4 ;; Next sin/cos pointer
	yloop_init  320			;; 5 iterations of 64
	yr4_b4cl_four_complex_djbfft_preload
;; Loop below reworked to avoid 4KB distances
;;b3b:	yr4_b4cl_four_complex_djbfft rsi, 64, dist64, 2*dist64, rdi, YMM_SCD2/4, 64
;;	yloop	64, b3b, -64*64+4*dist64, rdi, -64*YMM_SCD2/4 ;; Test loop counter, next source pointer, restore sin/cos pointer
b3b:	yr4_b4cl_four_complex_djbfft rsi, 4*64, dist64, 2*dist64, rdi, 4*YMM_SCD2/4, 16
	yloop	16, b3b, -16*4*64+64, rdi, -16*4*YMM_SCD2/4+YMM_SCD2/4 ;; Test loop counter, next source pointer, next sin/cos pointer
	yloop	4, b3b, -4*64+4*dist64, rdi, -4*YMM_SCD2/4 ;; Test loop counter, next source pointer, restore sin/cos pointer
	yloop	5, b3b, -5*4*dist64	;; Test loop counter, restore source pointer
	end_timer 6

;; Do FFT levels 8,9
;; On input the 64-byte cache lines hold these data values:
;;	0	+2560	+2560	+2560	64	+2560	+2560	+2560
;;	1
;;	...
;;	63
;;	128	...
;;	...
;;	2432	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+2560	+2560	+2560	16	+2560	+2560	+2560
;;	1
;;	...
;;	15
;;	32	...
;;	...
;;	2528	...

;; Do 1280 four_complex_fft operations
;;	distance between fft data elements is 16
;;	do 320 macros each processing 32 data values

	start_timer 7
	yloop_init  320			;; 20 iterations of 16
	yr4_b4cl_four_complex_djbfft_preload
b4b:	yr4_b4cl_four_complex_djbfft rsi, 64, 16*64, 32*64, rdi, 4*YMM_SCD2/4, 16
	yloop	16, b4b, -16*64+dist64, rdi, -16*4*YMM_SCD2/4 ;; Test loop counter, next source pointer, restore sin/cos pointer
	yloop	20, b4b, -20*dist64	;; Test loop counter, restore source pointer
	end_timer 7

;; Do FFT levels 10,11
;; On input the 64-byte cache lines hold these data values:
;;	0	+2560	+2560	+2560	16	+2560	+2560	+2560
;;	1
;;	...
;;	15
;;	32	...
;;	...
;;	2528	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+2560	+2560	+2560	4	+2560	+2560	+2560
;;	1
;;	...
;;	3
;;	8	...
;;	...
;;	2552	...

;; Do 1280 four_complex_fft operations
;;	distance between fft data elements is 4
;;	do 320 macros each processing 32 data values

	start_timer 8
	yloop_init  320			;; 80 iterations of 4
	yr4_b4cl_four_complex_djbfft_preload
b5b:	yr4_b4cl_four_complex_djbfft rsi, 64, 4*64, 8*64, rdi, 16*YMM_SCD2/4, 4
	yloop	4, b5b, -4*64+16*64, rdi, -4*16*YMM_SCD2/4 ;; Test loop counter, next source pointer, restore sin/cos pointer
	yloop	4, b5b, -4*16*64+dist64	;; Test loop counter, next source pointer
	yloop	20, b5b, -20*dist64	;; Test loop counter, restore source pointer
	end_timer 8

;; Do FFT levels 12,13
;; On input and output the 64-byte cache lines hold these data values:
;;	0	+2560	+2560	+2560	4	+2560	+2560	+2560
;;	1
;;	...
;;	3
;;	8	...
;;	...
;;	2552	...

;; Do 1280 four_complex_fft operations
;;	distance between fft data elements is 1
;;	do 320 macros each processing 32 data values

	cmp	ffttype, 2
	je	ymid2p
	ja	ymid3p

	start_timer 9
	yloop_init  320			;; 320 iterations
b6a:	yr4_4cl_four_complex_fft_final rsi, 4*64, 64, 2*64
	yloop	16, b6a, -16*4*64+dist64 ;; Test loop counter, next source pointer
	yloop	20, b6a			;; Test loop counter
	end_timer 9
	yfft_1_ret

ymid2p:	start_timer 9
	yloop_init  320			;; 320 iterations
	yr4_4cl_four_complex_with_square_preload
b6b:	yr4_4cl_four_complex_with_square rsi, 4*64, 64, 2*64, 16
	yloop	16, b6b, -16*4*64+dist64 ;; Test loop counter, next source pointer
	yloop	20, b6b			;; Test loop counter
	end_timer 9
	jmp	yfinish_unfft

ymid3p:	start_timer 9
x64	mov	r8, DEST2ARG
	mov	rbp, DIST_TO_MULSRCARG
	yloop_init  320			;; 320 iterations
	yr4_4cl_four_complex_with_mult_preload
b6c:	yr4_4cl_four_complex_with_mult rsi, 4*64, 64, 2*64
	yloop	16, b6c, -16*4*64+dist64 ;; Test loop counter, next source pointer
	yloop	20, b6c			;; Test loop counter
x64	add	rsi, r8
	end_timer 9
	jmp	yfinish_unfft

ymid4p:	start_timer 9
	mov	rbp, DIST_TO_MULSRCARG
	yloop_init  320			;; 320 iterations
	yr4_4cl_four_complex_with_mulf_preload
b6d:	yr4_4cl_four_complex_with_mulf rsi, 4*64, 64, 2*64
	yloop	16, b6d, -16*4*64+dist64 ;; Test loop counter, next source pointer
	yloop	20, b6d			;; Test loop counter
	end_timer 9
yfinish_unfft:
	bump	rsi, -20*dist64		;; Restore source pointer

;; Do inverse FFT levels 10,11
;; On input the 64-byte cache lines hold these data values:
;;	0	+2560	+2560	+2560	4	+2560	+2560	+2560
;;	1
;;	...
;;	3
;;	8	...
;;	...
;;	2552	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+2560	+2560	+2560	16	+2560	+2560	+2560
;;	1
;;	...
;;	15
;;	32	...
;;	...
;;	2528	...

;; Do 1280 four_complex_unfft operations
;;	distance between fft data elements is 4
;;	do 320 macros each processing 32 data values

	start_timer 10
	mov	rdi, sincos1		;; Load sin/cos pointer
	yloop_init  320			;; 80 iterations of 4
	yr4_b4cl_four_complex_djbunfft_preload
c5b:	yr4_b4cl_four_complex_djbunfft rsi, 64, 4*64, 8*64, rdi, 16*YMM_SCD2/4, 4
	yloop	4, c5b, -4*64+16*64, rdi, -4*16*YMM_SCD2/4 ;; Test loop counter, next source pointer, restore sine/cosine pointer
	yloop	4, c5b, -4*16*64+dist64	;; Test loop counter, next source pointer
	yloop	20, c5b, -20*dist64	;; Test loop counter, restore source pointer
	end_timer 10

;; Do inverse FFT levels 8,9
;; On input the 64-byte cache lines hold these data values:
;;	0	+2560	+2560	+2560	16	+2560	+2560	+2560
;;	1
;;	...
;;	15
;;	32	...
;;	...
;;	2528	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+2560	+2560	+2560	64	+2560	+2560	+2560
;;	1
;;	...
;;	63
;;	128	...
;;	...
;;	2432	...

;; Do 1280 four_complex_unfft operations
;;	distance between fft data elements is 16
;;	do 320 macros each processing 32 data values

	start_timer 11
	yloop_init  320			;; 20 iterations of 16
	yr4_b4cl_four_complex_djbunfft_preload
c4b:	yr4_b4cl_four_complex_djbunfft rsi, 64, 16*64, 32*64, rdi, 4*YMM_SCD2/4, 16
	yloop	16, c4b, -16*64+dist64, rdi, -16*4*YMM_SCD2/4 ;; Test loop counter, next source pointer, restore sine/cosine pointer
	yloop	20, c4b, -20*dist64	;; Test loop counter, restore source pointer
	end_timer 11

;; Do inverse FFT levels 6,7
;; On input the 64-byte cache lines hold these data values:
;;	0	+2560	+2560	+2560	64	+2560	+2560	+2560
;;	1
;;	...
;;	63
;;	128	...
;;	...
;;	2432	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+2560	+2560	+2560	256	+2560	+2560	+2560
;;	1
;;	...
;;	255
;;	512	...
;;	...
;;	2048	...

;; Do 1280 four_complex_unfft operations
;;	distance between fft data elements is 64
;;	do 320 macros each processing 32 data values

	start_timer 12
	yloop_init  320			;; 5 iterations of 64
	yr4_b4cl_four_complex_djbunfft_preload
;; Loop below reworked to avoid 4KB distances
;;c3b:	yr4_b4cl_four_complex_djbunfft rsi, 64, dist64, 2*dist64, rdi, YMM_SCD2/4, 64
;;	yloop	64, c3b, -64*64+4*dist64, rdi, -64*YMM_SCD2/4 ;; Test loop counter, next source pointer, restore sine/cosine pointer
c3b:	yr4_b4cl_four_complex_djbunfft rsi, 4*64, dist64, 2*dist64, rdi, 4*YMM_SCD2/4, 16
	yloop	16, c3b, -16*4*64+64, rdi, -16*4*YMM_SCD2/4+YMM_SCD2/4 ;; Test loop counter, next source pointer, next sine/cosine pointer
	yloop	4, c3b, -4*64+4*dist64, rdi, -4*YMM_SCD2/4 ;; Test loop counter, next source pointer, restore sine/cosine pointer
	yloop	5, c3b, -5*4*dist64	;; Test loop counter, restore source pointer
	end_timer 12

;; Do inverse FFT levels 4,5
;; On input the 64-byte cache lines hold these data values:
;;	0	+2560	+2560	+2560	256	+2560	+2560	+2560
;;	1
;;	...
;;	255
;;	512	...
;;	...
;;	2048	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+2560	+2560	+2560	1280	+2560	+2560	+2560
;;	1
;;	...
;;	1279

;; Do 1024 five_complex_unfft operations
;;	distance between fft data elements is 256
;;	do 256 macros each processing 40 data values

	start_timer 13
	bump	rdi, 64*YMM_SCD2/4	;; Restore sine/cosine pointer
	yloop_init  256			;; 256 iterations
	yr5_b5cl_five_complex_djbunfft_preload
c2b:	yr5_b5cl_five_complex_djbunfft rsi, 64, 4*dist64, rdi, YMM_SCD2/4, 64
	yloop	64, c2b, -64*64+dist64	;; Test loop counter, next source pointer
	yloop	4, c2b, -4*dist64	;; Test loop counter, restore source pointer
	end_timer 13

;; Do inverse FFT levels 1,2,3
;; On input the 64-byte cache lines hold these data values:
;;	0	+2560	+2560	+2560	1280	+2560	+2560	+2560
;;	1
;;	...
;;	1279
;; These are swizzled to:
;;	0	+320	+320	+320	1280	+320	+320	+320
;;	1	...
;;	...
;;	319
;;	2560	...
;;	5120	...
;;	7680	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+320	+320	+320	5K	+320	+320	+320
;;	1	...
;;	...
;;	319
;;	1280	...
;;	2560	...
;;	3840	...

;; Do 1280 four_complex_last_unfft operations
;;	distance between fft data elements is 1280
;;	do 320 macros each processing 32 data values

	start_timer 14
	yloop_init  320			;; 320 iterations
	yr4_s4cl_four_complex_last_unfft_preload
c1b:	yr4_s4cl_four_complex_last_unfft rsi, 64, 5*dist64, 10*dist64, rdi, YMM_SCD4, 64
	yloop	64, c1b, -64*64+dist64	;; Test loop counter, next source pointer
	yloop	5, c1b			;; Test loop counter
	end_timer 14

	ytwo_dest_adjust
	yfft_3_ret
	ENDM


;; Perform a 12K-element FFT.

yfft12K MACRO type
	LOCAL	ymid2, ymid3, ymid4, yfinish_unfft
	LOCAL	b1b, b2b, b3a, b3b, b4a, b4b, b5a, b5b, b6a, b6b, b6c, b6d
	LOCAL	c1b, c2b, c3a, c3b, c4a, c4b, c5a, c5b

;; Do a multiply with pre-FFTed inputs

	cmp	ffttype, 4
	je	ymid4

;; Do FFT levels 1,2,3
;; On input the 64-byte cache lines hold these data values:
;;	0	+384	+384	+384	6K	+384	+384	+384
;;	1
;;	...
;;	383
;;	1536	...
;;	3072	...
;;	4608	...
;; On output these are swizzled to:
;;	0	+3K	+3K	+3K	1536	+3K	+3K	+3K
;;	1
;;	...
;;	1535

;; Do 1536 eight_reals_fft operations
;;	distance between fft data elements is 1536
;;	do 384 macros each processing 32 data values

	start_timer 4
	mov	rdi, sincos1		;; Load sin/cos pointer
	bump	rdi, 512*YMM_SCD2+128*YMM_SCD3
	yloop_init  384			;; 384 iterations
	yr4_fs4cl_eight_reals_fft_preload
b1b:	yr4_fs4cl_eight_reals_fft rsi, 64, 6*dist64, 12*dist64, rdi, YMM_SCD3, 64
	yloop	64, b1b, -64*64+dist64	;; Test loop counter, next source pointer
	yloop	6, b1b, -6*dist64	;; Test loop counter, restore source pointer
	end_timer 4

;; Do FFT levels 4,5
;; On input the 64-byte cache lines hold these data values:
;;	0	+3K	+3K	+3K	1536	+3K	+3K	+3K
;;	1
;;	...
;;	1535
;; On output the 64-byte cache lines hold these data values:
;;	0	+3K	+3K	+3K	512	+3K	+3K	+3K
;;	1
;;	...
;;	511
;;	1K	...
;;	2K	...

;; Do 512 six_reals_fft operations
;; Do 1536 three_complex_fft operations
;;	distance between fft data elements is 512
;;	do 512 macros each processing 24 data values

	start_timer 5
	bump	rdi, -384*YMM_SCD3-512*YMM_SCD2 ;; Load sin/cos pointer
	yloop_init  512			;; 512 iterations
	yr3_3cl_six_reals_three_complex_djbfft_preload
b2b:	yr3_3cl_six_reals_three_complex_djbfft rsi, 64, 8*dist64, rdi, YMM_SCD2, 64
	yloop	64, b2b, -64*64+dist64	;; Test loop counter, next source pointer
	yloop	8, b2b, -8*dist64	;; Test loop counter, restore source pointer
	end_timer 5

;; Do FFT levels 6,7
;; On input the 64-byte cache lines hold these data values:
;;	0	+3K	+3K	+3K	512	+3K	+3K	+3K
;;	1
;;	...
;;	511
;;	1K	...
;;	2K	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+3K	+3K	+3K	128	+3K	+3K	+3K
;;	1
;;	...
;;	127
;;	256	...
;;	...
;;	2816	...

;; Do 128 eight_reals_fft operations
;; Do 1408 four_complex_fft operations
;;	distance between fft data elements is 128
;;	do 384 macros each processing 32 data values

	start_timer 6
	bump	rdi, -512*YMM_SCD2-128*YMM_SCD3 ;; Restore sin/cos pointer
	yloop_init  384			;; 3 iterations of 128
	yr4_4cl_eight_reals_four_complex_djbfft_preload
b3a:	yr4_4cl_eight_reals_four_complex_djbfft rsi, 64, 2*dist64, 4*dist64, rdi, YMM_SCD3, 64
	yloop	64, b3a, -64*64+dist64	;; Test loop counter, next source pointer
	yloop	2, b3a, -2*dist64+8*dist64, rdi, -128*YMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer

	loops_reset			;; 2 more iters of 128
	yr4_rb4cl_four_complex_djbfft_preload
b3b:	yr4_rb4cl_four_complex_djbfft rsi, 64, 2*dist64, 4*dist64, rdi, YMM_SCD3, 64
	yloop	64, b3b, -64*64+dist64	;; Test loop counter, next source pointer
	yloop	2, b3b, -2*dist64+8*dist64, rdi, -128*YMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer
	yloop	3, b3b, -3*8*dist64	;; Test loop counter, restore source pointer
	end_timer 6

;; Do FFT levels 8,9
;; On input the 64-byte cache lines hold these data values:
;;	0	+3K	+3K	+3K	128	+3K	+3K	+3K
;;	1
;;	...
;;	127
;;	256	...
;;	...
;;	2816	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+3K	+3K	+3K	32	+3K	+3K	+3K
;;	1
;;	...
;;	31
;;	64	...
;;	...
;;	3008	...

;; Do 32 eight_reals_fft operations
;; Do 1504 four_complex_fft operations
;;	distance between fft data elements is 32
;;	do 384 macros each processing 32 data values

	start_timer 7
	yloop_init  384			;; 12 iterations of 32
	yr4_4cl_eight_reals_four_complex_djbfft_preload
;; Loop below reworked to avoid 4KB distances
;;b4a:	yr4_4cl_eight_reals_four_complex_djbfft rsi, 64, 32*64, dist64, rdi, 4*YMM_SCD3, 32
;;	yloop	32, b4a, -32*64+2*dist64, rdi, -32*4*YMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer
b4a:	yr4_4cl_eight_reals_four_complex_djbfft rsi, 2*64, 32*64, dist64, rdi, 2*4*YMM_SCD3, 16
	yloop	16, b4a, -16*2*64+64, rdi, -16*2*4*YMM_SCD3+4*YMM_SCD3 ;; Test loop counter, next source pointer, next sin/cos pointer
	yloop	2, b4a, -2*64+2*dist64, rdi, -2*4*YMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer

	loops_reset			;; 11 more iters of 32
	yr4_rb4cl_four_complex_djbfft_preload
;; Loop below reworked to avoid 4KB distances
;;b4b:	yr4_rb4cl_four_complex_djbfft rsi, 64, 32*64, dist64, rdi, 4*YMM_SCD3, 32
;;	yloop	32, b4b, -32*64+2*dist64, rdi, -32*4*YMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer
b4b:	yr4_rb4cl_four_complex_djbfft rsi, 2*64, 32*64, dist64, rdi, 2*4*YMM_SCD3, 16
	yloop	16, b4b, -16*2*64+64, rdi, -16*2*4*YMM_SCD3+4*YMM_SCD3 ;; Test loop counter, next source pointer, next sin/cos pointer
	yloop	2, b4b, -2*64+2*dist64, rdi, -2*4*YMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer
	yloop	12, b4b, -12*2*dist64	;; Test loop counter, restore source pointer
	end_timer 7

;; Do FFT levels 10,11
;; On input the 64-byte cache lines hold these data values:
;;	0	+3K	+3K	+3K	32	+3K	+3K	+3K
;;	1
;;	...
;;	31
;;	64	...
;;	...
;;	3008	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+3K	+3K	+3K	8	+3K	+3K	+3K
;;	1
;;	...
;;	7
;;	16	...
;;	...
;;	3056	...

;; Do 8 eight_reals_fft operations
;; Do 1528 four_complex_fft operations
;;	distance between fft data elements is 8
;;	do 384 macros each processing 32 data values

	start_timer 8
	yloop_init  384			;; 48 iterations of 8
	yr4_4cl_eight_reals_four_complex_djbfft_preload
b5a:	yr4_4cl_eight_reals_four_complex_djbfft rsi, 64, 8*64, 16*64, rdi, 16*YMM_SCD3, 8
	yloop	8, b5a, -8*64+32*64, rdi, -8*16*YMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer

	loops_reset			;; 47 more iters of 8
	yr4_rb4cl_four_complex_djbfft_preload
b5b:	yr4_rb4cl_four_complex_djbfft rsi, 64, 8*64, 16*64, rdi, 16*YMM_SCD3, 8
	yloop	8, b5b, -8*64+32*64, rdi, -8*16*YMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer
	yloop	2, b5b, -2*32*64+dist64	;; Test loop counter, next source pointer
	yloop	24, b5b, -24*dist64	;; Test loop counter, restore source pointer
	end_timer 8

;; Do FFT levels 12,13
;; On input and output the 64-byte cache lines hold these data values:
;;	0	+3K	+3K	+3K	8	+3K	+3K	+3K
;;	1
;;	...
;;	7
;;	16	...
;;	...
;;	3056	...

;; Do 1 sixteen_reals_fft operation
;; Do 767 eight_complex_fft operations
;;	distance between fft data elements is 1
;;	do 192 macros each processing 64 data values

	cmp	ffttype, 2
	je	ymid2
	ja	ymid3

	start_timer 9
;;	yr8_8cl_sixteen_reals_eight_complex_fft_final rsi, 8*64, 64, 2*64, 4*64
	yr8_dispatch call, y8real_fft_final, 64, 2*64, 4*64
	yloop_init  192, 1, 8		;; 191 more iterations
b6a:	yr8_8cl_eight_complex_fft_final rsi, 8*64, 64, 2*64, 4*64
	yloop	8, b6a, -8*8*64+dist64	;; Test loop counter, next source pointer
	yloop	24, b6a			;; Test loop counter
	end_timer 9
	yfft_1_ret

ymid2:	start_timer 9
;;	yr8_8cl_sixteen_reals_eight_complex_with_square rsi, 8*64, 64, 2*64, 4*64
	yr8_dispatch call, y8real_square, 64, 2*64, 4*64
	yloop_init  192, 1, 8		;; 191 more iterations
	yr8_8cl_eight_complex_with_square_preload
b6b:	yr8_8cl_eight_complex_with_square rsi, 8*64, 64, 2*64, 4*64
	yloop	8, b6b, -8*8*64+dist64	;; Test loop counter, next source pointer
	yloop	24, b6b			;; Test loop counter
	end_timer 9
	jmp	yfinish_unfft

ymid3:	start_timer 9
x64	mov	r8, DEST2ARG
	mov	rbp, DIST_TO_MULSRCARG
;;	yr8_8cl_sixteen_reals_eight_complex_with_mult rsi, 8*64, 64, 2*64, 4*64
	yr8_dispatch call, y8real_mult, 64, 2*64, 4*64
	yloop_init  192, 1, 8		;; 191 more iterations
	yr8_8cl_eight_complex_with_mult_preload
b6c:	yr8_8cl_eight_complex_with_mult rsi, 8*64, 64, 2*64, 4*64
	yloop	8, b6c, -8*8*64+dist64	;; Test loop counter, next source pointer
	yloop	24, b6c			;; Test loop counter
x64	add	rsi, r8
	end_timer 9
	jmp	yfinish_unfft

ymid4:	start_timer 9
	mov	rbp, DIST_TO_MULSRCARG
;;	yr8_8cl_sixteen_reals_eight_complex_with_mulf rsi, 8*64, 64, 2*64, 4*64
	yr8_dispatch call, y8real_mulf, 64, 2*64, 4*64
	yloop_init  192, 1, 8		;; 191 more iterations
	yr8_8cl_eight_complex_with_mulf_preload
b6d:	yr8_8cl_eight_complex_with_mulf rsi, 8*64, 64, 2*64, 4*64
	yloop	8, b6d, -8*8*64+dist64	;; Test loop counter, next source pointer
	yloop	24, b6d			;; Test loop counter
	end_timer 9
yfinish_unfft:
	bump	rsi, -24*dist64		;; Restore source pointer

;; Do inverse FFT levels 10,11
;; On input the 64-byte cache lines hold these data values:
;;	0	+3K	+3K	+3K	8	+3K	+3K	+3K
;;	1
;;	...
;;	7
;;	16	...
;;	...
;;	3056	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+3K	+3K	+3K	32	+3K	+3K	+3K
;;	1
;;	...
;;	31
;;	64	...
;;	...
;;	3008	...

;; Do 8 eight_reals_unfft operations
;; Do 1528 four_complex_unfft operations
;;	distance between fft data elements is 8
;;	do 384 macros each processing 32 data values

	start_timer 10
	mov	rdi, sincos1		;; Load sin/cos pointer
	yloop_init  384			;; 48 iterations of 8
	yr4_4cl_eight_reals_four_complex_djbunfft_preload
c5a:	yr4_4cl_eight_reals_four_complex_djbunfft rsi, 64, 8*64, 16*64, rdi, 16*YMM_SCD3, 8
	yloop	8, c5a, -8*64+32*64, rdi, -8*16*YMM_SCD3 ;; Test loop counter, next source pointer, restore sine/cosine pointer

	loops_reset			;; 47 more iters of 8
	yr4_rb4cl_four_complex_djbunfft_preload
c5b:	yr4_rb4cl_four_complex_djbunfft rsi, 64, 8*64, 16*64, rdi, 16*YMM_SCD3, 8
	yloop	8, c5b, -8*64+32*64, rdi, -8*16*YMM_SCD3 ;; Test loop counter, next source pointer, restore sine/cosine pointer
	yloop	2, c5b, -2*32*64+dist64	;; Test loop counter, next source pointer
	yloop	24, c5b, -24*dist64	;; Test loop counter, restore source pointer
	end_timer 10

;; Do inverse FFT levels 8,9
;; On input the 64-byte cache lines hold these data values:
;;	0	+3K	+3K	+3K	32	+3K	+3K	+3K
;;	1
;;	...
;;	31
;;	64	...
;;	...
;;	3008	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+3K	+3K	+3K	128	+3K	+3K	+3K
;;	1
;;	...
;;	127
;;	256	...
;;	...
;;	2816	...

;; Do 32 eight_reals_unfft operations
;; Do 1496 four_complex_unfft operations
;;	distance between fft data elements is 32
;;	do 384 macros each processing 32 data values

	start_timer 11
	yloop_init  384			;; 12 iterations of 32
	yr4_4cl_eight_reals_four_complex_djbunfft_preload
;; Loop below reworked to avoid 4KB distances
;;c4a:	yr4_4cl_eight_reals_four_complex_djbunfft rsi, 64, 32*64, dist64, rdi, 4*YMM_SCD3, 32
;;	yloop	32, c4a, -32*64+2*dist64, rdi, -32*4*YMM_SCD3 ;; Test loop counter, next source pointer, restore sine/cosine pointer
c4a:	yr4_4cl_eight_reals_four_complex_djbunfft rsi, 2*64, 32*64, dist64, rdi, 2*4*YMM_SCD3, 16
	yloop	16, c4a, -16*2*64+64, rdi, -16*2*4*YMM_SCD3+4*YMM_SCD3 ;; Test loop counter, next source pointer, next sine/cosine pointer
	yloop	2, c4a, -2*64+2*dist64, rdi, -2*4*YMM_SCD3 ;; Test loop counter, next source pointer, restore sine/cosine pointer

	loops_reset			;; 11 more iters of 32
	yr4_rb4cl_four_complex_djbunfft_preload
;; Loop below reworked to avoid 4KB distances
;;c4b:	yr4_rb4cl_four_complex_djbunfft rsi, 64, 32*64, dist64, rdi, 4*YMM_SCD3, 32
;;	yloop	32, c4b, -32*64+2*dist64, rdi, -32*4*YMM_SCD3 ;; Test loop counter, next source pointer, restore sine/cosine pointer
c4b:	yr4_rb4cl_four_complex_djbunfft rsi, 2*64, 32*64, dist64, rdi, 2*4*YMM_SCD3, 16
	yloop	16, c4b, -16*2*64+64, rdi, -16*2*4*YMM_SCD3+4*YMM_SCD3 ;; Test loop counter, next source pointer, next sine/cosine pointer
	yloop	2, c4b, -2*64+2*dist64, rdi, -2*4*YMM_SCD3 ;; Test loop counter, next source pointer, restore sine/cosine pointer
	yloop	12, c4b, -12*2*dist64	;; Test loop counter, restore source pointer
	end_timer 11

;; Do inverse FFT levels 6,7
;; On input the 64-byte cache lines hold these data values:
;;	0	+3K	+3K	+3K	128	+3K	+3K	+3K
;;	1
;;	...
;;	127
;;	256	...
;;	...
;;	2816	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+3K	+3K	+3K	512	+3K	+3K	+3K
;;	1
;;	...
;;	511
;;	1K	...
;;	2K	...

;; Do 128 eight_reals_unfft operations
;; Do 1416 four_complex_unfft operations
;;	distance between fft data elements is 128
;;	do 384 macros each processing 32 data values

	start_timer 12
	yloop_init  384			;; 3 iterations of 128
	yr4_4cl_eight_reals_four_complex_djbunfft_preload
c3a:	yr4_4cl_eight_reals_four_complex_djbunfft rsi, 64, 2*dist64, 4*dist64, rdi, YMM_SCD3, 64
	yloop	64, c3a, -64*64+dist64	;; Test loop counter, next source pointer
	yloop	2, c3a, -2*dist64+8*dist64, rdi, -128*YMM_SCD3 ;; Test loop counter, next source pointer, restore sine/cosine pointer

	loops_reset			;; 2 more iters of 128
	yr4_rb4cl_four_complex_djbunfft_preload
c3b:	yr4_rb4cl_four_complex_djbunfft rsi, 64, 2*dist64, 4*dist64, rdi, YMM_SCD3, 64
	yloop	64, c3b, -64*64+dist64	;; Test loop counter, next source pointer
	yloop	2, c3b, -2*dist64+8*dist64, rdi, -128*YMM_SCD3 ;; Test loop counter, next source pointer, restore sine/cosine pointer
	yloop	3, c3b, -3*8*dist64	;; Test loop counter, restore source pointer
	end_timer 12

;; Do inverse FFT levels 4,5
;; On input the 64-byte cache lines hold these data values:
;;	0	+3K	+3K	+3K	512	+3K	+3K	+3K
;;	1
;;	...
;;	511
;;	1K	...
;;	2K	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+3K	+3K	+3K	1536	+3K	+3K	+3K
;;	1
;;	...
;;	1535

;; Do 512 six_reals_unfft operations
;; Do 1536 three_complex_unfft operations
;;	distance between fft data elements is 512
;;	do 512 macros each processing 24 data values

	start_timer 13
	bump	rdi, 128*YMM_SCD3	;; Next sine/cosine pointer
	yloop_init  512			;; 512 iterations
	yr3_3cl_six_reals_three_complex_djbunfft_preload
c2b:	yr3_3cl_six_reals_three_complex_djbunfft rsi, 64, 8*dist64, rdi, YMM_SCD2, 64
	yloop	64, c2b, -64*64+dist64	;; Test loop counter, next source pointer
	yloop	8, c2b, -8*dist64	;; Test loop counter, restore source pointer
	end_timer 13

;; Do inverse FFT levels 1,2,3
;; On input the 64-byte cache lines hold these data values:
;;	0	+3K	+3K	+3K	1536	+3K	+3K	+3K
;;	1
;;	...
;;	1535
;; These are swizzled to:
;;	0	+384	+384	+384	1536	+384	+384	+384
;;	1	...
;;	...
;;	383
;;	3K	...
;;	6K	...
;;	9K	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+384	+384	+384	6K	+384	+384	+384
;;	1	...
;;	...
;;	383
;;	1536	...
;;	3072	...
;;	4608	...

;; Do 1536 eight_reals_unfft operations
;;	distance between fft data elements is 1536
;;	do 384 macros each processing 32 data values

	start_timer 14
	yloop_init  384			;; 384 iterations
	yr4_s4cl_eight_reals_unfft_preload
c1b:	yr4_s4cl_eight_reals_unfft rsi, 64, 6*dist64, 12*dist64, rdi, YMM_SCD3, 64
	yloop	64, c1b, -64*64+dist64	;; Test loop counter, next source pointer
	yloop	6, c1b			;; Test loop counter
	end_timer 14

	ytwo_dest_adjust
	yfft_3_ret
	ENDM

;; Perform a 12K-element negacyclic FFT

yfft12Kp MACRO type
	LOCAL	ymid2p, ymid3p, ymid4p, yfinish_unfft
	LOCAL	b1b, b2b, b3b, b4b, b5b, b6a, b6b, b6c, b6d, c1b, c2b, c3b, c4b, c5b

;; Do a multiply with pre-FFTed inputs

	cmp	ffttype, 4
	je	ymid4p

;; Do FFT levels 1,2,3
;; On input the 64-byte cache lines hold these data values:
;;	0	+384	+384	+384	6K	+384	+384	+384
;;	1
;;	...
;;	383
;;	1536	...
;;	3072	...
;;	4608	...
;; On output these are swizzled to:
;;	0	+3K	+3K	+3K	1536	+3K	+3K	+3K
;;	1
;;	...
;;	1535

;; Do 1536 four_complex_first_fft operations
;;	distance between fft data elements is 1536
;;	do 384 macros each processing 32 data values

	start_timer 4
	mov	rdi, sincos1		;; Load sin/cos pointer
	bump	rdi, 512*YMM_SCD1/4+128*YMM_SCD2/4
	yloop_init  384			;; 384 iterations
	yr4_fs4cl_four_complex_first_djbfft_preload
b1b:	yr4_fs4cl_four_complex_first_djbfft rsi, 64, 6*dist64, 12*dist64, rdi, YMM_SCD4, 64
	yloop	64, b1b, -64*64+dist64	;; Test loop counter, next source pointer
	yloop	6, b1b, -6*dist64	;; Test loop counter, restore source pointer
	end_timer 4

;; Do FFT levels 4,5
;; On input the 64-byte cache lines hold these data values:
;;	0	+3K	+3K	+3K	1536	+3K	+3K	+3K
;;	1
;;	...
;;	1535
;; On output the 64-byte cache lines hold these data values:
;;	0	+3K	+3K	+3K	512	+3K	+3K	+3K
;;	1
;;	...
;;	511
;;	1K	...
;;	2K	...

;; Do 2048 three_complex_fft operations
;;	distance between fft data elements is 512
;;	do 512 macros each processing 24 data values

	start_timer 5
	bump	rdi, -384*YMM_SCD4-512*YMM_SCD1/4 ;; Load sin/cos pointer
	yloop_init  512			;; 512 iterations
	yr3_b3cl_three_complex_djbfft_preload
b2b:	yr3_b3cl_three_complex_djbfft rsi, 64, 8*dist64, rdi, YMM_SCD1/4, 64
	yloop	64, b2b, -64*64+dist64	;; Test loop counter, next source pointer
	yloop	8, b2b, -8*dist64	;; Test loop counter, restore source pointer
	end_timer 5

;; Do FFT levels 6,7
;; On input the 64-byte cache lines hold these data values:
;;	0	+3K	+3K	+3K	512	+3K	+3K	+3K
;;	1
;;	...
;;	511
;;	1K	...
;;	2K	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+3K	+3K	+3K	128	+3K	+3K	+3K
;;	1
;;	...
;;	127
;;	256	...
;;	...
;;	2816	...

;; Do 1536 four_complex_fft operations
;;	distance between fft data elements is 128
;;	do 384 macros each processing 32 data values

	start_timer 6
	bump	rdi, -512*YMM_SCD1/4-128*YMM_SCD2/4 ;; Next sin/cos pointer
	yloop_init  384			;; 3 iterations of 128
	yr4_b4cl_four_complex_djbfft_preload
b3b:	yr4_b4cl_four_complex_djbfft rsi, 64, 2*dist64, 4*dist64, rdi, YMM_SCD2/4, 64
	yloop	64, b3b, -64*64+dist64	;; Test loop counter, next source pointer
	yloop	2, b3b, -2*dist64+8*dist64, rdi, -128*YMM_SCD2/4 ;; Test loop counter, next source pointer, restore sin/cos pointer
	yloop	3, b3b, -3*8*dist64	;; Test loop counter, restore source pointer
	end_timer 6

;; Do FFT levels 8,9
;; On input the 64-byte cache lines hold these data values:
;;	0	+3K	+3K	+3K	128	+3K	+3K	+3K
;;	1
;;	...
;;	127
;;	256	...
;;	...
;;	2816	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+3K	+3K	+3K	32	+3K	+3K	+3K
;;	1
;;	...
;;	31
;;	64	...
;;	...
;;	3008	...

;; Do 1536 four_complex_fft operations
;;	distance between fft data elements is 32
;;	do 384 macros each processing 32 data values

	start_timer 7
	yloop_init  384			;; 12 iterations of 32
	yr4_b4cl_four_complex_djbfft_preload
;; Loop below reworked to avoid 4KB distances
;;b4b:	yr4_b4cl_four_complex_djbfft rsi, 64, 32*64, dist64, rdi, 4*YMM_SCD2/4, 32
;;	yloop	32, b4b, -32*64+2*dist64, rdi, -32*4*YMM_SCD2/4 ;; Test loop counter, next source pointer, restore sin/cos pointer
b4b:	yr4_b4cl_four_complex_djbfft rsi, 2*64, 32*64, dist64, rdi, 2*4*YMM_SCD2/4, 16
	yloop	16, b4b, -16*2*64+64, rdi, -16*2*4*YMM_SCD2/4+4*YMM_SCD2/4 ;; Test loop counter, next source pointer, next sin/cos pointer
	yloop	2, b4b, -2*64+2*dist64, rdi, -2*4*YMM_SCD2/4 ;; Test loop counter, next source pointer, restore sin/cos pointer
	yloop	12, b4b, -12*2*dist64	;; Test loop counter, restore source pointer
	end_timer 7

;; Do FFT levels 10,11
;; On input the 64-byte cache lines hold these data values:
;;	0	+3K	+3K	+3K	32	+3K	+3K	+3K
;;	1
;;	...
;;	31
;;	64	...
;;	...
;;	3008	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+3K	+3K	+3K	8	+3K	+3K	+3K
;;	1
;;	...
;;	7
;;	16	...
;;	...
;;	3056	...

;; Do 1536 four_complex_fft operations
;;	distance between fft data elements is 8
;;	do 384 macros each processing 32 data values

	start_timer 8
	yloop_init  384			;; 48 iterations of 8
	yr4_b4cl_four_complex_djbfft_preload
b5b:	yr4_b4cl_four_complex_djbfft rsi, 64, 8*64, 16*64, rdi, 16*YMM_SCD2/4, 8
	yloop	8, b5b, -8*64+32*64, rdi, -8*16*YMM_SCD2/4 ;; Test loop counter, next source pointer, restore sin/cos pointer
	yloop	2, b5b, -2*32*64+dist64	;; Test loop counter, next source pointer
	yloop	24, b5b, -24*dist64	;; Test loop counter, restore source pointer
	end_timer 8

;; Do FFT levels 12,13
;; On input and output the 64-byte cache lines hold these data values:
;;	0	+3K	+3K	+3K	8	+3K	+3K	+3K
;;	1
;;	...
;;	7
;;	16	...
;;	...
;;	3056	...

;; Do 768 eight_complex_fft operations
;;	distance between fft data elements is 1
;;	do 192 macros each processing 64 data values

	cmp	ffttype, 2
	je	ymid2p
	ja	ymid3p

	start_timer 9
	yloop_init  192			;; 192 iterations
b6a:	yr8_8cl_eight_complex_fft_final rsi, 8*64, 64, 2*64, 4*64
	yloop	8, b6a, -8*8*64+dist64	;; Test loop counter, next source pointer
	yloop	24, b6a			;; Test loop counter
	end_timer 9
	yfft_1_ret

ymid2p:	start_timer 9
	yloop_init  192			;; 192 iterations
	yr8_8cl_eight_complex_with_square_preload
b6b:	yr8_8cl_eight_complex_with_square rsi, 8*64, 64, 2*64, 4*64
	yloop	8, b6b, -8*8*64+dist64	;; Test loop counter, next source pointer
	yloop	24, b6b			;; Test loop counter
	end_timer 9
	jmp	yfinish_unfft

ymid3p:	start_timer 9
x64	mov	r8, DEST2ARG
	mov	rbp, DIST_TO_MULSRCARG
	yloop_init  192			;; 192 iterations
	yr8_8cl_eight_complex_with_mult_preload
b6c:	yr8_8cl_eight_complex_with_mult rsi, 8*64, 64, 2*64, 4*64
	yloop	8, b6c, -8*8*64+dist64	;; Test loop counter, next source pointer
	yloop	24, b6c			;; Test loop counter
x64	add	rsi, r8
	end_timer 9
	jmp	yfinish_unfft

ymid4p:	start_timer 9
	mov	rbp, DIST_TO_MULSRCARG
	yloop_init  192			;; 192 iterations
	yr8_8cl_eight_complex_with_mulf_preload
b6d:	yr8_8cl_eight_complex_with_mulf rsi, 8*64, 64, 2*64, 4*64
	yloop	8, b6d, -8*8*64+dist64	;; Test loop counter, next source pointer
	yloop	24, b6d			;; Test loop counter
	end_timer 9
yfinish_unfft:
	bump	rsi, -24*dist64		;; Restore source pointer

;; Do inverse FFT levels 10,11
;; On input the 64-byte cache lines hold these data values:
;;	0	+3K	+3K	+3K	8	+3K	+3K	+3K
;;	1
;;	...
;;	7
;;	16	...
;;	...
;;	3056	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+3K	+3K	+3K	32	+3K	+3K	+3K
;;	1
;;	...
;;	31
;;	64	...
;;	...
;;	3008	...

;; Do 1536 four_complex_unfft operations
;;	distance between fft data elements is 8
;;	do 384 macros each processing 32 data values

	start_timer 10
	mov	rdi, sincos1		;; Load sin/cos pointer
	yloop_init  384			;; 48 iterations of 8
	yr4_b4cl_four_complex_djbunfft_preload
c5b:	yr4_b4cl_four_complex_djbunfft rsi, 64, 8*64, 16*64, rdi, 16*YMM_SCD2/4, 8
	yloop	8, c5b, -8*64+32*64, rdi, -8*16*YMM_SCD2/4 ;; Test loop counter, next source pointer, restore sine/cosine pointer
	yloop	2, c5b, -2*32*64+dist64	;; Test loop counter, next source pointer
	yloop	24, c5b, -24*dist64	;; Test loop counter, restore source pointer
	end_timer 10

;; Do inverse FFT levels 8,9
;; On input the 64-byte cache lines hold these data values:
;;	0	+3K	+3K	+3K	32	+3K	+3K	+3K
;;	1
;;	...
;;	31
;;	64	...
;;	...
;;	3008	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+3K	+3K	+3K	128	+3K	+3K	+3K
;;	1
;;	...
;;	127
;;	256	...
;;	...
;;	2816	...

;; Do 1536 four_complex_unfft operations
;;	distance between fft data elements is 32
;;	do 384 macros each processing 32 data values

	start_timer 11
	yloop_init  384			;; 12 iterations of 32
	yr4_b4cl_four_complex_djbunfft_preload
;; Loop below reworked to avoid 4KB distances
;;c4b:	yr4_b4cl_four_complex_djbunfft rsi, 64, 32*64, dist64, rdi, 4*YMM_SCD2/4, 32
;;	yloop	32, c4b, -32*64+2*dist64, rdi, -32*4*YMM_SCD2/4 ;; Test loop counter, next source pointer, restore sine/cosine pointer
c4b:	yr4_b4cl_four_complex_djbunfft rsi, 2*64, 32*64, dist64, rdi, 2*4*YMM_SCD2/4, 16
	yloop	16, c4b, -16*2*64+64, rdi, -16*2*4*YMM_SCD2/4+4*YMM_SCD2/4 ;; Test loop counter, next source pointer, next sine/cosine pointer
	yloop	2, c4b, -2*64+2*dist64, rdi, -2*4*YMM_SCD2/4 ;; Test loop counter, next source pointer, restore sine/cosine pointer
	yloop	12, c4b, -12*2*dist64	;; Test loop counter, restore source pointer
	end_timer 11

;; Do inverse FFT levels 6,7
;; On input the 64-byte cache lines hold these data values:
;;	0	+3K	+3K	+3K	128	+3K	+3K	+3K
;;	1
;;	...
;;	127
;;	256	...
;;	...
;;	2816	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+3K	+3K	+3K	512	+3K	+3K	+3K
;;	1
;;	...
;;	511
;;	1K	...
;;	2K	...

;; Do 1536 four_complex_unfft operations
;;	distance between fft data elements is 128
;;	do 384 macros each processing 32 data values

	start_timer 12
	yloop_init  384			;; 3 iterations of 128
	yr4_b4cl_four_complex_djbunfft_preload
c3b:	yr4_b4cl_four_complex_djbunfft rsi, 64, 2*dist64, 4*dist64, rdi, YMM_SCD2/4, 64
	yloop	64, c3b, -64*64+dist64	;; Test loop counter, next source pointer
	yloop	2, c3b, -2*dist64+8*dist64, rdi, -128*YMM_SCD2/4 ;; Test loop counter, next source pointer, restore sine/cosine pointer
	yloop	3, c3b, -3*8*dist64	;; Test loop counter, restore source pointer
	end_timer 12

;; Do inverse FFT levels 4,5
;; On input the 64-byte cache lines hold these data values:
;;	0	+3K	+3K	+3K	512	+3K	+3K	+3K
;;	1
;;	...
;;	511
;;	1K	...
;;	2K	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+3K	+3K	+3K	1536	+3K	+3K	+3K
;;	1
;;	...
;;	1535

;; Do 2048 three_complex_unfft operations
;;	distance between fft data elements is 512
;;	do 512 macros each processing 24 data values

	start_timer 13
	bump	rdi, 128*YMM_SCD2/4	;; Restore sine/cosine pointer
	yloop_init  512			;; 512 iterations
	yr3_b3cl_three_complex_djbunfft_preload
c2b:	yr3_b3cl_three_complex_djbunfft rsi, 64, 8*dist64, rdi, YMM_SCD1/4, 64
	yloop	64, c2b, -64*64+dist64	;; Test loop counter, next source pointer
	yloop	8, c2b, -8*dist64	;; Test loop counter, restore source pointer
	end_timer 13

;; Do inverse FFT levels 1,2,3
;; On input the 64-byte cache lines hold these data values:
;;	0	+3K	+3K	+3K	1536	+3K	+3K	+3K
;;	1
;;	...
;;	1535
;; These are swizzled to:
;;	0	+384	+384	+384	1536	+384	+384	+384
;;	1	...
;;	...
;;	383
;;	3K	...
;;	6K	...
;;	9K	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+384	+384	+384	6K	+384	+384	+384
;;	1	...
;;	...
;;	383
;;	1536	...
;;	3072	...
;;	4608	...

;; Do 1536 four_complex_last_unfft operations
;;	distance between fft data elements is 1536
;;	do 384 macros each processing 32 data values

	start_timer 14
	yloop_init  384			;; 384 iterations
	yr4_s4cl_four_complex_last_unfft_preload
c1b:	yr4_s4cl_four_complex_last_unfft rsi, 64, 6*dist64, 12*dist64, rdi, YMM_SCD4, 64
	yloop	64, c1b, -64*64+dist64	;; Test loop counter, next source pointer
	yloop	6, c1b			;; Test loop counter
	end_timer 14

	ytwo_dest_adjust
	yfft_3_ret
	ENDM


;; Perform a 16K-element FFT.

yfft16K MACRO type
	LOCAL	ymid2, ymid3, ymid4, yfinish_unfft
	LOCAL	b1b, b2b, b3a, b3b, b4a, b4b, b5a, b5b, b6a, b6b, b6c, b6d
	LOCAL	c1b, c2b, c3a, c3b, c4a, c4b, c5a, c5b

;; Do a multiply with pre-FFTed inputs

	cmp	ffttype, 4
	je	ymid4

;; Do FFT levels 1,2,3
;; On input the 64-byte cache lines hold these data values:
;;	0	+512	+512	+512	8K	+512	+512	+512
;;	1
;;	...
;;	511
;;	2K	...
;;	4K	...
;;	6K	...
;; On output these are swizzled to:
;;	0	+4K	+4K	+4K	2K	+4K	+4K	+4K
;;	1
;;	...
;;	2047

;; Do 2048 eight_reals_fft operations
;;	distance between fft data elements is 2048
;;	do 512 macros each processing 32 data values

	start_timer 4
	mov	rdi, sincos1		;; Load sin/cos pointer
	bump	rdi, 512*YMM_SCD3
	yloop_init  512			;; 512 iterations
	yr4_fs4cl_eight_reals_fft_preload
b1b:	yr4_fs4cl_eight_reals_fft rsi, 64, 8*dist64, 16*dist64, rdi, YMM_SCD3, 64
	yloop	64, b1b, -64*64+dist64	;; Test loop counter, next source pointer
	yloop	8, b1b, -8*dist64	;; Test loop counter, restore source pointer
	end_timer 4

;; Do FFT levels 4,5
;; On input the 64-byte cache lines hold these data values:
;;	0	+4K	+4K	+4K	2K	+4K	+4K	+4K
;;	1
;;	...
;;	2047
;; On output the 64-byte cache lines hold these data values:
;;	0	+4K	+4K	+4K	512	+4K	+4K	+4K
;;	1
;;	...
;;	511
;;	1K	...
;;	2K	...
;;	3K	...

;; Do 512 eight_reals_fft operations
;; Do 1536 four_complex_fft operations
;;	distance between fft data elements is 512
;;	do 512 macros each processing 32 data values

	start_timer 5
	bump	rdi, -512*YMM_SCD3-512*YMM_SCD3 ;; Load sin/cos pointer
	yloop_init  512			;; 512 iterations
	yr4_4cl_eight_reals_four_complex_djbfft_preload
b2b:	yr4_4cl_eight_reals_four_complex_djbfft rsi, 64, 8*dist64, 16*dist64, rdi, YMM_SCD3, 64
	yloop	64, b2b, -64*64+dist64	;; Test loop counter, next source pointer
	yloop	8, b2b, -8*dist64, rdi, -512*YMM_SCD3 ;; Test loop counter, restore source pointer, restore sin/cos pointer
	end_timer 5

;; Do FFT levels 6,7
;; On input the 64-byte cache lines hold these data values:
;;	0	+4K	+4K	+4K	512	+4K	+4K	+4K
;;	1
;;	...
;;	511
;;	1K	...
;;	2K	...
;;	3K	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+4K	+4K	+4K	128	+4K	+4K	+4K
;;	1
;;	...
;;	127
;;	256	...
;;	...
;;	3840	...

;; Do 128 eight_reals_fft operations
;; Do 1920 four_complex_fft operations
;;	distance between fft data elements is 128
;;	do 512 macros each processing 32 data values

	start_timer 6
	yloop_init  512			;; 4 iterations of 128
	yr4_4cl_eight_reals_four_complex_djbfft_preload
b3a:	yr4_4cl_eight_reals_four_complex_djbfft rsi, 64, 2*dist64, 4*dist64, rdi, 4*YMM_SCD3, 64
	yloop	64, b3a, -64*64+dist64	;; Test loop counter, next source pointer
	yloop	2, b3a, -2*dist64+8*dist64, rdi, -128*4*YMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer

	loops_reset			;; 3 more iters of 128
	yr4_rb4cl_four_complex_djbfft_preload
b3b:	yr4_rb4cl_four_complex_djbfft rsi, 64, 2*dist64, 4*dist64, rdi, 4*YMM_SCD3, 64
	yloop	64, b3b, -64*64+dist64	;; Test loop counter, next source pointer
	yloop	2, b3b, -2*dist64+8*dist64, rdi, -128*4*YMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer
	yloop	4, b3b, -4*8*dist64	;; Test loop counter, restore source pointer
	end_timer 6

;; Do FFT levels 8,9
;; On input the 64-byte cache lines hold these data values:
;;	0	+4K	+4K	+4K	128	+4K	+4K	+4K
;;	1
;;	...
;;	127
;;	256	...
;;	...
;;	3840	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+4K	+4K	+4K	32	+4K	+4K	+4K
;;	1
;;	...
;;	31
;;	64	...
;;	...
;;	4032	...

;; Do 32 eight_reals_fft operations
;; Do 2016 four_complex_fft operations
;;	distance between fft data elements is 32
;;	do 512 macros each processing 32 data values

	start_timer 7
	yloop_init  512			;; 16 iterations of 32
	yr4_4cl_eight_reals_four_complex_djbfft_preload
;; Loop below reworked to avoid 4KB distances
;;b4a:	yr4_4cl_eight_reals_four_complex_djbfft rsi, 64, 32*64, dist64, rdi, 16*YMM_SCD3, 32
;;	yloop	32, b4a, -32*64+2*dist64, rdi, -32*16*YMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer
b4a:	yr4_4cl_eight_reals_four_complex_djbfft rsi, 2*64, 32*64, dist64, rdi, 2*16*YMM_SCD3, 16
	yloop	16, b4a, -16*2*64+64, rdi, -16*2*16*YMM_SCD3+16*YMM_SCD3 ;; Test loop counter, next source pointer, next sin/cos pointer
	yloop	2, b4a, -2*64+2*dist64, rdi, -2*16*YMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer

	loops_reset			;; 15 more iters of 32
	yr4_rb4cl_four_complex_djbfft_preload
;; Loop below reworked to avoid 4KB distances
;;b4b:	yr4_rb4cl_four_complex_djbfft rsi, 64, 32*64, dist64, rdi, 16*YMM_SCD3, 32
;;	yloop	32, b4b, -32*64+2*dist64, rdi, -32*16*YMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer
b4b:	yr4_rb4cl_four_complex_djbfft rsi, 2*64, 32*64, dist64, rdi, 2*16*YMM_SCD3, 16
	yloop	16, b4b, -16*2*64+64, rdi, -16*2*16*YMM_SCD3+16*YMM_SCD3 ;; Test loop counter, next source pointer, next sin/cos pointer
	yloop	2, b4b, -2*64+2*dist64, rdi, -2*16*YMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer
	yloop	16, b4b, -16*2*dist64	;; Test loop counter, restore source pointer
	end_timer 7

;; Do FFT levels 10,11
;; On input the 64-byte cache lines hold these data values:
;;	0	+4K	+4K	+4K	32	+4K	+4K	+4K
;;	1
;;	...
;;	31
;;	64	...
;;	...
;;	4032	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+4K	+4K	+4K	8	+4K	+4K	+4K
;;	1
;;	...
;;	7
;;	16	...
;;	...
;;	4080	...

;; Do 8 eight_reals_fft operations
;; Do 2040 four_complex_fft operations
;;	distance between fft data elements is 8
;;	do 512 macros each processing 32 data values

	start_timer 8
	yloop_init  512			;; 64 iterations of 8
	yr4_4cl_eight_reals_four_complex_djbfft_preload
b5a:	yr4_4cl_eight_reals_four_complex_djbfft rsi, 64, 8*64, 16*64, rdi, 64*YMM_SCD3, 8
	yloop	8, b5a, -8*64+32*64, rdi, -8*64*YMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer

	loops_reset			;; 63 more iters of 8
	yr4_rb4cl_four_complex_djbfft_preload
b5b:	yr4_rb4cl_four_complex_djbfft rsi, 64, 8*64, 16*64, rdi, 64*YMM_SCD3, 8
	yloop	8, b5b, -8*64+32*64, rdi, -8*64*YMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer
	yloop	2, b5b, -2*32*64+dist64	;; Test loop counter, next source pointer
	yloop	32, b5b, -32*dist64	;; Test loop counter, restore source pointer
	end_timer 8

;; Do FFT levels 12,13
;; On input and output the 64-byte cache lines hold these data values:
;;	0	+4K	+4K	+4K	8	+4K	+4K	+4K
;;	1
;;	...
;;	7
;;	16	...
;;	...
;;	4080	...

;; Do 1 sixteen_reals_fft operation
;; Do 1023 eight_complex_fft operations
;;	distance between fft data elements is 1
;;	do 256 macros each processing 64 data values

	cmp	ffttype, 2
	je	ymid2
	ja	ymid3

	start_timer 9
;;	yr8_8cl_sixteen_reals_eight_complex_fft_final rsi, 8*64, 64, 2*64, 4*64
	yr8_dispatch call, y8real_fft_final, 64, 2*64, 4*64
	yloop_init  256, 1, 8		;; 255 more iterations
b6a:	yr8_8cl_eight_complex_fft_final rsi, 8*64, 64, 2*64, 4*64
	yloop	8, b6a, -8*8*64+dist64	;; Test loop counter, next source pointer
	yloop	32, b6a			;; Test loop counter
	end_timer 9
	yfft_1_ret

ymid2:	start_timer 9
;;	yr8_8cl_sixteen_reals_eight_complex_with_square rsi, 8*64, 64, 2*64, 4*64
	yr8_dispatch call, y8real_square, 64, 2*64, 4*64
	yloop_init  256, 1, 8		;; 255 more iterations
	yr8_8cl_eight_complex_with_square_preload
b6b:	yr8_8cl_eight_complex_with_square rsi, 8*64, 64, 2*64, 4*64
	yloop	8, b6b, -8*8*64+dist64	;; Test loop counter, next source pointer
	yloop	32, b6b			;; Test loop counter
	end_timer 9
	jmp	yfinish_unfft

ymid3:	start_timer 9
x64	mov	r8, DEST2ARG
	mov	rbp, DIST_TO_MULSRCARG
;;	yr8_8cl_sixteen_reals_eight_complex_with_mult rsi, 8*64, 64, 2*64, 4*64
	yr8_dispatch call, y8real_mult, 64, 2*64, 4*64
	yloop_init  256, 1, 8		;; 255 more iterations
	yr8_8cl_eight_complex_with_mult_preload
b6c:	yr8_8cl_eight_complex_with_mult rsi, 8*64, 64, 2*64, 4*64
	yloop	8, b6c, -8*8*64+dist64	;; Test loop counter, next source pointer
	yloop	32, b6c			;; Test loop counter
x64	add	rsi, r8
	end_timer 9
	jmp	yfinish_unfft

ymid4:	start_timer 9
	mov	rbp, DIST_TO_MULSRCARG
;;	yr8_8cl_sixteen_reals_eight_complex_with_mulf rsi, 8*64, 64, 2*64, 4*64
	yr8_dispatch call, y8real_mulf, 64, 2*64, 4*64
	yloop_init  256, 1, 8		;; 255 more iterations
	yr8_8cl_eight_complex_with_mulf_preload
b6d:	yr8_8cl_eight_complex_with_mulf rsi, 8*64, 64, 2*64, 4*64
	yloop	8, b6d, -8*8*64+dist64	;; Test loop counter, next source pointer
	yloop	32, b6d			;; Test loop counter
	end_timer 9
yfinish_unfft:
	bump	rsi, -32*dist64		;; Restore source pointer

;; Do inverse FFT levels 10,11
;; On input the 64-byte cache lines hold these data values:
;;	0	+4K	+4K	+4K	8	+4K	+4K	+4K
;;	1
;;	...
;;	7
;;	16	...
;;	...
;;	4080	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+4K	+4K	+4K	32	+4K	+4K	+4K
;;	1
;;	...
;;	31
;;	64	...
;;	...
;;	4032	...

;; Do 8 eight_reals_unfft operations
;; Do 2040 four_complex_unfft operations
;;	distance between fft data elements is 8
;;	do 512 macros each processing 32 data values

	start_timer 10
	mov	rdi, sincos1		;; Load sin/cos pointer
	yloop_init  512			;; 64 iterations of 8
	yr4_4cl_eight_reals_four_complex_djbunfft_preload
c5a:	yr4_4cl_eight_reals_four_complex_djbunfft rsi, 64, 8*64, 16*64, rdi, 64*YMM_SCD3, 8
	yloop	8, c5a, -8*64+32*64, rdi, -8*64*YMM_SCD3 ;; Test loop counter, next source pointer, restore sine/cosine pointer

	loops_reset			;; 63 more iters of 8
	yr4_rb4cl_four_complex_djbunfft_preload
c5b:	yr4_rb4cl_four_complex_djbunfft rsi, 64, 8*64, 16*64, rdi, 64*YMM_SCD3, 8
	yloop	8, c5b, -8*64+32*64, rdi, -8*64*YMM_SCD3 ;; Test loop counter, next source pointer, restore sine/cosine pointer
	yloop	2, c5b, -2*32*64+dist64	;; Test loop counter, next source pointer
	yloop	32, c5b, -32*dist64	;; Test loop counter, restore source pointer
	end_timer 10

;; Do inverse FFT levels 8,9
;; On input the 64-byte cache lines hold these data values:
;;	0	+4K	+4K	+4K	32	+4K	+4K	+4K
;;	1
;;	...
;;	31
;;	64	...
;;	...
;;	4032	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+4K	+4K	+4K	128	+4K	+4K	+4K
;;	1
;;	...
;;	127
;;	256	...
;;	...
;;	3840	...

;; Do 32 eight_reals_unfft operations
;; Do 2016 four_complex_unfft operations
;;	distance between fft data elements is 32
;;	do 512 macros each processing 32 data values

	start_timer 11
	yloop_init  512			;; 16 iterations of 32
	yr4_4cl_eight_reals_four_complex_djbunfft_preload
;; Loop below reworked to avoid 4KB distances
;;c4a:	yr4_4cl_eight_reals_four_complex_djbunfft rsi, 64, 32*64, dist64, rdi, 16*YMM_SCD3, 32
;;	yloop	32, c4a, -32*64+2*dist64, rdi, -32*16*YMM_SCD3 ;; Test loop counter, next source pointer, restore sine/cosine pointer
c4a:	yr4_4cl_eight_reals_four_complex_djbunfft rsi, 2*64, 32*64, dist64, rdi, 2*16*YMM_SCD3, 16
	yloop	16, c4a, -16*2*64+64, rdi, -16*2*16*YMM_SCD3+16*YMM_SCD3 ;; Test loop counter, next source pointer, next sine/cosine pointer
	yloop	2, c4a, -2*64+2*dist64, rdi, -2*16*YMM_SCD3 ;; Test loop counter, next source pointer, restore sine/cosine pointer

	loops_reset			;; 15 more iters of 32
	yr4_rb4cl_four_complex_djbunfft_preload
;; Loop below reworked to avoid 4KB distances
;;c4b:	yr4_rb4cl_four_complex_djbunfft rsi, 64, 32*64, dist64, rdi, 16*YMM_SCD3, 32
;;	yloop	32, c4b, -32*64+2*dist64, rdi, -32*16*YMM_SCD3 ;; Test loop counter, next source pointer, restore sine/cosine pointer
c4b:	yr4_rb4cl_four_complex_djbunfft rsi, 2*64, 32*64, dist64, rdi, 2*16*YMM_SCD3, 16
	yloop	16, c4b, -16*2*64+64, rdi, -16*2*16*YMM_SCD3+16*YMM_SCD3 ;; Test loop counter, next source pointer, next sine/cosine pointer
	yloop	2, c4b, -2*64+2*dist64, rdi, -2*16*YMM_SCD3 ;; Test loop counter, next source pointer, restore sine/cosine pointer
	yloop	16, c4b, -16*2*dist64	;; Test loop counter, restore source pointer
	end_timer 11

;; Do inverse FFT levels 6,7
;; On input the 64-byte cache lines hold these data values:
;;	0	+4K	+4K	+4K	128	+4K	+4K	+4K
;;	1
;;	...
;;	127
;;	256	...
;;	...
;;	3840	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+4K	+4K	+4K	512	+4K	+4K	+4K
;;	1
;;	...
;;	511
;;	1K	...
;;	2K	...
;;	3K	...

;; Do 128 eight_reals_unfft operations
;; Do 3920 four_complex_unfft operations
;;	distance between fft data elements is 128
;;	do 512 macros each processing 32 data values

	start_timer 12
	yloop_init  512			;; 4 iterations of 128
	yr4_4cl_eight_reals_four_complex_djbunfft_preload
c3a:	yr4_4cl_eight_reals_four_complex_djbunfft rsi, 64, 2*dist64, 4*dist64, rdi, 4*YMM_SCD3, 64
	yloop	64, c3a, -64*64+dist64	;; Test loop counter, next source pointer
	yloop	2, c3a, -2*dist64+8*dist64, rdi, -128*4*YMM_SCD3 ;; Test loop counter, next source pointer, restore sine/cosine pointer

	loops_reset			;; 3 more iters of 128
	yr4_rb4cl_four_complex_djbunfft_preload
c3b:	yr4_rb4cl_four_complex_djbunfft rsi, 64, 2*dist64, 4*dist64, rdi, 4*YMM_SCD3, 64
	yloop	64, c3b, -64*64+dist64	;; Test loop counter, next source pointer
	yloop	2, c3b, -2*dist64+8*dist64, rdi, -128*4*YMM_SCD3 ;; Test loop counter, next source pointer, restore sine/cosine pointer
	yloop	4, c3b, -4*8*dist64	;; Test loop counter, restore source pointer
	end_timer 12

;; Do inverse FFT levels 4,5
;; On input the 64-byte cache lines hold these data values:
;;	0	+4K	+4K	+4K	512	+4K	+4K	+4K
;;	1
;;	...
;;	511
;;	1K	...
;;	2K	...
;;	3K	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+4K	+4K	+4K	2K	+4K	+4K	+4K
;;	1
;;	...
;;	2047

;; Do 512 eight_reals_unfft operations
;; Do 1536 four_complex_unfft operations
;;	distance between fft data elements is 512
;;	do 512 macros each processing 32 data values

	start_timer 13
	yloop_init  512			;; 512 iterations
	yr4_4cl_eight_reals_four_complex_djbunfft_preload
c2b:	yr4_4cl_eight_reals_four_complex_djbunfft rsi, 64, 8*dist64, 16*dist64, rdi, YMM_SCD3, 64
	yloop	64, c2b, -64*64+dist64	;; Test loop counter, next source pointer
	yloop	8, c2b, -8*dist64	;; Test loop counter, restore source pointer
	end_timer 13

;; Do inverse FFT levels 1,2,3
;; On input the 64-byte cache lines hold these data values:
;;	0	+4K	+4K	+4K	2K	+4K	+4K	+4K
;;	1
;;	...
;;	2047
;; These are swizzled to:
;;	0	+512	+512	+512	2K	+512	+512	+512
;;	1	...
;;	...
;;	511
;;	4K	...
;;	8K	...
;;	12K	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+512	+512	+512	8K	+512	+512	+512
;;	1	...
;;	...
;;	511
;;	2K	...
;;	4K	...
;;	6K	...

;; Do 2048 eight_reals_unfft operations
;;	distance between fft data elements is 2048
;;	do 512 macros each processing 32 data values

	start_timer 14
	yloop_init  512			;; 512 iterations
	yr4_s4cl_eight_reals_unfft_preload
c1b:	yr4_s4cl_eight_reals_unfft rsi, 64, 8*dist64, 16*dist64, rdi, YMM_SCD3, 64
	yloop	64, c1b, -64*64+dist64	;; Test loop counter, next source pointer
	yloop	8, c1b			;; Test loop counter
	end_timer 14

	ytwo_dest_adjust
	yfft_3_ret
	ENDM

;; Perform a 16K-element negacyclic FFT

yfft16Kp MACRO type
	LOCAL	ymid2p, ymid3p, ymid4p, yfinish_unfft
	LOCAL	b1b, b2b, b3b, b4b, b5b, b6a, b6b, b6c, b6d, c1b, c2b, c3b, c4b, c5b

;; Do a multiply with pre-FFTed inputs

	cmp	ffttype, 4
	je	ymid4p

;; Do FFT levels 1,2,3
;; On input the 64-byte cache lines hold these data values:
;;	0	+512	+512	+512	8K	+512	+512	+512
;;	1
;;	...
;;	511
;;	2K	...
;;	4K	...
;;	6K	...
;; On output these are swizzled to:
;;	0	+4K	+4K	+4K	2K	+4K	+4K	+4K
;;	1
;;	...
;;	2047

;; Do 2048 four_complex_first_fft operations
;;	distance between fft data elements is 2048
;;	do 512 macros each processing 32 data values

	start_timer 4
	mov	rdi, sincos1		;; Load sin/cos pointer
	bump	rdi, 512*YMM_SCD2/4
	yloop_init  512			;; 512 iterations
	yr4_fs4cl_four_complex_first_djbfft_preload
b1b:	yr4_fs4cl_four_complex_first_djbfft rsi, 64, 8*dist64, 16*dist64, rdi, YMM_SCD4, 64
	yloop	64, b1b, -64*64+dist64	;; Test loop counter, next source pointer
	yloop	8, b1b, -8*dist64	;; Test loop counter, restore source pointer
	end_timer 4

;; Do FFT levels 4,5
;; On input the 64-byte cache lines hold these data values:
;;	0	+4K	+4K	+4K	2K	+4K	+4K	+4K
;;	1
;;	...
;;	2047
;; On output the 64-byte cache lines hold these data values:
;;	0	+4K	+4K	+4K	512	+4K	+4K	+4K
;;	1
;;	...
;;	511
;;	1K	...
;;	2K	...
;;	3K	...

;; Do 2048 four_complex_fft operations
;;	distance between fft data elements is 512
;;	do 512 macros each processing 32 data values

	start_timer 5
	bump	rdi, -512*YMM_SCD4-512*YMM_SCD2/4 ;; Load sin/cos pointer
	yloop_init  512			;; 512 iterations
	yr4_b4cl_four_complex_djbfft_preload
b2b:	yr4_b4cl_four_complex_djbfft rsi, 64, 8*dist64, 16*dist64, rdi, YMM_SCD2/4, 64
	yloop	64, b2b, -64*64+dist64	;; Test loop counter, next source pointer
	yloop	8, b2b, -8*dist64, rdi, -512*YMM_SCD2/4 ;; Test loop counter, restore source pointer, restore sin/cos pointer
	end_timer 5

;; Do FFT levels 6,7
;; On input the 64-byte cache lines hold these data values:
;;	0	+4K	+4K	+4K	512	+4K	+4K	+4K
;;	1
;;	...
;;	511
;;	1K	...
;;	2K	...
;;	3K	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+4K	+4K	+4K	128	+4K	+4K	+4K
;;	1
;;	...
;;	127
;;	256	...
;;	...
;;	3840	...

;; Do 2048 four_complex_fft operations
;;	distance between fft data elements is 128
;;	do 512 macros each processing 32 data values

	start_timer 6
	yloop_init  512			;; 4 iterations of 128
	yr4_b4cl_four_complex_djbfft_preload
b3b:	yr4_b4cl_four_complex_djbfft rsi, 64, 2*dist64, 4*dist64, rdi, 4*YMM_SCD2/4, 64
	yloop	64, b3b, -64*64+dist64	;; Test loop counter, next source pointer
	yloop	2, b3b, -2*dist64+8*dist64, rdi, -128*4*YMM_SCD2/4 ;; Test loop counter, next source pointer, restore sin/cos pointer
	yloop	4, b3b, -4*8*dist64	;; Test loop counter, restore source pointer
	end_timer 6

;; Do FFT levels 8,9
;; On input the 64-byte cache lines hold these data values:
;;	0	+4K	+4K	+4K	128	+4K	+4K	+4K
;;	1
;;	...
;;	127
;;	256	...
;;	...
;;	3840	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+4K	+4K	+4K	32	+4K	+4K	+4K
;;	1
;;	...
;;	31
;;	64	...
;;	...
;;	4032	...

;; Do 2048 four_complex_fft operations
;;	distance between fft data elements is 32
;;	do 512 macros each processing 32 data values

	start_timer 7
	yloop_init  512			;; 16 iterations of 32
	yr4_b4cl_four_complex_djbfft_preload
;; Loop below reworked to avoid 4KB distances
;;b4b:	yr4_b4cl_four_complex_djbfft rsi, 64, 32*64, dist64, rdi, 16*YMM_SCD2/4, 32
;;	yloop	32, b4b, -32*64+2*dist64, rdi, -32*16*YMM_SCD2/4 ;; Test loop counter, next source pointer, restore sin/cos pointer
b4b:	yr4_b4cl_four_complex_djbfft rsi, 2*64, 32*64, dist64, rdi, 2*16*YMM_SCD2/4, 16
	yloop	16, b4b, -16*2*64+64, rdi, -16*2*16*YMM_SCD2/4+16*YMM_SCD2/4 ;; Test loop counter, next source pointer, next sin/cos pointer
	yloop	2, b4b, -2*64+2*dist64, rdi, -2*16*YMM_SCD2/4 ;; Test loop counter, next source pointer, restore sin/cos pointer
	yloop	16, b4b, -16*2*dist64	;; Test loop counter, restore source pointer
	end_timer 7

;; Do FFT levels 10,11
;; On input the 64-byte cache lines hold these data values:
;;	0	+4K	+4K	+4K	32	+4K	+4K	+4K
;;	1
;;	...
;;	31
;;	64	...
;;	...
;;	4032	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+4K	+4K	+4K	8	+4K	+4K	+4K
;;	1
;;	...
;;	7
;;	16	...
;;	...
;;	4080	...

;; Do 2048 four_complex_fft operations
;;	distance between fft data elements is 8
;;	do 512 macros each processing 32 data values

	start_timer 8
	yloop_init  512			;; 64 iterations of 8
	yr4_b4cl_four_complex_djbfft_preload
b5b:	yr4_b4cl_four_complex_djbfft rsi, 64, 8*64, 16*64, rdi, 64*YMM_SCD2/4, 8
	yloop	8, b5b, -8*64+32*64, rdi, -8*64*YMM_SCD2/4 ;; Test loop counter, next source pointer, restore sin/cos pointer
	yloop	2, b5b, -2*32*64+dist64	;; Test loop counter, next source pointer
	yloop	32, b5b, -32*dist64	;; Test loop counter, restore source pointer
	end_timer 8

;; Do FFT levels 12,13
;; On input and output the 64-byte cache lines hold these data values:
;;	0	+4K	+4K	+4K	8	+4K	+4K	+4K
;;	1
;;	...
;;	7
;;	16	...
;;	...
;;	4080	...

;; Do 1024 eight_complex_fft operations
;;	distance between fft data elements is 1
;;	do 256 macros each processing 64 data values

	cmp	ffttype, 2
	je	ymid2p
	ja	ymid3p

	start_timer 9
	yloop_init  256			;; 256 iterations
b6a:	yr8_8cl_eight_complex_fft_final rsi, 8*64, 64, 2*64, 4*64
	yloop	8, b6a, -8*8*64+dist64	;; Test loop counter, next source pointer
	yloop	32, b6a			;; Test loop counter
	end_timer 9
	yfft_1_ret

ymid2p:	start_timer 9
	yloop_init  256			;; 256 iterations
	yr8_8cl_eight_complex_with_square_preload
b6b:	yr8_8cl_eight_complex_with_square rsi, 8*64, 64, 2*64, 4*64
	yloop	8, b6b, -8*8*64+dist64	;; Test loop counter, next source pointer
	yloop	32, b6b			;; Test loop counter
	end_timer 9
	jmp	yfinish_unfft

ymid3p:	start_timer 9
x64	mov	r8, DEST2ARG
	mov	rbp, DIST_TO_MULSRCARG
	yloop_init  256			;; 256 iterations
	yr8_8cl_eight_complex_with_mult_preload
b6c:	yr8_8cl_eight_complex_with_mult rsi, 8*64, 64, 2*64, 4*64
	yloop	8, b6c, -8*8*64+dist64	;; Test loop counter, next source pointer
	yloop	32, b6c			;; Test loop counter
x64	add	rsi, r8
	end_timer 9
	jmp	yfinish_unfft

ymid4p:	start_timer 9
	mov	rbp, DIST_TO_MULSRCARG
	yloop_init  256			;; 256 iterations
	yr8_8cl_eight_complex_with_mulf_preload
b6d:	yr8_8cl_eight_complex_with_mulf rsi, 8*64, 64, 2*64, 4*64
	yloop	8, b6d, -8*8*64+dist64	;; Test loop counter, next source pointer
	yloop	32, b6d			;; Test loop counter
	end_timer 9
yfinish_unfft:
	bump	rsi, -32*dist64		;; Restore source pointer

;; Do inverse FFT levels 10,11
;; On input the 64-byte cache lines hold these data values:
;;	0	+4K	+4K	+4K	8	+4K	+4K	+4K
;;	1
;;	...
;;	7
;;	16	...
;;	...
;;	4080	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+4K	+4K	+4K	32	+4K	+4K	+4K
;;	1
;;	...
;;	31
;;	64	...
;;	...
;;	4032	...

;; Do 2048 four_complex_unfft operations
;;	distance between fft data elements is 8
;;	do 512 macros each processing 32 data values

	start_timer 10
	mov	rdi, sincos1		;; Load sin/cos pointer
	yloop_init  512			;; 64 iterations of 8
	yr4_b4cl_four_complex_djbunfft_preload
c5b:	yr4_b4cl_four_complex_djbunfft rsi, 64, 8*64, 16*64, rdi, 64*YMM_SCD2/4, 8
	yloop	8, c5b, -8*64+32*64, rdi, -8*64*YMM_SCD2/4 ;; Test loop counter, next source pointer, restore sine/cosine pointer
	yloop	2, c5b, -2*32*64+dist64	;; Test loop counter, next source pointer
	yloop	32, c5b, -32*dist64	;; Test loop counter, restore source pointer
	end_timer 10

;; Do inverse FFT levels 8,9
;; On input the 64-byte cache lines hold these data values:
;;	0	+4K	+4K	+4K	32	+4K	+4K	+4K
;;	1
;;	...
;;	31
;;	64	...
;;	...
;;	4032	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+4K	+4K	+4K	128	+4K	+4K	+4K
;;	1
;;	...
;;	127
;;	256	...
;;	...
;;	3840	...

;; Do 2048 four_complex_unfft operations
;;	distance between fft data elements is 32
;;	do 512 macros each processing 32 data values

	start_timer 11
	yloop_init  512			;; 16 iterations of 32
	yr4_b4cl_four_complex_djbunfft_preload
;; Loop below reworked to avoid 4KB distances
;;c4b:	yr4_b4cl_four_complex_djbunfft rsi, 64, 32*64, dist64, rdi, 16*YMM_SCD2/4, 32
;;	yloop	32, c4b, -32*64+2*dist64, rdi, -32*16*YMM_SCD2/4 ;; Test loop counter, next source pointer, restore sine/cosine pointer
c4b:	yr4_b4cl_four_complex_djbunfft rsi, 2*64, 32*64, dist64, rdi, 2*16*YMM_SCD2/4, 16
	yloop	16, c4b, -16*2*64+64, rdi, -16*2*16*YMM_SCD2/4+16*YMM_SCD2/4 ;; Test loop counter, next source pointer, next sine/cosine pointer
	yloop	2, c4b, -2*64+2*dist64, rdi, -2*16*YMM_SCD2/4 ;; Test loop counter, next source pointer, restore sine/cosine pointer
	yloop	16, c4b, -16*2*dist64	;; Test loop counter, restore source pointer
	end_timer 11

;; Do inverse FFT levels 6,7
;; On input the 64-byte cache lines hold these data values:
;;	0	+4K	+4K	+4K	128	+4K	+4K	+4K
;;	1
;;	...
;;	127
;;	256	...
;;	...
;;	3840	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+4K	+4K	+4K	512	+4K	+4K	+4K
;;	1
;;	...
;;	511
;;	1K	...
;;	2K	...
;;	3K	...

;; Do 2048 four_complex_unfft operations
;;	distance between fft data elements is 128
;;	do 512 macros each processing 32 data values

	start_timer 12
	yloop_init  512			;; 4 iterations of 128
	yr4_b4cl_four_complex_djbunfft_preload
c3b:	yr4_b4cl_four_complex_djbunfft rsi, 64, 2*dist64, 4*dist64, rdi, 4*YMM_SCD2/4, 64
	yloop	64, c3b, -64*64+dist64	;; Test loop counter, next source pointer
	yloop	2, c3b, -2*dist64+8*dist64, rdi, -128*4*YMM_SCD2/4 ;; Test loop counter, next source pointer, restore sine/cosine pointer
	yloop	4, c3b, -4*8*dist64	;; Test loop counter, restore source pointer
	end_timer 12

;; Do inverse FFT levels 4,5
;; On input the 64-byte cache lines hold these data values:
;;	0	+4K	+4K	+4K	512	+4K	+4K	+4K
;;	1
;;	...
;;	511
;;	1K	...
;;	2K	...
;;	3K	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+4K	+4K	+4K	2K	+4K	+4K	+4K
;;	1
;;	...
;;	2047

;; Do 2048 four_complex_unfft operations
;;	distance between fft data elements is 512
;;	do 512 macros each processing 32 data values

	start_timer 13
	yloop_init  512			;; 512 iterations
	yr4_b4cl_four_complex_djbunfft_preload
c2b:	yr4_b4cl_four_complex_djbunfft rsi, 64, 8*dist64, 16*dist64, rdi, YMM_SCD2/4, 64
	yloop	64, c2b, -64*64+dist64	;; Test loop counter, next source pointer
	yloop	8, c2b, -8*dist64	;; Test loop counter, restore source pointer
	end_timer 13

;; Do inverse FFT levels 1,2,3
;; On input the 64-byte cache lines hold these data values:
;;	0	+4K	+4K	+4K	2K	+4K	+4K	+4K
;;	1
;;	...
;;	2047
;; These are swizzled to:
;;	0	+512	+512	+512	2K	+512	+512	+512
;;	1	...
;;	...
;;	511
;;	4K	...
;;	8K	...
;;	12K	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+512	+512	+512	8K	+512	+512	+512
;;	1	...
;;	...
;;	511
;;	2K	...
;;	4K	...
;;	6K	...

;; Do 2048 four_complex_last_unfft operations
;;	distance between fft data elements is 2048
;;	do 512 macros each processing 32 data values

	start_timer 14
	yloop_init  512			;; 512 iterations
	yr4_s4cl_four_complex_last_unfft_preload
c1b:	yr4_s4cl_four_complex_last_unfft rsi, 64, 8*dist64, 16*dist64, rdi, YMM_SCD4, 64
	yloop	64, c1b, -64*64+dist64	;; Test loop counter, next source pointer
	yloop	8, c1b			;; Test loop counter
	end_timer 14

	ytwo_dest_adjust
	yfft_3_ret
	ENDM


;; Perform a 20K-element FFT.

yfft20K MACRO type
	LOCAL	ymid2, ymid3, ymid4, yfinish_unfft
	LOCAL	b1b, b2b, b3a, b3b, b4a, b4b, b5a, b5b, b6a, b6b, b6c, b6d
	LOCAL	c1b, c2b, c3a, c3b, c4a, c4b, c5a, c5b

;; Do a multiply with pre-FFTed inputs

	cmp	ffttype, 4
	je	ymid4

;; Do FFT levels 1,2,3
;; On input the 64-byte cache lines hold these data values:
;;	0	+640	+640	+640	10K	+640	+640	+640
;;	1
;;	...
;;	639
;;	2560	...
;;	5120	...
;;	7680	...
;; On output these are swizzled to:
;;	0	+5K	+5K	+5K	2560	+5K	+5K	+5K
;;	1
;;	...
;;	2559

;; Do 2560 eight_reals_fft operations
;;	distance between fft data elements is 2560
;;	do 640 macros each processing 32 data values

	start_timer 4
	mov	rdi, sincos1		;; Load sin/cos pointer
	bump	rdi, 512*YMM_SCD4+128*YMM_SCD3
	yloop_init  640			;; 640 iterations
	yr4_fs4cl_eight_reals_fft_preload
b1b:	yr4_fs4cl_eight_reals_fft rsi, 64, 10*dist64, 20*dist64, rdi, YMM_SCD3, 64
	yloop	64, b1b, -64*64+dist64	;; Test loop counter, next source pointer
	yloop	10, b1b, -10*dist64	;; Test loop counter, restore source pointer
	end_timer 4

;; Do FFT levels 4,5
;; On input the 64-byte cache lines hold these data values:
;;	0	+5K	+5K	+5K	2560	+5K	+5K	+5K
;;	1
;;	...
;;	2559
;; On output the 64-byte cache lines hold these data values:
;;	0	+5K	+5K	+5K	512	+5K	+5K	+5K
;;	1
;;	...
;;	511
;;	1K	...
;;	....
;;	4K	...

;; Do 512 ten_reals_fft operations
;; Do 1536 five_complex_fft operations
;;	distance between fft data elements is 512
;;	do 512 macros each processing 40 data values

	start_timer 5
	bump	rdi, -640*YMM_SCD3-512*YMM_SCD4 ;; Load sin/cos pointer
	yloop_init  512			;; 512 iterations
	yr5_5cl_ten_reals_five_complex_djbfft_preload
b2b:	yr5_5cl_ten_reals_five_complex_djbfft rsi, 64, 8*dist64, rdi, YMM_SCD4, 64
	yloop	64, b2b, -64*64+dist64	;; Test loop counter, next source pointer
	yloop	8, b2b, -8*dist64	;; Test loop counter, restore source pointer
	end_timer 5

;; Do FFT levels 6,7
;; On input the 64-byte cache lines hold these data values:
;;	0	+5K	+5K	+5K	512	+5K	+5K	+5K
;;	1
;;	...
;;	511
;;	1K	...
;;	....
;;	4K	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+5K	+5K	+5K	128	+5K	+5K	+5K
;;	1
;;	...
;;	127
;;	256	...
;;	...
;;	4864	...

;; Do 128 eight_reals_fft operations
;; Do 2432 four_complex_fft operations
;;	distance between fft data elements is 128
;;	do 640 macros each processing 32 data values

	start_timer 6
	bump	rdi, -512*YMM_SCD4-128*YMM_SCD3 ;; Restore sin/cos pointer
	yloop_init  640			;; 5 iterations of 128
	yr4_4cl_eight_reals_four_complex_djbfft_preload
b3a:	yr4_4cl_eight_reals_four_complex_djbfft rsi, 64, 2*dist64, 4*dist64, rdi, YMM_SCD3, 64
	yloop	64, b3a, -64*64+dist64	;; Test loop counter, next source pointer
	yloop	2, b3a, -2*dist64+8*dist64, rdi, -128*YMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer

	loops_reset			;; 4 more iters of 128
	yr4_rb4cl_four_complex_djbfft_preload
b3b:	yr4_rb4cl_four_complex_djbfft rsi, 64, 2*dist64, 4*dist64, rdi, YMM_SCD3, 64
	yloop	64, b3b, -64*64+dist64	;; Test loop counter, next source pointer
	yloop	2, b3b, -2*dist64+8*dist64, rdi, -128*YMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer
	yloop	5, b3b, -5*8*dist64	;; Test loop counter, restore source pointer
	end_timer 6

;; Do FFT levels 8,9
;; On input the 64-byte cache lines hold these data values:
;;	0	+5K	+5K	+5K	128	+5K	+5K	+5K
;;	1
;;	...
;;	127
;;	256	...
;;	...
;;	4864	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+5K	+5K	+5K	32	+5K	+5K	+5K
;;	1
;;	...
;;	31
;;	64	...
;;	...
;;	5056	...

;; Do 32 eight_reals_fft operations
;; Do 2528 four_complex_fft operations
;;	distance between fft data elements is 32
;;	do 640 macros each processing 32 data values

	start_timer 7
	yloop_init  640			;; 20 iterations of 32
	yr4_4cl_eight_reals_four_complex_djbfft_preload
;; Loop below reworked to avoid 4KB distances
;;b4a:	yr4_4cl_eight_reals_four_complex_djbfft rsi, 64, 32*64, dist64, rdi, 4*YMM_SCD3, 32
;;	yloop	32, b4a, -32*64+2*dist64, rdi, -32*4*YMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer
b4a:	yr4_4cl_eight_reals_four_complex_djbfft rsi, 2*64, 32*64, dist64, rdi, 2*4*YMM_SCD3, 16
	yloop	16, b4a, -16*2*64+64, rdi, -16*2*4*YMM_SCD3+4*YMM_SCD3 ;; Test loop counter, next source pointer, next sin/cos pointer
	yloop	2, b4a, -2*64+2*dist64, rdi, -2*4*YMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer

	loops_reset			;; 19 more iters of 32
	yr4_rb4cl_four_complex_djbfft_preload
;; Loop below reworked to avoid 4KB distances
;;b4b:	yr4_rb4cl_four_complex_djbfft rsi, 64, 32*64, dist64, rdi, 4*YMM_SCD3, 32
;;	yloop	32, b4b, -32*64+2*dist64, rdi, -32*4*YMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer
b4b:	yr4_rb4cl_four_complex_djbfft rsi, 2*64, 32*64, dist64, rdi, 2*4*YMM_SCD3, 16
	yloop	16, b4b, -16*2*64+64, rdi, -16*2*4*YMM_SCD3+4*YMM_SCD3 ;; Test loop counter, next source pointer, next sin/cos pointer
	yloop	2, b4b, -2*64+2*dist64, rdi, -2*4*YMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer
	yloop	20, b4b, -20*2*dist64	;; Test loop counter, restore source pointer
	end_timer 7

;; Do FFT levels 10,11
;; On input the 64-byte cache lines hold these data values:
;;	0	+5K	+5K	+5K	32	+5K	+5K	+5K
;;	1
;;	...
;;	31
;;	64	...
;;	...
;;	5056	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+5K	+5K	+5K	8	+5K	+5K	+5K
;;	1
;;	...
;;	7
;;	16	...
;;	...
;;	5004	...

;; Do 8 eight_reals_fft operations
;; Do 2552 four_complex_fft operations
;;	distance between fft data elements is 8
;;	do 640 macros each processing 32 data values

	start_timer 8
	yloop_init  640			;; 80 iterations of 8
	yr4_4cl_eight_reals_four_complex_djbfft_preload
b5a:	yr4_4cl_eight_reals_four_complex_djbfft rsi, 64, 8*64, 16*64, rdi, 16*YMM_SCD3, 8
	yloop	8, b5a, -8*64+32*64, rdi, -8*16*YMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer

	loops_reset			;; 79 more iters of 8
	yr4_rb4cl_four_complex_djbfft_preload
b5b:	yr4_rb4cl_four_complex_djbfft rsi, 64, 8*64, 16*64, rdi, 16*YMM_SCD3, 8
	yloop	8, b5b, -8*64+32*64, rdi, -8*16*YMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer
	yloop	2, b5b, -2*32*64+dist64	;; Test loop counter, next source pointer
	yloop	40, b5b, -40*dist64	;; Test loop counter, restore source pointer
	end_timer 8

;; Do FFT levels 12,13
;; On input and output the 64-byte cache lines hold these data values:
;;	0	+5K	+5K	+5K	8	+5K	+5K	+5K
;;	1
;;	...
;;	7
;;	16	...
;;	...
;;	5004	...

;; Do 1 sixteen_reals_fft operation
;; Do 1279 eight_complex_fft operations
;;	distance between fft data elements is 1
;;	do 320 macros each processing 64 data values

	cmp	ffttype, 2
	je	ymid2
	ja	ymid3

	start_timer 9
;;	yr8_8cl_sixteen_reals_eight_complex_fft_final rsi, 8*64, 64, 2*64, 4*64
	yr8_dispatch call, y8real_fft_final, 64, 2*64, 4*64
	yloop_init  320, 1, 8		;; 319 more iterations
b6a:	yr8_8cl_eight_complex_fft_final rsi, 8*64, 64, 2*64, 4*64
	yloop	8, b6a, -8*8*64+dist64	;; Test loop counter, next source pointer
	yloop	40, b6a			;; Test loop counter
	end_timer 9
	yfft_1_ret

ymid2:	start_timer 9
;;	yr8_8cl_sixteen_reals_eight_complex_with_square rsi, 8*64, 64, 2*64, 4*64
	yr8_dispatch call, y8real_square, 64, 2*64, 4*64
	yloop_init  320, 1, 8		;; 319 more iterations
	yr8_8cl_eight_complex_with_square_preload
b6b:	yr8_8cl_eight_complex_with_square rsi, 8*64, 64, 2*64, 4*64
	yloop	8, b6b, -8*8*64+dist64	;; Test loop counter, next source pointer
	yloop	40, b6b			;; Test loop counter
	end_timer 9
	jmp	yfinish_unfft

ymid3:	start_timer 9
x64	mov	r8, DEST2ARG
	mov	rbp, DIST_TO_MULSRCARG
;;	yr8_8cl_sixteen_reals_eight_complex_with_mult rsi, 8*64, 64, 2*64, 4*64
	yr8_dispatch call, y8real_mult, 64, 2*64, 4*64
	yloop_init  320, 1, 8		;; 319 more iterations
	yr8_8cl_eight_complex_with_mult_preload
b6c:	yr8_8cl_eight_complex_with_mult rsi, 8*64, 64, 2*64, 4*64
	yloop	8, b6c, -8*8*64+dist64	;; Test loop counter, next source pointer
	yloop	40, b6c			;; Test loop counter
x64	add	rsi, r8
	end_timer 9
	jmp	yfinish_unfft

ymid4:	start_timer 9
	mov	rbp, DIST_TO_MULSRCARG
;;	yr8_8cl_sixteen_reals_eight_complex_with_mulf rsi, 8*64, 64, 2*64, 4*64
	yr8_dispatch call, y8real_mulf, 64, 2*64, 4*64
	yloop_init  320, 1, 8		;; 319 more iterations
	yr8_8cl_eight_complex_with_mulf_preload
b6d:	yr8_8cl_eight_complex_with_mulf rsi, 8*64, 64, 2*64, 4*64
	yloop	8, b6d, -8*8*64+dist64	;; Test loop counter, next source pointer
	yloop	40, b6d			;; Test loop counter
	end_timer 9
yfinish_unfft:
	bump	rsi, -40*dist64		;; Restore source pointer

;; Do inverse FFT levels 10,11
;; On input the 64-byte cache lines hold these data values:
;;	0	+5K	+5K	+5K	8	+5K	+5K	+5K
;;	1
;;	...
;;	7
;;	16	...
;;	...
;;	5004	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+5K	+5K	+5K	32	+5K	+5K	+5K
;;	1
;;	...
;;	31
;;	64	...
;;	...
;;	5056	...

;; Do 8 eight_reals_unfft operations
;; Do 2552 four_complex_unfft operations
;;	distance between fft data elements is 8
;;	do 640 macros each processing 32 data values

	start_timer 10
	mov	rdi, sincos1		;; Load sin/cos pointer
	yloop_init  640			;; 80 iterations of 8
	yr4_4cl_eight_reals_four_complex_djbunfft_preload
c5a:	yr4_4cl_eight_reals_four_complex_djbunfft rsi, 64, 8*64, 16*64, rdi, 16*YMM_SCD3, 8
	yloop	8, c5a, -8*64+32*64, rdi, -8*16*YMM_SCD3 ;; Test loop counter, next source pointer, restore sine/cosine pointer

	loops_reset			;; 79 more iters of 8
	yr4_rb4cl_four_complex_djbunfft_preload
c5b:	yr4_rb4cl_four_complex_djbunfft rsi, 64, 8*64, 16*64, rdi, 16*YMM_SCD3, 8
	yloop	8, c5b, -8*64+32*64, rdi, -8*16*YMM_SCD3 ;; Test loop counter, next source pointer, restore sine/cosine pointer
	yloop	2, c5b, -2*32*64+dist64	;; Test loop counter, next source pointer
	yloop	40, c5b, -40*dist64	;; Test loop counter, restore source pointer
	end_timer 10

;; Do inverse FFT levels 8,9
;; On input the 64-byte cache lines hold these data values:
;;	0	+5K	+5K	+5K	32	+5K	+5K	+5K
;;	1
;;	...
;;	31
;;	64	...
;;	...
;;	5056	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+5K	+5K	+5K	128	+5K	+5K	+5K
;;	1
;;	...
;;	127
;;	256	...
;;	...
;;	4864	...

;; Do 32 eight_reals_unfft operations
;; Do 2528 four_complex_unfft operations
;;	distance between fft data elements is 32
;;	do 640 macros each processing 32 data values

	start_timer 11
	yloop_init  640			;; 20 iterations of 32
	yr4_4cl_eight_reals_four_complex_djbunfft_preload
;; Loop below reworked to avoid 4KB distances
;;c4a:	yr4_4cl_eight_reals_four_complex_djbunfft rsi, 64, 32*64, dist64, rdi, 4*YMM_SCD3, 32
;;	yloop	32, c4a, -32*64+2*dist64, rdi, -32*4*YMM_SCD3 ;; Test loop counter, next source pointer, restore sine/cosine pointer
c4a:	yr4_4cl_eight_reals_four_complex_djbunfft rsi, 2*64, 32*64, dist64, rdi, 2*4*YMM_SCD3, 16
	yloop	16, c4a, -16*2*64+64, rdi, -16*2*4*YMM_SCD3+4*YMM_SCD3 ;; Test loop counter, next source pointer, next sine/cosine pointer
	yloop	2, c4a, -2*64+2*dist64, rdi, -2*4*YMM_SCD3 ;; Test loop counter, next source pointer, restore sine/cosine pointer

	loops_reset			;; 19 more iters of 32
	yr4_rb4cl_four_complex_djbunfft_preload
;; Loop below reworked to avoid 4KB distances
;;c4b:	yr4_rb4cl_four_complex_djbunfft rsi, 64, 32*64, dist64, rdi, 4*YMM_SCD3, 32
;;	yloop	32, c4b, -32*64+2*dist64, rdi, -32*4*YMM_SCD3 ;; Test loop counter, next source pointer, restore sine/cosine pointer
c4b:	yr4_rb4cl_four_complex_djbunfft rsi, 2*64, 32*64, dist64, rdi, 2*4*YMM_SCD3, 16
	yloop	16, c4b, -16*2*64+64, rdi, -16*2*4*YMM_SCD3+4*YMM_SCD3 ;; Test loop counter, next source pointer, next sine/cosine pointer
	yloop	2, c4b, -2*64+2*dist64, rdi, -2*4*YMM_SCD3 ;; Test loop counter, next source pointer, restore sine/cosine pointer
	yloop	20, c4b, -20*2*dist64	;; Test loop counter, restore source pointer
	end_timer 11

;; Do inverse FFT levels 6,7
;; On input the 64-byte cache lines hold these data values:
;;	0	+5K	+5K	+5K	128	+5K	+5K	+5K
;;	1
;;	...
;;	127
;;	256	...
;;	...
;;	4864	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+5K	+5K	+5K	512	+5K	+5K	+5K
;;	1
;;	...
;;	511
;;	1K	...
;;	...
;;	4K	...

;; Do 128 eight_reals_unfft operations
;; Do 2432 four_complex_unfft operations
;;	distance between fft data elements is 128
;;	do 640 macros each processing 32 data values

	start_timer 12
	yloop_init  640			;; 5 iterations of 128
	yr4_4cl_eight_reals_four_complex_djbunfft_preload
c3a:	yr4_4cl_eight_reals_four_complex_djbunfft rsi, 64, 2*dist64, 4*dist64, rdi, YMM_SCD3, 64
	yloop	64, c3a, -64*64+dist64	;; Test loop counter, next source pointer
	yloop	2, c3a, -2*dist64+8*dist64, rdi, -128*YMM_SCD3 ;; Test loop counter, next source pointer, restore sine/cosine pointer

	loops_reset			;; 4 more iters of 128
	yr4_rb4cl_four_complex_djbunfft_preload
c3b:	yr4_rb4cl_four_complex_djbunfft rsi, 64, 2*dist64, 4*dist64, rdi, YMM_SCD3, 64
	yloop	64, c3b, -64*64+dist64	;; Test loop counter, next source pointer
	yloop	2, c3b, -2*dist64+8*dist64, rdi, -128*YMM_SCD3 ;; Test loop counter, next source pointer, restore sine/cosine pointer
	yloop	5, c3b, -5*8*dist64	;; Test loop counter, restore source pointer
	end_timer 12

;; Do inverse FFT levels 4,5
;; On input the 64-byte cache lines hold these data values:
;;	0	+5K	+5K	+5K	512	+5K	+5K	+5K
;;	1
;;	...
;;	511
;;	1K	...
;;	...
;;	4K	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+5K	+5K	+5K	2560	+5K	+5K	+5K
;;	1
;;	...
;;	2559

;; Do 512 ten_reals_unfft operations
;; Do 1536 five_complex_unfft operations
;;	distance between fft data elements is 512
;;	do 512 macros each processing 40 data values

	start_timer 13
	bump	rdi, 128*YMM_SCD3	;; Next sine/cosine pointer
	yloop_init  512			;; 512 iterations
	yr5_5cl_ten_reals_five_complex_djbunfft_preload
c2b:	yr5_5cl_ten_reals_five_complex_djbunfft rsi, 64, 8*dist64, rdi, YMM_SCD4, 64
	yloop	64, c2b, -64*64+dist64	;; Test loop counter, next source pointer
	yloop	8, c2b, -8*dist64	;; Test loop counter, restore source pointer
	end_timer 13

;; Do inverse FFT levels 1,2,3
;; On input the 64-byte cache lines hold these data values:
;;	0	+5K	+5K	+5K	2560	+5K	+5K	+5K
;;	1
;;	...
;;	2559
;; These are swizzled to:
;;	0	+640	+640	+640	2560	+640	+640	+640
;;	1	...
;;	...
;;	639
;;	5K	...
;;	10K	...
;;	15K	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+640	+640	+640	10K	+640	+640	+640
;;	1	...
;;	...
;;	639
;;	2560	...
;;	5120	...
;;	7680	...

;; Do 2560 eight_reals_unfft operations
;;	distance between fft data elements is 2560
;;	do 640 macros each processing 32 data values

	start_timer 14
	yloop_init  640			;; 640 iterations
	yr4_s4cl_eight_reals_unfft_preload
c1b:	yr4_s4cl_eight_reals_unfft rsi, 64, 10*dist64, 20*dist64, rdi, YMM_SCD3, 64
	yloop	64, c1b, -64*64+dist64	;; Test loop counter, next source pointer
	yloop	10, c1b			;; Test loop counter
	end_timer 14

	ytwo_dest_adjust
	yfft_3_ret
	ENDM

;; Perform a 20K-element negacyclic FFT

yfft20Kp MACRO type
	LOCAL	ymid2p, ymid3p, ymid4p, yfinish_unfft
	LOCAL	b1b, b2b, b3b, b4b, b5b, b6a, b6b, b6c, b6d, c1b, c2b, c3b, c4b, c5b

;; Do a multiply with pre-FFTed inputs

	cmp	ffttype, 4
	je	ymid4p

;; Do FFT levels 1,2,3
;; On input the 64-byte cache lines hold these data values:
;;	0	+640	+640	+640	10K	+640	+640	+640
;;	1
;;	...
;;	639
;;	2560	...
;;	5120	...
;;	7680	...
;; On output these are swizzled to:
;;	0	+5K	+5K	+5K	2560	+5K	+5K	+5K
;;	1
;;	...
;;	2559

;; Do 2560 four_complex_first_fft operations
;;	distance between fft data elements is 2560
;;	do 640 macros each processing 32 data values

	start_timer 4
	mov	rdi, sincos1		;; Load sin/cos pointer
	bump	rdi, 512*YMM_SCD2/4+128*YMM_SCD2/4
	yloop_init  640			;; 640 iterations
	yr4_fs4cl_four_complex_first_djbfft_preload
b1b:	yr4_fs4cl_four_complex_first_djbfft rsi, 64, 10*dist64, 20*dist64, rdi, YMM_SCD4, 64
	yloop	64, b1b, -64*64+dist64	;; Test loop counter, next source pointer
	yloop	10, b1b, -10*dist64	;; Test loop counter, restore source pointer
	end_timer 4

;; Do FFT levels 4,5
;; On input the 64-byte cache lines hold these data values:
;;	0	+5K	+5K	+5K	2560	+5K	+5K	+5K
;;	1
;;	...
;;	2559
;; On output the 64-byte cache lines hold these data values:
;;	0	+5K	+5K	+5K	512	+5K	+5K	+5K
;;	1
;;	...
;;	511
;;	1K	...
;;	....
;;	4K	...

;; Do 2048 five_complex_fft operations
;;	distance between fft data elements is 512
;;	do 512 macros each processing 40 data values

	start_timer 5
	bump	rdi, -640*YMM_SCD4-512*YMM_SCD2/4 ;; Load sin/cos pointer
	yloop_init  512			;; 512 iterations
	yr5_b5cl_five_complex_djbfft_preload
b2b:	yr5_b5cl_five_complex_djbfft rsi, 64, 8*dist64, rdi, YMM_SCD2/4, 64
	yloop	64, b2b, -64*64+dist64	;; Test loop counter, next source pointer
	yloop	8, b2b, -8*dist64	;; Test loop counter, restore source pointer
	end_timer 5

;; Do FFT levels 6,7
;; On input the 64-byte cache lines hold these data values:
;;	0	+5K	+5K	+5K	512	+5K	+5K	+5K
;;	1
;;	...
;;	511
;;	1K	...
;;	....
;;	4K	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+5K	+5K	+5K	128	+5K	+5K	+5K
;;	1
;;	...
;;	127
;;	256	...
;;	...
;;	4864	...

;; Do 2560 four_complex_fft operations
;;	distance between fft data elements is 128
;;	do 640 macros each processing 32 data values

	start_timer 6
	bump	rdi, -512*YMM_SCD2/4-128*YMM_SCD2/4 ;; Next sin/cos pointer
	yloop_init  640			;; 5 iterations of 128
	yr4_b4cl_four_complex_djbfft_preload
b3b:	yr4_b4cl_four_complex_djbfft rsi, 64, 2*dist64, 4*dist64, rdi, YMM_SCD2/4, 64
	yloop	64, b3b, -64*64+dist64	;; Test loop counter, next source pointer
	yloop	2, b3b, -2*dist64+8*dist64, rdi, -128*YMM_SCD2/4 ;; Test loop counter, next source pointer, restore sin/cos pointer
	yloop	5, b3b, -5*8*dist64	;; Test loop counter, restore source pointer
	end_timer 6

;; Do FFT levels 8,9
;; On input the 64-byte cache lines hold these data values:
;;	0	+5K	+5K	+5K	128	+5K	+5K	+5K
;;	1
;;	...
;;	127
;;	256	...
;;	...
;;	4864	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+5K	+5K	+5K	32	+5K	+5K	+5K
;;	1
;;	...
;;	31
;;	64	...
;;	...
;;	5056	...

;; Do 2560 four_complex_fft operations
;;	distance between fft data elements is 32
;;	do 640 macros each processing 32 data values

	start_timer 7
	yloop_init  640			;; 20 iterations of 32
	yr4_b4cl_four_complex_djbfft_preload
;; Loop below reworked to avoid 4KB distances
;;b4b:	yr4_b4cl_four_complex_djbfft rsi, 64, 32*64, dist64, rdi, 4*YMM_SCD2/4, 32
;;	yloop	32, b4b, -32*64+2*dist64, rdi, -32*4*YMM_SCD2/4 ;; Test loop counter, next source pointer, restore sin/cos pointer
b4b:	yr4_b4cl_four_complex_djbfft rsi, 2*64, 32*64, dist64, rdi, 2*4*YMM_SCD2/4, 16
	yloop	16, b4b, -16*2*64+64, rdi, -16*2*4*YMM_SCD2/4+4*YMM_SCD2/4 ;; Test loop counter, next source pointer, next sin/cos pointer
	yloop	2, b4b, -2*64+2*dist64, rdi, -2*4*YMM_SCD2/4 ;; Test loop counter, next source pointer, restore sin/cos pointer
	yloop	20, b4b, -20*2*dist64	;; Test loop counter, restore source pointer
	end_timer 7

;; Do FFT levels 10,11
;; On input the 64-byte cache lines hold these data values:
;;	0	+5K	+5K	+5K	32	+5K	+5K	+5K
;;	1
;;	...
;;	31
;;	64	...
;;	...
;;	5056	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+5K	+5K	+5K	8	+5K	+5K	+5K
;;	1
;;	...
;;	7
;;	16	...
;;	...
;;	5004	...

;; Do 2560 four_complex_fft operations
;;	distance between fft data elements is 8
;;	do 640 macros each processing 32 data values

	start_timer 8
	yloop_init  640			;; 80 iterations of 8
	yr4_b4cl_four_complex_djbfft_preload
b5b:	yr4_b4cl_four_complex_djbfft rsi, 64, 8*64, 16*64, rdi, 16*YMM_SCD2/4, 8
	yloop	8, b5b, -8*64+32*64, rdi, -8*16*YMM_SCD2/4 ;; Test loop counter, next source pointer, restore sin/cos pointer
	yloop	2, b5b, -2*32*64+dist64	;; Test loop counter, next source pointer
	yloop	40, b5b, -40*dist64	;; Test loop counter, restore source pointer
	end_timer 8

;; Do FFT levels 12,13
;; On input and output the 64-byte cache lines hold these data values:
;;	0	+5K	+5K	+5K	8	+5K	+5K	+5K
;;	1
;;	...
;;	7
;;	16	...
;;	...
;;	5004	...

;; Do 1280 eight_complex_fft operations
;;	distance between fft data elements is 1
;;	do 320 macros each processing 64 data values

	cmp	ffttype, 2
	je	ymid2p
	ja	ymid3p

	start_timer 9
	yloop_init  320			;; 320 iterations
b6a:	yr8_8cl_eight_complex_fft_final rsi, 8*64, 64, 2*64, 4*64
	yloop	8, b6a, -8*8*64+dist64	;; Test loop counter, next source pointer
	yloop	40, b6a			;; Test loop counter
	end_timer 9
	yfft_1_ret

ymid2p:	start_timer 9
	yloop_init  320			;; 320 iterations
	yr8_8cl_eight_complex_with_square_preload
b6b:	yr8_8cl_eight_complex_with_square rsi, 8*64, 64, 2*64, 4*64
	yloop	8, b6b, -8*8*64+dist64	;; Test loop counter, next source pointer
	yloop	40, b6b			;; Test loop counter
	end_timer 9
	jmp	yfinish_unfft

ymid3p:	start_timer 9
x64	mov	r8, DEST2ARG
	mov	rbp, DIST_TO_MULSRCARG
	yloop_init  320			;; 320 iterations
	yr8_8cl_eight_complex_with_mult_preload
b6c:	yr8_8cl_eight_complex_with_mult rsi, 8*64, 64, 2*64, 4*64
	yloop	8, b6c, -8*8*64+dist64	;; Test loop counter, next source pointer
	yloop	40, b6c			;; Test loop counter
x64	add	rsi, r8
	end_timer 9
	jmp	yfinish_unfft

ymid4p:	start_timer 9
	mov	rbp, DIST_TO_MULSRCARG
	yloop_init  320			;; 320 iterations
	yr8_8cl_eight_complex_with_mulf_preload
b6d:	yr8_8cl_eight_complex_with_mulf rsi, 8*64, 64, 2*64, 4*64
	yloop	8, b6d, -8*8*64+dist64	;; Test loop counter, next source pointer
	yloop	40, b6d			;; Test loop counter
	end_timer 9
yfinish_unfft:
	bump	rsi, -40*dist64		;; Restore source pointer

;; Do inverse FFT levels 10,11
;; On input the 64-byte cache lines hold these data values:
;;	0	+5K	+5K	+5K	8	+5K	+5K	+5K
;;	1
;;	...
;;	7
;;	16	...
;;	...
;;	5004	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+5K	+5K	+5K	32	+5K	+5K	+5K
;;	1
;;	...
;;	31
;;	64	...
;;	...
;;	5056	...

;; Do 2560 four_complex_unfft operations
;;	distance between fft data elements is 8
;;	do 640 macros each processing 32 data values

	start_timer 10
	mov	rdi, sincos1		;; Load sin/cos pointer
	yloop_init  640			;; 80 iterations of 8
	yr4_b4cl_four_complex_djbunfft_preload
c5b:	yr4_b4cl_four_complex_djbunfft rsi, 64, 8*64, 16*64, rdi, 16*YMM_SCD2/4, 8
	yloop	8, c5b, -8*64+32*64, rdi, -8*16*YMM_SCD2/4 ;; Test loop counter, next source pointer, restore sine/cosine pointer
	yloop	2, c5b, -2*32*64+dist64	;; Test loop counter, next source pointer
	yloop	40, c5b, -40*dist64	;; Test loop counter, restore source pointer
	end_timer 10

;; Do inverse FFT levels 8,9
;; On input the 64-byte cache lines hold these data values:
;;	0	+5K	+5K	+5K	32	+5K	+5K	+5K
;;	1
;;	...
;;	31
;;	64	...
;;	...
;;	5056	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+5K	+5K	+5K	128	+5K	+5K	+5K
;;	1
;;	...
;;	127
;;	256	...
;;	...
;;	4864	...

;; Do 2560 four_complex_unfft operations
;;	distance between fft data elements is 32
;;	do 640 macros each processing 32 data values

	start_timer 11
	yloop_init  640			;; 20 iterations of 32
	yr4_b4cl_four_complex_djbunfft_preload
;; Loop below reworked to avoid 4KB distances
;;c4b:	yr4_b4cl_four_complex_djbunfft rsi, 64, 32*64, dist64, rdi, 4*YMM_SCD2/4, 32
;;	yloop	32, c4b, -32*64+2*dist64, rdi, -32*4*YMM_SCD2/4 ;; Test loop counter, next source pointer, restore sine/cosine pointer
c4b:	yr4_b4cl_four_complex_djbunfft rsi, 2*64, 32*64, dist64, rdi, 2*4*YMM_SCD2/4, 16
	yloop	16, c4b, -16*2*64+64, rdi, -16*2*4*YMM_SCD2/4+4*YMM_SCD2/4 ;; Test loop counter, next source pointer, next sine/cosine pointer
	yloop	2, c4b, -2*64+2*dist64, rdi, -2*4*YMM_SCD2/4 ;; Test loop counter, next source pointer, restore sine/cosine pointer
	yloop	20, c4b, -20*2*dist64	;; Test loop counter, restore source pointer
	end_timer 11

;; Do inverse FFT levels 6,7
;; On input the 64-byte cache lines hold these data values:
;;	0	+5K	+5K	+5K	128	+5K	+5K	+5K
;;	1
;;	...
;;	127
;;	256	...
;;	...
;;	4864	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+5K	+5K	+5K	512	+5K	+5K	+5K
;;	1
;;	...
;;	511
;;	1K	...
;;	...
;;	4K	...

;; Do 2560 four_complex_unfft operations
;;	distance between fft data elements is 128
;;	do 640 macros each processing 32 data values

	start_timer 12
	yloop_init  640			;; 5 iterations of 128
	yr4_b4cl_four_complex_djbunfft_preload
c3b:	yr4_b4cl_four_complex_djbunfft rsi, 64, 2*dist64, 4*dist64, rdi, YMM_SCD2/4, 64
	yloop	64, c3b, -64*64+dist64	;; Test loop counter, next source pointer
	yloop	2, c3b, -2*dist64+8*dist64, rdi, -128*YMM_SCD2/4 ;; Test loop counter, next source pointer, restore sine/cosine pointer
	yloop	5, c3b, -5*8*dist64	;; Test loop counter, restore source pointer
	end_timer 12

;; Do inverse FFT levels 4,5
;; On input the 64-byte cache lines hold these data values:
;;	0	+5K	+5K	+5K	512	+5K	+5K	+5K
;;	1
;;	...
;;	511
;;	1K	...
;;	...
;;	4K	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+5K	+5K	+5K	2560	+5K	+5K	+5K
;;	1
;;	...
;;	2559

;; Do 2048 five_complex_unfft operations
;;	distance between fft data elements is 512
;;	do 512 macros each processing 40 data values

	start_timer 13
	bump	rdi, 128*YMM_SCD2/4	;; Restore sine/cosine pointer
	yloop_init  512			;; 512 iterations
	yr5_b5cl_five_complex_djbunfft_preload
c2b:	yr5_b5cl_five_complex_djbunfft rsi, 64, 8*dist64, rdi, YMM_SCD2/4, 64
	yloop	64, c2b, -64*64+dist64	;; Test loop counter, next source pointer
	yloop	8, c2b, -8*dist64	;; Test loop counter, restore source pointer
	end_timer 13

;; Do inverse FFT levels 1,2,3
;; On input the 64-byte cache lines hold these data values:
;;	0	+5K	+5K	+5K	2560	+5K	+5K	+5K
;;	1
;;	...
;;	2559
;; These are swizzled to:
;;	0	+640	+640	+640	2560	+640	+640	+640
;;	1	...
;;	...
;;	639
;;	5K	...
;;	10K	...
;;	15K	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+640	+640	+640	10K	+640	+640	+640
;;	1	...
;;	...
;;	639
;;	2560	...
;;	5120	...
;;	7680	...

;; Do 2560 four_complex_last_unfft operations
;;	distance between fft data elements is 2560
;;	do 640 macros each processing 32 data values

	start_timer 14
	yloop_init  640			;; 640 iterations
	yr4_s4cl_four_complex_last_unfft_preload
c1b:	yr4_s4cl_four_complex_last_unfft rsi, 64, 10*dist64, 20*dist64, rdi, YMM_SCD4, 64
	yloop	64, c1b, -64*64+dist64	;; Test loop counter, next source pointer
	yloop	10, c1b			;; Test loop counter
	end_timer 14

	ytwo_dest_adjust
	yfft_3_ret
	ENDM


;; Perform a 18K-element FFT.

yfft18K MACRO type
	LOCAL	ymid2, ymid3, ymid4, yfinish_unfft
	LOCAL	b1b, b2b, b3a, b3b, b4a, b4b, b5a, b5b, b6a, b6b, b7a, b7b, b7c, b7d
	LOCAL	c1b, c2b, c3a, c3b, c4a, c4b, c5a, c5b, c6a, c6b

;; Do a multiply with pre-FFTed inputs

	cmp	ffttype, 4
	je	ymid4

;; Do FFT levels 1,2,3
;; On input the 64-byte cache lines hold these data values:
;;	0	+576	+576	+576	9K	+576	+576	+576
;;	1
;;	...
;;	575
;;	2304	...
;;	4608	...
;;	6912	...
;; On output these are swizzled to:
;;	0	+4.5K	+4.5K	+4.5K	2304	+4.5K	+4.5K	+4.5K
;;	1
;;	...
;;	2303

;; Do 2304 eight_reals_fft operations
;;	distance between fft data elements is 2304
;;	do 576 macros each processing 32 data values

	start_timer 4
	mov	rdi, sincos1		;; Load sin/cos pointer
	bump	rdi, 768*YMM_SCD2+64*YMM_SCD3
	yloop_init  576			;; 576 iterations
	yr4_fs4cl_eight_reals_fft_preload
b1b:	yr4_fs4cl_eight_reals_fft rsi, 64, 9*dist64, 18*dist64, rdi, YMM_SCD3, 64
	yloop	64, b1b, -64*64+dist64	;; Test loop counter, next source pointer
	yloop	9, b1b, -9*dist64	;; Test loop counter, restore source pointer
	end_timer 4

;; Do FFT levels 4,5
;; On input the 64-byte cache lines hold these data values:
;;	0	+4.5K	+4.5K	+4.5K	2304	+4.5K	+4.5K	+4.5K
;;	1
;;	...
;;	2303
;; On output the 64-byte cache lines hold these data values:
;;	0	+4.5K	+4.5K	+4.5K	768	+4.5K	+4.5K	+4.5K
;;	1
;;	...
;;	767
;;	1536	...
;;	3072	...

;; Do 768 six_reals_fft operations
;; Do 2304 three_complex_fft operations
;;	distance between fft data elements is 768
;;	do 768 macros each processing 24 data values

	start_timer 5
	bump	rdi, -576*YMM_SCD3-768*YMM_SCD2 ;; Load sin/cos pointer
	yloop_init  768			;; 768 iterations
	yr3_3cl_six_reals_three_complex_djbfft_preload
b2b:	yr3_3cl_six_reals_three_complex_djbfft rsi, 64, 12*dist64, rdi, YMM_SCD2, 64
	yloop	64, b2b, -64*64+dist64	;; Test loop counter, next source pointer
	yloop	12, b2b, -12*dist64, rdi, -768*YMM_SCD2 ;; Test loop counter, restore source pointer, restore sin/cos pointer
	end_timer 5

;; Do FFT levels 6,7
;; On input the 64-byte cache lines hold these data values:
;;	0	+4.5K	+4.5K	+4.5K	768	+4.5K	+4.5K	+4.5K
;;	1
;;	...
;;	767
;;	1536	...
;;	2304	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+4.5K	+4.5K	+4.5K	256	+4.5K	+4.5K	+4.5K
;;	1
;;	...
;;	255
;;	512	...
;;	...
;;	4096	...

;; Do 256 six_reals_fft operations
;; Do 2816 three_complex_fft operations
;;	distance between fft data elements is 256
;;	do 768 macros each processing 24 data values

	start_timer 6
	yloop_init  768			;; 3 iterations of 256
	yr3_3cl_six_reals_three_complex_djbfft_preload
b3a:	yr3_3cl_six_reals_three_complex_djbfft rsi, 64, 4*dist64, rdi, 3*YMM_SCD2, 64
	yloop	64, b3a, -64*64+dist64	;; Test loop counter, next source pointer
	yloop	4, b3a, -4*dist64+12*dist64, rdi, -256*3*YMM_SCD2 ;; Test loop counter, next source pointer, restore sin/cos pointer

	loops_reset			;; 2 more iters of 256
	yr3_rb3cl_three_complex_djbfft_preload
b3b:	yr3_rb3cl_three_complex_djbfft rsi, 64, 4*dist64, rdi, 3*YMM_SCD2, 64
	yloop	64, b3b, -64*64+dist64	;; Test loop counter, next source pointer
	yloop	4, b3b, -4*dist64+12*dist64, rdi, -256*3*YMM_SCD2 ;; Test loop counter, next source pointer, restore sin/cos pointer
	yloop	3, b3b, -3*12*dist64	;; Test loop counter, restore source pointer
	end_timer 6

;; Do FFT levels 8,9
;; On input the 64-byte cache lines hold these data values:
;;	0	+4.5K	+4.5K	+4.5K	256	+4.5K	+4.5K	+4.5K
;;	1
;;	...
;;	255
;;	512	...
;;	...
;;	4096	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+4.5K	+4.5K	+4.5K	64	+4.5K	+4.5K	+4.5K
;;	1
;;	...
;;	63
;;	128	...
;;	...
;;	4480	...

;; Do 64 eight_reals_fft operations
;; Do 2240 four_complex_fft operations
;;	distance between fft data elements is 64
;;	do 576 macros each processing 32 data values

	start_timer 7
	bump	rdi, -64*YMM_SCD3	;; Next sin/cos pointer
	yloop_init  576			;; 9 iterations of 64
	yr4_4cl_eight_reals_four_complex_djbfft_preload
;; Loop below reworked to avoid 4KB distances
;;b4a:	yr4_4cl_eight_reals_four_complex_djbfft rsi, 64, dist64, 2*dist64, rdi, YMM_SCD3, 64
;;	yloop	64, b4a, -64*64+4*dist64, rdi, -64*YMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer
b4a:	yr4_4cl_eight_reals_four_complex_djbfft rsi, 4*64, dist64, 2*dist64, rdi, 4*YMM_SCD3, 16
	yloop	16, b4a, -16*4*64+64, rdi, -16*4*YMM_SCD3+YMM_SCD3 ;; Test loop counter, next source pointer, next sin/cos pointer
	yloop	4, b4a, -4*64+4*dist64, rdi, -4*YMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer

	loops_reset			;; 8 more iters of 64
	yr4_rb4cl_four_complex_djbfft_preload
;; Loop below reworked to avoid 4KB distances
;;b4b:	yr4_rb4cl_four_complex_djbfft rsi, 64, dist64, 2*dist64, rdi, YMM_SCD3, 64
;;	yloop	64, b4b, -64*64+4*dist64, rdi, -64*YMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer
b4b:	yr4_rb4cl_four_complex_djbfft rsi, 4*64, dist64, 2*dist64, rdi, 4*YMM_SCD3, 16
	yloop	16, b4b, -16*4*64+64, rdi, -16*4*YMM_SCD3+YMM_SCD3 ;; Test loop counter, next source pointer, next sin/cos pointer
	yloop	4, b4b, -4*64+4*dist64, rdi, -4*YMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer
	yloop	9, b4b, -9*4*dist64	;; Test loop counter, restore source pointer
	end_timer 7

;; Do FFT levels 10,11
;; On input the 64-byte cache lines hold these data values:
;;	0	+4.5K	+4.5K	+4.5K	64	+4.5K	+4.5K	+4.5K
;;	1
;;	...
;;	63
;;	128	...
;;	...
;;	4480	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+4.5K	+4.5K	+4.5K	16	+4.5K	+4.5K	+4.5K
;;	1
;;	...
;;	15
;;	32	...
;;	...
;;	4576	...

;; Do 16 eight_reals_fft operations
;; Do 2288 four_complex_fft operations
;;	distance between fft data elements is 16
;;	do 576 macros each processing 32 data values

	start_timer 8
	yloop_init  576			;; 36 iterations of 16
	yr4_4cl_eight_reals_four_complex_djbfft_preload
b5a:	yr4_4cl_eight_reals_four_complex_djbfft rsi, 64, 16*64, 32*64, rdi, 4*YMM_SCD3, 16
	yloop	16, b5a, -16*64+dist64, rdi, -16*4*YMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer

	loops_reset			;; 35 more iters of 16
	yr4_rb4cl_four_complex_djbfft_preload
b5b:	yr4_rb4cl_four_complex_djbfft rsi, 64, 16*64, 32*64, rdi, 4*YMM_SCD3, 16
	yloop	16, b5b, -16*64+dist64, rdi, -16*4*YMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer
	yloop	36, b5b, -36*dist64	;; Test loop counter, restore source pointer
	end_timer 8

;; Do FFT levels 12,13
;; On input the 64-byte cache lines hold these data values:
;;	0	+4.5K	+4.5K	+4.5K	16	+4.5K	+4.5K	+4.5K
;;	1
;;	...
;;	15
;;	32	...
;;	...
;;	4576	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+4.5K	+4.5K	+4.5K	4	+4.5K	+4.5K	+4.5K
;;	1
;;	...
;;	3
;;	8	...
;;	...
;;	4600	...

;; Do 4 eight_reals_fft operations
;; Do 2300 four_complex_fft operations
;;	distance between fft data elements is 4
;;	do 576 macros each processing 32 data values

	start_timer 9
	yloop_init  576			;; 144 iterations of 4
	yr4_4cl_eight_reals_four_complex_djbfft_preload
b6a:	yr4_4cl_eight_reals_four_complex_djbfft rsi, 64, 4*64, 8*64, rdi, 16*YMM_SCD3, 4
	yloop	4, b6a, -4*64+16*64, rdi, -4*16*YMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer

	loops_reset			;; 143 more iters of 4
	yr4_rb4cl_four_complex_djbfft_preload
b6b:	yr4_rb4cl_four_complex_djbfft rsi, 64, 4*64, 8*64, rdi, 16*YMM_SCD3, 4
	yloop	4, b6b, -4*64+16*64, rdi, -4*16*YMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer
	yloop	4, b6b, -4*16*64+dist64	;; Test loop counter, next source pointer
	yloop	36, b6b, -36*dist64	;; Test loop counter, restore source pointer
	end_timer 9

;; Do FFT levels 14,15
;; On input and output the 64-byte cache lines hold these data values:
;;	0	+4.5K	+4.5K	+4.5K	4	+4.5K	+4.5K	+4.5K
;;	1
;;	...
;;	3
;;	8	...
;;	...
;;	4600	...

;; Do 1 eight_reals_fft operation
;; Do 2303 four_complex_fft operations
;;	distance between fft data elements is 1
;;	do 576 macros each processing 32 data values

	cmp	ffttype, 2
	je	ymid2
	ja	ymid3

	start_timer 10
;;	yr4_4cl_eight_reals_four_complex_fft_final rsi, 4*64, 64, 2*64
	yr4_dispatch call, yreal_fft_final, 64, 2*64
	yloop_init  576, 1, 16		;; 575 more iterations
b7a:	yr4_4cl_four_complex_fft_final rsi, 4*64, 64, 2*64
	yloop	16, b7a, -16*4*64+dist64 ;; Test loop counter, next source pointer
	yloop	36, b7a			;; Test loop counter
	end_timer 10
	yfft_1_ret

ymid2:	start_timer 10
;;	yr4_4cl_eight_reals_four_complex_with_square rsi, 4*64, 64, 2*64
	yr4_dispatch call, yreal_square, 64, 2*64
	yloop_init  576, 1, 16		;; 575 more iterations
	yr4_4cl_four_complex_with_square_preload
b7b:	yr4_4cl_four_complex_with_square rsi, 4*64, 64, 2*64, 1
	yloop	16, b7b, -16*4*64+dist64 ;; Test loop counter, next source pointer
	yloop	36, b7b			;; Test loop counter
	end_timer 10
	jmp	yfinish_unfft

ymid3:	start_timer 10
x64	mov	r8, DEST2ARG
	mov	rbp, DIST_TO_MULSRCARG
;;	yr4_4cl_eight_reals_four_complex_with_mult rsi, 4*64, 64, 2*64
	yr4_dispatch call, yreal_mult, 64, 2*64
	yloop_init  576, 1, 16		;; 575 more iterations
	yr4_4cl_four_complex_with_mult_preload
b7c:	yr4_4cl_four_complex_with_mult rsi, 4*64, 64, 2*64
	yloop	16, b7c, -16*4*64+dist64 ;; Test loop counter, next source pointer
	yloop	36, b7c			;; Test loop counter
x64	add	rsi, r8
	end_timer 10
	jmp	yfinish_unfft

ymid4:	start_timer 10
	mov	rbp, DIST_TO_MULSRCARG
;;	yr4_4cl_eight_reals_four_complex_with_mulf rsi, 4*64, 64, 2*64
	yr4_dispatch call, yreal_mulf, 64, 2*64
	yloop_init  576, 1, 16		;; 575 more iterations
	yr4_4cl_four_complex_with_mulf_preload
b7d:	yr4_4cl_four_complex_with_mulf rsi, 4*64, 64, 2*64
	yloop	16, b7d, -16*4*64+dist64 ;; Test loop counter, next source pointer
	yloop	36, b7d			;; Test loop counter
	end_timer 10
yfinish_unfft:
	bump	rsi, -36*dist64		;; Restore source pointer

;; Do inverse FFT levels 12,13
;; On input the 64-byte cache lines hold these data values:
;;	0	+4.5K	+4.5K	+4.5K	4	+4.5K	+4.5K	+4.5K
;;	1
;;	...
;;	3
;;	8	...
;;	...
;;	4600	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+4.5K	+4.5K	+4.5K	16	+4.5K	+4.5K	+4.5K
;;	1
;;	...
;;	15
;;	32	...
;;	...
;;	4576	...

;; Do 4 eight_reals_unfft operations
;; Do 2300 four_complex_unfft operations
;;	distance between fft data elements is 4
;;	do 576 macros each processing 32 data values

	start_timer 11
	mov	rdi, sincos1		;; Load sin/cos pointer
	yloop_init  576			;; 144 iterations of 4
	yr4_4cl_eight_reals_four_complex_djbunfft_preload
c6a:	yr4_4cl_eight_reals_four_complex_djbunfft rsi, 64, 4*64, 8*64, rdi, 16*YMM_SCD3, 4
	yloop	4, c6a, -4*64+16*64, rdi, -4*16*YMM_SCD3 ;; Test loop counter, next source pointer, restore sine/cosine pointer

	loops_reset			;; 143 more iters of 4
	yr4_rb4cl_four_complex_djbunfft_preload
c6b:	yr4_rb4cl_four_complex_djbunfft rsi, 64, 4*64, 8*64, rdi, 16*YMM_SCD3, 4
	yloop	4, c6b, -4*64+16*64, rdi, -4*16*YMM_SCD3 ;; Test loop counter, next source pointer, restore sine/cosine pointer
	yloop	4, c6b, -4*16*64+dist64	;; Test loop counter, next source pointer
	yloop	36, c6b, -36*dist64	;; Test loop counter, restore source pointer
	end_timer 11

;; Do inverse FFT levels 10,11
;; On input the 64-byte cache lines hold these data values:
;;	0	+4.5K	+4.5K	+4.5K	16	+4.5K	+4.5K	+4.5K
;;	1
;;	...
;;	15
;;	32	...
;;	...
;;	4576	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+4.5K	+4.5K	+4.5K	64	+4.5K	+4.5K	+4.5K
;;	1
;;	...
;;	63
;;	128	...
;;	...
;;	4480	...

;; Do 16 eight_reals_unfft operations
;; Do 2288 four_complex_unfft operations
;;	distance between fft data elements is 16
;;	do 576 macros each processing 32 data values

	start_timer 12
	yloop_init  576			;; 36 iterations of 16
	yr4_4cl_eight_reals_four_complex_djbunfft_preload
c5a:	yr4_4cl_eight_reals_four_complex_djbunfft rsi, 64, 16*64, 32*64, rdi, 4*YMM_SCD3, 16
	yloop	16, c5a, -16*64+dist64, rdi, -16*4*YMM_SCD3 ;; Test loop counter, next source pointer, restore sine/cosine pointer

	loops_reset			;; 47 more iters of 16
	yr4_rb4cl_four_complex_djbunfft_preload
c5b:	yr4_rb4cl_four_complex_djbunfft rsi, 64, 16*64, 32*64, rdi, 4*YMM_SCD3, 16
	yloop	16, c5b, -16*64+dist64, rdi, -16*4*YMM_SCD3 ;; Test loop counter, next source pointer, restore sine/cosine pointer
	yloop	36, c5b, -36*dist64	;; Test loop counter, restore source pointer
	end_timer 12

;; Do inverse FFT levels 8,9
;; On input the 64-byte cache lines hold these data values:
;;	0	+4.5K	+4.5K	+4.5K	64	+4.5K	+4.5K	+4.5K
;;	1
;;	...
;;	63
;;	128	...
;;	...
;;	4480	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+4.5K	+4.5K	+4.5K	256	+4.5K	+4.5K	+4.5K
;;	1
;;	...
;;	255
;;	512	...
;;	...
;;	4096	...

;; Do 64 eight_reals_unfft operations
;; Do 2240 four_complex_unfft operations
;;	distance between fft data elements is 64
;;	do 576 macros each processing 32 data values

	start_timer 13
	yloop_init  576			;; 9 iterations of 64
	yr4_4cl_eight_reals_four_complex_djbunfft_preload
;; Loop below reworked to avoid 4KB distances
;;c4a:	yr4_4cl_eight_reals_four_complex_djbunfft rsi, 64, dist64, 2*dist64, rdi, YMM_SCD3, 64
;;	yloop	64, c4a, -64*64+4*dist64, rdi, -64*YMM_SCD3 ;; Test loop counter, next source pointer, restore sine/cosine pointer
c4a:	yr4_4cl_eight_reals_four_complex_djbunfft rsi, 4*64, dist64, 2*dist64, rdi, 4*YMM_SCD3, 16
	yloop	16, c4a, -16*4*64+64, rdi, -16*4*YMM_SCD3+YMM_SCD3 ;; Test loop counter, next source pointer, next sine/cosine pointer
	yloop	4, c4a, -4*64+4*dist64, rdi, -4*YMM_SCD3 ;; Test loop counter, next source pointer, restore sine/cosine pointer

	loops_reset			;; 8 more iters of 64
	yr4_rb4cl_four_complex_djbunfft_preload
;; Loop below reworked to avoid 4KB distances
;;c4b:	yr4_rb4cl_four_complex_djbunfft rsi, 64, dist64, 2*dist64, rdi, YMM_SCD3, 64
;;	yloop	64, c4b, -64*64+4*dist64, rdi, -64*YMM_SCD3 ;; Test loop counter, next source pointer, restore sine/cosine pointer
c4b:	yr4_rb4cl_four_complex_djbunfft rsi, 4*64, dist64, 2*dist64, rdi, 4*YMM_SCD3, 16
	yloop	16, c4b, -16*4*64+64, rdi, -16*4*YMM_SCD3+YMM_SCD3 ;; Test loop counter, next source pointer, next sine/cosine pointer
	yloop	4, c4b, -4*64+4*dist64, rdi, -4*YMM_SCD3 ;; Test loop counter, next source pointer, restore sine/cosine pointer
	yloop	9, c4b, -9*4*dist64	;; Test loop counter, restore source pointer
	end_timer 13

;; Do inverse FFT levels 6,7
;; On input the 64-byte cache lines hold these data values:
;;	0	+4.5K	+4.5K	+4.5K	256	+4.5K	+4.5K	+4.5K
;;	1
;;	...
;;	255
;;	512	...
;;	...
;;	4096	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+4.5K	+4.5K	+4.5K	768	+4.5K	+4.5K	+4.5K
;;	1
;;	...
;;	767
;;	1536	...
;;	3072	...

;; Do 256 six_reals_unfft operations
;; Do 2816 three_complex_unfft operations
;;	distance between fft data elements is 256
;;	do 768 macros each processing 24 data values

	start_timer 14
	bump	rdi, 64*YMM_SCD3	;; Next sine/cosine pointer
	yloop_init  768			;; 3 iterations of 256
	yr3_3cl_six_reals_three_complex_djbunfft_preload
c3a:	yr3_3cl_six_reals_three_complex_djbunfft rsi, 64, 4*dist64, rdi, 3*YMM_SCD2, 64
	yloop	64, c3a, -64*64+dist64	;; Test loop counter, next source pointer
	yloop	4, c3a, -4*dist64+12*dist64, rdi, -256*3*YMM_SCD2 ;; Test loop counter, next source pointer, restore sine/cosine pointer

	loops_reset			;; 2 more iters of 256
	yr3_rb3cl_three_complex_djbunfft_preload
c3b:	yr3_rb3cl_three_complex_djbunfft rsi, 64, 4*dist64, rdi, 3*YMM_SCD2, 64
	yloop	64, c3b, -64*64+dist64	;; Test loop counter, next source pointer
	yloop	4, c3b, -4*dist64+12*dist64, rdi, -256*3*YMM_SCD2 ;; Test loop counter, next source pointer, restore sine/cosine pointer
	yloop	3, c3b, -3*12*dist64	;; Test loop counter, restore source pointer
	end_timer 14

;; Do inverse FFT levels 4,5
;; On input the 64-byte cache lines hold these data values:
;;	0	+4.5K	+4.5K	+4.5K	768	+4.5K	+4.5K	+4.5K
;;	1
;;	...
;;	767
;;	1536	...
;;	3072	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+4.5K	+4.5K	+4.5K	2304	+4.5K	+4.5K	+4.5K
;;	1
;;	...
;;	2303

;; Do 768 six_reals_unfft operations
;; Do 2304 three_complex_unfft operations
;;	distance between fft data elements is 768
;;	do 768 macros each processing 24 data values

	start_timer 15
	yloop_init  768			;; 768 iterations
	yr3_3cl_six_reals_three_complex_djbunfft_preload
c2b:	yr3_3cl_six_reals_three_complex_djbunfft rsi, 64, 12*dist64, rdi, YMM_SCD2, 64
	yloop	64, c2b, -64*64+dist64	;; Test loop counter, next source pointer
	yloop	12, c2b, -12*dist64	;; Test loop counter, restore source pointer
	end_timer 15

;; Do inverse FFT levels 1,2,3
;; On input the 64-byte cache lines hold these data values:
;;	0	+4.5K	+4.5K	+4.5K	2304	+4.5K	+4.5K	+4.5K
;;	1
;;	...
;;	2303
;; These are swizzled to:
;;	0	+576	+576	+576	2304	+576	+576	+576
;;	1	...
;;	...
;;	575
;;	4.5K	...
;;	9K	...
;;	13.5K	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+576	+576	+576	9K	+576	+576	+576
;;	1	...
;;	...
;;	575
;;	2304	...
;;	4608	...
;;	6912	...

;; Do 2304 eight_reals_unfft operations
;;	distance between fft data elements is 2304
;;	do 576 macros each processing 32 data values

	start_timer 16
	yloop_init  576			;; 576 iterations
	yr4_s4cl_eight_reals_unfft_preload
c1b:	yr4_s4cl_eight_reals_unfft rsi, 64, 9*dist64, 18*dist64, rdi, YMM_SCD3, 64
	yloop	64, c1b, -64*64+dist64	;; Test loop counter, next source pointer
	yloop	9, c1b			;; Test loop counter
	end_timer 16

	ytwo_dest_adjust
	yfft_3_ret
	ENDM

;; Perform a 18K-element negacyclic FFT

yfft18Kp MACRO type
	LOCAL	ymid2p, ymid3p, ymid4p, yfinish_unfft
	LOCAL	b1b, b2b, b3b, b4b, b5b, b6b, b7a, b7b, b7c, b7d, c1b, c2b, c3b, c4b, c5b, c6b

;; Do a multiply with pre-FFTed inputs

	cmp	ffttype, 4
	je	ymid4p

;; Do FFT levels 1,2,3
;; On input the 64-byte cache lines hold these data values:
;;	0	+576	+576	+576	9K	+576	+576	+576
;;	1
;;	...
;;	575
;;	2304	...
;;	4608	...
;;	6912	...
;; On output these are swizzled to:
;;	0	+4.5K	+4.5K	+4.5K	2304	+4.5K	+4.5K	+4.5K
;;	1
;;	...
;;	2303

;; Do 2304 four_complex_first_fft operations
;;	distance between fft data elements is 2304
;;	do 576 macros each processing 32 data values

	start_timer 4
	mov	rdi, sincos1		;; Load sin/cos pointer
	bump	rdi, 768*YMM_SCD1/4+64*YMM_SCD2/4
	yloop_init  576			;; 576 iterations
	yr4_fs4cl_four_complex_first_djbfft_preload
b1b:	yr4_fs4cl_four_complex_first_djbfft rsi, 64, 9*dist64, 18*dist64, rdi, YMM_SCD4, 64
	yloop	64, b1b, -64*64+dist64	;; Test loop counter, next source pointer
	yloop	9, b1b, -9*dist64	;; Test loop counter, restore source pointer
	end_timer 4

;; Do FFT levels 4,5
;; On input the 64-byte cache lines hold these data values:
;;	0	+4.5K	+4.5K	+4.5K	2304	+4.5K	+4.5K	+4.5K
;;	1
;;	...
;;	2303
;; On output the 64-byte cache lines hold these data values:
;;	0	+4.5K	+4.5K	+4.5K	768	+4.5K	+4.5K	+4.5K
;;	1
;;	...
;;	767
;;	1536	...
;;	3072	...

;; Do 3072 three_complex_fft operations
;;	distance between fft data elements is 768
;;	do 768 macros each processing 24 data values

	start_timer 5
	bump	rdi, -576*YMM_SCD4-768*YMM_SCD1/4 ;; Next sin/cos pointer
	yloop_init  768			;; 768 iterations
	yr3_b3cl_three_complex_djbfft_preload
b2b:	yr3_b3cl_three_complex_djbfft rsi, 64, 12*dist64, rdi, YMM_SCD1/4, 64
	yloop	64, b2b, -64*64+dist64	;; Test loop counter, next source pointer
	yloop	12, b2b, -12*dist64, rdi, -768*YMM_SCD1/4 ;; Test loop counter, restore source pointer, restore sin/cos pointer
	end_timer 5

;; Do FFT levels 6,7
;; On input the 64-byte cache lines hold these data values:
;;	0	+4.5K	+4.5K	+4.5K	768	+4.5K	+4.5K	+4.5K
;;	1
;;	...
;;	767
;;	1536	...
;;	2304	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+4.5K	+4.5K	+4.5K	256	+4.5K	+4.5K	+4.5K
;;	1
;;	...
;;	255
;;	512	...
;;	...
;;	4096	...

;; Do 3072 three_complex_fft operations
;;	distance between fft data elements is 256
;;	do 768 macros each processing 24 data values

	start_timer 6
	yloop_init  768			;; 3 iterations of 256
	yr3_b3cl_three_complex_djbfft_preload
b3b:	yr3_b3cl_three_complex_djbfft rsi, 64, 4*dist64, rdi, 3*YMM_SCD1/4, 64
	yloop	64, b3b, -64*64+dist64	;; Test loop counter, next source pointer
	yloop	4, b3b, -4*dist64+12*dist64, rdi, -256*3*YMM_SCD1/4 ;; Test loop counter, next source pointer, restore sin/cos pointer
	yloop	3, b3b, -3*12*dist64	;; Test loop counter, restore source pointer
	end_timer 6

;; Do FFT levels 8,9
;; On input the 64-byte cache lines hold these data values:
;;	0	+4.5K	+4.5K	+4.5K	256	+4.5K	+4.5K	+4.5K
;;	1
;;	...
;;	255
;;	512	...
;;	...
;;	4096	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+4.5K	+4.5K	+4.5K	64	+4.5K	+4.5K	+4.5K
;;	1
;;	...
;;	63
;;	128	...
;;	...
;;	4480	...

;; Do 2304 four_complex_fft operations
;;	distance between fft data elements is 64
;;	do 576 macros each processing 32 data values

	start_timer 7
	bump	rdi, -64*YMM_SCD2/4	;; Next sin/cos pointer
	yloop_init  576			;; 9 iterations of 64
	yr4_b4cl_four_complex_djbfft_preload
;; Loop below reworked to avoid 4KB distances
;;b4b:	yr4_b4cl_four_complex_djbfft rsi, 64, dist64, 2*dist64, rdi, YMM_SCD2/4, 64
;;	yloop	64, b4b, -64*64+4*dist64, rdi, -64*YMM_SCD2/4 ;; Test loop counter, next source pointer, restore sin/cos pointer
b4b:	yr4_b4cl_four_complex_djbfft rsi, 4*64, dist64, 2*dist64, rdi, 4*YMM_SCD2/4, 16
	yloop	16, b4b, -16*4*64+64, rdi, -16*4*YMM_SCD2/4+YMM_SCD2/4 ;; Test loop counter, next source pointer, next sin/cos pointer
	yloop	4, b4b, -4*64+4*dist64, rdi, -4*YMM_SCD2/4 ;; Test loop counter, next source pointer, restore sin/cos pointer
	yloop	9, b4b, -9*4*dist64	;; Test loop counter, restore source pointer
	end_timer 7

;; Do FFT levels 10,11
;; On input the 64-byte cache lines hold these data values:
;;	0	+4.5K	+4.5K	+4.5K	64	+4.5K	+4.5K	+4.5K
;;	1
;;	...
;;	63
;;	128	...
;;	...
;;	4480	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+4.5K	+4.5K	+4.5K	16	+4.5K	+4.5K	+4.5K
;;	1
;;	...
;;	15
;;	32	...
;;	...
;;	4576	...

;; Do 2304 four_complex_fft operations
;;	distance between fft data elements is 16
;;	do 576 macros each processing 32 data values

	start_timer 8
	yloop_init  576			;; 36 iterations of 16
	yr4_b4cl_four_complex_djbfft_preload
b5b:	yr4_b4cl_four_complex_djbfft rsi, 64, 16*64, 32*64, rdi, 4*YMM_SCD2/4, 16
	yloop	16, b5b, -16*64+dist64, rdi, -16*4*YMM_SCD2/4 ;; Test loop counter, next source pointer, restore sin/cos pointer
	yloop	36, b5b, -36*dist64	;; Test loop counter, restore source pointer
	end_timer 8

;; Do FFT levels 12,13
;; On input the 64-byte cache lines hold these data values:
;;	0	+4.5K	+4.5K	+4.5K	16	+4.5K	+4.5K	+4.5K
;;	1
;;	...
;;	15
;;	32	...
;;	...
;;	4576	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+4.5K	+4.5K	+4.5K	4	+4.5K	+4.5K	+4.5K
;;	1
;;	...
;;	3
;;	8	...
;;	...
;;	4600	...

;; Do 2304 four_complex_fft operations
;;	distance between fft data elements is 4
;;	do 576 macros each processing 32 data values

	start_timer 9
	yloop_init  576			;; 144 iterations of 4
	yr4_b4cl_four_complex_djbfft_preload
b6b:	yr4_b4cl_four_complex_djbfft rsi, 64, 4*64, 8*64, rdi, 16*YMM_SCD2/4, 4
	yloop	4, b6b, -4*64+16*64, rdi, -4*16*YMM_SCD2/4 ;; Test loop counter, next source pointer, restore sin/cos pointer
	yloop	4, b6b, -4*16*64+dist64	;; Test loop counter, next source pointer
	yloop	36, b6b, -36*dist64	;; Test loop counter, restore source pointer
	end_timer 9

;; Do FFT levels 14,15
;; On input and output the 64-byte cache lines hold these data values:
;;	0	+4.5K	+4.5K	+4.5K	4	+4.5K	+4.5K	+4.5K
;;	1
;;	...
;;	3
;;	8	...
;;	...
;;	4600	...

;; Do 2304 four_complex_fft operations
;;	distance between fft data elements is 1
;;	do 576 macros each processing 32 data values

	cmp	ffttype, 2
	je	ymid2p
	ja	ymid3p

	start_timer 10
	yloop_init  576			;; 576 iterations
b7a:	yr4_4cl_four_complex_fft_final rsi, 4*64, 64, 2*64
	yloop	16, b7a, -16*4*64+dist64 ;; Test loop counter, next source pointer
	yloop	36, b7a			;; Test loop counter
	end_timer 10
	yfft_1_ret

ymid2p:	start_timer 10
	yloop_init  576			;; 576 iterations
	yr4_4cl_four_complex_with_square_preload
b7b:	yr4_4cl_four_complex_with_square rsi, 4*64, 64, 2*64, 16
	yloop	16, b7b, -16*4*64+dist64 ;; Test loop counter, next source pointer
	yloop	36, b7b			;; Test loop counter
	end_timer 10
	jmp	yfinish_unfft

ymid3p:	start_timer 10
x64	mov	r8, DEST2ARG
	mov	rbp, DIST_TO_MULSRCARG
	yloop_init  576			;; 576 iterations
	yr4_4cl_four_complex_with_mult_preload
b7c:	yr4_4cl_four_complex_with_mult rsi, 4*64, 64, 2*64
	yloop	16, b7c, -16*4*64+dist64 ;; Test loop counter, next source pointer
	yloop	36, b7c			;; Test loop counter
x64	add	rsi, r8
	end_timer 10
	jmp	yfinish_unfft

ymid4p:	start_timer 10
	mov	rbp, DIST_TO_MULSRCARG
	yloop_init  576			;; 576 iterations
	yr4_4cl_four_complex_with_mulf_preload
b7d:	yr4_4cl_four_complex_with_mulf rsi, 4*64, 64, 2*64
	yloop	16, b7d, -16*4*64+dist64 ;; Test loop counter, next source pointer
	yloop	36, b7d			;; Test loop counter
	end_timer 10
yfinish_unfft:
	bump	rsi, -36*dist64		;; Restore source pointer

;; Do inverse FFT levels 12,13
;; On input the 64-byte cache lines hold these data values:
;;	0	+4.5K	+4.5K	+4.5K	4	+4.5K	+4.5K	+4.5K
;;	1
;;	...
;;	3
;;	8	...
;;	...
;;	4600	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+4.5K	+4.5K	+4.5K	16	+4.5K	+4.5K	+4.5K
;;	1
;;	...
;;	15
;;	32	...
;;	...
;;	4576	...

;; Do 2304 four_complex_unfft operations
;;	distance between fft data elements is 4
;;	do 576 macros each processing 32 data values

	start_timer 11
	mov	rdi, sincos1		;; Load sin/cos pointer
	yloop_init  576			;; 144 iterations of 4
	yr4_b4cl_four_complex_djbunfft_preload
c6b:	yr4_b4cl_four_complex_djbunfft rsi, 64, 4*64, 8*64, rdi, 16*YMM_SCD2/4, 4
	yloop	4, c6b, -4*64+16*64, rdi, -4*16*YMM_SCD2/4 ;; Test loop counter, next source pointer, restore sine/cosine pointer
	yloop	4, c6b, -4*16*64+dist64	;; Test loop counter, next source pointer
	yloop	36, c6b, -36*dist64	;; Test loop counter, restore source pointer
	end_timer 11

;; Do inverse FFT levels 10,11
;; On input the 64-byte cache lines hold these data values:
;;	0	+4.5K	+4.5K	+4.5K	16	+4.5K	+4.5K	+4.5K
;;	1
;;	...
;;	15
;;	32	...
;;	...
;;	4576	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+4.5K	+4.5K	+4.5K	64	+4.5K	+4.5K	+4.5K
;;	1
;;	...
;;	63
;;	128	...
;;	...
;;	4480	...

;; Do 2304 four_complex_unfft operations
;;	distance between fft data elements is 16
;;	do 576 macros each processing 32 data values

	start_timer 12
	yloop_init  576			;; 36 iterations of 16
	yr4_b4cl_four_complex_djbunfft_preload
c5b:	yr4_b4cl_four_complex_djbunfft rsi, 64, 16*64, 32*64, rdi, 4*YMM_SCD2/4, 16
	yloop	16, c5b, -16*64+dist64, rdi, -16*4*YMM_SCD2/4 ;; Test loop counter, next source pointer, restore sine/cosine pointer
	yloop	36, c5b, -36*dist64	;; Test loop counter, restore source pointer
	end_timer 12

;; Do inverse FFT levels 8,9
;; On input the 64-byte cache lines hold these data values:
;;	0	+4.5K	+4.5K	+4.5K	64	+4.5K	+4.5K	+4.5K
;;	1
;;	...
;;	63
;;	128	...
;;	...
;;	4480	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+4.5K	+4.5K	+4.5K	256	+4.5K	+4.5K	+4.5K
;;	1
;;	...
;;	255
;;	512	...
;;	...
;;	4096	...

;; Do 2304 four_complex_unfft operations
;;	distance between fft data elements is 64
;;	do 576 macros each processing 32 data values

	start_timer 13
	yloop_init  576			;; 9 iterations of 64
	yr4_b4cl_four_complex_djbunfft_preload
;; Loop below reworked to avoid 4KB distances
;;c4b:	yr4_b4cl_four_complex_djbunfft rsi, 64, dist64, 2*dist64, rdi, YMM_SCD2/4, 64
;;	yloop	64, c4b, -64*64+4*dist64, rdi, -64*YMM_SCD2/4 ;; Test loop counter, next source pointer, restore sine/cosine pointer
c4b:	yr4_b4cl_four_complex_djbunfft rsi, 4*64, dist64, 2*dist64, rdi, 4*YMM_SCD2/4, 16
	yloop	16, c4b, -16*4*64+64, rdi, -16*4*YMM_SCD2/4+YMM_SCD2/4 ;; Test loop counter, next source pointer, next sine/cosine pointer
	yloop	4, c4b, -4*64+4*dist64, rdi, -4*YMM_SCD2/4 ;; Test loop counter, next source pointer, restore sine/cosine pointer
	yloop	9, c4b, -9*4*dist64	;; Test loop counter, restore source pointer
	end_timer 13

;; Do inverse FFT levels 6,7
;; On input the 64-byte cache lines hold these data values:
;;	0	+4.5K	+4.5K	+4.5K	256	+4.5K	+4.5K	+4.5K
;;	1
;;	...
;;	255
;;	512	...
;;	...
;;	4096	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+4.5K	+4.5K	+4.5K	768	+4.5K	+4.5K	+4.5K
;;	1
;;	...
;;	767
;;	1536	...
;;	3072	...

;; Do 3072 three_complex_unfft operations
;;	distance between fft data elements is 256
;;	do 768 macros each processing 32 data values

	start_timer 14
	bump	rdi, 64*YMM_SCD2/4	;; Next sine/cosine pointer
	yloop_init  768			;; 3 iterations of 256
	yr3_b3cl_three_complex_djbunfft_preload
c3b:	yr3_b3cl_three_complex_djbunfft rsi, 64, 4*dist64, rdi, 3*YMM_SCD1/4, 64
	yloop	64, c3b, -64*64+dist64	;; Test loop counter, next source pointer
	yloop	4, c3b, -4*dist64+12*dist64, rdi, -256*3*YMM_SCD1/4 ;; Test loop counter, next source pointer, restore sine/cosine pointer
	yloop	3, c3b, -3*12*dist64	;; Test loop counter, restore source pointer
	end_timer 14

;; Do inverse FFT levels 4,5
;; On input the 64-byte cache lines hold these data values:
;;	0	+4.5K	+4.5K	+4.5K	768	+4.5K	+4.5K	+4.5K
;;	1
;;	...
;;	767
;;	1536	...
;;	3072	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+4.5K	+4.5K	+4.5K	2304	+4.5K	+4.5K	+4.5K
;;	1
;;	...
;;	2303

;; Do 3072 three_complex_unfft operations
;;	distance between fft data elements is 768
;;	do 768 macros each processing 24 data values

	start_timer 15
	yloop_init  768			;; 768 iterations
	yr3_b3cl_three_complex_djbunfft_preload
c2b:	yr3_b3cl_three_complex_djbunfft rsi, 64, 12*dist64, rdi, YMM_SCD1/4, 64
	yloop	64, c2b, -64*64+dist64	;; Test loop counter, next source pointer
	yloop	12, c2b, -12*dist64	;; Test loop counter, restore source pointer
	end_timer 15

;; Do inverse FFT levels 1,2,3
;; On input the 64-byte cache lines hold these data values:
;;	0	+4.5K	+4.5K	+4.5K	2304	+4.5K	+4.5K	+4.5K
;;	1
;;	...
;;	2303
;; These are swizzled to:
;;	0	+576	+576	+576	2304	+576	+576	+576
;;	1	...
;;	...
;;	575
;;	4.5K	...
;;	9K	...
;;	13.5K	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+576	+576	+576	9K	+576	+576	+576
;;	1	...
;;	...
;;	575
;;	2304	...
;;	4608	...
;;	6912	...

;; Do 2304 four_complex_last_unfft operations
;;	distance between fft data elements is 2304
;;	do 576 macros each processing 32 data values

	start_timer 16
	yloop_init  576			;; 576 iterations
	yr4_s4cl_four_complex_last_unfft_preload
c1b:	yr4_s4cl_four_complex_last_unfft rsi, 64, 9*dist64, 18*dist64, rdi, YMM_SCD4, 64
	yloop	64, c1b, -64*64+dist64	;; Test loop counter, next source pointer
	yloop	9, c1b			;; Test loop counter
	end_timer 16

	ytwo_dest_adjust
	yfft_3_ret
	ENDM


;; Perform a 24K-element FFT.

yfft24K MACRO type
	LOCAL	ymid2, ymid3, ymid4, yfinish_unfft
	LOCAL	b1b, b2b, b3a, b3b, b4a, b4b, b5a, b5b, b6a, b6b, b7a, b7b, b7c, b7d
	LOCAL	c1b, c2b, c3a, c3b, c4a, c4b, c5a, c5b, c6a, c6b

;; Do a multiply with pre-FFTed inputs

	cmp	ffttype, 4
	je	ymid4

;; Do FFT levels 1,2,3
;; On input the 64-byte cache lines hold these data values:
;;	0	+768	+768	+768	12K	+768	+768	+768
;;	1
;;	...
;;	767
;;	3K	...
;;	6K	...
;;	9K	...
;; On output these are swizzled to:
;;	0	+6K	+6K	+6K	3K	+6K	+6K	+6K
;;	1
;;	...
;;	3071

;; Do 3072 eight_reals_fft operations
;;	distance between fft data elements is 3072
;;	do 768 macros each processing 32 data values

	start_timer 4
	mov	rdi, sincos1		;; Load sin/cos pointer
	bump	rdi, 1024*YMM_SCD2+256*YMM_SCD3
	yloop_init  768			;; 768 iterations
	yr4_fs4cl_eight_reals_fft_preload
b1b:	yr4_fs4cl_eight_reals_fft rsi, 64, 12*dist64, 24*dist64, rdi, YMM_SCD3, 64
	yloop	64, b1b, -64*64+dist64	;; Test loop counter, next source pointer
	yloop	12, b1b, -12*dist64	;; Test loop counter, restore source pointer
	end_timer 4

;; Do FFT levels 4,5
;; On input the 64-byte cache lines hold these data values:
;;	0	+6K	+6K	+6K	3K	+6K	+6K	+6K
;;	1
;;	...
;;	3071
;; On output the 64-byte cache lines hold these data values:
;;	0	+6K	+6K	+6K	1K	+6K	+6K	+6K
;;	1
;;	...
;;	1023
;;	2K	...
;;	4K	...

;; Do 1024 six_reals_fft operations
;; Do 3072 three_complex_fft operations
;;	distance between fft data elements is 1024
;;	do 1024 macros each processing 24 data values

	start_timer 5
	bump	rdi, -768*YMM_SCD3-1024*YMM_SCD2 ;; Load sin/cos pointer
	yloop_init  1024			;; 1024 iterations
	yr3_3cl_six_reals_three_complex_djbfft_preload
b2b:	yr3_3cl_six_reals_three_complex_djbfft rsi, 64, 16*dist64, rdi, YMM_SCD2, 64
	yloop	64, b2b, -64*64+dist64	;; Test loop counter, next source pointer
	yloop	16, b2b, -16*dist64	;; Test loop counter, restore source pointer
	end_timer 5

;; Do FFT levels 6,7
;; On input the 64-byte cache lines hold these data values:
;;	0	+6K	+6K	+6K	1K	+6K	+6K	+6K
;;	1
;;	...
;;	1023
;;	2K	...
;;	4K	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+6K	+6K	+6K	256	+6K	+6K	+6K
;;	1
;;	...
;;	255
;;	512	...
;;	...
;;	5632	...

;; Do 256 eight_reals_fft operations
;; Do 2816 four_complex_fft operations
;;	distance between fft data elements is 256
;;	do 768 macros each processing 32 data values

	start_timer 6
	bump	rdi, -1024*YMM_SCD2-256*YMM_SCD3 ;; Next sin/cos pointer
	yloop_init  768			;; 3 iterations of 256
	yr4_4cl_eight_reals_four_complex_djbfft_preload
b3a:	yr4_4cl_eight_reals_four_complex_djbfft rsi, 64, 4*dist64, 8*dist64, rdi, YMM_SCD3, 64
	yloop	64, b3a, -64*64+dist64	;; Test loop counter, next source pointer
	yloop	4, b3a, -4*dist64+16*dist64, rdi, -256*YMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer

	loops_reset			;; 2 more iters of 256
	yr4_rb4cl_four_complex_djbfft_preload
b3b:	yr4_rb4cl_four_complex_djbfft rsi, 64, 4*dist64, 8*dist64, rdi, YMM_SCD3, 64
	yloop	64, b3b, -64*64+dist64	;; Test loop counter, next source pointer
	yloop	4, b3b, -4*dist64+16*dist64, rdi, -256*YMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer
	yloop	3, b3b, -3*16*dist64	;; Test loop counter, restore source pointer
	end_timer 6

;; Do FFT levels 8,9
;; On input the 64-byte cache lines hold these data values:
;;	0	+6K	+6K	+6K	256	+6K	+6K	+6K
;;	1
;;	...
;;	255
;;	512	...
;;	...
;;	5632	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+6K	+6K	+6K	64	+6K	+6K	+6K
;;	1
;;	...
;;	63
;;	128	...
;;	...
;;	6016	...

;; Do 64 eight_reals_fft operations
;; Do 3008 four_complex_fft operations
;;	distance between fft data elements is 64
;;	do 768 macros each processing 32 data values

	start_timer 7
	yloop_init  768			;; 12 iterations of 64
	yr4_4cl_eight_reals_four_complex_djbfft_preload
;; Loop below reworked to avoid 4KB distances
;;b4a:	yr4_4cl_eight_reals_four_complex_djbfft rsi, 64, dist64, 2*dist64, rdi, 4*YMM_SCD3, 64
;;	yloop	64, b4a, -64*64+4*dist64, rdi, -64*4*YMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer
b4a:	yr4_4cl_eight_reals_four_complex_djbfft rsi, 4*64, dist64, 2*dist64, rdi, 4*4*YMM_SCD3, 16
	yloop	16, b4a, -16*4*64+64, rdi, -16*4*4*YMM_SCD3+4*YMM_SCD3 ;; Test loop counter, next source pointer, next sin/cos pointer
	yloop	4, b4a, -4*64+4*dist64, rdi, -4*4*YMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer

	loops_reset			;; 11 more iters of 64
	yr4_rb4cl_four_complex_djbfft_preload
;; Loop below reworked to avoid 4KB distances
;;b4b:	yr4_rb4cl_four_complex_djbfft rsi, 64, dist64, 2*dist64, rdi, 4*YMM_SCD3, 64
;;	yloop	64, b4b, -64*64+4*dist64, rdi, -64*4*YMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer
b4b:	yr4_rb4cl_four_complex_djbfft rsi, 4*64, dist64, 2*dist64, rdi, 4*4*YMM_SCD3, 16
	yloop	16, b4b, -16*4*64+64, rdi, -16*4*4*YMM_SCD3+4*YMM_SCD3 ;; Test loop counter, next source pointer, next sin/cos pointer
	yloop	4, b4b, -4*64+4*dist64, rdi, -4*4*YMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer
	yloop	12, b4b, -12*4*dist64	;; Test loop counter, restore source pointer
	end_timer 7

;; Do FFT levels 10,11
;; On input the 64-byte cache lines hold these data values:
;;	0	+6K	+6K	+6K	64	+6K	+6K	+6K
;;	1
;;	...
;;	63
;;	128	...
;;	...
;;	6016	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+6K	+6K	+6K	16	+6K	+6K	+6K
;;	1
;;	...
;;	15
;;	32	...
;;	...
;;	6112	...

;; Do 16 eight_reals_fft operations
;; Do 3056 four_complex_fft operations
;;	distance between fft data elements is 16
;;	do 768 macros each processing 32 data values

	start_timer 8
	yloop_init  768			;; 48 iterations of 16
	yr4_4cl_eight_reals_four_complex_djbfft_preload
b5a:	yr4_4cl_eight_reals_four_complex_djbfft rsi, 64, 16*64, 32*64, rdi, 16*YMM_SCD3, 16
	yloop	16, b5a, -16*64+dist64, rdi, -16*16*YMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer

	loops_reset			;; 47 more iters of 16
	yr4_rb4cl_four_complex_djbfft_preload
b5b:	yr4_rb4cl_four_complex_djbfft rsi, 64, 16*64, 32*64, rdi, 16*YMM_SCD3, 16
	yloop	16, b5b, -16*64+dist64, rdi, -16*16*YMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer
	yloop	48, b5b, -48*dist64	;; Test loop counter, restore source pointer
	end_timer 8

;; Do FFT levels 12,13
;; On input the 64-byte cache lines hold these data values:
;;	0	+6K	+6K	+6K	16	+6K	+6K	+6K
;;	1
;;	...
;;	15
;;	32	...
;;	...
;;	6112	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+6K	+6K	+6K	4	+6K	+6K	+6K
;;	1
;;	...
;;	3
;;	8	...
;;	...
;;	6136	...

;; Do 4 eight_reals_fft operations
;; Do 3068 four_complex_fft operations
;;	distance between fft data elements is 4
;;	do 768 macros each processing 32 data values

	start_timer 9
	yloop_init  768			;; 192 iterations of 4
	yr4_4cl_eight_reals_four_complex_djbfft_preload
b6a:	yr4_4cl_eight_reals_four_complex_djbfft rsi, 64, 4*64, 8*64, rdi, 64*YMM_SCD3, 4
	yloop	4, b6a, -4*64+16*64, rdi, -4*64*YMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer

	loops_reset			;; 191 more iters of 4
	yr4_rb4cl_four_complex_djbfft_preload
b6b:	yr4_rb4cl_four_complex_djbfft rsi, 64, 4*64, 8*64, rdi, 64*YMM_SCD3, 4
	yloop	4, b6b, -4*64+16*64, rdi, -4*64*YMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer
	yloop	4, b6b, -4*16*64+dist64	;; Test loop counter, next source pointer
	yloop	48, b6b, -48*dist64	;; Test loop counter, restore source pointer
	end_timer 9

;; Do FFT levels 14,15
;; On input and output the 64-byte cache lines hold these data values:
;;	0	+6K	+6K	+6K	4	+6K	+6K	+6K
;;	1
;;	...
;;	3
;;	8	...
;;	...
;;	6136	...

;; Do 1 eight_reals_fft operation
;; Do 3071 four_complex_fft operations
;;	distance between fft data elements is 1
;;	do 767 macros each processing 32 data values

	cmp	ffttype, 2
	je	ymid2
	ja	ymid3

	start_timer 10
;;	yr4_4cl_eight_reals_four_complex_fft_final rsi, 4*64, 64, 2*64
	yr4_dispatch call, yreal_fft_final, 64, 2*64
	yloop_init  768, 1, 16		;; 767 more iterations
b7a:	yr4_4cl_four_complex_fft_final rsi, 4*64, 64, 2*64
	yloop	16, b7a, -16*4*64+dist64 ;; Test loop counter, next source pointer
	yloop	48, b7a			;; Test loop counter
	end_timer 10
	yfft_1_ret

ymid2:	start_timer 10
;;	yr4_4cl_eight_reals_four_complex_with_square rsi, 4*64, 64, 2*64
	yr4_dispatch call, yreal_square, 64, 2*64
	yloop_init  768, 1, 16		;; 767 more iterations
	yr4_4cl_four_complex_with_square_preload
b7b:	yr4_4cl_four_complex_with_square rsi, 4*64, 64, 2*64, 1
	yloop	16, b7b, -16*4*64+dist64 ;; Test loop counter, next source pointer
	yloop	48, b7b			;; Test loop counter
	end_timer 10
	jmp	yfinish_unfft

ymid3:	start_timer 10
x64	mov	r8, DEST2ARG
	mov	rbp, DIST_TO_MULSRCARG
;;	yr4_4cl_eight_reals_four_complex_with_mult rsi, 4*64, 64, 2*64
	yr4_dispatch call, yreal_mult, 64, 2*64
	yloop_init  768, 1, 16		;; 767 more iterations
	yr4_4cl_four_complex_with_mult_preload
b7c:	yr4_4cl_four_complex_with_mult rsi, 4*64, 64, 2*64
	yloop	16, b7c, -16*4*64+dist64 ;; Test loop counter, next source pointer
	yloop	48, b7c			;; Test loop counter
x64	add	rsi, r8
	end_timer 10
	jmp	yfinish_unfft

ymid4:	start_timer 10
	mov	rbp, DIST_TO_MULSRCARG
;;	yr4_4cl_eight_reals_four_complex_with_mulf rsi, 4*64, 64, 2*64
	yr4_dispatch call, yreal_mulf, 64, 2*64
	yloop_init  768, 1, 16		;; 767 more iterations
	yr4_4cl_four_complex_with_mulf_preload
b7d:	yr4_4cl_four_complex_with_mulf rsi, 4*64, 64, 2*64
	yloop	16, b7d, -16*4*64+dist64 ;; Test loop counter, next source pointer
	yloop	48, b7d			;; Test loop counter
	end_timer 10
yfinish_unfft:
	bump	rsi, -48*dist64		;; Restore source pointer

;; Do inverse FFT levels 12,13
;; On input the 64-byte cache lines hold these data values:
;;	0	+6K	+6K	+6K	4	+6K	+6K	+6K
;;	1
;;	...
;;	3
;;	8	...
;;	...
;;	6136	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+6K	+6K	+6K	16	+6K	+6K	+6K
;;	1
;;	...
;;	15
;;	32	...
;;	...
;;	6112	...

;; Do 4 eight_reals_unfft operations
;; Do 3068 four_complex_unfft operations
;;	distance between fft data elements is 4
;;	do 768 macros each processing 32 data values

	start_timer 11
	mov	rdi, sincos1		;; Load sin/cos pointer
	yloop_init  768			;; 192 iterations of 4
	yr4_4cl_eight_reals_four_complex_djbunfft_preload
c6a:	yr4_4cl_eight_reals_four_complex_djbunfft rsi, 64, 4*64, 8*64, rdi, 64*YMM_SCD3,4 
	yloop	4, c6a, -4*64+16*64, rdi, -4*64*YMM_SCD3 ;; Test loop counter, next source pointer, restore sine/cosine pointer

	loops_reset			;; 191 more iters of 4
	yr4_rb4cl_four_complex_djbunfft_preload
c6b:	yr4_rb4cl_four_complex_djbunfft rsi, 64, 4*64, 8*64, rdi, 64*YMM_SCD3, 4
	yloop	4, c6b, -4*64+16*64, rdi, -4*64*YMM_SCD3 ;; Test loop counter, next source pointer, restore sine/cosine pointer
	yloop	4, c6b, -4*16*64+dist64	;; Test loop counter, next source pointer
	yloop	48, c6b, -48*dist64	;; Test loop counter, restore source pointer
	end_timer 11

;; Do inverse FFT levels 10,11
;; On input the 64-byte cache lines hold these data values:
;;	0	+6K	+6K	+6K	16	+6K	+6K	+6K
;;	1
;;	...
;;	15
;;	32	...
;;	...
;;	6112	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+6K	+6K	+6K	64	+6K	+6K	+6K
;;	1
;;	...
;;	63
;;	128	...
;;	...
;;	6016	...

;; Do 16 eight_reals_unfft operations
;; Do 3056 four_complex_unfft operations
;;	distance between fft data elements is 16
;;	do 768 macros each processing 32 data values

	start_timer 12
	yloop_init  768			;; 48 iterations of 16
	yr4_4cl_eight_reals_four_complex_djbunfft_preload
c5a:	yr4_4cl_eight_reals_four_complex_djbunfft rsi, 64, 16*64, 32*64, rdi, 16*YMM_SCD3, 16
	yloop	16, c5a, -16*64+dist64, rdi, -16*16*YMM_SCD3 ;; Test loop counter, next source pointer, restore sine/cosine pointer

	loops_reset			;; 47 more iters of 16
	yr4_rb4cl_four_complex_djbunfft_preload
c5b:	yr4_rb4cl_four_complex_djbunfft rsi, 64, 16*64, 32*64, rdi, 16*YMM_SCD3, 16
	yloop	16, c5b, -16*64+dist64, rdi, -16*16*YMM_SCD3 ;; Test loop counter, next source pointer, restore sine/cosine pointer
	yloop	48, c5b, -48*dist64	;; Test loop counter, restore source pointer
	end_timer 12

;; Do inverse FFT levels 8,9
;; On input the 64-byte cache lines hold these data values:
;;	0	+6K	+6K	+6K	64	+6K	+6K	+6K
;;	1
;;	...
;;	63
;;	128	...
;;	...
;;	6016	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+6K	+6K	+6K	256	+6K	+6K	+6K
;;	1
;;	...
;;	255
;;	512	...
;;	...
;;	5632	...

;; Do 64 eight_reals_unfft operations
;; Do 3008 four_complex_unfft operations
;;	distance between fft data elements is 64
;;	do 768 macros each processing 32 data values

	start_timer 13
	yloop_init  768			;; 12 iterations of 64
	yr4_4cl_eight_reals_four_complex_djbunfft_preload
;; Loop below reworked to avoid 4KB distances
;;c4a:	yr4_4cl_eight_reals_four_complex_djbunfft rsi, 64, dist64, 2*dist64, rdi, 4*YMM_SCD3, 64
;;	yloop	64, c4a, -64*64+4*dist64, rdi, -64*4*YMM_SCD3 ;; Test loop counter, next source pointer, restore sine/cosine pointer
c4a:	yr4_4cl_eight_reals_four_complex_djbunfft rsi, 4*64, dist64, 2*dist64, rdi, 4*4*YMM_SCD3, 16
	yloop	16, c4a, -16*4*64+64, rdi, -16*4*4*YMM_SCD3+4*YMM_SCD3 ;; Test loop counter, next source pointer, next sine/cosine pointer
	yloop	4, c4a, -4*64+4*dist64, rdi, -4*4*YMM_SCD3 ;; Test loop counter, next source pointer, restore sine/cosine pointer

	loops_reset			;; 11 more iters of 64
	yr4_rb4cl_four_complex_djbunfft_preload
;; Loop below reworked to avoid 4KB distances
;;c4b:	yr4_rb4cl_four_complex_djbunfft rsi, 64, dist64, 2*dist64, rdi, 4*YMM_SCD3, 64
;;	yloop	64, c4b, -64*64+4*dist64, rdi, -64*4*YMM_SCD3 ;; Test loop counter, next source pointer, restore sine/cosine pointer
c4b:	yr4_rb4cl_four_complex_djbunfft rsi, 4*64, dist64, 2*dist64, rdi, 4*4*YMM_SCD3, 16
	yloop	16, c4b, -16*4*64+64, rdi, -16*4*4*YMM_SCD3+4*YMM_SCD3 ;; Test loop counter, next source pointer, next sine/cosine pointer
	yloop	4, c4b, -4*64+4*dist64, rdi, -4*4*YMM_SCD3 ;; Test loop counter, next source pointer, restore sine/cosine pointer
	yloop	12, c4b, -12*4*dist64	;; Test loop counter, restore source pointer
	end_timer 13

;; Do inverse FFT levels 6,7
;; On input the 64-byte cache lines hold these data values:
;;	0	+6K	+6K	+6K	256	+6K	+6K	+6K
;;	1
;;	...
;;	255
;;	512	...
;;	...
;;	5632	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+6K	+6K	+6K	1K	+6K	+6K	+6K
;;	1
;;	...
;;	1023
;;	2K	...
;;	4K	...

;; Do 256 eight_reals_unfft operations
;; Do 2816 four_complex_unfft operations
;;	distance between fft data elements is 256
;;	do 768 macros each processing 32 data values

	start_timer 14
	yloop_init  768			;; 3 iterations of 256
	yr4_4cl_eight_reals_four_complex_djbunfft_preload
c3a:	yr4_4cl_eight_reals_four_complex_djbunfft rsi, 64, 4*dist64, 8*dist64, rdi, YMM_SCD3, 64
	yloop	64, c3a, -64*64+dist64	;; Test loop counter, next source pointer
	yloop	4, c3a, -4*dist64+16*dist64, rdi, -256*YMM_SCD3 ;; Test loop counter, next source pointer, restore sine/cosine pointer

	loops_reset			;; 2 more iters of 256
	yr4_rb4cl_four_complex_djbunfft_preload
c3b:	yr4_rb4cl_four_complex_djbunfft rsi, 64, 4*dist64, 8*dist64, rdi, YMM_SCD3, 64
	yloop	64, c3b, -64*64+dist64	;; Test loop counter, next source pointer
	yloop	4, c3b, -4*dist64+16*dist64, rdi, -256*YMM_SCD3 ;; Test loop counter, next source pointer, restore sine/cosine pointer
	yloop	3, c3b, -3*16*dist64	;; Test loop counter, restore source pointer
	end_timer 14

;; Do inverse FFT levels 4,5
;; On input the 64-byte cache lines hold these data values:
;;	0	+6K	+6K	+6K	1K	+6K	+6K	+6K
;;	1
;;	...
;;	1023
;;	2K	...
;;	4K	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+6K	+6K	+6K	3K	+6K	+6K	+6K
;;	1
;;	...
;;	3071

;; Do 1024 six_reals_unfft operations
;; Do 3072 three_complex_unfft operations
;;	distance between fft data elements is 1024
;;	do 1024 macros each processing 24 data values

	start_timer 15
	bump	rdi, 256*YMM_SCD3	;; Next sine/cosine pointer
	yloop_init  1024			;; 1024 iterations
	yr3_3cl_six_reals_three_complex_djbunfft_preload
c2b:	yr3_3cl_six_reals_three_complex_djbunfft rsi, 64, 16*dist64, rdi, YMM_SCD2, 64
	yloop	64, c2b, -64*64+dist64	;; Test loop counter, next source pointer
	yloop	16, c2b, -16*dist64	;; Test loop counter, restore source pointer
	end_timer 15

;; Do inverse FFT levels 1,2,3
;; On input the 64-byte cache lines hold these data values:
;;	0	+6K	+6K	+6K	3K	+6K	+6K	+6K
;;	1
;;	...
;;	3071
;; These are swizzled to:
;;	0	+768	+768	+768	3K	+768	+768	+768
;;	1	...
;;	...
;;	767
;;	6K	...
;;	12K	...
;;	18K	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+768	+768	+768	12K	+768	+768	+768
;;	1	...
;;	...
;;	767
;;	3K	...
;;	6K	...
;;	9K	...

;; Do 3072 eight_reals_unfft operations
;;	distance between fft data elements is 3072
;;	do 768 macros each processing 32 data values

	start_timer 16
	yloop_init  768			;; 768 iterations
	yr4_s4cl_eight_reals_unfft_preload
c1b:	yr4_s4cl_eight_reals_unfft rsi, 64, 12*dist64, 24*dist64, rdi, YMM_SCD3, 64
	yloop	64, c1b, -64*64+dist64	;; Test loop counter, next source pointer
	yloop	12, c1b			;; Test loop counter
	end_timer 16

	ytwo_dest_adjust
	yfft_3_ret
	ENDM

;; Perform a 24K-element negacyclic FFT

yfft24Kp MACRO type
	LOCAL	ymid2p, ymid3p, ymid4p, yfinish_unfft
	LOCAL	b1b, b2b, b3b, b4b, b5b, b6b, b7a, b7b, b7c, b7d, c1b, c2b, c3b, c4b, c5b, c6b

;; Do a multiply with pre-FFTed inputs

	cmp	ffttype, 4
	je	ymid4p

;; Do FFT levels 1,2,3
;; On input the 64-byte cache lines hold these data values:
;;	0	+768	+768	+768	12K	+768	+768	+768
;;	1
;;	...
;;	767
;;	3K	...
;;	6K	...
;;	9K	...
;; On output these are swizzled to:
;;	0	+6K	+6K	+6K	3K	+6K	+6K	+6K
;;	1
;;	...
;;	3071

;; Do 3072 four_complex_first_fft operations
;;	distance between fft data elements is 3072
;;	do 768 macros each processing 32 data values

	start_timer 4
	mov	rdi, sincos1		;; Load sin/cos pointer
	bump	rdi, 1024*YMM_SCD1/4+256*YMM_SCD2/4
	yloop_init  768			;; 768 iterations
	yr4_fs4cl_four_complex_first_djbfft_preload
b1b:	yr4_fs4cl_four_complex_first_djbfft rsi, 64, 12*dist64, 24*dist64, rdi, YMM_SCD4, 64
	yloop	64, b1b, -64*64+dist64	;; Test loop counter, next source pointer
	yloop	12, b1b, -12*dist64	;; Test loop counter, restore source pointer
	end_timer 4

;; Do FFT levels 4,5
;; On input the 64-byte cache lines hold these data values:
;;	0	+6K	+6K	+6K	3K	+6K	+6K	+6K
;;	1
;;	...
;;	3071
;; On output the 64-byte cache lines hold these data values:
;;	0	+6K	+6K	+6K	1K	+6K	+6K	+6K
;;	1
;;	...
;;	1023
;;	2K	...
;;	4K	...

;; Do 4096 three_complex_fft operations
;;	distance between fft data elements is 1024
;;	do 1024 macros each processing 24 data values

	start_timer 5
	bump	rdi, -768*YMM_SCD4-1024*YMM_SCD1/4 ;; Next sin/cos pointer
	yloop_init  1024			;; 1024 iterations
	yr3_b3cl_three_complex_djbfft_preload
b2b:	yr3_b3cl_three_complex_djbfft rsi, 64, 16*dist64, rdi, YMM_SCD1/4, 64
	yloop	64, b2b, -64*64+dist64	;; Test loop counter, next source pointer
	yloop	16, b2b, -16*dist64	;; Test loop counter, restore source pointer
	end_timer 5

;; Do FFT levels 6,7
;; On input the 64-byte cache lines hold these data values:
;;	0	+6K	+6K	+6K	1K	+6K	+6K	+6K
;;	1
;;	...
;;	1023
;;	2K	...
;;	4K	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+6K	+6K	+6K	256	+6K	+6K	+6K
;;	1
;;	...
;;	255
;;	512	...
;;	...
;;	5632	...

;; Do 3072 four_complex_fft operations
;;	distance between fft data elements is 256
;;	do 768 macros each processing 32 data values

	start_timer 6
	bump	rdi, -1024*YMM_SCD1/4-256*YMM_SCD2/4 ;; Next sin/cos pointer
	yloop_init  768			;; 3 iterations of 256
	yr4_b4cl_four_complex_djbfft_preload
b3b:	yr4_b4cl_four_complex_djbfft rsi, 64, 4*dist64, 8*dist64, rdi, YMM_SCD2/4, 64
	yloop	64, b3b, -64*64+dist64	;; Test loop counter, next source pointer
	yloop	4, b3b, -4*dist64+16*dist64, rdi, -256*YMM_SCD2/4 ;; Test loop counter, next source pointer, restore sin/cos pointer
	yloop	3, b3b, -3*16*dist64	;; Test loop counter, restore source pointer
	end_timer 6

;; Do FFT levels 8,9
;; On input the 64-byte cache lines hold these data values:
;;	0	+6K	+6K	+6K	256	+6K	+6K	+6K
;;	1
;;	...
;;	255
;;	512	...
;;	...
;;	5632	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+6K	+6K	+6K	64	+6K	+6K	+6K
;;	1
;;	...
;;	63
;;	128	...
;;	...
;;	6016	...

;; Do 3072 four_complex_fft operations
;;	distance between fft data elements is 64
;;	do 768 macros each processing 32 data values

	start_timer 7
	yloop_init  768			;; 12 iterations of 64
	yr4_b4cl_four_complex_djbfft_preload
;; Loop below reworked to avoid 4KB distances
;;b4b:	yr4_b4cl_four_complex_djbfft rsi, 64, dist64, 2*dist64, rdi, 4*YMM_SCD2/4, 64
;;	yloop	64, b4b, -64*64+4*dist64, rdi, -64*4*YMM_SCD2/4 ;; Test loop counter, next source pointer, restore sin/cos pointer
b4b:	yr4_b4cl_four_complex_djbfft rsi, 4*64, dist64, 2*dist64, rdi, 4*4*YMM_SCD2/4, 16
	yloop	16, b4b, -16*4*64+64, rdi, -16*4*4*YMM_SCD2/4+4*YMM_SCD2/4 ;; Test loop counter, next source pointer, next sin/cos pointer
	yloop	4, b4b, -4*64+4*dist64, rdi, -4*4*YMM_SCD2/4 ;; Test loop counter, next source pointer, restore sin/cos pointer
	yloop	12, b4b, -12*4*dist64	;; Test loop counter, restore source pointer
	end_timer 7

;; Do FFT levels 10,11
;; On input the 64-byte cache lines hold these data values:
;;	0	+6K	+6K	+6K	64	+6K	+6K	+6K
;;	1
;;	...
;;	63
;;	128	...
;;	...
;;	6016	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+6K	+6K	+6K	16	+6K	+6K	+6K
;;	1
;;	...
;;	15
;;	32	...
;;	...
;;	6112	...

;; Do 3072 four_complex_fft operations
;;	distance between fft data elements is 16
;;	do 768 macros each processing 32 data values

	start_timer 8
	yloop_init  768			;; 48 iterations of 16
	yr4_b4cl_four_complex_djbfft_preload
b5b:	yr4_b4cl_four_complex_djbfft rsi, 64, 16*64, 32*64, rdi, 16*YMM_SCD2/4, 16
	yloop	16, b5b, -16*64+dist64, rdi, -16*16*YMM_SCD2/4 ;; Test loop counter, next source pointer, restore sin/cos pointer
	yloop	48, b5b, -48*dist64	;; Test loop counter, restore source pointer
	end_timer 8

;; Do FFT levels 12,13
;; On input the 64-byte cache lines hold these data values:
;;	0	+6K	+6K	+6K	16	+6K	+6K	+6K
;;	1
;;	...
;;	15
;;	32	...
;;	...
;;	6112	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+6K	+6K	+6K	4	+6K	+6K	+6K
;;	1
;;	...
;;	3
;;	8	...
;;	...
;;	6136	...

;; Do 3072 four_complex_fft operations
;;	distance between fft data elements is 4
;;	do 768 macros each processing 32 data values

	start_timer 9
	yloop_init  768			;; 192 iterations of 4
	yr4_b4cl_four_complex_djbfft_preload
b6b:	yr4_b4cl_four_complex_djbfft rsi, 64, 4*64, 8*64, rdi, 64*YMM_SCD2/4, 4
	yloop	4, b6b, -4*64+16*64, rdi, -4*64*YMM_SCD2/4 ;; Test loop counter, next source pointer, restore sin/cos pointer
	yloop	4, b6b, -4*16*64+dist64	;; Test loop counter, next source pointer
	yloop	48, b6b, -48*dist64	;; Test loop counter, restore source pointer
	end_timer 9

;; Do FFT levels 14,15
;; On input and output the 64-byte cache lines hold these data values:
;;	0	+6K	+6K	+6K	4	+6K	+6K	+6K
;;	1
;;	...
;;	3
;;	8	...
;;	...
;;	6136	...

;; Do 3072 four_complex_fft operations
;;	distance between fft data elements is 1
;;	do 768 macros each processing 32 data values

	cmp	ffttype, 2
	je	ymid2p
	ja	ymid3p

	start_timer 10
	yloop_init  768			;; 768 iterations
b7a:	yr4_4cl_four_complex_fft_final rsi, 4*64, 64, 2*64
	yloop	16, b7a, -16*4*64+dist64 ;; Test loop counter, next source pointer
	yloop	48, b7a			;; Test loop counter
	end_timer 10
	yfft_1_ret

ymid2p:	start_timer 10
	yloop_init  768			;; 768 iterations
	yr4_4cl_four_complex_with_square_preload
b7b:	yr4_4cl_four_complex_with_square rsi, 4*64, 64, 2*64, 16
	yloop	16, b7b, -16*4*64+dist64 ;; Test loop counter, next source pointer
	yloop	48, b7b			;; Test loop counter
	end_timer 10
	jmp	yfinish_unfft

ymid3p:	start_timer 10
x64	mov	r8, DEST2ARG
	mov	rbp, DIST_TO_MULSRCARG
	yloop_init  768			;; 768 iterations
	yr4_4cl_four_complex_with_mult_preload
b7c:	yr4_4cl_four_complex_with_mult rsi, 4*64, 64, 2*64
	yloop	16, b7c, -16*4*64+dist64 ;; Test loop counter, next source pointer
	yloop	48, b7c			;; Test loop counter
x64	add	rsi, r8
	end_timer 10
	jmp	yfinish_unfft

ymid4p:	start_timer 10
	mov	rbp, DIST_TO_MULSRCARG
	yloop_init  768			;; 768 iterations
	yr4_4cl_four_complex_with_mulf_preload
b7d:	yr4_4cl_four_complex_with_mulf rsi, 4*64, 64, 2*64
	yloop	16, b7d, -16*4*64+dist64 ;; Test loop counter, next source pointer
	yloop	48, b7d			;; Test loop counter
	end_timer 10
yfinish_unfft:
	bump	rsi, -48*dist64		;; Restore source pointer

;; Do inverse FFT levels 12,13
;; On input the 64-byte cache lines hold these data values:
;;	0	+6K	+6K	+6K	4	+6K	+6K	+6K
;;	1
;;	...
;;	3
;;	8	...
;;	...
;;	6136	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+6K	+6K	+6K	16	+6K	+6K	+6K
;;	1
;;	...
;;	15
;;	32	...
;;	...
;;	6112	...

;; Do 3072 four_complex_unfft operations
;;	distance between fft data elements is 4
;;	do 768 macros each processing 32 data values

	start_timer 11
	mov	rdi, sincos1		;; Load sin/cos pointer
	yloop_init  768			;; 192 iterations of 4
	yr4_b4cl_four_complex_djbunfft_preload
c6b:	yr4_b4cl_four_complex_djbunfft rsi, 64, 4*64, 8*64, rdi, 64*YMM_SCD2/4, 4
	yloop	4, c6b, -4*64+16*64, rdi, -4*64*YMM_SCD2/4 ;; Test loop counter, next source pointer, restore sine/cosine pointer
	yloop	4, c6b, -4*16*64+dist64	;; Test loop counter, next source pointer
	yloop	48, c6b, -48*dist64	;; Test loop counter, restore source pointer
	end_timer 11

;; Do inverse FFT levels 10,11
;; On input the 64-byte cache lines hold these data values:
;;	0	+6K	+6K	+6K	16	+6K	+6K	+6K
;;	1
;;	...
;;	15
;;	32	...
;;	...
;;	6112	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+6K	+6K	+6K	64	+6K	+6K	+6K
;;	1
;;	...
;;	63
;;	128	...
;;	...
;;	6016	...

;; Do 3072 four_complex_unfft operations
;;	distance between fft data elements is 16
;;	do 768 macros each processing 32 data values

	start_timer 12
	yloop_init  768			;; 48 iterations of 16
	yr4_b4cl_four_complex_djbunfft_preload
c5b:	yr4_b4cl_four_complex_djbunfft rsi, 64, 16*64, 32*64, rdi, 16*YMM_SCD2/4, 16
	yloop	16, c5b, -16*64+dist64, rdi, -16*16*YMM_SCD2/4 ;; Test loop counter, next source pointer, restore sine/cosine pointer
	yloop	48, c5b, -48*dist64	;; Test loop counter, restore source pointer
	end_timer 12

;; Do inverse FFT levels 8,9
;; On input the 64-byte cache lines hold these data values:
;;	0	+6K	+6K	+6K	64	+6K	+6K	+6K
;;	1
;;	...
;;	63
;;	128	...
;;	...
;;	6016	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+6K	+6K	+6K	256	+6K	+6K	+6K
;;	1
;;	...
;;	255
;;	512	...
;;	...
;;	5632	...

;; Do 3072 four_complex_unfft operations
;;	distance between fft data elements is 64
;;	do 768 macros each processing 32 data values

	start_timer 13
	yloop_init  768			;; 12 iterations of 64
	yr4_b4cl_four_complex_djbunfft_preload
;; Loop below reworked to avoid 4KB distances
;;c4b:	yr4_b4cl_four_complex_djbunfft rsi, 64, dist64, 2*dist64, rdi, 4*YMM_SCD2/4, 64
;;	yloop	64, c4b, -64*64+4*dist64, rdi, -64*4*YMM_SCD2/4 ;; Test loop counter, next source pointer, restore sine/cosine pointer
c4b:	yr4_b4cl_four_complex_djbunfft rsi, 4*64, dist64, 2*dist64, rdi, 4*4*YMM_SCD2/4, 16
	yloop	16, c4b, -16*4*64+64, rdi, -16*4*4*YMM_SCD2/4+4*YMM_SCD2/4 ;; Test loop counter, next source pointer, next sine/cosine pointer
	yloop	4, c4b, -4*64+4*dist64, rdi, -4*4*YMM_SCD2/4 ;; Test loop counter, next source pointer, restore sine/cosine pointer
	yloop	12, c4b, -12*4*dist64	;; Test loop counter, restore source pointer
	end_timer 13

;; Do inverse FFT levels 6,7
;; On input the 64-byte cache lines hold these data values:
;;	0	+6K	+6K	+6K	256	+6K	+6K	+6K
;;	1
;;	...
;;	255
;;	512	...
;;	...
;;	5632	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+6K	+6K	+6K	1K	+6K	+6K	+6K
;;	1
;;	...
;;	1023
;;	2K	...
;;	4K	...

;; Do 3072 four_complex_unfft operations
;;	distance between fft data elements is 256
;;	do 768 macros each processing 32 data values

	start_timer 14
	yloop_init  768			;; 3 iterations of 256
	yr4_b4cl_four_complex_djbunfft_preload
c3b:	yr4_b4cl_four_complex_djbunfft rsi, 64, 4*dist64, 8*dist64, rdi, YMM_SCD2/4, 64
	yloop	64, c3b, -64*64+dist64	;; Test loop counter, next source pointer
	yloop	4, c3b, -4*dist64+16*dist64, rdi, -256*YMM_SCD2/4 ;; Test loop counter, next source pointer, restore sine/cosine pointer
	yloop	3, c3b, -3*16*dist64	;; Test loop counter, restore source pointer
	end_timer 14

;; Do inverse FFT levels 4,5
;; On input the 64-byte cache lines hold these data values:
;;	0	+6K	+6K	+6K	1K	+6K	+6K	+6K
;;	1
;;	...
;;	1023
;;	2K	...
;;	4K	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+6K	+6K	+6K	3K	+6K	+6K	+6K
;;	1
;;	...
;;	3071

;; Do 4096 three_complex_unfft operations
;;	distance between fft data elements is 1024
;;	do 1024 macros each processing 24 data values

	start_timer 15
	bump	rdi, 256*YMM_SCD2/4	;; Next sine/cosine pointer
	yloop_init  1024			;; 1024 iterations
	yr3_b3cl_three_complex_djbunfft_preload
c2b:	yr3_b3cl_three_complex_djbunfft rsi, 64, 16*dist64, rdi, YMM_SCD1/4, 64
	yloop	64, c2b, -64*64+dist64	;; Test loop counter, next source pointer
	yloop	16, c2b, -16*dist64	;; Test loop counter, restore source pointer
	end_timer 15

;; Do inverse FFT levels 1,2,3
;; On input the 64-byte cache lines hold these data values:
;;	0	+6K	+6K	+6K	3K	+6K	+6K	+6K
;;	1
;;	...
;;	3071
;; These are swizzled to:
;;	0	+768	+768	+768	3K	+768	+768	+768
;;	1	...
;;	...
;;	767
;;	6K	...
;;	12K	...
;;	18K	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+768	+768	+768	12K	+768	+768	+768
;;	1	...
;;	...
;;	767
;;	3K	...
;;	6K	...
;;	9K	...

;; Do 3072 four_complex_last_unfft operations
;;	distance between fft data elements is 3072
;;	do 768 macros each processing 32 data values

	start_timer 16
	yloop_init  768			;; 768 iterations
	yr4_s4cl_four_complex_last_unfft_preload
c1b:	yr4_s4cl_four_complex_last_unfft rsi, 64, 12*dist64, 24*dist64, rdi, YMM_SCD4, 64
	yloop	64, c1b, -64*64+dist64	;; Test loop counter, next source pointer
	yloop	12, c1b			;; Test loop counter
	end_timer 16

	ytwo_dest_adjust
	yfft_3_ret
	ENDM


;; Perform a 32K-element FFT.

yfft32K MACRO type
	LOCAL	ymid2, ymid3, ymid4, yfinish_unfft
	LOCAL	b1b, b2b, b3a, b3b, b4a, b4b, b5a, b5b, b6a, b6b, b7a, b7b, b7c, b7d
	LOCAL	c1b, c2b, c3a, c3b, c4a, c4b, c5a, c5b, c6a, c6b

;; Do a multiply with pre-FFTed inputs

	cmp	ffttype, 4
	je	ymid4

;; Do FFT levels 1,2,3
;; On input the 64-byte cache lines hold these data values:
;;	0	1K	2K	3K	16K	+1K	+1K	+1K
;;	1
;;	...
;;	1023
;;	4K	...
;;	8K	...
;;	12K	...
;; On output these are swizzled to:
;;	0	8K	16K	24K	4K	+8K	+8K	+8K
;;	1
;;	...
;;	4095

;; Do 4096 eight_reals_fft operations
;;	distance between fft data elements is 4096
;;	do 1024 macros each processing 32 data values

	start_timer 4
	mov	rdi, sincos1		;; Load sin/cos pointer
	bump	rdi, 1024*YMM_SCD3
	yloop_init  1024		;; 1024 iterations
	yr4_fs4cl_eight_reals_fft_preload
b1b:	yr4_fs4cl_eight_reals_fft rsi, 64, 16*dist64, 32*dist64, rdi, YMM_SCD3, 64
	yloop	64, b1b, -64*64+dist64	;; Test loop counter, next source pointer
	yloop	16, b1b, -16*dist64	;; Test loop counter, restore source pointer
	end_timer 4

;; Do FFT levels 4,5
;; On input the 64-byte cache lines hold these data values:
;;	0	8K	16K	24K	4K	+8K	+8K	+8K
;;	1
;;	...
;;	4095
;; On output the 64-byte cache lines hold these data values:
;;	0	8K	16K	24K	1K	+8K	+8K	+8K
;;	1
;;	...
;;	1023
;;	2K	...
;;	4K	...
;;	6K	...

;; Do 1024 eight_reals_fft operations
;; Do 3072 four_complex_fft operations
;;	distance between fft data elements is 1024
;;	do 1024 macros each processing 32 data values

	start_timer 5
	bump	rdi, -1024*YMM_SCD3-1024*YMM_SCD3 ;; Load sin/cos pointer
	yloop_init  1024		;; 1024 iterations
	yr4_4cl_eight_reals_four_complex_djbfft_preload
b2b:	yr4_4cl_eight_reals_four_complex_djbfft rsi, 64, 16*dist64, 32*dist64, rdi, YMM_SCD3, 64
	yloop	64, b2b, -64*64+dist64	;; Test loop counter, next source pointer
	yloop	16, b2b, -16*dist64, rdi, -1024*YMM_SCD3 ;; Test loop counter, restore source pointer, restore sin/cos pointer
	end_timer 5

;; Do FFT levels 6,7
;; On input the 64-byte cache lines hold these data values:
;;	0	8K	16K	24K	1K	+8K	+8K	+8K
;;	1
;;	...
;;	1023
;;	2K	...
;;	4K	...
;;	6K	...
;; On output the 64-byte cache lines hold these data values:
;;	0	8K	16K	24K	256	+8K	+8K	+8K
;;	1
;;	...
;;	255
;;	512	...
;;	...
;;	7680	...

;; Do 256 eight_reals_fft operations
;; Do 3840 four_complex_fft operations
;;	distance between fft data elements is 256
;;	do 1024 macros each processing 32 data values

	start_timer 6
	yloop_init  1024		;; 4 iterations of 256
	yr4_4cl_eight_reals_four_complex_djbfft_preload
b3a:	yr4_4cl_eight_reals_four_complex_djbfft rsi, 64, 4*dist64, 8*dist64, rdi, 4*YMM_SCD3, 64
	yloop	64, b3a, -64*64+dist64	;; Test loop counter, next source pointer
	yloop	4, b3a, -4*dist64+16*dist64, rdi, -256*4*YMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer

	loops_reset			;; 3 more iters of 256
	yr4_rb4cl_four_complex_djbfft_preload
b3b:	yr4_rb4cl_four_complex_djbfft rsi, 64, 4*dist64, 8*dist64, rdi, 4*YMM_SCD3, 64
	yloop	64, b3b, -64*64+dist64	;; Test loop counter, next source pointer
	yloop	4, b3b, -4*dist64+16*dist64, rdi, -256*4*YMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer
	yloop	4, b3b, -4*16*dist64	;; Test loop counter, restore source pointer
	end_timer 6

;; Do FFT levels 8,9
;; On input the 64-byte cache lines hold these data values:
;;	0	8K	16K	24K	256	+8K	+8K	+8K
;;	1
;;	...
;;	255
;;	512	...
;;	...
;;	7680	...
;; On output the 64-byte cache lines hold these data values:
;;	0	8K	16K	24K	64	+8K	+8K	+8K
;;	1
;;	...
;;	63
;;	128	...
;;	...
;;	8064	...

;; Do 64 eight_reals_fft operations
;; Do 4032 four_complex_fft operations
;;	distance between fft data elements is 64
;;	do 1024 macros each processing 32 data values

	start_timer 7
	yloop_init  1024		;; 16 iterations of 64
	yr4_4cl_eight_reals_four_complex_djbfft_preload
;; Loop below reworked to avoid 4KB distances
;;b4a:	yr4_4cl_eight_reals_four_complex_djbfft rsi, 64, dist64, 2*dist64, rdi, 16*YMM_SCD3, 64
;;	yloop	64, b4a, -64*64+4*dist64, rdi, -64*16*YMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer
b4a:	yr4_4cl_eight_reals_four_complex_djbfft rsi, 4*64, dist64, 2*dist64, rdi, 4*16*YMM_SCD3, 16
	yloop	16, b4a, -16*4*64+64, rdi, -16*4*16*YMM_SCD3+16*YMM_SCD3 ;; Test loop counter, next source pointer, next sin/cos pointer
	yloop	4, b4a, -4*64+4*dist64, rdi, -4*16*YMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer

	loops_reset			;; 15 more iters of 64
	yr4_rb4cl_four_complex_djbfft_preload
;; Loop below reworked to avoid 4KB distances
;;b4b:	yr4_rb4cl_four_complex_djbfft rsi, 64, dist64, 2*dist64, rdi, 16*YMM_SCD3, 64
;;	yloop	64, b4b, -64*64+4*dist64, rdi, -64*16*YMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer
b4b:	yr4_rb4cl_four_complex_djbfft rsi, 4*64, dist64, 2*dist64, rdi, 4*16*YMM_SCD3, 16
	yloop	16, b4b, -16*4*64+64, rdi, -16*4*16*YMM_SCD3+16*YMM_SCD3 ;; Test loop counter, next source pointer, next sin/cos pointer
	yloop	4, b4b, -4*64+4*dist64, rdi, -4*16*YMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer
	yloop	16, b4b, -16*4*dist64	;; Test loop counter, restore source pointer
	end_timer 7

;; Do FFT levels 10,11
;; On input the 64-byte cache lines hold these data values:
;;	0	8K	16K	24K	64	+8K	+8K	+8K
;;	1
;;	...
;;	63
;;	128	...
;;	...
;;	8064	...
;; On output the 64-byte cache lines hold these data values:
;;	0	8K	16K	24K	16	+8K	+8K	+8K
;;	1
;;	...
;;	15
;;	32	...
;;	...
;;	8160	...

;; Do 16 eight_reals_fft operations
;; Do 4080 four_complex_fft operations
;;	distance between fft data elements is 16
;;	do 1024 macros each processing 32 data values

	start_timer 8
	yloop_init  1024		;; 64 iterations of 16
	yr4_4cl_eight_reals_four_complex_djbfft_preload
b5a:	yr4_4cl_eight_reals_four_complex_djbfft rsi, 64, 16*64, 32*64, rdi, 64*YMM_SCD3, 16
	yloop	16, b5a, -16*64+dist64, rdi, -16*64*YMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer

	loops_reset			;; 63 more iters of 16
	yr4_rb4cl_four_complex_djbfft_preload
b5b:	yr4_rb4cl_four_complex_djbfft rsi, 64, 16*64, 32*64, rdi, 64*YMM_SCD3, 16
	yloop	16, b5b, -16*64+dist64, rdi, -16*64*YMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer
	yloop	64, b5b, -64*dist64	;; Test loop counter, restore source pointer
	end_timer 8

;; Do FFT levels 12,13
;; On input the 64-byte cache lines hold these data values:
;;	0	8K	16K	24K	16	+8K	+8K	+8K
;;	1
;;	...
;;	15
;;	32	...
;;	...
;;	8160	...
;; On output the 64-byte cache lines hold these data values:
;;	0	8K	16K	24K	4	+8K	+8K	+8K
;;	1
;;	...
;;	3
;;	8	...
;;	...
;;	8184	...

;; Do 4 eight_reals_fft operations
;; Do 4092 four_complex_fft operations
;;	distance between fft data elements is 4
;;	do 1024 macros each processing 32 data values

	start_timer 9
	yloop_init  1024		;; 256 iterations of 4
	yr4_4cl_eight_reals_four_complex_djbfft_preload
b6a:	yr4_4cl_eight_reals_four_complex_djbfft rsi, 64, 4*64, 8*64, rdi, 256*YMM_SCD3, 4
	yloop	4, b6a, -4*64+16*64, rdi, -4*256*YMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer

	loops_reset			;; 255 more iters of 4
	yr4_rb4cl_four_complex_djbfft_preload
b6b:	yr4_rb4cl_four_complex_djbfft rsi, 64, 4*64, 8*64, rdi, 256*YMM_SCD3, 4
	yloop	4, b6b, -4*64+16*64, rdi, -4*256*YMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer
	yloop	4, b6b, -4*16*64+dist64	;; Test loop counter, next source pointer
	yloop	64, b6b, -64*dist64	;; Test loop counter, restore source pointer
	end_timer 9

;; Do FFT levels 14,15
;; On input and output the 64-byte cache lines hold these data values:
;;	0	8K	16K	24K	4	+8K	+8K	+8K
;;	1
;;	...
;;	3
;;	8	...
;;	...
;;	8184	...

;; Do 1 eight_reals_fft operation
;; Do 1023 four_complex_fft operations
;;	distance between fft data elements is 1
;;	do 256 macros each processing 32 data values

	cmp	ffttype, 2
	je	ymid2
	ja	ymid3

	start_timer 10
;;	yr4_4cl_eight_reals_four_complex_fft_final rsi, 4*64, 64, 2*64
	yr4_dispatch call, yreal_fft_final, 64, 2*64
	yloop_init  1024, 1, 16		;; 1023 more iterations
b7a:	yr4_4cl_four_complex_fft_final rsi, 4*64, 64, 2*64
	yloop	16, b7a, -16*4*64+dist64 ;; Test loop counter, next source pointer
	yloop	64, b7a			;; Test loop counter
	end_timer 10
	yfft_1_ret

ymid2:	start_timer 10
;;	yr4_4cl_eight_reals_four_complex_with_square rsi, 4*64, 64, 2*64
	yr4_dispatch call, yreal_square, 64, 2*64
	yloop_init  1024, 1, 16		;; 1023 more iterations
	yr4_4cl_four_complex_with_square_preload
b7b:	yr4_4cl_four_complex_with_square rsi, 4*64, 64, 2*64, 1
	yloop	16, b7b, -16*4*64+dist64 ;; Test loop counter, next source pointer
	yloop	64, b7b			;; Test loop counter
	end_timer 10
	jmp	yfinish_unfft

ymid3:	start_timer 10
x64	mov	r8, DEST2ARG
	mov	rbp, DIST_TO_MULSRCARG
;;	yr4_4cl_eight_reals_four_complex_with_mult rsi, 4*64, 64, 2*64
	yr4_dispatch call, yreal_mult, 64, 2*64
	yloop_init  1024, 1, 16		;; 1023 more iterations
	yr4_4cl_four_complex_with_mult_preload
b7c:	yr4_4cl_four_complex_with_mult rsi, 4*64, 64, 2*64
	yloop	16, b7c, -16*4*64+dist64 ;; Test loop counter, next source pointer
	yloop	64, b7c			;; Test loop counter
x64	add	rsi, r8
	end_timer 10
	jmp	yfinish_unfft

ymid4:	start_timer 10
	mov	rbp, DIST_TO_MULSRCARG
;;	yr4_4cl_eight_reals_four_complex_with_mulf rsi, 4*64, 64, 2*64
	yr4_dispatch call, yreal_mulf, 64, 2*64
	yloop_init  1024, 1, 16		;; 1023 more iterations
	yr4_4cl_four_complex_with_mulf_preload
b7d:	yr4_4cl_four_complex_with_mulf rsi, 4*64, 64, 2*64
	yloop	16, b7d, -16*4*64+dist64 ;; Test loop counter, next source pointer
	yloop	64, b7d			;; Test loop counter
	end_timer 10
yfinish_unfft:
	bump	rsi, -64*dist64		;; Restore source pointer

;; Do inverse FFT levels 12,13
;; On input the 64-byte cache lines hold these data values:
;;	0	8K	16K	24K	4	+8K	+8K	+8K
;;	1
;;	...
;;	3
;;	8	...
;;	...
;;	8184	...
;; On output the 64-byte cache lines hold these data values:
;;	0	8K	16K	24K	16	+8K	+8K	+8K
;;	1
;;	...
;;	15
;;	32	...
;;	...
;;	8160	...

;; Do 4 eight_reals_unfft operations
;; Do 4092 four_complex_unfft operations
;;	distance between fft data elements is 4
;;	do 1024 macros each processing 32 data values

	start_timer 11
	mov	rdi, sincos1		;; Load sin/cos pointer
	yloop_init  1024		;; 256 iterations of 4
	yr4_4cl_eight_reals_four_complex_djbunfft_preload
c6a:	yr4_4cl_eight_reals_four_complex_djbunfft rsi, 64, 4*64, 8*64, rdi, 256*YMM_SCD3, 4
	yloop	4, c6a, -4*64+16*64, rdi, -4*256*YMM_SCD3 ;; Test loop counter, next source pointer, restore sine/cosine pointer

	loops_reset			;; 255 more iters of 4
	yr4_rb4cl_four_complex_djbunfft_preload
c6b:	yr4_rb4cl_four_complex_djbunfft rsi, 64, 4*64, 8*64, rdi, 256*YMM_SCD3, 4
	yloop	4, c6b, -4*64+16*64, rdi, -4*256*YMM_SCD3 ;; Test loop counter, next source pointer, restore sine/cosine pointer
	yloop	4, c6b, -4*16*64+dist64	;; Test loop counter, next source pointer
	yloop	64, c6b, -64*dist64	;; Test loop counter, restore source pointer
	end_timer 11

;; Do inverse FFT levels 10,11
;; On input the 64-byte cache lines hold these data values:
;;	0	8K	16K	24K	16	+8K	+8K	+8K
;;	1
;;	...
;;	15
;;	32	...
;;	...
;;	8160	...
;; On output the 64-byte cache lines hold these data values:
;;	0	8K	16K	24K	64	+8K	+8K	+8K
;;	1
;;	...
;;	63
;;	128	...
;;	...
;;	8064	...

;; Do 16 eight_reals_unfft operations
;; Do 4080 four_complex_unfft operations
;;	distance between fft data elements is 16
;;	do 1024 macros each processing 32 data values

	start_timer 12
	yloop_init  1024		;; 64 iterations of 16
	yr4_4cl_eight_reals_four_complex_djbunfft_preload
c5a:	yr4_4cl_eight_reals_four_complex_djbunfft rsi, 64, 16*64, 32*64, rdi, 64*YMM_SCD3, 16
	yloop	16, c5a, -16*64+dist64, rdi, -16*64*YMM_SCD3 ;; Test loop counter, next source pointer, restore sine/cosine pointer

	loops_reset			;; 63 more iters of 16
	yr4_rb4cl_four_complex_djbunfft_preload
c5b:	yr4_rb4cl_four_complex_djbunfft rsi, 64, 16*64, 32*64, rdi, 64*YMM_SCD3, 16
	yloop	16, c5b, -16*64+dist64, rdi, -16*64*YMM_SCD3 ;; Test loop counter, next source pointer, restore sine/cosine pointer
	yloop	64, c5b, -64*dist64	;; Test loop counter, restore source pointer
	end_timer 12

;; Do inverse FFT levels 8,9
;; On input the 64-byte cache lines hold these data values:
;;	0	8K	16K	24K	64	+8K	+8K	+8K
;;	1
;;	...
;;	63
;;	128	...
;;	...
;;	8064	...
;; On output the 64-byte cache lines hold these data values:
;;	0	8K	16K	24K	256	+8K	+8K	+8K
;;	1
;;	...
;;	255
;;	512	...
;;	...
;;	7680	...

;; Do 64 eight_reals_unfft operations
;; Do 4032 four_complex_unfft operations
;;	distance between fft data elements is 64
;;	do 1024 macros each processing 32 data values

	start_timer 13
	yloop_init  1024		;; 16 iterations of 64
	yr4_4cl_eight_reals_four_complex_djbunfft_preload
;; Loop below reworked to avoid 4KB distances
;;c4a:	yr4_4cl_eight_reals_four_complex_djbunfft rsi, 64, dist64, 2*dist64, rdi, 16*YMM_SCD3, 64
;;	yloop	64, c4a, -64*64+4*dist64, rdi, -64*16*YMM_SCD3 ;; Test loop counter, next source pointer, restore sine/cosine pointer
c4a:	yr4_4cl_eight_reals_four_complex_djbunfft rsi, 4*64, dist64, 2*dist64, rdi, 4*16*YMM_SCD3, 16
	yloop	16, c4a, -16*4*64+64, rdi, -16*4*16*YMM_SCD3+16*YMM_SCD3 ;; Test loop counter, next source pointer, next sine/cosine pointer
	yloop	4, c4a, -4*64+4*dist64, rdi, -4*16*YMM_SCD3 ;; Test loop counter, next source pointer, restore sine/cosine pointer

	loops_reset			;; 15 more iters of 64
	yr4_rb4cl_four_complex_djbunfft_preload
;; Loop below reworked to avoid 4KB distances
;;c4b:	yr4_rb4cl_four_complex_djbunfft rsi, 64, dist64, 2*dist64, rdi, 16*YMM_SCD3, 64
;;	yloop	64, c4b, -64*64+4*dist64, rdi, -64*16*YMM_SCD3 ;; Test loop counter, next source pointer, restore sine/cosine pointer
c4b:	yr4_rb4cl_four_complex_djbunfft rsi, 4*64, dist64, 2*dist64, rdi, 4*16*YMM_SCD3, 16
	yloop	16, c4b, -16*4*64+64, rdi, -16*4*16*YMM_SCD3+16*YMM_SCD3 ;; Test loop counter, next source pointer, next sine/cosine pointer
	yloop	4, c4b, -4*64+4*dist64, rdi, -4*16*YMM_SCD3 ;; Test loop counter, next source pointer, restore sine/cosine pointer
	yloop	16, c4b, -16*4*dist64	;; Test loop counter, restore source pointer
	end_timer 13

;; Do inverse FFT levels 6,7
;; On input the 64-byte cache lines hold these data values:
;;	0	8K	16K	24K	256	+8K	+8K	+8K
;;	1
;;	...
;;	255
;;	512	...
;;	...
;;	7680	...
;; On output the 64-byte cache lines hold these data values:
;;	0	8K	16K	24K	1K	+8K	+8K	+8K
;;	1
;;	...
;;	1023
;;	2K	...
;;	4K	...
;;	6K	...

;; Do 256 eight_reals_unfft operations
;; Do 3840 four_complex_unfft operations
;;	distance between fft data elements is 256
;;	do 1024 macros each processing 32 data values

	start_timer 14
	yloop_init  1024		;; 4 iterations of 256
	yr4_4cl_eight_reals_four_complex_djbunfft_preload
c3a:	yr4_4cl_eight_reals_four_complex_djbunfft rsi, 64, 4*dist64, 8*dist64, rdi, 4*YMM_SCD3, 64
	yloop	64, c3a, -64*64+dist64	;; Test loop counter, next source pointer
	yloop	4, c3a, -4*dist64+16*dist64, rdi, -256*4*YMM_SCD3 ;; Test loop counter, next source pointer, restore sine/cosine pointer

	loops_reset			;; 3 more iters of 256
	yr4_rb4cl_four_complex_djbunfft_preload
c3b:	yr4_rb4cl_four_complex_djbunfft rsi, 64, 4*dist64, 8*dist64, rdi, 4*YMM_SCD3, 64
	yloop	64, c3b, -64*64+dist64	;; Test loop counter, next source pointer
	yloop	4, c3b, -4*dist64+16*dist64, rdi, -256*4*YMM_SCD3 ;; Test loop counter, next source pointer, restore sine/cosine pointer
	yloop	4, c3b, -4*16*dist64	;; Test loop counter, restore source pointer
	end_timer 14

;; Do inverse FFT levels 4,5
;; On input the 64-byte cache lines hold these data values:
;;	0	8K	16K	24K	1K	+8K	+8K	+8K
;;	1
;;	...
;;	1023
;;	2K	...
;;	4K	...
;;	6K	...
;; On output the 64-byte cache lines hold these data values:
;;	0	8K	16K	24K	4K	+8K	+8K	+8K
;;	1
;;	...
;;	4095

;; Do 1024 eight_reals_unfft operations
;; Do 3072 four_complex_unfft operations
;;	distance between fft data elements is 1024
;;	do 1024 macros each processing 32 data values

	start_timer 15
	yloop_init  1024		;; 1024 iterations
	yr4_4cl_eight_reals_four_complex_djbunfft_preload
c2b:	yr4_4cl_eight_reals_four_complex_djbunfft rsi, 64, 16*dist64, 32*dist64, rdi, YMM_SCD3, 64
	yloop	64, c2b, -64*64+dist64	;; Test loop counter, next source pointer
	yloop	16, c2b, -16*dist64	;; Test loop counter, restore source pointer
	end_timer 15

;; Do inverse FFT levels 1,2,3
;; On input the 64-byte cache lines hold these data values:
;;	0	8K	16K	24K	4K	+8K	+8K	+8K
;;	1
;;	...
;;	4095
;; These are swizzled to:
;;	0	1K	2K	3K	4K	+1K	+1K	+1K
;;	1	...
;;	...
;;	1023
;;	8K	...
;;	16K	...
;;	24K	...
;; On output the 64-byte cache lines hold these data values:
;;	0	1K	2K	3K	16K	+1K	+1K	+1K
;;	1	...
;;	...
;;	1023
;;	4K	...
;;	8K	...
;;	12K	...

;; Do 4096 eight_reals_unfft operations
;;	distance between fft data elements is 4096
;;	do 1024 macros each processing 32 data values

	start_timer 16
	yloop_init  1024		;; 1024 iterations
	yr4_s4cl_eight_reals_unfft_preload
c1b:	yr4_s4cl_eight_reals_unfft rsi, 64, 16*dist64, 32*dist64, rdi, YMM_SCD3, 64
	yloop	64, c1b, -64*64+dist64	;; Test loop counter, next source pointer
	yloop	16, c1b			;; Test loop counter
	end_timer 16

	ytwo_dest_adjust
	yfft_3_ret
	ENDM

;; Perform a 32768-element negacyclic FFT

yfft32Kp MACRO type
	LOCAL	ymid2p, ymid3p, ymid4p, yfinish_unfft
	LOCAL	b1b, b2b, b3b, b4b, b5b, b6b, b7a, b7b, b7c, b7d, c1b, c2b, c3b, c4b, c5b, c6b

;; Do a multiply with pre-FFTed inputs

	cmp	ffttype, 4
	je	ymid4p

;; Do FFT levels 1,2,3
;; On input the 64-byte cache lines hold these data values:
;;	0	1K	2K	3K	16K	+1K	+1K	+1K
;;	1
;;	...
;;	1023
;;	4K	...
;;	8K	...
;;	12K	...
;; On output these are swizzled to:
;;	0	8K	16K	24K	4K	+8K	+8K	+8K
;;	1
;;	...
;;	4095

;; Do 4096 four_complex_first_fft operations
;;	distance between fft data elements is 4096
;;	do 1024 macros each processing 32 data values

	start_timer 4
	mov	rdi, sincos1		;; Load sin/cos pointer
	bump	rdi, 1024*YMM_SCD2/4
	yloop_init  1024		;; 1024 iterations
	yr4_fs4cl_four_complex_first_djbfft_preload
b1b:	yr4_fs4cl_four_complex_first_djbfft rsi, 64, 16*dist64, 32*dist64, rdi, YMM_SCD4, 64
	yloop	64, b1b, -64*64+dist64	;; Test loop counter, next source pointer
	yloop	16, b1b, -16*dist64	;; Test loop counter, restore source pointer
	end_timer 4

;; Do FFT levels 4,5
;; On input the 64-byte cache lines hold these data values:
;;	0	8K	16K	24K	4K	+8K	+8K	+8K
;;	1
;;	...
;;	4095
;; On output the 64-byte cache lines hold these data values:
;;	0	8K	16K	24K	1K	+8K	+8K	+8K
;;	1
;;	...
;;	1023
;;	2K	...
;;	4K	...
;;	6K	...

;; Do 4096 four_complex_fft operations
;;	distance between fft data elements is 1024
;;	do 1024 macros each processing 32 data values

	start_timer 5
	bump	rdi, -1024*YMM_SCD4-1024*YMM_SCD2/4 ;; Load sin/cos pointer
	yloop_init  1024		;; 1024 iterations
	yr4_b4cl_four_complex_djbfft_preload
b2b:	yr4_b4cl_four_complex_djbfft rsi, 64, 16*dist64, 32*dist64, rdi, YMM_SCD2/4, 64
	yloop	64, b2b, -64*64+dist64	;; Test loop counter, next source pointer
	yloop	16, b2b, -16*dist64, rdi, -1024*YMM_SCD2/4 ;; Test loop counter, restore source pointer, restore sin/cos pointer
	end_timer 5

;; Do FFT levels 6,7
;; On input the 64-byte cache lines hold these data values:
;;	0	8K	16K	24K	1K	+8K	+8K	+8K
;;	1
;;	...
;;	1023
;;	2K	...
;;	4K	...
;;	6K	...
;; On output the 64-byte cache lines hold these data values:
;;	0	8K	16K	24K	256	+8K	+8K	+8K
;;	1
;;	...
;;	255
;;	512	...
;;	...
;;	7680	...

;; Do 4096 four_complex_fft operations
;;	distance between fft data elements is 256
;;	do 1024 macros each processing 32 data values

	start_timer 6
	yloop_init  1024		;; 4 iterations of 256
	yr4_b4cl_four_complex_djbfft_preload
b3b:	yr4_b4cl_four_complex_djbfft rsi, 64, 4*dist64, 8*dist64, rdi, 4*YMM_SCD2/4, 64
	yloop	64, b3b, -64*64+dist64	;; Test loop counter, next source pointer
	yloop	4, b3b, -4*dist64+16*dist64, rdi, -256*4*YMM_SCD2/4 ;; Test loop counter, next source pointer, restore sin/cos pointer
	yloop	4, b3b, -4*16*dist64	;; Test loop counter, restore source pointer
	end_timer 6

;; Do FFT levels 8,9
;; On input the 64-byte cache lines hold these data values:
;;	0	8K	16K	24K	256	+8K	+8K	+8K
;;	1
;;	...
;;	255
;;	512	...
;;	...
;;	7680	...
;; On output the 64-byte cache lines hold these data values:
;;	0	8K	16K	24K	64	+8K	+8K	+8K
;;	1
;;	...
;;	63
;;	128	...
;;	...
;;	8064	...

;; Do 4096 four_complex_fft operations
;;	distance between fft data elements is 64
;;	do 1024 macros each processing 32 data values

	start_timer 7
	yloop_init  1024		;; 16 iterations of 64
	yr4_b4cl_four_complex_djbfft_preload
;; Loop below reworked to avoid 4KB distances
;;b4b:	yr4_b4cl_four_complex_djbfft rsi, 64, dist64, 2*dist64, rdi, 16*YMM_SCD2/4, 64
;;	yloop	64, b4b, -64*64+4*dist64, rdi, -64*16*YMM_SCD2/4 ;; Test loop counter, next source pointer, restore sin/cos pointer
b4b:	yr4_b4cl_four_complex_djbfft rsi, 4*64, dist64, 2*dist64, rdi, 4*16*YMM_SCD2/4, 16
	yloop	16, b4b, -16*4*64+64, rdi, -16*4*16*YMM_SCD2/4+16*YMM_SCD2/4 ;; Test loop counter, next source pointer, next sin/cos pointer
	yloop	4, b4b, -4*64+4*dist64, rdi, -4*16*YMM_SCD2/4 ;; Test loop counter, next source pointer, restore sin/cos pointer
	yloop	16, b4b, -16*4*dist64	;; Test loop counter, restore source pointer
	end_timer 7

;; Do FFT levels 10,11
;; On input the 64-byte cache lines hold these data values:
;;	0	8K	16K	24K	64	+8K	+8K	+8K
;;	1
;;	...
;;	63
;;	128	...
;;	...
;;	8064	...
;; On output the 64-byte cache lines hold these data values:
;;	0	8K	16K	24K	16	+8K	+8K	+8K
;;	1
;;	...
;;	15
;;	32	...
;;	...
;;	8160	...

;; Do 4096 four_complex_fft operations
;;	distance between fft data elements is 16
;;	do 1024 macros each processing 32 data values

	start_timer 8
	yloop_init  1024		;; 64 iterations of 16
	yr4_b4cl_four_complex_djbfft_preload
b5b:	yr4_b4cl_four_complex_djbfft rsi, 64, 16*64, 32*64, rdi, 64*YMM_SCD2/4, 16
	yloop	16, b5b, -16*64+dist64, rdi, -16*64*YMM_SCD2/4 ;; Test loop counter, next source pointer, restore sin/cos pointer
	yloop	64, b5b, -64*dist64	;; Test loop counter, restore source pointer
	end_timer 8

;; Do FFT levels 12,13
;; On input the 64-byte cache lines hold these data values:
;;	0	8K	16K	24K	16	+8K	+8K	+8K
;;	1
;;	...
;;	15
;;	32	...
;;	...
;;	8160	...
;; On output the 64-byte cache lines hold these data values:
;;	0	8K	16K	24K	4	+8K	+8K	+8K
;;	1
;;	...
;;	3
;;	8	...
;;	...
;;	8184	...

;; Do 4096 four_complex_fft operations
;;	distance between fft data elements is 4
;;	do 1024 macros each processing 32 data values

	start_timer 9
	yloop_init  1024		;; 256 iterations of 4
	yr4_b4cl_four_complex_djbfft_preload
b6b:	yr4_b4cl_four_complex_djbfft rsi, 64, 4*64, 8*64, rdi, 256*YMM_SCD2/4, 4
	yloop	4, b6b, -4*64+16*64, rdi, -4*256*YMM_SCD2/4 ;; Test loop counter, next source pointer, restore sin/cos pointer
	yloop	4, b6b, -4*16*64+dist64	;; Test loop counter, next source pointer
	yloop	64, b6b, -64*dist64	;; Test loop counter, restore source pointer
	end_timer 9

;; Do FFT levels 14,15
;; On input and output the 64-byte cache lines hold these data values:
;;	0	8K	16K	24K	4	+8K	+8K	+8K
;;	1
;;	...
;;	3
;;	8	...
;;	...
;;	8184	...

;; Do 4096 four_complex_fft operations
;;	distance between fft data elements is 1
;;	do 1024 macros each processing 32 data values

	cmp	ffttype, 2
	je	ymid2p
	ja	ymid3p

	start_timer 10
	yloop_init  1024		;; 1024 iterations
b7a:	yr4_4cl_four_complex_fft_final rsi, 4*64, 64, 2*64
	yloop	16, b7a, -16*4*64+dist64 ;; Test loop counter, next source pointer
	yloop	64, b7a			;; Test loop counter
	end_timer 10
	yfft_1_ret

ymid2p:	start_timer 10
	yloop_init  1024		;; 1024 iterations
	yr4_4cl_four_complex_with_square_preload
b7b:	yr4_4cl_four_complex_with_square rsi, 4*64, 64, 2*64, 16
	yloop	16, b7b, -16*4*64+dist64 ;; Test loop counter, next source pointer
	yloop	64, b7b			;; Test loop counter
	end_timer 10
	jmp	yfinish_unfft

ymid3p:	start_timer 10
x64	mov	r8, DEST2ARG
	mov	rbp, DIST_TO_MULSRCARG
	yloop_init  1024		;; 1024 iterations
	yr4_4cl_four_complex_with_mult_preload
b7c:	yr4_4cl_four_complex_with_mult rsi, 4*64, 64, 2*64
	yloop	16, b7c, -16*4*64+dist64 ;; Test loop counter, next source pointer
	yloop	64, b7c			;; Test loop counter
x64	add	rsi, r8
	end_timer 10
	jmp	yfinish_unfft

ymid4p:	start_timer 10
	mov	rbp, DIST_TO_MULSRCARG
	yloop_init  1024		;; 1024 iterations
	yr4_4cl_four_complex_with_mulf_preload
b7d:	yr4_4cl_four_complex_with_mulf rsi, 4*64, 64, 2*64
	yloop	16, b7d, -16*4*64+dist64 ;; Test loop counter, next source pointer
	yloop	64, b7d			;; Test loop counter
	end_timer 10
yfinish_unfft:
	bump	rsi, -64*dist64		;; Restore source pointer

;; Do inverse FFT levels 12,13
;; On input the 64-byte cache lines hold these data values:
;;	0	8K	16K	24K	4	+8K	+8K	+8K
;;	1
;;	...
;;	3
;;	8	...
;;	...
;;	8184	...
;; On output the 64-byte cache lines hold these data values:
;;	0	8K	16K	24K	16	+8K	+8K	+8K
;;	1
;;	...
;;	15
;;	32	...
;;	...
;;	8160	...

;; Do 4096 four_complex_unfft operations
;;	distance between fft data elements is 4
;;	do 1024 macros each processing 32 data values

	start_timer 11
	mov	rdi, sincos1		;; Load sin/cos pointer
	yloop_init  1024		;; 256 iterations of 4
	yr4_b4cl_four_complex_djbunfft_preload
c6b:	yr4_b4cl_four_complex_djbunfft rsi, 64, 4*64, 8*64, rdi, 256*YMM_SCD2/4, 4
	yloop	4, c6b, -4*64+16*64, rdi, -4*256*YMM_SCD2/4 ;; Test loop counter, next source pointer, restore sine/cosine pointer
	yloop	4, c6b, -4*16*64+dist64	;; Test loop counter, next source pointer
	yloop	64, c6b, -64*dist64	;; Test loop counter, restore source pointer
	end_timer 11

;; Do inverse FFT levels 10,11
;; On input the 64-byte cache lines hold these data values:
;;	0	8K	16K	24K	16	+8K	+8K	+8K
;;	1
;;	...
;;	15
;;	32	...
;;	...
;;	8160	...
;; On output the 64-byte cache lines hold these data values:
;;	0	8K	16K	24K	64	+8K	+8K	+8K
;;	1
;;	...
;;	63
;;	128	...
;;	...
;;	8064	...

;; Do 4096 four_complex_unfft operations
;;	distance between fft data elements is 16
;;	do 1024 macros each processing 32 data values

	start_timer 12
	yloop_init  1024		;; 64 iterations of 16
	yr4_b4cl_four_complex_djbunfft_preload
c5b:	yr4_b4cl_four_complex_djbunfft rsi, 64, 16*64, 32*64, rdi, 64*YMM_SCD2/4, 16
	yloop	16, c5b, -16*64+dist64, rdi, -16*64*YMM_SCD2/4 ;; Test loop counter, next source pointer, restore sine/cosine pointer
	yloop	64, c5b, -64*dist64	;; Test loop counter, restore source pointer
	end_timer 12

;; Do inverse FFT levels 8,9
;; On input the 64-byte cache lines hold these data values:
;;	0	8K	16K	24K	64	+8K	+8K	+8K
;;	1
;;	...
;;	63
;;	128	...
;;	...
;;	8064	...
;; On output the 64-byte cache lines hold these data values:
;;	0	8K	16K	24K	256	+8K	+8K	+8K
;;	1
;;	...
;;	255
;;	512	...
;;	...
;;	7680	...

;; Do 4096 four_complex_unfft operations
;;	distance between fft data elements is 64
;;	do 1024 macros each processing 32 data values

	start_timer 13
	yloop_init  1024		;; 16 iterations of 64
	yr4_b4cl_four_complex_djbunfft_preload
;; Loop below reworked to avoid 4KB distances
;;c4b:	yr4_b4cl_four_complex_djbunfft rsi, 64, dist64, 2*dist64, rdi, 16*YMM_SCD2/4, 64
;;	yloop	64, c4b, -64*64+4*dist64, rdi, -64*16*YMM_SCD2/4 ;; Test loop counter, next source pointer, restore sine/cosine pointer
c4b:	yr4_b4cl_four_complex_djbunfft rsi, 4*64, dist64, 2*dist64, rdi, 4*16*YMM_SCD2/4, 16
	yloop	16, c4b, -16*4*64+64, rdi, -16*4*16*YMM_SCD2/4+16*YMM_SCD2/4 ;; Test loop counter, next source pointer, next sine/cosine pointer
	yloop	4, c4b, -4*64+4*dist64, rdi, -4*16*YMM_SCD2/4 ;; Test loop counter, next source pointer, restore sine/cosine pointer
	yloop	16, c4b, -16*4*dist64	;; Test loop counter, restore source pointer
	end_timer 13

;; Do inverse FFT levels 6,7
;; On input the 64-byte cache lines hold these data values:
;;	0	8K	16K	24K	256	+8K	+8K	+8K
;;	1
;;	...
;;	255
;;	512	...
;;	...
;;	7680	...
;; On output the 64-byte cache lines hold these data values:
;;	0	8K	16K	24K	1K	+8K	+8K	+8K
;;	1
;;	...
;;	1023
;;	2K	...
;;	4K	...
;;	6K	...

;; Do 4096 four_complex_unfft operations
;;	distance between fft data elements is 256
;;	do 1024 macros each processing 32 data values

	start_timer 14
	yloop_init  1024		;; 4 iterations of 256
	yr4_b4cl_four_complex_djbunfft_preload
c3b:	yr4_b4cl_four_complex_djbunfft rsi, 64, 4*dist64, 8*dist64, rdi, 4*YMM_SCD2/4, 64
	yloop	64, c3b, -64*64+dist64	;; Test loop counter, next source pointer
	yloop	4, c3b, -4*dist64+16*dist64, rdi, -256*4*YMM_SCD2/4 ;; Test loop counter, next source pointer, restore sine/cosine pointer
	yloop	4, c3b, -4*16*dist64	;; Test loop counter, restore source pointer
	end_timer 14

;; Do inverse FFT levels 4,5
;; On input the 64-byte cache lines hold these data values:
;;	0	8K	16K	24K	1K	+8K	+8K	+8K
;;	1
;;	...
;;	1023
;;	2K	...
;;	4K	...
;;	6K	...
;; On output the 64-byte cache lines hold these data values:
;;	0	8K	16K	24K	4K	+8K	+8K	+8K
;;	1
;;	...
;;	4095

;; Do 4096 four_complex_unfft operations
;;	distance between fft data elements is 1024
;;	do 1024 macros each processing 32 data values

	start_timer 15
	yloop_init  1024		;; 1024 iterations
	yr4_b4cl_four_complex_djbunfft_preload
c2b:	yr4_b4cl_four_complex_djbunfft rsi, 64, 16*dist64, 32*dist64, rdi, YMM_SCD2/4, 64
	yloop	64, c2b, -64*64+dist64	;; Test loop counter, next source pointer
	yloop	16, c2b, -16*dist64	;; Test loop counter, restore source pointer
	end_timer 15

;; Do inverse FFT levels 1,2,3
;; On input the 64-byte cache lines hold these data values:
;;	0	8K	16K	24K	4K	+8K	+8K	+8K
;;	1
;;	...
;;	4095
;; These are swizzled to:
;;	0	1K	2K	3K	4K	+1K	+1K	+1K
;;	1	...
;;	...
;;	1023
;;	8K	...
;;	16K	...
;;	24K	...
;; On output the 64-byte cache lines hold these data values:
;;	0	1K	2K	3K	16K	+1K	+1K	+1K
;;	1	...
;;	...
;;	1023
;;	4K	...
;;	8K	...
;;	12K	...

;; Do 4096 four_complex_last_unfft operations
;;	distance between fft data elements is 4096
;;	do 1024 macros each processing 32 data values

	start_timer 16
	yloop_init  1024		;; 1024 iterations
	yr4_s4cl_four_complex_last_unfft_preload
c1b:	yr4_s4cl_four_complex_last_unfft rsi, 64, 16*dist64, 32*dist64, rdi, YMM_SCD4, 64
	yloop	64, c1b, -64*64+dist64	;; Test loop counter, next source pointer
	yloop	16, c1b			;; Test loop counter
	end_timer 16

	ytwo_dest_adjust
	yfft_3_ret
	ENDM
