; Copyright 2017-2024 - Mersenne Research, Inc.  All rights reserved
; Author:  George Woltman
; Email: woltman@alum.mit.edu
;

; ********************************************************
; ********************************************************
; ********************  FFT MACROS  **********************
; ********************************************************
; ********************************************************

;; Wrapper macro that takes two reals, swizzles data, and applies weights.  Output will be suitable for use by a pass 2 FFT macro.
;; NOTE: The r1/r9 col multiplier is 1.0 thus there are no fudges to apply to R1/R9

zsf_onepass_real_fft_wrapper_preload MACRO
	mov	eax, 00110011b			;; For vblendmpd during swizzle
	kmovw	k7, eax
	vpmovzxbq zmm31, ZMM_PERMUTE1		;; zmm31 = 8+4 0+4 8+6 0+6 8+0 0+0 8+2 0+2 [msw at left]
	vpmovzxbq zmm30, ZMM_PERMUTE2		;; zmm30 = 8+5 0+5 8+7 0+7 8+1 0+1 8+3 0+3
	vbroadcastsd zmm29, ZMM_ONE_OVER_B
	ENDM
zsf_onepass_real_fft_wrapper MACRO srcreg,srcinc,d1,d2,d4,colreg,colinc,maskreg,maskinc,grpreg,grpinc,maxrpt,L1pt,L1pd
	vmovapd zmm28, [grpreg+0*64]		;; group multiplier for R1-R8
	vmulpd	zmm2, zmm28, [srcreg+rbx+d1]	;; apply the group multiplier to R2			; 1-4		n 5
	vmulpd	zmm14, zmm28, [srcreg+rbx]	;; apply the group multiplier to R1			; 1-4		n 23

	vmulpd	zmm10, zmm28, [srcreg+rbx+d2]	;; apply the group multiplier to R3			; 2-5		n 6
	vmulpd	zmm6, zmm28, [srcreg+rbx+d2+d1]	;; apply the group multiplier to R4			; 2-5		n 7

	vmulpd	zmm16, zmm28, [srcreg+rbx+d4]	;; apply the group multiplier to R5			; 3-6		n 8
	vmulpd	zmm8, zmm28, [srcreg+rbx+d4+d1]	;; apply the group multiplier to R6			; 3-6		n 9

	vmulpd	zmm12, zmm28, [srcreg+rbx+d4+d2];; apply the group multiplier to R7			; 4-7		n 10
	vmulpd	zmm4, zmm28, [srcreg+rbx+d4+d2+d1];; apply the group multiplier to R8			; 4-7		n 11

	vmovapd zmm20, [grpreg+1*64]		;; group multiplier for R9-R16
	vbroadcastsd zmm28, [colreg+1*8]	;; Load col multiplier for R2/R10
	vmulpd	zmm2, zmm2, zmm28		;; apply the col multiplier to R2			; 5-8		n 13
	vmulpd	zmm21, zmm20, zmm28		;; unfudged multiplier for R10				; 5-8		n 14

	vbroadcastsd zmm28, [colreg+2*8]	;; Load col multiplier for R3/R11
	vmulpd	zmm10, zmm10, zmm28		;; apply the col multiplier to R3			; 6-9		n 15
	vmulpd	zmm22, zmm20, zmm28		;; unfudged multiplier for R11				; 6-9		n 16

	vbroadcastsd zmm28, [colreg+3*8]	;; Load col multiplier for R4/R12
	vmulpd	zmm6, zmm6, zmm28		;; apply the col multiplier to R4			; 7-10		n 17
	vmulpd	zmm23, zmm20, zmm28		;; unfudged multiplier for R12				; 7-10		n 18

	vbroadcastsd zmm28, [colreg+4*8]	;; Load col multiplier for R5/R13
	vmulpd	zmm16, zmm16, zmm28		;; apply the col multiplier to R5			; 8-11		n 19
	vmulpd	zmm24, zmm20, zmm28		;; unfudged multiplier for R13				; 8-11		n 20

	vbroadcastsd zmm28, [colreg+5*8]	;; Load col multiplier for R6/R14
	vmulpd	zmm8, zmm8, zmm28		;; apply the col multiplier to R6			; 9-12		n 21
	vmulpd	zmm25, zmm20, zmm28		;; unfudged multiplier for R14				; 9-12		n 22

	vbroadcastsd zmm28, [colreg+6*8]	;; Load col multiplier for R7/R15
	vmulpd	zmm12, zmm12, zmm28		;; apply the col multiplier to R7			; 10-13		n 26
	vmulpd	zmm26, zmm20, zmm28		;; unfudged multiplier for R15				; 10-13		n 26

	mov	r13, [maskreg+0*8]		;; Load uncompressed fudge flags
	vbroadcastsd zmm28, [colreg+7*8]	;; Load col multiplier for R8/R16
	kmovw	k3, r13d			;; Load R2 and R10 fudge factor mask			; 11		n 12
	vmulpd	zmm4, zmm4, zmm28		;; apply the col multiplier to R8			; 11-14		n 27
	shr	r13, 16				;; Next 16 bits of fudge flags

	kshiftrw k4, k3, 8			;; R10's fudge						; 12		n 14
	vmulpd	zmm27, zmm20, zmm28		;; unfudged multiplier for R16				; 12-15		n 27
	mov	r14, [maskreg+1*8]		;; Load uncompressed fudge flags
	bump	maskreg, maskinc

	kmovw	k1, r13d			;; Load R3 and R11 fudge factor mask			; 13		n 14
	vmulpd	zmm2{k3}, zmm2, zmm29		;; apply fudge to R2					; 13-16		n 28
	shr	r13, 16				;; Next 16 bits of fudge flags

	kshiftrw k2, k1, 8			;; R11's fudge						; 14		n 16
	vmulpd	zmm21{k4}, zmm21, zmm29		;; fudged multiplier for R10				; 14-17		n 24
	bump	grpreg, grpinc

	kmovw	k3, r13d			;; Load R4 and R12 fudge factor mask			; 15		n 16
	vmulpd	zmm10{k1}, zmm10, zmm29		;; apply fudge to R3					; 15-18		n 25
	shr	r13, 16				;; Next 16 bits of fudge flags
	bump	colreg, colinc

	kshiftrw k4, k3, 8			;; R12's fudge						; 16		n 18
	vmulpd	zmm22{k2}, zmm22, zmm29		;; fudged multiplier for R11				; 16-19		n 25

	kmovw	k1, r13d			;; Load R5 and R13 fudge factor mask			; 17		n 18
	vmulpd	zmm6{k3}, zmm6, zmm29		;; apply fudge to R4					; 17-20		n 25
	vmovapd	zmm17, [srcreg+rbx+64]		;; r9

	kshiftrw k2, k1, 8			;; R13's fudge						; 18		n 20
	vmulpd	zmm23{k4}, zmm23, zmm29		;; fudged multiplier for R12				; 18-21		n 25
	vmovapd	zmm18, [srcreg+rbx+d1+64]	;; r10

	kmovw	k3, r14d			;; Load R6 and R14 fudge factor mask			; 19		n 20
	vmulpd	zmm16{k1}, zmm16, zmm29		;; apply fudge to R5					; 19-22		n 31
	shr	r14, 16				;; Next 16 bits of fudge flags
	vmovapd	zmm0, [srcreg+rbx+d2+64]	;; r11

	kshiftrw k4, k3, 8			;; R14's fudge						; 20		n 22
	vmulpd	zmm24{k2}, zmm24, zmm29		;; fudged multiplier for R13				; 20-23		n 31
	vmovapd	zmm19, [srcreg+rbx+d2+d1+64]	;; r12

	kmovw	k1, r14d			;; Load R7 and R15 fudge factor mask			; 21		n 22
	vmulpd	zmm8{k3}, zmm8, zmm29		;; apply fudge to R6					; 21-24		n 32
	shr	r14, 16				;; Next 16 bits of fudge flags
	L1prefetch srcreg+L1pd, L1pt

	kshiftrw k2, k1, 8			;; R15's fudge						; 22		n 26
	vmulpd	zmm25{k4}, zmm25, zmm29		;; fudged multiplier for R14				; 22-25		n 32
	L1prefetch srcreg+64+L1pd, L1pt

	kmovw	k3, r14d			;; Load R8 and R16 fudge factor mask			; 23		n 27
	zfmaddpd zmm13, zmm17, zmm20, zmm14	;; r1+r9*group_mult (new "R1")				; 23-26		n 29
	L1prefetch srcreg+d1+L1pd, L1pt

	kshiftrw k4, k3, 8			;; R16's fudge						; 24		n 27
	zfmaddpd zmm1, zmm18, zmm21, zmm2	;; r2+r10*fudged_mult (new "R2")			; 24-27		n 29
	L1prefetch srcreg+d1+64+L1pd, L1pt

	zfmaddpd zmm9, zmm0, zmm22, zmm10	;; r3+r11*fudged_mult (new "R3")			; 25-28		n 31
	zfmaddpd zmm5, zmm19, zmm23, zmm6	;; r4+r12*fudged_mult (new "R4")			; 25-28		n 31
	L1prefetch srcreg+d2+L1pd, L1pt

	vmulpd	zmm12{k1}, zmm12, zmm29		;; apply fudge to R7					; 26-29		n 33
	vmulpd	zmm26{k2}, zmm26, zmm29		;; fudged multiplier for R15				; 26-29		n 33
	L1prefetch srcreg+d2+64+L1pd, L1pt

	vmulpd	zmm4{k3}, zmm4, zmm29		;; apply fudge to R8					; 27-30		n 34
	vmulpd	zmm27{k4}, zmm27, zmm29		;; fudged multiplier for R16				; 27-30		n 34
	L1prefetch srcreg+d2+d1+L1pd, L1pt

	zfnmaddpd zmm14, zmm17, zmm20, zmm14	;; r1-r9*group_mult (new "I1")				; 28-31		n 33
	zfnmaddpd zmm2, zmm18, zmm21, zmm2	;; r2-r10*fudged_mult (new "I2")			; 28-31		n 33
	L1prefetch srcreg+d2+d1+64+L1pd, L1pt

	vshufpd	zmm21, zmm13, zmm1, 00000000b	;; r2_6 r1_6 r2_4 r1_4 r2_2 r1_2 r2_0 r1_0		; 29		n 39
	vshufpd	zmm13, zmm13, zmm1, 11111111b	;; r2_7 r1_7 r2_5 r1_5 r2_3 r1_3 r2_1 r1_1		; 30		n 41
	L1prefetch srcreg+d4+L1pd, L1pt

	zfnmaddpd zmm10, zmm0, zmm22, zmm10	;; r3-r11*fudged_mult (new "I3")			; 29-32		n 35
	zfnmaddpd zmm6, zmm19, zmm23, zmm6	;; r4-r12*fudged_mult (new "I4")			; 30-33		n 35
	vmovapd	zmm0, [srcreg+rbx+d4+64]	;; r13

	zperm2pd zmm1, zmm31, zmm9, zmm5	;; r4_4 r3_4 r4_6 r3_6 r4_0 r3_0 r4_2 r3_2		; 31-33		n 39
	zperm2pd zmm9, zmm30, zmm9, zmm5	;; r4_5 r3_5 r4_7 r3_7 r4_1 r3_1 r4_3 r3_3		; 32-34		n 41
	vmovapd	zmm17, [srcreg+rbx+d4+d1+64]	;; r14

	zfmaddpd zmm15, zmm0, zmm24, zmm16	;; r5+r13*fudged_mult (new "R5")			; 31-34		n 37
	zfmaddpd zmm7, zmm17, zmm25, zmm8	;; r6+r14*fudged_mult (new "R6")			; 32-35		n 37
	vmovapd	zmm20, [srcreg+rbx+d4+d2+64]	;; r15

	vshufpd	zmm19, zmm14, zmm2, 00000000b	;; i2_6 i1_6 i2_4 i1_4 i2_2 i1_2 i2_0 i1_0		; 33		n 47
	vshufpd	zmm14, zmm14, zmm2, 11111111b	;; i2_7 i1_7 i2_5 i1_5 i2_3 i1_3 i2_1 i1_1		; 34		n 51
	vmovapd	zmm18, [srcreg+rbx+d4+d2+d1+64]	;; r16

	zfmaddpd zmm11, zmm20, zmm26, zmm12	;; r7+r15*fudged_mult (new "R7")			; 33-36		n 39
	zfmaddpd zmm3, zmm18, zmm27, zmm4	;; r8+r16*fudged_mult (new "R8")			; 34-37		n 39
	L1prefetch srcreg+d4+64+L1pd, L1pt

	zperm2pd zmm2, zmm31, zmm10, zmm6	;; i4_4 i3_4 i4_6 i3_6 i4_0 i3_0 i4_2 i3_2		; 35-37		n 47
	zperm2pd zmm10, zmm30, zmm10, zmm6	;; i4_5 i3_5 i4_7 i3_7 i4_1 i3_1 i4_3 i3_3		; 36-38		n 51
	L1prefetch srcreg+d4+d1+L1pd, L1pt

	zfnmaddpd zmm16, zmm0, zmm24, zmm16	;; r5-r13*fudged_mult (new "I5")			; 35-38		n 41
	zfnmaddpd zmm8, zmm17, zmm25, zmm8	;; r6-r14*fudged_mult (new "I6")			; 36-39		n 41
	L1prefetch srcreg+d4+d1+64+L1pd, L1pt

	vshufpd	zmm22, zmm15, zmm7, 00000000b	;; r6_6 r5_6 r6_4 r5_4 r6_2 r5_2 r6_0 r5_0		; 37		n 43
	vshufpd	zmm15, zmm15, zmm7, 11111111b	;; r6_7 r5_7 r6_5 r5_5 r6_3 r5_3 r6_1 r5_1		; 38		n 45
	L1prefetch srcreg+d4+d2+L1pd, L1pt

	zfnmaddpd zmm12, zmm20, zmm26, zmm12	;; r7-r15*fudged_mult (new "I7")			; 37-40		n 43
	zfnmaddpd zmm4, zmm18, zmm27, zmm4	;; r8-r16*fudged_mult (new "I8")			; 38-41		n 43
	L1prefetch srcreg+d4+d2+64+L1pd, L1pt

	zperm2pd zmm7, zmm31, zmm11, zmm3	;; r8_4 r7_4 r8_6 r7_6 r8_0 r7_0 r8_2 r7_2		; 39-41		n 43
	zperm2pd zmm11, zmm30, zmm11, zmm3	;; r8_5 r7_5 r8_7 r7_7 r8_1 r7_1 r8_3 r7_3		; 40-42		n 45
	L1prefetch srcreg+d4+d2+d1+L1pd, L1pt

	vblendmpd zmm5{k7}, zmm1, zmm21		;; r4_4 r3_4 r2_4 r1_4 r4_0 r3_0 r2_0 r1_0		; 39		n 45
	vblendmpd zmm21{k7}, zmm21, zmm1	;; r2_6 r1_6 r4_6 r3_6 r2_2 r1_2 r4_2 r3_2		; 40		n 47
	L1prefetch srcreg+d4+d2+d1+64+L1pd, L1pt

	vshufpd	zmm23, zmm16, zmm8, 00000000b	;; i6_6 i5_6 i6_4 i5_4 i6_2 i5_2 i6_0 i5_0		; 41		n 49
	vshufpd	zmm16, zmm16, zmm8, 11111111b	;; i6_7 i5_7 i6_5 i5_5 i6_3 i5_3 i6_1 i5_1		; 42		n 53

	vblendmpd zmm1{k7}, zmm9, zmm13		;; r4_5 r3_5 r2_5 r1_5 r4_1 r3_1 r2_1 r1_1		; 41		n 49
	vblendmpd zmm13{k7}, zmm13, zmm9	;; r2_7 r1_7 r4_7 r3_7 r2_3 r1_3 r4_3 r3_3		; 42		n 51

	zperm2pd zmm8, zmm31, zmm12, zmm4	;; i8_4 i7_4 i8_6 i7_6 i8_0 i7_0 i8_2 i7_2		; 43-45		n 49
	zperm2pd zmm12, zmm30, zmm12, zmm4	;; i8_5 i7_5 i8_7 i7_7 i8_1 i7_1 i8_3 i7_3		; 44-46		n 59

	vblendmpd zmm3{k7}, zmm7, zmm22		;; r8_4 r7_4 r6_4 r5_4 r8_0 r7_0 r6_0 r5_0		; 43		n 45
	vblendmpd zmm22{k7}, zmm22, zmm7	;; r6_6 r5_6 r8_6 r7_6 r6_2 r5_2 r8_2 r7_2		; 44		n 47

	vshuff64x2 zmm18, zmm5, zmm3, 01000100b	;; r8_0 r7_0 r6_0 r5_0 r4_0 r3_0 r2_0 r1_0		; 45-47
	vshuff64x2 zmm5, zmm5, zmm3, 11101110b	;; r8_4 r7_4 r6_4 r5_4 r4_4 r3_4 r2_4 r1_4		; 46-48

	vblendmpd zmm7{k7}, zmm11, zmm15	;; r8_5 r7_5 r6_5 r6_5 r8_1 r7_1 r6_1 r5_1		; 45		n 49
	vblendmpd zmm15{k7}, zmm15, zmm11	;; r6_7 r5_7 r8_7 r7_7 r6_3 r5_3 r8_3 r7_3		; 46		n 51

	vshuff64x2 zmm3, zmm21, zmm22, 00010001b ;; r8_2 r7_2 r6_2 r5_2 r4_2 r3_2 r2_2 r1_2		; 47-49
	vshuff64x2 zmm21, zmm21, zmm22, 10111011b;; r8_6 r7_6 r6_6 r5_6 r4_6 r3_6 r2_6 r1_6		; 48-50

	vblendmpd zmm6{k7}, zmm2, zmm19		;; i4_4 i3_4 i2_4 i1_4 i4_0 i3_0 i2_0 i1_0		; 47		n 53
	vblendmpd zmm19{k7}, zmm19, zmm2	;; i2_6 i1_6 i4_6 i3_6 i2_2 i1_2 i4_2 i3_2		; 48		n 55
	zstore	[srcreg], zmm18										; 48

	vshuff64x2 zmm9, zmm1, zmm7, 01000100b ;; r8_1 r7_1 r6_1 r5_1 r4_1 r3_1 r2_1 r1_1		; 49-51
	vshuff64x2 zmm1, zmm1, zmm7, 11101110b ;; r8_5 r7_5 r6_5 r5_5 r4_5 r3_5 r2_5 r1_5		; 50-52
	zstore	[srcreg+d4], zmm5									; 49

	vblendmpd zmm4{k7}, zmm8, zmm23		;; i8_4 i7_4 i6_4 i5_4 i8_0 i7_0 i6_0 i5_0		; 49		n 53
	vblendmpd zmm23{k7}, zmm23, zmm8	;; i6_6 i5_6 i8_6 i7_6 i6_2 i5_2 i8_2 i7_2		; 50		n 55
	zstore	[srcreg+d2], zmm3									; 50

	vshuff64x2 zmm7, zmm13, zmm15, 00010001b;; r8_3 r7_3 r6_3 r5_3 r4_3 r3_3 r2_3 r1_3		; 51-53
	vshuff64x2 zmm13, zmm13, zmm15, 10111011b ;; r8_7 r7_7 r6_7 r5_7 r4_7 r3_7 r2_7 r1_7		; 52-54
	zstore	[srcreg+d4+d2], zmm21									; 51

	vblendmpd zmm2{k7}, zmm10, zmm14	;; i4_5 i3_5 i2_5 i1_5 i4_1 i3_1 i2_1 i1_1		; 51		n 57
	vblendmpd zmm14{k7}, zmm14, zmm10	;; i2_7 i1_7 i4_7 i3_7 i2_3 i1_3 i4_3 i3_3		; 52		n 59
	zstore	[srcreg+d1], zmm9									; 52

	vshuff64x2 zmm20, zmm6, zmm4, 01000100b ;; i8_0 i7_0 i6_0 i5_0 i4_0 i3_0 i2_0 i1_0		; 53-55
	vshuff64x2 zmm6, zmm6, zmm4, 11101110b ;; i8_4 i7_4 i6_4 i5_4 i4_4 i3_4 i2_4 i1_4		; 54-56
	zstore	[srcreg+d4+d1], zmm1									; 53

	vblendmpd zmm8{k7}, zmm12, zmm16	;; i8_5 i7_5 i6_5 i5_5 i8_1 i7_1 i6_1 i5_1		; 53		n 57
	vblendmpd zmm16{k7}, zmm16, zmm12	;; i6_7 i5_7 i8_7 i7_7 i6_3 i5_3 i8_3 i7_3		; 54		n 59
	zstore	[srcreg+d2+d1], zmm7									; 54

	vshuff64x2 zmm4, zmm19, zmm23, 00010001b ;; i8_2 i7_2 i6_2 i5_2 i4_2 i3_2 i2_2 i1_2		; 55-57
	zstore	[srcreg+d4+d2+d1], zmm13								; 55

	vshuff64x2 zmm19, zmm19, zmm23, 10111011b ;; i8_6 i7_6 i6_6 i5_6 i4_6 i3_6 i2_6 i1_6		; 56-58
	zstore	[srcreg+64], zmm20									; 56

	vshuff64x2 zmm10, zmm2, zmm8, 01000100b ;; i8_1 i7_1 i6_1 i5_1 i4_1 i3_1 i2_1 i1_1		; 57-59
	zstore	[srcreg+d4+64], zmm6									; 57

	vshuff64x2 zmm2, zmm2, zmm8, 11101110b ;; i8_5 i7_5 i6_5 i5_5 i4_5 i3_5 i2_5 i1_5		; 58-60
	zstore	[srcreg+d2+64], zmm4									; 58

	vshuff64x2 zmm8, zmm14, zmm16, 00010001b ;; i8_3 i7_3 i6_3 i5_3 i4_3 i3_3 i2_3 i1_3		; 59-61
	zstore	[srcreg+d4+d2+64], zmm19								; 59

	vshuff64x2 zmm14, zmm14, zmm16, 10111011b ;; i8_7 i7_7 i6_7 i5_7 i4_7 i3_7 i2_7 i1_7		; 60-62
	zstore	[srcreg+d1+64], zmm10									; 60

	zstore	[srcreg+d4+d1+64], zmm2									; 61
	zstore	[srcreg+d2+d1+64], zmm8									; 62
	zstore	[srcreg+d4+d2+d1+64], zmm14								; 63
	bump	srcreg, srcinc
	ENDM


;; Wrapper macro that takes two reals, swizzles data, and applies weights.  Output will be suitable for use by a pass 2 FFT macro.
;; This version does more loads so that we can use vblendmpd and vbroadcastf64x4 instead of vshuff64x2.
;; The former can run on ports 0 & 5 whereas the latter only runs on port 5.
;; NOTE: The r1/r9 inverse col multiplier is 1.0 thus there are no fudges to apply to R1/R9

zs_onepass_real_unfft_wrapper_preload MACRO
	mov	r14d, 11110000b
	kmovw	k6, r14d
	knotw	k7, k6
	vbroadcastsd zmm28, ZMM_B
	ENDM
zs_onepass_real_unfft_wrapper MACRO srcreg,srcinc,d1,d2,d4,colreg,colinc,maskreg,maskinc,maxrpt,L1pt,L1pd
	vmovapd	zmm15, [srcreg]				;; d0_7	d0_6 d0_5 d0_4 d0_3 d0_2 d0_1 d0_0
	vmovapd	zmm16, [srcreg+d1]			;; d1_7	d1_6 d1_5 d1_4 d1_3 d1_2 d1_1 d1_0
	vbroadcastf64x4	zmm15{k6}, [srcreg+d2]		;; d2_3 d2_2 d2_1 d2_0 d0_3 d0_2 d0_1 d0_0		; 1		n 2
	vbroadcastf64x4	zmm16{k6}, [srcreg+d2+d1]	;; d3_3 d3_2 d3_1 d3_0 d1_3 d1_2 d1_1 d1_0		; 1		n 2

	vmovapd	zmm2, [srcreg+d4]			;; d4_7	d4_6 d4_5 d4_4 d4_3 d4_2 d4_1 d4_0
	vshufpd	zmm5, zmm15, zmm16, 00000000b		;; d3_2 d2_2 d3_0 d2_0 d1_2 d0_2 d1_0 d0_0		; 2		n 10
	vbroadcastf64x4	zmm2{k6}, [srcreg+d4+d2]	;; d6_3 d6_2 d6_1 d6_0 d4_3 d4_2 d4_1 d4_0		; 2		n 4

	vmovapd	zmm8, [srcreg+d4+d1]			;; d5_7	d5_6 d5_5 d5_4 d5_3 d5_2 d5_1 d5_0
	vshufpd	zmm16, zmm15, zmm16, 11111111b		;; d3_3 d2_3 d3_1 d2_1 d1_3 d0_3 d1_1 d0_1		; 3		n 12
	vbroadcastf64x4	zmm8{k6}, [srcreg+d4+d2+d1]	;; d7_3 d7_2 d7_1 d7_0 d5_3 d5_2 d5_1 d5_0		; 3		n 4

	vmovapd	zmm3, [srcreg+d2]			;; d2_7	d2_6 d2_5 d2_4 d2_3 d2_2 d2_1 d2_0
	vmovupd	zmm17, [srcreg+32]			;; (+64 d0_3 d0_2 d0_1 d0_0) d0_7 d0_6 d0_5 d0_4
	vshufpd	zmm15, zmm2, zmm8, 00000000b		;; d7_2 d6_2 d7_0 d6_0 d5_2 d4_2 d5_0 d4_0		; 4		n 10
	vblendmpd zmm3{k6}, zmm17, zmm3			;; d2_7 d2_6 d2_5 d2_4 d0_7 d0_6 d0_5 d0_4		; 4		n 6

	vmovapd	zmm31, [srcreg+d2+d1]			;; d3_7	d3_6 d3_5 d3_4 d3_3 d3_2 d3_1 d3_0
	vmovupd	zmm18, [srcreg+d1+32]			;; (+64 d1_3 d1_2 d1_1 d1_0) d1_7 d1_6 d1_5 d1_4
	vshufpd	zmm8, zmm2, zmm8, 11111111b		;; d7_3 d6_3 d7_1 d6_1 d5_3 d4_3 d5_1 d4_1		; 5		n 12
	vblendmpd zmm31{k6}, zmm18, zmm31		;; d3_7 d3_6 d3_5 d3_4 d1_7 d1_6 d1_5 d1_4		; 5		n 6

	vmovapd	zmm7, [srcreg+d4+d2]			;; d6_7	d6_6 d6_5 d6_4 d6_3 d6_2 d6_1 d6_0
	vmovupd	zmm19, [srcreg+d4+32]			;; (+64 d4_3 d4_2 d4_1 d4_0) d4_7 d4_6 d4_5 d4_4
	vshufpd	zmm2, zmm3, zmm31, 00000000b		;; d3_6 d2_6 d3_4 d2_4 d1_6 d0_6 d1_4 d0_4		; 6		n 14
	vblendmpd zmm7{k6}, zmm19, zmm7			;; d6_7	d6_6 d6_5 d6_4 d4_7 d4_6 d4_5 d4_4		; 6		n 8

	vmovapd	zmm14, [srcreg+d4+d2+d1]		;; d7_7	d7_6 d7_5 d7_4 d7_3 d7_2 d7_1 d7_0
	vmovupd zmm29, [srcreg+d4+d1+32]		;; (+64 d5_3 d5_2 d5_1 d5_0) d5_7 d5_6 d5_5 d5_4
	vshufpd	zmm31, zmm3, zmm31, 11111111b		;; d3_7 d2_7 d3_5 d2_5 d1_7 d0_7 d1_5 d0_5		; 7		n 16
	vblendmpd zmm14{k6}, zmm29, zmm14		;; d7_7	d7_6 d7_5 d7_4 d5_7 d5_6 d5_5 d5_4		; 7		n 8

	vmovapd	zmm9, [srcreg+d2+64]			;; (+64 d2_7 d2_6 d2_5 d2_4 d2_3 d2_2 d2_1 d2_0)
	vshufpd	zmm3, zmm7, zmm14, 00000000b		;; d7_6 d6_6 d7_4 d6_4 d5_6 d4_6 d5_4 d4_4		; 8		n 14
	vblendmpd zmm9{k6}, zmm9, zmm17			;; (+64 d0_3 d0_2 d0_1 d0_0 d2_3 d2_2 d2_1 d2_0)	; 8		n 18

	vmovapd	zmm11, [srcreg+d2+d1+64]		;; (+64 d3_7 d3_6 d3_5 d3_4 d3_3 d3_2 d3_1 d3_0)
	vshufpd	zmm14, zmm7, zmm14, 11111111b		;; d7_7 d6_7 d7_5 d6_5 d5_7 d4_7 d5_5 d4_5		; 9		n 16
	vblendmpd zmm11{k6}, zmm11, zmm18		;; (+64 d1_3 d1_2 d1_1 d1_0 d3_3 d3_2 d3_1 d3_0)	; 9		n 18

	vmovapd	zmm13, [srcreg+d4+d2+64]		;; (+64 d6_7 d6_6 d6_5 d6_4 d6_3 d6_2 d6_1 d6_0)
	vshuff64x2 zmm7, zmm5, zmm15, 10001000b		;; d7_0 d6_0 d5_0 d4_0 d3_0 d2_0 d1_0 d0_0 (R1)		; 10-12		n 16
	vblendmpd zmm13{k6}, zmm13, zmm19		;; (+64 d4_3 d4_2 d4_1 d4_0 d6_3 d6_2 d6_1 d6_0)	; 10		n 20

	vmovapd	zmm12, [srcreg+d4+d2+d1+64]		;; (+64 d7_7 d7_6 d7_5 d7_4 d7_3 d7_2 d7_1 d7_0)
	vshuff64x2 zmm15, zmm5, zmm15, 11011101b	;; d7_2 d6_2 d5_2 d4_2 d3_2 d2_2 d1_2 d0_2 (R3)		; 11-13		n 16
	vblendmpd zmm12{k6}, zmm12, zmm29		;; (+64 d5_3 d5_2 d5_1 d5_0 d7_3 d7_2 d7_1 d7_0)	; 11		n 20

	vmovapd	zmm10, [srcreg+64]			;; (+64 d0_7 d0_6 d0_5 d0_4 d0_3 d0_2 d0_1 d0_0)
	vshuff64x2 zmm5, zmm16, zmm8, 10001000b		;; d7_1 d6_1 d5_1 d4_1 d3_1 d2_1 d1_1 d0_1 (R2)		; 12-14		n 17
	vbroadcastf64x4	zmm10{k7}, [srcreg+d2+64+32]	;; (+64 d0_7 d0_6 d0_5 d0_4 d2_7 d2_6 d2_5 d2_4)	; 12		n 22

	vmovapd	zmm30, [srcreg+d1+64]			;; (+64 d1_7 d1_6 d1_5 d1_4 d1_3 d1_2 d1_1 d1_0)
	vshuff64x2 zmm8, zmm16, zmm8, 11011101b		;; d7_3 d6_3 d5_3 d4_3 d3_3 d2_3 d1_3 d0_3 (R4)		; 13-15		n 18
	vbroadcastf64x4	zmm30{k7}, [srcreg+d2+d1+64+32]	;; (+64 d1_7 d1_6 d1_5 d1_4 d3_7 d3_6 d3_5 d3_4)	; 13		n 22

	vmovapd	zmm1, [srcreg+d4+64]			;; (+64 d4_7 d4_6 d4_5 d4_4 d4_3 d4_2 d4_1 d4_0)
	vshuff64x2 zmm16, zmm2, zmm3, 10001000b		;; d7_4	d6_4 d5_4 d4_4 d3_4 d2_4 d1_4 d0_4 (R5)		; 14-16		n 19
	vbroadcastf64x4	zmm1{k7}, [srcreg+d4+d2+64+32]	;; (+64 d4_7 d4_6 d4_5 d4_4 d6_7 d6_6 d6_5 d6_4)	; 14		n 24

	vmovapd	zmm6, [srcreg+d4+d1+64]			;; (+64 d5_7 d5_6 d5_5 d5_4 d5_3 d5_2 d5_1 d5_0)
	vshuff64x2 zmm3, zmm2, zmm3, 11011101b		;; d7_6 d6_6 d5_6 d4_6 d3_6 d2_6 d1_6 d0_6 (R7)		; 15-17		n 20
	vbroadcastf64x4	zmm6{k7}, [srcreg+d4+d2+d1+64+32];;(+64 d5_7 d5_6 d5_5 d5_4 d7_7 d7_6 d7_5 d7_4)	; 15		n 24

	vbroadcastsd zmm22, [colreg+2*8]		;; Load the inverse col multiplier for R3/R11
	vshuff64x2 zmm2, zmm31, zmm14, 10001000b	;; d7_5 d6_5 d5_5 d4_5 d3_5 d2_5 d1_5 d0_5 (R6)		; 16-18		n 21
	vmulpd	zmm15, zmm15, zmm22			;; apply the inverse col multiplier to R3		; 16-19		n 37

	vbroadcastsd zmm21, [colreg+1*8]		;; Load the inverse col multiplier for R2/R10
	vshuff64x2 zmm14, zmm31, zmm14, 11011101b	;; d7_7 d6_7 d5_7 d4_7 d3_7 d2_7 d1_7 d0_7 (R8)		; 17-19		n 22
	vmulpd	zmm5, zmm5, zmm21			;; apply the inverse col multiplier to R2		; 17-20		n 36

	vbroadcastsd zmm23, [colreg+3*8]		;; Load the inverse col multiplier for R4/R12
	vshufpd	zmm0, zmm9, zmm11, 00000000b		;; d1_2 d0_2 d1_0 d0_0 d3_2 d2_2 d3_0 d2_0		; 18		n 26
	vmulpd	zmm8, zmm8, zmm23			;; apply the inverse col multiplier to R4		; 18-21		n 38

	vbroadcastsd zmm24, [colreg+4*8]		;; Load the inverse col multiplier for R5/R13
	vshufpd	zmm11, zmm9, zmm11, 11111111b		;; d1_3 d0_3 d1_1 d0_1 d3_3 d2_3 d3_1 d2_1		; 19		n 32
	vmulpd	zmm16, zmm16, zmm24			;; apply the inverse col multiplier to R5		; 19-22		n 39

	vbroadcastsd zmm26, [colreg+6*8]		;; Load the inverse col multiplier for R7/R15
	vshufpd	zmm9, zmm13, zmm12, 00000000b		;; d5_2 d4_2 d5_0 d4_0 d7_2 d6_2 d7_0 d6_0		; 20		n 26
	vmulpd	zmm3, zmm3, zmm26			;; apply the inverse col multiplier to R7		; 20-23		n 44

	vbroadcastsd zmm25, [colreg+5*8]		;; Load the inverse col multiplier for R6/R14
	vshufpd	zmm12, zmm13, zmm12, 11111111b		;; d5_3 d4_3 d5_1 d4_1 d7_3 d6_3 d7_1 d6_1		; 21		n 32
	vmulpd	zmm2, zmm2, zmm25			;; apply the inverse col multiplier to R6		; 21-24		n 43

	vbroadcastsd zmm27, [colreg+7*8]		;; Load the inverse col multiplier for R8/R16
	vshufpd	zmm13, zmm10, zmm30, 00000000b		;; d1_6 d0_6 d1_4 d0_4 d3_6 d2_6 d3_4 d2_4		; 22		n 28
	vmulpd	zmm14, zmm14, zmm27			;; apply the inverse col multiplier to R8		; 22-25		n 45

	vbroadcastsd zmm20, [colreg+0*8]		;; Load the inverse col multiplier for R1/R9
	vshufpd	zmm30, zmm10, zmm30, 11111111b		;; d1_7 d0_7 d1_5 d0_5 d3_7 d2_7 d3_5 d2_5		; 23		n 30
	vmulpd	zmm7, zmm7, zmm20			;; apply the inverse col multiplier to R1		; 23-26
	mov	r13, [maskreg+0*8]			;; Load uncompressed fudge flags

	vshufpd	zmm10, zmm1, zmm6, 00000000b		;; d5_6 d4_6 d5_4 d4_4 d7_6 d6_6 d7_4 d6_4		; 24		n 28
	mov	r14, [maskreg+1*8]			;; Load uncompressed fudge flags

	vshufpd	zmm6, zmm1, zmm6, 11111111b		;; d5_7 d4_7 d5_5 d4_5 d7_7 d6_7 d7_5 d6_5		; 25		n 30
	L1prefetchw srcreg+L1pd, L1pt
	bump	colreg, colinc

	vshuff64x2 zmm1, zmm0, zmm9, 00100010b		;; d7_0 d6_0 d5_0 d4_0 d3_0 d2_0 d1_0 d0_0 (R9)		; 26-28		n 29
	L1prefetchw srcreg+64+L1pd, L1pt
	bump	maskreg, maskinc

	vshuff64x2 zmm9, zmm0, zmm9, 01110111b		;; d7_2 d6_2 d5_2 d4_2 d3_2 d2_2 d1_2 d0_2 (R11)	; 27-29		n 30
	zstore	[srcreg], zmm7				;; Save R1						; 27
	L1prefetchw srcreg+d1+L1pd, L1pt

	vshuff64x2 zmm0, zmm13, zmm10, 00100010b	;; d7_4 d6_4 d5_4 d4_4 d3_4 d2_4 d1_4 d0_4 (R13)	; 28-30		n 31
	L1prefetchw srcreg+d1+64+L1pd, L1pt

	vshuff64x2 zmm10, zmm13, zmm10, 01110111b	;; d7_6 d6_6 d5_6 d4_6 d3_6 d2_6 d1_6 d0_6 (R15)	; 29-31		n 32
	vmulpd	zmm1, zmm1, zmm20			;; apply the inverse col multiplier to R11		; 29-32
	L1prefetchw srcreg+d2+L1pd, L1pt

	vshuff64x2 zmm13, zmm30, zmm6, 00100010b	;; d7_5 d6_5 d5_5 d4_5 d3_5 d2_5 d1_5 d0_5 (R14)	; 30-32		n 33
	vmulpd	zmm9, zmm9, zmm22			;; apply the inverse col multiplier to R11		; 30-33		n 40
	L1prefetchw srcreg+d2+64+L1pd, L1pt

	vshuff64x2 zmm6, zmm30, zmm6, 01110111b		;; d7_7 d6_7 d5_7 d4_7 d3_7 d2_7 d1_7 d0_7 (R16)	; 31-33		n 34
	vmulpd	zmm0, zmm0, zmm24			;; apply the inverse col multiplier to R13		; 31-34		n 45
	L1prefetchw srcreg+d2+d1+L1pd, L1pt

	vshuff64x2 zmm4, zmm11, zmm12, 00100010b	;; d7_1 d6_1 d5_1 d4_1 d3_1 d2_1 d1_1 d0_1 (R10)	; 32-34		n 41
	vmulpd	zmm10, zmm10, zmm26			;; apply the inverse col multiplier to R15		; 32-35		n 47
	L1prefetchw srcreg+d2+d1+64+L1pd, L1pt

	vshuff64x2 zmm12, zmm11, zmm12, 01110111b	;; d7_3 d6_3 d5_3 d4_3 d3_3 d2_3 d1_3 d0_3 (R12)	; 33-35		n 44
	vmulpd	zmm13, zmm13, zmm25			;; apply the inverse col multiplier to R14		; 33-36		n 46
	zstore	[srcreg+64], zmm1			;; Save R9						; 33
	L1prefetchw srcreg+d4+L1pd, L1pt

	kmovw	k2, r13d				;; Load R2 and R10 fudge factor mask			; 34		n 35
	vmulpd	zmm6, zmm6, zmm27			;; apply the inverse col multiplier to R16		; 34-37		n 48
	shr	r13, 16					;; Next 16 bits of fudge flags
	L1prefetchw srcreg+d4+64+L1pd, L1pt

	kmovw	k3, r13d				;; Load R3 and R11 fudge factor mask			; 35		n 36
	vmulpd	zmm5{k2}, zmm5, zmm28			;; apply fudge multiplier for R2			; 35-38
	shr	r13, 16					;; Next 16 bits of fudge flags
	L1prefetchw srcreg+d4+d1+L1pd, L1pt

	kmovw	k4, r13d				;; Load R4 and R12 fudge factor mask			; 36		n 37
	vmulpd	zmm15{k3}, zmm15, zmm28			;; apply fudge multiplier for R3			; 36-39
	shr	r13, 16					;; Next 16 bits of fudge flags
	L1prefetchw srcreg+d4+d1+64+L1pd, L1pt

	kmovw	k5, r13d				;; Load R5 and R13 fudge factor mask			; 37		n 38
	vmulpd	zmm8{k4}, zmm8, zmm28			;; apply fudge multiplier for R4			; 37-40
	L1prefetchw srcreg+d4+d2+L1pd, L1pt

	kshiftrw k2, k2, 8				;; R10's fudge						; 38		n 39
	vmulpd	zmm16{k5}, zmm16, zmm28			;; apply fudge multiplier for R5			; 38-41
	L1prefetchw srcreg+d4+d2+64+L1pd, L1pt

	kshiftrw k3, k3, 8				;; R11's fudge						; 39		n 40
	vmulpd	zmm4{k2}, zmm4, zmm28			;; apply fudge multiplier for R10			; 39-42		n 50
	L1prefetchw srcreg+d4+d2+d1+L1pd, L1pt
	zstore	[srcreg+d1], zmm5			;; Save R2						; 39

	kmovw	k1, r14d				;; Load R6 and R14 fudge factor mask			; 40		n 41
	vmulpd	zmm9{k3}, zmm9, zmm28			;; apply fudge multiplier for R11			; 40-43
	shr	r14, 16					;; Next 16 bits of fudge flags
	L1prefetchw srcreg+d4+d2+d1+64+L1pd, L1pt
	zstore	[srcreg+d2], zmm15			;; Save R3						; 40

	kmovw	k2, r14d				;; Load R7 and R15 fudge factor mask			; 41		n 42
	vmulpd	zmm2{k1}, zmm2, zmm28			;; apply fudge multiplier for R6			; 41-44
	shr	r14, 16					;; Next 16 bits of fudge flags
	zstore	[srcreg+d2+d1], zmm8			;; Save R4						; 41

	kmovw	k3, r14d				;; Load R8 and R16 fudge factor mask			; 42		n 43
	vmulpd	zmm3{k2}, zmm3, zmm28			;; apply fudge multiplier for R7			; 42-45
	zstore	[srcreg+d4], zmm16			;; Save R5						; 42

	kshiftrw k4, k4, 8				;; R12's fudge						; 43		n 44
	vmulpd	zmm14{k3}, zmm14, zmm28			;; apply fudge multiplier for R8			; 43-46

	kshiftrw k5, k5, 8				;; R13's fudge						; 44		n 45
	vmulpd	zmm12{k4}, zmm12, zmm28			;; apply fudge multiplier for R12			; 44-47		n 49
 	zstore	[srcreg+d2+64], zmm9			;; Save R11						; 44

	kshiftrw k1, k1, 8				;; R14's fudge						; 45		n 46
	vmulpd	zmm0{k5}, zmm0, zmm28			;; apply fudge multiplier for R13			; 45-48
	zstore	[srcreg+d4+d1], zmm2			;; Save R6						; 45

	kshiftrw k2, k2, 8				;; R15's fudge						; 46		n 47
	vmulpd	zmm13{k1}, zmm13, zmm28			;; apply fudge multiplier for R14			; 46-49
	zstore	[srcreg+d4+d2], zmm3			;; Save R7						; 46

	kshiftrw k3, k3, 8				;; R16's fudge						; 47		n 48
	vmulpd	zmm10{k2}, zmm10, zmm28			;; apply fudge multiplier for R15			; 47-50
	zstore	[srcreg+d4+d2+d1], zmm14		;; Save R8						; 47

	vmulpd	zmm6{k3}, zmm6, zmm28			;; apply fudge multiplier for R16			; 48-51
	vmulpd	zmm4, zmm4, zmm21			;; apply the inverse col multiplier to R10		; 48-51

	vmulpd	zmm12, zmm12, zmm23			;; apply the inverse col multiplier to R12		; 49-52
	zstore	[srcreg+d4+64], zmm0			;; Save R13						; 49

	zstore	[srcreg+d4+d1+64], zmm13		;; Save R14						; 50
	zstore	[srcreg+d4+d2+64], zmm10		;; Save R15						; 51
	zstore	[srcreg+d4+d2+d1+64], zmm6		;; Save R16						; 52
	zstore	[srcreg+d1+64], zmm4			;; Save R10						; 52+1
	zstore	[srcreg+d2+d1+64], zmm12		;; Save R12						; 53+1
	bump	srcreg, srcinc
	ENDM


;; Wrapper macro that takes two negacyclic reals, swizzles data, and applies weights.  Output will be suitable for use by a pass 2 FFT macro.
;; NOTE: The r1/i1 col multiplier is 1.0 thus there are no fudges to apply to R1/I1.  Also the column multiplier has been
;; pre-applied to the complex-multiplier sine value.

zsf_onepass_complex_fft_wrapper_preload MACRO
	mov	eax, 00110011b			;; For vblendmpd during swizzle
	kmovw	k7, eax
	vpmovzxbq zmm31, ZMM_PERMUTE1		;; zmm31 = 8+4 0+4 8+6 0+6 8+0 0+0 8+2 0+2 [msw at left]
	vpmovzxbq zmm30, ZMM_PERMUTE2		;; zmm30 = 8+5 0+5 8+7 0+7 8+1 0+1 8+3 0+3
	vbroadcastsd zmm29, ZMM_ONE_OVER_B
	ENDM
zsf_onepass_complex_fft_wrapper MACRO srcreg,srcinc,d1,d2,d4,pmreg,pminc,maskreg,maskinc,grpreg,grpinc,maxrpt,L1pt,L1pd
	vmovapd zmm28, [grpreg+0*64]			;; group multiplier for R1-R8
	vmulpd	zmm14, zmm28, [srcreg+rbx]		;; apply group multiplier to R1			; 1-4		n 9
	vmulpd	zmm2, zmm28, [srcreg+rbx+d1]		;; apply group multiplier to R2			; 1-4		n 10

	vmulpd	zmm10, zmm28, [srcreg+rbx+d2]		;; apply group multiplier to R3			; 2-5		n 11
	vmulpd	zmm6, zmm28, [srcreg+rbx+d2+d1]		;; apply group multiplier to R4			; 2-5		n 12

	vmulpd	zmm16, zmm28, [srcreg+rbx+d4]		;; apply group multiplier to R5			; 3-6		n 13
	vmulpd	zmm8, zmm28, [srcreg+rbx+d4+d1]		;; apply group multiplier to R6			; 3-6		n 14

	vmulpd	zmm12, zmm28, [srcreg+rbx+d4+d2]	;; apply group multiplier to R7			; 4-7		n 15
	vmulpd	zmm4, zmm28, [srcreg+rbx+d4+d2+d1]	;; apply group multiplier to R8			; 4-7		n 16

	vmovapd zmm27, [grpreg+1*64]			;; group multiplier for I1-I8
	vmulpd	zmm20, zmm27, [srcreg+rbx+64]		;; apply group multiplier to I1			; 5-8		n 9
	vmulpd	zmm21, zmm27, [srcreg+rbx+d1+64]	;; apply group multiplier to I2			; 5-8		n 10

	vmulpd	zmm22, zmm27, [srcreg+rbx+d2+64]	;; apply group multiplier to I3			; 6-9		n 11
	vmulpd	zmm23, zmm27, [srcreg+rbx+d2+d1+64]	;; apply group multiplier to I4			; 6-9		n 12

	vmulpd	zmm24, zmm27, [srcreg+rbx+d4+64]	;; apply group multiplier to I5			; 7-10		n 13
	vmulpd	zmm25, zmm27, [srcreg+rbx+d4+d1+64]	;; apply group multiplier to I6			; 7-10		n 14

	vmulpd	zmm26, zmm27, [srcreg+rbx+d4+d2+64]	;; apply group multiplier to I7			; 8-11		n 15
	vmulpd	zmm27, zmm27, [srcreg+rbx+d4+d2+d1+64]	;; apply group multiplier to I8			; 8-11		n 17
	bump	grpreg, grpinc

	vmovapd zmm9, [pmreg+0*128]		;; premultiplier sine for R1/I1
	vmulpd	zmm14, zmm14, zmm9		;; apply col multiplier * premultiplier sine to R1	; 9-12		n 30
	vmulpd	zmm20, zmm20, zmm9		;; apply col multiplier * premultiplier sine to I1	; 9-12		n 30

	vmovapd zmm1, [pmreg+1*128]		;; premultiplier sine for R2/I2
	vmulpd	zmm2, zmm2, zmm1		;; apply col multiplier * premultiplier sine to R2	; 10-13		n 18
	vmulpd	zmm21, zmm21, zmm1		;; apply col multiplier * premultiplier sine to I2	; 10-13		n 19

	vmovapd zmm11, [pmreg+2*128]		;; premultiplier sine for R3/I3
	vmulpd	zmm10, zmm10, zmm11		;; apply col multiplier * premultiplier sine to R3	; 11-14		n 20
	vmulpd	zmm22, zmm22, zmm11		;; apply col multiplier * premultiplier sine to I3	; 11-14		n 21

	vmovapd zmm3, [pmreg+3*128]		;; premultiplier sine for R4/I4
	vmulpd	zmm6, zmm6, zmm3		;; apply col multiplier * premultiplier sine to R4	; 12-15		n 22
	vmulpd	zmm23, zmm23, zmm3		;; apply col multiplier * premultiplier sine to I4	; 12-15		n 23

	vmovapd zmm13, [pmreg+4*128]		;; premultiplier sine for R5/I5
	vmulpd	zmm16, zmm16, zmm13		;; apply col multiplier * premultiplier sine to R5	; 13-16		n 24
	vmulpd	zmm24, zmm24, zmm13		;; apply col multiplier * premultiplier sine to I5	; 13-16		n 25

	vmovapd zmm5, [pmreg+5*128]		;; premultiplier sine for R6/I6
	vmulpd	zmm8, zmm8, zmm5		;; apply col multiplier * premultiplier sine to R6	; 14-17		n 26
	vmulpd	zmm25, zmm25, zmm5		;; apply col multiplier * premultiplier sine to I6	; 14-17		n 27

	vmovapd zmm15, [pmreg+6*128]		;; premultiplier sine for R7/I7
	vmulpd	zmm12, zmm12, zmm15		;; apply col multiplier * premultiplier sine to R7	; 15-18		n 28
	vmulpd	zmm26, zmm26, zmm15		;; apply col multiplier * premultiplier sine to I7	; 15-18		n 29

	mov	r14, [maskreg+0*8]		;; Load uncompressed fudge flags
	vmovapd zmm7, [pmreg+7*128]		;; premultiplier sine for R8/I8
	kmovw	k3, r14d			;; Load R2 and I2 fudge factor mask			; 16		n 17
	vmulpd	zmm4, zmm4, zmm7		;; apply col multiplier * premultiplier sine to R8	; 16-19		n 32
	shr	r14, 16				;; Next 16 bits of fudge flags

	kshiftrw k4, k3, 8			;; I2's fudge						; 17		n 19
	vmulpd	zmm27, zmm27, zmm7		;; apply col multiplier * premultiplier sine to I8	; 17-20		n 32
	mov	r12, [maskreg+1*8]		;; Load uncompressed fudge flags
	bump	maskreg, maskinc

	kmovw	k1, r14d			;; Load R3 and I3 fudge factor mask			; 18		n 19
	vmulpd	zmm2{k3}, zmm2, zmm29		;; apply fudge to R2					; 18-21		n 30
	shr	r14, 16				;; Next 16 bits of fudge flags
	vmovapd zmm28, [pmreg+0*128+64]		;; premultiplier cosine/sine for R1/I1

	kshiftrw k2, k1, 8			;; I3's fudge						; 19		n 21
	vmulpd	zmm21{k4}, zmm21, zmm29		;; apply fudge to I2					; 19-22		n 30
	vmovapd zmm18, [pmreg+1*128+64]		;; premultiplier cosine/sine for R2/I2

	kmovw	k3, r14d			;; Load R4 and I4 fudge factor mask			; 20		n 21
	vmulpd	zmm10{k1}, zmm10, zmm29		;; apply fudge to R3					; 20-23		n 31
	shr	r14, 16				;; Next 16 bits of fudge flags
	vmovapd zmm19, [pmreg+2*128+64]		;; premultiplier cosine/sine for R3/I3

	kshiftrw k4, k3, 8			;; I4's fudge						; 21		n 23
	vmulpd	zmm22{k2}, zmm22, zmm29		;; apply fudge to I3					; 21-24		n 31
	vmovapd zmm17, [pmreg+3*128+64]		;; premultiplier cosine/sine for R4/I4

	kmovw	k1, r14d			;; Load R5 and I5 fudge factor mask			; 22		n 23
	vmulpd	zmm6{k3}, zmm6, zmm29		;; apply fudge to R4					; 22-25		n 31
	vmovapd zmm0, [pmreg+4*128+64]		;; premultiplier cosine/sine for R5/I5

	kshiftrw k2, k1, 8			;; I5's fudge						; 23		n 25
	vmulpd	zmm23{k4}, zmm23, zmm29		;; apply fudge to I4					; 23-26		n 31
	vmovapd zmm11, [pmreg+5*128+64]		;; premultiplier cosine/sine for R6/I6

	kmovw	k3, r12d			;; Load R6 and I6 fudge factor mask			; 24		n 25
	vmulpd	zmm16{k1}, zmm16, zmm29		;; apply fudge to R5					; 24-27		n 36
	shr	r12, 16				;; Next 16 bits of fudge flags
	vmovapd zmm13, [pmreg+6*128+64]		;; premultiplier cosine/sine for R7/I7

	kshiftrw k4, k3, 8			;; I6's fudge						; 25		n 27
	vmulpd	zmm24{k2}, zmm24, zmm29		;; apply fudge to I5					; 25-28		n 36
	vmovapd zmm15, [pmreg+7*128+64]		;; premultiplier cosine/sine for R8/I8

	kmovw	k1, r12d			;; Load R7 and I7 fudge factor mask			; 26		n 27
	vmulpd	zmm8{k3}, zmm8, zmm29		;; apply fudge to R6					; 26-29		n 37
	shr	r12, 16				;; Next 16 bits of fudge flags
	L1prefetch srcreg+L1pd, L1pt

	kshiftrw k2, k1, 8			;; I7's fudge						; 27		n 29
	vmulpd	zmm25{k4}, zmm25, zmm29		;; apply fudge to I6					; 27-30		n 37
	bump	pmreg, pminc
	L1prefetch srcreg+64+L1pd, L1pt

	kmovw	k3, r12d			;; Load R8 and I8 fudge factor mask			; 28		n 29
	vmulpd	zmm12{k1}, zmm12, zmm29		;; apply fudge to R7					; 28-31		n 38
	L1prefetch srcreg+d1+L1pd, L1pt

	kshiftrw k4, k3, 8			;; I8's fudge						; 29		n 32
	vmulpd	zmm26{k2}, zmm26, zmm29		;; apply fudge to I7					; 29-32		n 38
	L1prefetch srcreg+d1+64+L1pd, L1pt

	zfmsubpd zmm1, zmm14, zmm28, zmm20	;; R1 * cosine - I1 (new R1)				; 30-33		n 34
	zfmsubpd zmm3, zmm2, zmm18, zmm21	;; R2 * cosine - I2 (new R2)				; 30-33		n 34
	L1prefetch srcreg+d2+L1pd, L1pt

	zfmsubpd zmm5, zmm10, zmm19, zmm22	;; R3 * cosine - I3 (new R3)				; 31-34		n 36
	zfmsubpd zmm7, zmm6, zmm17, zmm23	;; R4 * cosine - I4 (new R4)				; 31-34		n 36
	L1prefetch srcreg+d2+64+L1pd, L1pt

	vmulpd	zmm4{k3}, zmm4, zmm29		;; apply fudge to R8					; 32-35		n 39
	vmulpd	zmm27{k4}, zmm27, zmm29		;; apply fudge to I8					; 32-35		n 39
	L1prefetch srcreg+d2+d1+L1pd, L1pt

	zfmaddpd zmm20, zmm20, zmm28, zmm14	;; I1 * cosine + R1 (new I1)				; 33-36		n 38
	zfmaddpd zmm21, zmm21, zmm18, zmm2	;; I2 * cosine + R2 (new I2)				; 33-36		n 38
	L1prefetch srcreg+d2+d1+64+L1pd, L1pt

	vshufpd	zmm9, zmm1, zmm3, 00000000b	;; r2_6 r1_6 r2_4 r1_4 r2_2 r1_2 r2_0 r1_0		; 34		n 44
	vshufpd	zmm1, zmm1, zmm3, 11111111b	;; r2_7 r1_7 r2_5 r1_5 r2_3 r1_3 r2_1 r1_1		; 35		n 46
	L1prefetch srcreg+d4+L1pd, L1pt

	zfmaddpd zmm22, zmm22, zmm19, zmm10	;; I3 * cosine + R3 (new I3)				; 34-37		n 40
	zfmaddpd zmm23, zmm23, zmm17, zmm6	;; I4 * cosine + R4 (new I4)				; 35-38		n 40
	L1prefetch srcreg+d4+64+L1pd, L1pt

	zperm2pd zmm3, zmm31, zmm5, zmm7	;; r4_4 r3_4 r4_6 r3_6 r4_0 r3_0 r4_2 r3_2		; 36-38		n 44
	zperm2pd zmm5, zmm30, zmm5, zmm7	;; r4_5 r3_5 r4_7 r3_7 r4_1 r3_1 r4_3 r3_3		; 37-39		n 46
	L1prefetch srcreg+d4+d1+L1pd, L1pt

	zfmsubpd zmm6, zmm16, zmm0, zmm24	;; R5 * cosine - I5 (new R5)				; 36-39		n 42
	zfmsubpd zmm10, zmm8, zmm11, zmm25	;; R6 * cosine - I6 (new R6)				; 37-40		n 42
	L1prefetch srcreg+d4+d1+64+L1pd, L1pt

	vshufpd	zmm17, zmm20, zmm21, 00000000b	;; i2_6 i1_6 i2_4 i1_4 i2_2 i1_2 i2_0 i1_0		; 38		n 52
	vshufpd	zmm20, zmm20, zmm21, 11111111b	;; i2_7 i1_7 i2_5 i1_5 i2_3 i1_3 i2_1 i1_1		; 39		n 56
	L1prefetch srcreg+d4+d2+L1pd, L1pt

	zfmsubpd zmm14, zmm12, zmm13, zmm26	;; R7 * cosine - I7 (new R7)				; 38-41		n 44
	zfmsubpd zmm18, zmm4, zmm15, zmm27	;; R8 * cosine - I8 (new R8)				; 39-42		n 44
	L1prefetch srcreg+d4+d2+64+L1pd, L1pt

	zperm2pd zmm21, zmm31, zmm22, zmm23	;; i4_4 i3_4 i4_6 i3_6 i4_0 i3_0 i4_2 i3_2		; 40-42		n 52
	zperm2pd zmm22, zmm30, zmm22, zmm23	;; i4_5 i3_5 i4_7 i3_7 i4_1 i3_1 i4_3 i3_3		; 41-43		n 56
	L1prefetch srcreg+d4+d2+d1+L1pd, L1pt

	zfmaddpd zmm24, zmm24, zmm0, zmm16	;; I5 * cosine + R5 (new I5)				; 40-43		n 46
	zfmaddpd zmm25, zmm25, zmm11, zmm8	;; I6 * cosine + R6 (new I6)				; 41-44		n 46
	L1prefetch srcreg+d4+d2+d1+64+L1pd, L1pt

	vshufpd	zmm8, zmm6, zmm10, 00000000b	;; r6_6 r5_6 r6_4 r5_4 r6_2 r5_2 r6_0 r5_0		; 42		n 48
	vshufpd	zmm6, zmm6, zmm10, 11111111b	;; r6_7 r5_7 r6_5 r5_5 r6_3 r5_3 r6_1 r5_1		; 43		n 50

	zfmaddpd zmm26, zmm26, zmm13, zmm12	;; I7 * cosine + R7 (new I7)				; 42-45		n 48
	zfmaddpd zmm27, zmm27, zmm15, zmm4	;; I8 * cosine + R8 (new I8)				; 43-46		n 48

	zperm2pd zmm10, zmm31, zmm14, zmm18	;; r8_4 r7_4 r8_6 r7_6 r8_0 r7_0 r8_2 r7_2		; 44-46		n 48
	zperm2pd zmm14, zmm30, zmm14, zmm18	;; r8_5 r7_5 r8_7 r7_7 r8_1 r7_1 r8_3 r7_3		; 45-47		n 50

	vblendmpd zmm7{k7}, zmm3, zmm9		;; r4_4 r3_4 r2_4 r1_4 r4_0 r3_0 r2_0 r1_0		; 44		n 50
	vblendmpd zmm9{k7}, zmm9, zmm3		;; r2_6 r1_6 r4_6 r3_6 r2_2 r1_2 r4_2 r3_2		; 45		n 52

	vshufpd	zmm4, zmm24, zmm25, 00000000b	;; i6_6 i5_6 i6_4 i5_4 i6_2 i5_2 i6_0 i5_0		; 46		n 54
	vshufpd	zmm24, zmm24, zmm25, 11111111b	;; i6_7 i5_7 i6_5 i5_5 i6_3 i5_3 i6_1 i5_1		; 47		n 58

	vblendmpd zmm3{k7}, zmm5, zmm1		;; r4_5 r3_5 r2_5 r1_5 r4_1 r3_1 r2_1 r1_1		; 46		n 54
	vblendmpd zmm1{k7}, zmm1, zmm5		;; r2_7 r1_7 r4_7 r3_7 r2_3 r1_3 r4_3 r3_3		; 47		n 56

	zperm2pd zmm25, zmm31, zmm26, zmm27	;; i8_4 i7_4 i8_6 i7_6 i8_0 i7_0 i8_2 i7_2		; 48-50		n 54
	zperm2pd zmm26, zmm30, zmm26, zmm27	;; i8_5 i7_5 i8_7 i7_7 i8_1 i7_1 i8_3 i7_3		; 49-51		n 64

	vblendmpd zmm18{k7}, zmm10, zmm8	;; r8_4 r7_4 r6_4 r5_4 r8_0 r7_0 r6_0 r5_0		; 48		n 50
	vblendmpd zmm8{k7}, zmm8, zmm10		;; r6_6 r5_6 r8_6 r7_6 r6_2 r5_2 r8_2 r7_2		; 49		n 52

	vshuff64x2 zmm15, zmm7, zmm18, 01000100b;; r8_0 r7_0 r6_0 r5_0 r4_0 r3_0 r2_0 r1_0		; 50-52
	vshuff64x2 zmm7, zmm7, zmm18, 11101110b ;; r8_4 r7_4 r6_4 r5_4 r4_4 r3_4 r2_4 r1_4		; 51-53

	vblendmpd zmm10{k7}, zmm14, zmm6	;; r8_5 r7_5 r6_5 r6_5 r8_1 r7_1 r6_1 r5_1		; 50		n 54
	vblendmpd zmm6{k7}, zmm6, zmm14		;; r6_7 r5_7 r8_7 r7_7 r6_3 r5_3 r8_3 r7_3		; 51		n 56

	vshuff64x2 zmm18, zmm9, zmm8, 00010001b ;; r8_2 r7_2 r6_2 r5_2 r4_2 r3_2 r2_2 r1_2		; 52-54
	vshuff64x2 zmm9, zmm9, zmm8, 10111011b	;; r8_6 r7_6 r6_6 r5_6 r4_6 r3_6 r2_6 r1_6		; 53-55

	vblendmpd zmm23{k7}, zmm21, zmm17	;; i4_4 i3_4 i2_4 i1_4 i4_0 i3_0 i2_0 i1_0		; 52		n 58
	vblendmpd zmm17{k7}, zmm17, zmm21	;; i2_6 i1_6 i4_6 i3_6 i2_2 i1_2 i4_2 i3_2		; 53		n 60
	zstore	[srcreg], zmm15										; 53

	vshuff64x2 zmm5, zmm3, zmm10, 01000100b ;; r8_1 r7_1 r6_1 r5_1 r4_1 r3_1 r2_1 r1_1		; 54-56
	vshuff64x2 zmm3, zmm3, zmm10, 11101110b ;; r8_5 r7_5 r6_5 r5_5 r4_5 r3_5 r2_5 r1_5		; 55-57
	zstore	[srcreg+d4], zmm7									; 54

	vblendmpd zmm27{k7}, zmm25, zmm4	;; i8_4 i7_4 i6_4 i5_4 i8_0 i7_0 i6_0 i5_0		; 54		n 58
	vblendmpd zmm4{k7}, zmm4, zmm25		;; i6_6 i5_6 i8_6 i7_6 i6_2 i5_2 i8_2 i7_2		; 55		n 60
	zstore	[srcreg+d2], zmm18									; 55

	vshuff64x2 zmm10, zmm1, zmm6, 00010001b;; r8_3 r7_3 r6_3 r5_3 r4_3 r3_3 r2_3 r1_3		; 56-58
	vshuff64x2 zmm1, zmm1, zmm6, 10111011b ;; r8_7 r7_7 r6_7 r5_7 r4_7 r3_7 r2_7 r1_7		; 57-59
	zstore	[srcreg+d4+d2], zmm9									; 56

	vblendmpd zmm21{k7}, zmm22, zmm20	;; i4_5 i3_5 i2_5 i1_5 i4_1 i3_1 i2_1 i1_1		; 56		n 62
	vblendmpd zmm20{k7}, zmm20, zmm22	;; i2_7 i1_7 i4_7 i3_7 i2_3 i1_3 i4_3 i3_3		; 57		n 64
	zstore	[srcreg+d1], zmm5									; 57

	vshuff64x2 zmm0, zmm23, zmm27, 01000100b ;; i8_0 i7_0 i6_0 i5_0 i4_0 i3_0 i2_0 i1_0		; 58-60
	vshuff64x2 zmm23, zmm23, zmm27, 11101110b ;; i8_4 i7_4 i6_4 i5_4 i4_4 i3_4 i2_4 i1_4		; 59-61
	zstore	[srcreg+d4+d1], zmm3									; 58

	vblendmpd zmm25{k7}, zmm26, zmm24	;; i8_5 i7_5 i6_5 i5_5 i8_1 i7_1 i6_1 i5_1		; 58		n 62
	vblendmpd zmm24{k7}, zmm24, zmm26	;; i6_7 i5_7 i8_7 i7_7 i6_3 i5_3 i8_3 i7_3		; 59		n 64
	zstore	[srcreg+d2+d1], zmm10									; 59

	vshuff64x2 zmm27, zmm17, zmm4, 00010001b ;; i8_2 i7_2 i6_2 i5_2 i4_2 i3_2 i2_2 i1_2		; 60-62
	zstore	[srcreg+d4+d2+d1], zmm1									; 60

	vshuff64x2 zmm17, zmm17, zmm4, 10111011b ;; i8_6 i7_6 i6_6 i5_6 i4_6 i3_6 i2_6 i1_6		; 61-63
	zstore	[srcreg+64], zmm0									; 61

	vshuff64x2 zmm22, zmm21, zmm25, 01000100b ;; i8_1 i7_1 i6_1 i5_1 i4_1 i3_1 i2_1 i1_1		; 62-64
	zstore	[srcreg+d4+64], zmm23									; 62

	vshuff64x2 zmm21, zmm21, zmm25, 11101110b ;; i8_5 i7_5 i6_5 i5_5 i4_5 i3_5 i2_5 i1_5		; 63-65
	zstore	[srcreg+d2+64], zmm27									; 63

	vshuff64x2 zmm25, zmm20, zmm24, 00010001b ;; i8_3 i7_3 i6_3 i5_3 i4_3 i3_3 i2_3 i1_3		; 64-66
	zstore	[srcreg+d4+d2+64], zmm17								; 64

	vshuff64x2 zmm20, zmm20, zmm24, 10111011b ;; i8_7 i7_7 i6_7 i5_7 i4_7 i3_7 i2_7 i1_7		; 65-67
	zstore	[srcreg+d1+64], zmm22									; 65

	zstore	[srcreg+d4+d1+64], zmm21								; 66
	zstore	[srcreg+d2+d1+64], zmm25								; 67
	zstore	[srcreg+d4+d2+d1+64], zmm20								; 68
	bump	srcreg, srcinc
	ENDM


;; Wrapper macro that takes two negacyclic reals, swizzles data, and applies weights.  Output will be suitable for normalize.
;; NOTE: The r1/i1 inverse column multiplier is 1.0 thus there are no fudges to apply to R1/I1

zs_onepass_complex_unfft_wrapper_preload MACRO
	mov	r14d, 11110000b
	kmovw	k6, r14d
	knotw	k7, k6
	vbroadcastsd zmm28, ZMM_B
	ENDM
zs_onepass_complex_unfft_wrapper MACRO srcreg,srcinc,d1,d2,d4,pmreg,pminc,colreg,colinc,maskreg,maskinc,maxrpt,L1pt,L1pd
	vmovapd zmm20, [pmreg+0*128]			;; premultiplier sine for R1/I1
	vmulpd	zmm20, zmm20, [colreg+0*8]{1to8}	;; apply the col multiplier to sine for R1/I1		; x17-20
	vmovapd zmm21, [pmreg+1*128]			;; premultiplier sine for R2/I2
	vmulpd	zmm21, zmm21, [colreg+1*8]{1to8}	;; apply the col multiplier to sine for R2/I2		; x17-20
	vmovapd zmm22, [pmreg+2*128]			;; premultiplier sine for R3/I3
	vmulpd	zmm22, zmm22, [colreg+2*8]{1to8}	;; apply the col multiplier to sine for R3/I3		; x17-20
	vmovapd zmm23, [pmreg+3*128]			;; premultiplier sine for R4/I4
	vmulpd	zmm23, zmm23, [colreg+3*8]{1to8}	;; apply the col multiplier to sine for R4/I4		; x17-20
	vmovapd zmm24, [pmreg+4*128]			;; premultiplier sine for R5/I5
	vmulpd	zmm24, zmm24, [colreg+4*8]{1to8}	;; apply the col multiplier to sine for R5/I5		; x17-20
	vmovapd zmm25, [pmreg+5*128]			;; premultiplier sine for R6/I6
	vmulpd	zmm25, zmm25, [colreg+5*8]{1to8}	;; apply the col multiplier to sine for R6/I6		; x17-20
	vmovapd zmm26, [pmreg+6*128]			;; premultiplier sine for R7/I7
	vmulpd	zmm26, zmm26, [colreg+6*8]{1to8}	;; apply the col multiplier to sine for R7/I7		; x17-20
	vmovapd zmm27, [pmreg+7*128]			;; premultiplier sine for R8/I8
	vmulpd	zmm27, zmm27, [colreg+7*8]{1to8}	;; apply the col multiplier to sine for R8/I8		; x17-20
	bump	colreg, colinc

	vmovapd	zmm15, [srcreg]				;; d0_7	d0_6 d0_5 d0_4 d0_3 d0_2 d0_1 d0_0
	vmovapd	zmm16, [srcreg+d1]			;; d1_7	d1_6 d1_5 d1_4 d1_3 d1_2 d1_1 d1_0
	vbroadcastf64x4	zmm15{k6}, [srcreg+d2]		;; d2_3 d2_2 d2_1 d2_0 d0_3 d0_2 d0_1 d0_0		; 1		n 2
	vbroadcastf64x4	zmm16{k6}, [srcreg+d2+d1]	;; d3_3 d3_2 d3_1 d3_0 d1_3 d1_2 d1_1 d1_0		; 1		n 2

	vmovapd	zmm2, [srcreg+d4]			;; d4_7	d4_6 d4_5 d4_4 d4_3 d4_2 d4_1 d4_0
	vshufpd	zmm5, zmm15, zmm16, 00000000b		;; d3_2 d2_2 d3_0 d2_0 d1_2 d0_2 d1_0 d0_0		; 2		n 10
	vbroadcastf64x4	zmm2{k6}, [srcreg+d4+d2]	;; d6_3 d6_2 d6_1 d6_0 d4_3 d4_2 d4_1 d4_0		; 2		n 4

	vmovapd	zmm8, [srcreg+d4+d1]			;; d5_7	d5_6 d5_5 d5_4 d5_3 d5_2 d5_1 d5_0
	vshufpd	zmm16, zmm15, zmm16, 11111111b		;; d3_3 d2_3 d3_1 d2_1 d1_3 d0_3 d1_1 d0_1		; 3		n 12
	vbroadcastf64x4	zmm8{k6}, [srcreg+d4+d2+d1]	;; d7_3 d7_2 d7_1 d7_0 d5_3 d5_2 d5_1 d5_0		; 3		n 4

	vmovapd	zmm3, [srcreg+d2]			;; d2_7	d2_6 d2_5 d2_4 d2_3 d2_2 d2_1 d2_0
	vmovupd	zmm17, [srcreg+32]			;; (+64 d0_3 d0_2 d0_1 d0_0) d0_7 d0_6 d0_5 d0_4
	vshufpd	zmm15, zmm2, zmm8, 00000000b		;; d7_2 d6_2 d7_0 d6_0 d5_2 d4_2 d5_0 d4_0		; 4		n 10
	vblendmpd zmm3{k6}, zmm17, zmm3			;; d2_7 d2_6 d2_5 d2_4 d0_7 d0_6 d0_5 d0_4		; 4		n 6

	vmovapd	zmm31, [srcreg+d2+d1]			;; d3_7	d3_6 d3_5 d3_4 d3_3 d3_2 d3_1 d3_0
	vmovupd	zmm18, [srcreg+d1+32]			;; (+64 d1_3 d1_2 d1_1 d1_0) d1_7 d1_6 d1_5 d1_4
	vshufpd	zmm8, zmm2, zmm8, 11111111b		;; d7_3 d6_3 d7_1 d6_1 d5_3 d4_3 d5_1 d4_1		; 5		n 12
	vblendmpd zmm31{k6}, zmm18, zmm31		;; d3_7 d3_6 d3_5 d3_4 d1_7 d1_6 d1_5 d1_4		; 5		n 6

	vmovapd	zmm7, [srcreg+d4+d2]			;; d6_7	d6_6 d6_5 d6_4 d6_3 d6_2 d6_1 d6_0
	vmovupd	zmm19, [srcreg+d4+32]			;; (+64 d4_3 d4_2 d4_1 d4_0) d4_7 d4_6 d4_5 d4_4
	vshufpd	zmm2, zmm3, zmm31, 00000000b		;; d3_6 d2_6 d3_4 d2_4 d1_6 d0_6 d1_4 d0_4		; 6		n 14
	vblendmpd zmm7{k6}, zmm19, zmm7			;; d6_7	d6_6 d6_5 d6_4 d4_7 d4_6 d4_5 d4_4		; 6		n 8

	vmovapd	zmm14, [srcreg+d4+d2+d1]		;; d7_7	d7_6 d7_5 d7_4 d7_3 d7_2 d7_1 d7_0
	vmovupd zmm29, [srcreg+d4+d1+32]		;; (+64 d5_3 d5_2 d5_1 d5_0) d5_7 d5_6 d5_5 d5_4
	vshufpd	zmm31, zmm3, zmm31, 11111111b		;; d3_7 d2_7 d3_5 d2_5 d1_7 d0_7 d1_5 d0_5		; 7		n 20
	vblendmpd zmm14{k6}, zmm29, zmm14		;; d7_7	d7_6 d7_5 d7_4 d5_7 d5_6 d5_5 d5_4		; 7		n 8

	vmovapd	zmm9, [srcreg+d2+64]			;; (+64 d2_7 d2_6 d2_5 d2_4 d2_3 d2_2 d2_1 d2_0)
	vshufpd	zmm3, zmm7, zmm14, 00000000b		;; d7_6 d6_6 d7_4 d6_4 d5_6 d4_6 d5_4 d4_4		; 8		n 14
	vblendmpd zmm9{k6}, zmm9, zmm17			;; (+64 d0_3 d0_2 d0_1 d0_0 d2_3 d2_2 d2_1 d2_0)	; 8		n 16

	vmovapd	zmm11, [srcreg+d2+d1+64]		;; (+64 d3_7 d3_6 d3_5 d3_4 d3_3 d3_2 d3_1 d3_0)
	vshufpd	zmm14, zmm7, zmm14, 11111111b		;; d7_7 d6_7 d7_5 d6_5 d5_7 d4_7 d5_5 d4_5		; 9		n 20
	vblendmpd zmm11{k6}, zmm11, zmm18		;; (+64 d1_3 d1_2 d1_1 d1_0 d3_3 d3_2 d3_1 d3_0)	; 9		n 16

	vmovapd	zmm13, [srcreg+d4+d2+64]		;; (+64 d6_7 d6_6 d6_5 d6_4 d6_3 d6_2 d6_1 d6_0)
	vshuff64x2 zmm7, zmm5, zmm15, 10001000b		;; d7_0 d6_0 d5_0 d4_0 d3_0 d2_0 d1_0 d0_0 (R1)		; 10-12		n 16
	vblendmpd zmm13{k6}, zmm13, zmm19		;; (+64 d4_3 d4_2 d4_1 d4_0 d6_3 d6_2 d6_1 d6_0)	; 10		n 17

	vmovapd	zmm12, [srcreg+d4+d2+d1+64]		;; (+64 d7_7 d7_6 d7_5 d7_4 d7_3 d7_2 d7_1 d7_0)
	vshuff64x2 zmm15, zmm5, zmm15, 11011101b	;; d7_2 d6_2 d5_2 d4_2 d3_2 d2_2 d1_2 d0_2 (R3)		; 11-13		n 17
	vblendmpd zmm12{k6}, zmm12, zmm29		;; (+64 d5_3 d5_2 d5_1 d5_0 d7_3 d7_2 d7_1 d7_0)	; 11		n 17

	vmovapd	zmm10, [srcreg+64]			;; (+64 d0_7 d0_6 d0_5 d0_4 d0_3 d0_2 d0_1 d0_0)
	vshuff64x2 zmm5, zmm16, zmm8, 10001000b		;; d7_1 d6_1 d5_1 d4_1 d3_1 d2_1 d1_1 d0_1 (R2)		; 12-14		n 18
	vbroadcastf64x4	zmm10{k7}, [srcreg+d2+64+32]	;; (+64 d0_7 d0_6 d0_5 d0_4 d2_7 d2_6 d2_5 d2_4)	; 12		n 26

	vmovapd	zmm30, [srcreg+d1+64]			;; (+64 d1_7 d1_6 d1_5 d1_4 d1_3 d1_2 d1_1 d1_0)
	vshuff64x2 zmm8, zmm16, zmm8, 11011101b		;; d7_3 d6_3 d5_3 d4_3 d3_3 d2_3 d1_3 d0_3 (R4)		; 13-15		n 19
	vbroadcastf64x4	zmm30{k7}, [srcreg+d2+d1+64+32]	;; (+64 d1_7 d1_6 d1_5 d1_4 d3_7 d3_6 d3_5 d3_4)	; 13		n 26

	vmovapd	zmm1, [srcreg+d4+64]			;; (+64 d4_7 d4_6 d4_5 d4_4 d4_3 d4_2 d4_1 d4_0)
	vshuff64x2 zmm16, zmm2, zmm3, 10001000b		;; d7_4	d6_4 d5_4 d4_4 d3_4 d2_4 d1_4 d0_4 (R5)		; 14-16		n 20
	vbroadcastf64x4	zmm1{k7}, [srcreg+d4+d2+64+32]	;; (+64 d4_7 d4_6 d4_5 d4_4 d6_7 d6_6 d6_5 d6_4)	; 14		n 27

	vmovapd	zmm6, [srcreg+d4+d1+64]			;; (+64 d5_7 d5_6 d5_5 d5_4 d5_3 d5_2 d5_1 d5_0)
	vshuff64x2 zmm3, zmm2, zmm3, 11011101b		;; d7_6 d6_6 d5_6 d4_6 d3_6 d2_6 d1_6 d0_6 (R7)		; 15-17		n 21
	vbroadcastf64x4	zmm6{k7}, [srcreg+d4+d2+d1+64+32];;(+64 d5_7 d5_6 d5_5 d5_4 d7_7 d7_6 d7_5 d7_4)	; 15		n 27

	vshufpd	zmm0, zmm9, zmm11, 00000000b		;; d1_2 d0_2 d1_0 d0_0 d3_2 d2_2 d3_0 d2_0		; 16		n 18
	vmulpd	zmm7, zmm7, zmm20			;; apply col multiplier * premultiplier sine to R1	; 16-19		n 26

	vshufpd	zmm17, zmm13, zmm12, 00000000b		;; d5_2 d4_2 d5_0 d4_0 d7_2 d6_2 d7_0 d6_0		; 17		n 18
	vmulpd	zmm15, zmm15, zmm22			;; apply col multiplier * premultiplier sine to R3	; 17-20		n 30

	vshuff64x2 zmm18, zmm0, zmm17, 00100010b	;; d7_0 d6_0 d5_0 d4_0 d3_0 d2_0 d1_0 d0_0 (I1)		; 18-20		n 22
	vmulpd	zmm5, zmm5, zmm21			;; apply col multiplier * premultiplier sine to R2	; 18-21		n 34

	vshuff64x2 zmm17, zmm0, zmm17, 01110111b	;; d7_2 d6_2 d5_2 d4_2 d3_2 d2_2 d1_2 d0_2 (I3)		; 19-21		n 23
	vmulpd	zmm8, zmm8, zmm23			;; apply col multiplier * premultiplier sine to R4	; 19-22		n 35

	vshuff64x2 zmm2, zmm31, zmm14, 10001000b	;; d7_5 d6_5 d5_5 d4_5 d3_5 d2_5 d1_5 d0_5 (R6)		; 20-22		n 24
	vmulpd	zmm16, zmm16, zmm24			;; apply col multiplier * premultiplier sine to R5	; 20-23		n 37

	vshuff64x2 zmm14, zmm31, zmm14, 11011101b	;; d7_7 d6_7 d5_7 d4_7 d3_7 d2_7 d1_7 d0_7 (R8)		; 21-23		n 25
	vmulpd	zmm3, zmm3, zmm26			;; apply col multiplier * premultiplier sine to R7	; 21-24		n 38

	vshufpd	zmm11, zmm9, zmm11, 11111111b		;; d1_3 d0_3 d1_1 d0_1 d3_3 d2_3 d3_1 d2_1		; 22		n 24
	vmulpd	zmm18, zmm18, zmm20			;; apply col multiplier * premultiplier sine to I1	; 22-25		n 26

	vshufpd	zmm12, zmm13, zmm12, 11111111b		;; d5_3 d4_3 d5_1 d4_1 d7_3 d6_3 d7_1 d6_1		; 23		n 24
	vmulpd	zmm17, zmm17, zmm22			;; apply col multiplier * premultiplier sine to I3	; 23-26		n 30

	vshuff64x2 zmm0, zmm11, zmm12, 00100010b	;; d7_1 d6_1 d5_1 d4_1 d3_1 d2_1 d1_1 d0_1 (I2)		; 24-26		n 27
	vmulpd	zmm2, zmm2, zmm25			;; apply col multiplier * premultiplier sine to R6	; 24-27		n 39

	vshuff64x2 zmm12, zmm11, zmm12, 01110111b	;; d7_3 d6_3 d5_3 d4_3 d3_3 d2_3 d1_3 d0_3 (I4)		; 25-27		n 28
	vmulpd	zmm14, zmm14, zmm27			;; apply col multiplier * premultiplier sine to R8	; 25-28		n 40

	vshufpd	zmm11, zmm10, zmm30, 00000000b		;; d1_6 d0_6 d1_4 d0_4 d3_6 d2_6 d3_4 d2_4		; 26		n 28
	vmovapd zmm20, [pmreg+0*128+64]			;; premultiplier cosine/sine for R1/I1
	zfmaddpd zmm4, zmm7, zmm20, zmm18		;; R1 * cosine + I1 (new R1)				; 26-29
	L1prefetchw srcreg+L1pd, L1pt

	vshufpd	zmm19, zmm1, zmm6, 00000000b		;; d5_6 d4_6 d5_4 d4_4 d7_6 d6_6 d7_4 d6_4		; 27		n 28
	vmulpd	zmm0, zmm0, zmm21			;; apply col multiplier * premultiplier sine to I2	; 27-30		n 34
	L1prefetchw srcreg+64+L1pd, L1pt

	vshuff64x2 zmm9, zmm11, zmm19, 00100010b	;; d7_4 d6_4 d5_4 d4_4 d3_4 d2_4 d1_4 d0_4 (I5)		; 28-30		n 31
	vmulpd	zmm12, zmm12, zmm23			;; apply col multiplier * premultiplier sine to I4	; 28-31		n 35
	L1prefetchw srcreg+d1+L1pd, L1pt

	vshuff64x2 zmm19, zmm11, zmm19, 01110111b	;; d7_6 d6_6 d5_6 d4_6 d3_6 d2_6 d1_6 d0_6 (I7)		; 29-31		n 32
	zfmsubpd zmm18, zmm18, zmm20, zmm7		;; I1 * cosine - R1 (new I1)				; 29-32
	L1prefetchw srcreg+d1+64+L1pd, L1pt

	vshufpd	zmm30, zmm10, zmm30, 11111111b		;; d1_7 d0_7 d1_5 d0_5 d3_7 d2_7 d3_5 d2_5		; 30		n 32
	vmovapd zmm21, [pmreg+2*128+64]			;; premultiplier cosine/sine for R3/I3
	zfmaddpd zmm7, zmm15, zmm21, zmm17		;; R3 * cosine + I3 (new R3)				; 30-33		n 43
	zstore	[srcreg], zmm4				;; Save R1						; 30
	L1prefetchw srcreg+d2+L1pd, L1pt

	vshufpd	zmm6, zmm1, zmm6, 11111111b		;; d5_7 d4_7 d5_5 d4_5 d7_7 d6_7 d7_5 d6_5		; 31		n 32
	vmulpd	zmm9, zmm9, zmm24			;; apply col multiplier * premultiplier sine to I5	; 31-34		n 37
	L1prefetchw srcreg+d2+64+L1pd, L1pt

	vshuff64x2 zmm11, zmm30, zmm6, 00100010b	;; d7_5 d6_5 d5_5 d4_5 d3_5 d2_5 d1_5 d0_5 (I6)		; 32-34		n 35
	vmulpd	zmm19, zmm19, zmm26			;; apply col multiplier * premultiplier sine to I7	; 32-35		n 38
	L1prefetchw srcreg+d2+d1+L1pd, L1pt

	vshuff64x2 zmm6, zmm30, zmm6, 01110111b		;; d7_7 d6_7 d5_7 d4_7 d3_7 d2_7 d1_7 d0_7 (I8)		; 33-35		n 36
	zfmsubpd zmm17, zmm17, zmm21, zmm15		;; I3 * cosine - R3 (new I3)				; 33-36		n 47
	zstore	[srcreg+64], zmm18			;; Save I1						; 33
	L1prefetchw srcreg+d2+d1+64+L1pd, L1pt

	vmovapd zmm24, [pmreg+1*128+64]			;; premultiplier cosine/sine for R2/I2
	zfmaddpd zmm15, zmm5, zmm24, zmm0		;; R2 * cosine + I2 (new R2)				; 34-37		n 43
	zfmsubpd zmm0, zmm0, zmm24, zmm5		;; I2 * cosine - R2 (new I2)				; 34-37		n 46
	L1prefetchw srcreg+d4+L1pd, L1pt

	vmulpd	zmm11, zmm11, zmm25			;; apply col multiplier * premultiplier sine to I6	; 35-38		n 39
	vmovapd zmm23, [pmreg+3*128+64]			;; premultiplier cosine/sine for R4/I4
	zfmaddpd zmm5, zmm8, zmm23, zmm12		;; R4 * cosine + I4 (new R4)				; 35-38		n 44
	L1prefetchw srcreg+d4+64+L1pd, L1pt

	vmulpd	zmm6, zmm6, zmm27			;; apply col multiplier * premultiplier sine to I8	; 36-39		n 41
	zfmsubpd zmm12, zmm12, zmm23, zmm8		;; I4 * cosine - R4 (new I4)				; 36-39		n 48
	L1prefetchw srcreg+d4+d1+L1pd, L1pt

	vmovapd zmm26, [pmreg+4*128+64]			;; premultiplier cosine/sine for R5/I5
	zfmaddpd zmm8, zmm16, zmm26, zmm9		;; R5 * cosine + I5 (new R5)				; 37-40		n 46
	zfmsubpd zmm9, zmm9, zmm26, zmm16		;; I5 * cosine - R5 (new I5)				; 37-40		n 49
	L1prefetchw srcreg+d4+d1+64+L1pd, L1pt

	vmovapd zmm25, [pmreg+6*128+64]			;; premultiplier cosine/sine for R7/I7
	zfmaddpd zmm16, zmm3, zmm25, zmm19		;; R7 * cosine + I7 (new R7)				; 38-41		n 51
	zfmsubpd zmm19, zmm19, zmm25, zmm3		;; I7 * cosine - R7 (new I7)				; 38-41		n 54
	L1prefetchw srcreg+d4+d2+L1pd, L1pt

	vmovapd zmm22, [pmreg+5*128+64]			;; premultiplier cosine/sine for R6/I6
	zfmaddpd zmm3, zmm2, zmm22, zmm11		;; R6 * cosine + I6 (new R6)				; 39-42		n 52
	zfmsubpd zmm11, zmm11, zmm22, zmm2		;; I6 * cosine - R6 (new I6)				; 39-42		n 53
	L1prefetchw srcreg+d4+d2+64+L1pd, L1pt

	mov	r14, [maskreg+0*8]			;; Load uncompressed fudge flags
	vmovapd zmm27, [pmreg+7*128+64]			;; premultiplier cosine/sine for R8/I8
	kmovw	k2, r14d				;; Load R2 and I2 fudge factor mask			; 40		n 42
	zfmaddpd zmm2, zmm14, zmm27, zmm6		;; R8 * cosine + I8 (new R8)				; 40-43		n 52
	shr	r14, 16					;; Next 16 bits of fudge flags

	kmovw	k3, r14d				;; Load R3 and I3 fudge factor mask			; 41		n 43
	zfmsubpd zmm6, zmm6, zmm27, zmm14		;; I8 * cosine - R8 (new I8)				; 41-44		n 54
	shr	r14, 16					;; Next 16 bits of fudge flags
	mov	r12, [maskreg+1*8]			;; Load uncompressed fudge flags
	bump	pmreg, pminc

	kmovw	k4, r14d				;; Load R4 and I4 fudge factor mask			; 42		n 44
	vmulpd	zmm15{k2}, zmm15, zmm28			;; apply fudge multiplier for R2			; 42-45
	shr	r14, 16					;; Next 16 bits of fudge flags
	L1prefetchw srcreg+d4+d2+d1+L1pd, L1pt

	kmovw	k5, r14d				;; Load R5 and I5 fudge factor mask			; 43		n 45
	vmulpd	zmm7{k3}, zmm7, zmm28			;; apply fudge multiplier for R3			; 43-46
	L1prefetchw srcreg+d4+d2+d1+64+L1pd, L1pt
	bump	maskreg, maskinc

	kshiftrw k2, k2, 8				;; I2's fudge						; 44		n 46
	vmulpd	zmm5{k4}, zmm5, zmm28			;; apply fudge multiplier for R4			; 44-47

	kshiftrw k3, k3, 8				;; I3's fudge						; 45		n 47
	vmulpd	zmm8{k5}, zmm8, zmm28			;; apply fudge multiplier for R5			; 45-48

	kshiftrw k4, k4, 8				;; I4's fudge						; 46		n 48
	vmulpd	zmm0{k2}, zmm0, zmm28			;; apply fudge multiplier for I2			; 46-49
	zstore	[srcreg+d1], zmm15			;; Save R2						; 46

	kshiftrw k5, k5, 8				;; I5's fudge						; 47		n 49
	vmulpd	zmm17{k3}, zmm17, zmm28			;; apply fudge multiplier for I3			; 47-50
	zstore	[srcreg+d2], zmm7			;; Save R3						; 47

	kmovw	k1, r12d				;; Load R6 and I6 fudge factor mask			; 48		n 50
	vmulpd	zmm12{k4}, zmm12, zmm28			;; apply fudge multiplier for I4			; 48-51
	shr	r12, 16					;; Next 16 bits of fudge flags
	zstore	[srcreg+d2+d1], zmm5			;; Save R4						; 48

	kmovw	k2, r12d				;; Load R7 and I7 fudge factor mask			; 49		n 51
	vmulpd	zmm9{k5}, zmm9, zmm28			;; apply fudge multiplier for I5			; 49-52
	shr	r12, 16					;; Next 16 bits of fudge flags
	zstore	[srcreg+d4], zmm8			;; Save R5						; 49

	kmovw	k3, r12d				;; Load R8 and I8 fudge factor mask			; 50		n 52
	vmulpd	zmm3{k1}, zmm3, zmm28			;; apply fudge multiplier for R6			; 50-53
	zstore	[srcreg+d1+64], zmm0			;; Save I2						; 50

	kshiftrw k1, k1, 8				;; I6's fudge						; 51		n 53
	vmulpd	zmm16{k2}, zmm16, zmm28			;; apply fudge multiplier for R7			; 51-54
 	zstore	[srcreg+d2+64], zmm17			;; Save I3						; 51

	kshiftrw k2, k2, 8				;; I7's fudge						; 52		n 54
	vmulpd	zmm2{k3}, zmm2, zmm28			;; apply fudge multiplier for R8			; 52-55
	zstore	[srcreg+d2+d1+64], zmm12		;; Save I4						; 52

	kshiftrw k3, k3, 8				;; I8's fudge						; 53		n 54
	vmulpd	zmm11{k1}, zmm11, zmm28			;; apply fudge multiplier for I6			; 53-56
	zstore	[srcreg+d4+64], zmm9			;; Save I5						; 53

	vmulpd	zmm19{k2}, zmm19, zmm28			;; apply fudge multiplier for I7			; 54-57
	vmulpd	zmm6{k3}, zmm6, zmm28			;; apply fudge multiplier for I8			; 54-57
	zstore	[srcreg+d4+d1], zmm3			;; Save R6						; 54

	zstore	[srcreg+d4+d2], zmm16			;; Save R7						; 55
	zstore	[srcreg+d4+d2+d1], zmm2			;; Save R8						; 56
	zstore	[srcreg+d4+d1+64], zmm11		;; Save I6						; 57
	zstore	[srcreg+d4+d2+64], zmm19		;; Save I7						; 58
	zstore	[srcreg+d4+d2+d1+64], zmm6		;; Save I8						; 58+1
	bump	srcreg, srcinc
	ENDM


; ********************************************************
; ********************************************************
; ********************  FFT MACROS  **********************
; ********************************************************
; ********************************************************

;; Custom coded one pass AVX-512 FFTs
;;
;; These ffts are all done in a single pass.  On entry the memory layout is
;; quite simple.  It is assumed each cache line is 64 bytes or 8 doubles.

;; These macros support 4 "types".  Type 1 if forward FFT only.
;; Type 2 is forward FFT, square, inverse FFT.  Type 3 is forward
;; FFT, multiply, inverse FFT.  Type 4 is multiply, and inverse FFT.

;; rsi = destination pointer
;; DIST_TO_FFTSRCARG = rbx = distance from rsi to source (zero for types 1, 2)
;; DIST_TO_MULSRCARG = distance from rsi to multiplier (zero for types 2, 3)

;;;
;;;  One macro for each FFT length
;;;

;; Perform a 128-element FFT

zfft128 MACRO type
	LOCAL	zmid

;; Check for a multiply with pre-FFTed inputs

	cmp	ffttype, 4
	je	zmid

;; Do FFT levels 1,2,3,4
;; On input the 128-byte cache line pairs hold these data values:
;;	0	+1	...	64	+1	...
;;	8	...
;;	16	...
;;	...
;; On output the 128-byte cache line pairs hold these data values:
;;	0	+1	...	8	+1	...
;;	16	...
;;	32	...
;;	...
;; These are swizzled to:
;;	0	+16	...	8	+16	...
;;	1	...
;;	2	...
;;	...

;; Do 8 sixteen_reals_fft operations
;;	distance between fft data elements is 8
;;	do 1 macro processing 128 data values

	start_timer 4
	mov	rdi, sincos1		;; Load sin/cos data ptr
	mov	r9, norm_col_mults	;; Load weights pointer
	zr8fs_sixteen_reals_first_fft_preload
	zr8fs_sixteen_reals_first_fft rsi, 0, 128, 2*128, 4*128, rdi, 0, r9, 0, 1, L1PREFETCH_NONE
	end_timer 4

;; Do FFT levels 5,6,7
;; On input and output the 128-byte cache line pairs hold these data values:
;;	0	+16	...	8	+16	...
;;	1	...
;;	2	...
;;	...

;; Do 1 sixteen_reals_fft operation
;; Do 7 eight_complex_fft operations
;;	distance between fft data elements is 1
;;	do 1 macro processing 128 data values

zmid:	start_timer 5
	mov	rcx, 1
	call	zreal_onepass_middle
	end_timer 5

;; Do inverse FFT levels 1,2,3,4
;; On input the 128-byte cache line pairs hold these data values:
;;	0	+16	...	8	+16	...
;;	1	...
;;	2	...
;;	...
;; These are swizzled to:
;;	0	+1	...	8	+1	...
;;	16	...
;;	32	...
;;	...
;; On output the 128-byte cache line pairs hold these data values:
;;	0	+1	...	64	+1	...
;;	8	...
;;	16	...
;;	...

;; Do 8 sixteen_reals_unfft operations
;;	distance between fft data elements is 8
;;	do 1 macro processing 128 data values

	start_timer 6
	mov	rdi, sincos1		;; Load sin/cos data ptr
	zr8s_sixteen_reals_last_unfft_preload
	zr8s_sixteen_reals_last_unfft rsi, 0, 128, 2*128, 4*128, rdi, 0, 1, L1PREFETCH_NONE
	end_timer 6

	zfft_3_ret
	ENDM

;; Perform a 128-element negacyclic FFT

zfft128p MACRO type
	LOCAL	zmid

;; Do a multiply with pre-FFTed inputs

	cmp	ffttype, 4
	je	zmid

;; Do FFT levels 1,2,3,4
;; On input the 128-byte cache line pairs hold these data values:
;;	0	+1	...	64	+1	...
;;	8	...
;;	16	...
;;	...
;; On output the 128-byte cache line pairs hold these data values:
;;	0	+1	...	8	+1	...
;;	16	...
;;	32	...
;;	...
;; These are swizzled to:
;;	0	+16	...	8	+16	...
;;	1	...
;;	2	...
;;	...

;; Do 8 eight_complex_first_fft operations
;;	distance between fft data elements is 8
;;	do 1 macro processing 128 data values

	start_timer 4
	mov	rdi, sincos1		;; Load sin/cos pointer
	mov	r9, norm_col_mults	;; Load weights pointer
	zr8fs_eight_complex_first_fft_preload
	zr8fs_eight_complex_first_fft rsi, 0, 128, 2*128, 4*128, rdi, 0, r9, 0, 1, L1PREFETCH_NONE
	end_timer 4

;; Do FFT levels 5,6,7
;; On input and output the 128-byte cache line pairs hold these data values:
;;	0	+16	...	8	+16	...
;;	1	...
;;	2	...
;;	...

;; Do 8 eight_complex_fft operations
;;	distance between fft data elements is 1
;;	do 1 macro processing 128 data values

zmid:	start_timer 5
	mov	rcx, 1
	call	zcomplex_onepass_middle
	end_timer 5

;; Do inverse FFT levels 1,2,3,4
;; On input the 128-byte cache line pairs hold these data values:
;;	0	+16	...	8	+16	...
;;	1	...
;;	2	...
;;	...
;; These are swizzled to:
;;	0	+1	...	8	+1	...
;;	16	...
;;	32	...
;;	...
;; On output the 128-byte cache line pairs hold these data values:
;;	0	+1	...	64	+1	...
;;	8	...
;;	16	...
;;	...

;; Do 8 eight_complex_last_unfft operations
;;	distance between fft data elements is 8
;;	do 1 macro processing 128 data values

	start_timer 6
	mov	rdi, sincos1		;; Load sin/cos pointer
	zr8s_eight_complex_last_unfft_preload
	zr8s_eight_complex_last_unfft rsi, 0, 128, 2*128, 4*128, rdi, 0, 1, L1PREFETCH_NONE
	end_timer 6

	zfft_3_ret
	ENDM


;; Perform a 256-element FFT

zfft256 MACRO type
	LOCAL	b1b, b2b, zmid, c2b, c1b

;; Check for a multiply with pre-FFTed inputs

	cmp	ffttype, 4
	je	zmid

;; Do FFT levels 1,2,3,4
;; On input the 128-byte cache line pairs hold these data values:
;;	0	+2	...	128	+2	...
;;	1	...
;;	16	...
;;	32	...
;;	...
;; On output the 128-byte cache line pairs hold these data values:
;;	0	+2	...	16	+2	...
;;	1	...
;;	32	...
;;	64	...
;;	...
;; These are swizzled to:
;;	0	+32	...	16	+32	...
;;	...
;;	15	...

;; Do 16 sixteen_reals_fft operations
;;	distance between fft data elements is 16
;;	do 2 macros processing 128 data values

	start_timer 4
	mov	rdi, sincos1		;; Load sin/cos data ptr
	bump	rdi, 8*ZMM_SCD1
	mov	r9, norm_col_mults	;; Load weights pointer
	zloop_init 2			;; 2 iterations
	zr8fs_sixteen_reals_first_fft_preload
b1b:	zr8fs_sixteen_reals_first_fft rsi, 128, 2*128, 4*128, 8*128, rdi, ZMM_SCD7, r9, 8*128, 2, L1PREFETCH_NONE
	zloop	2, b1b, -2*128		;; Test loop counter, restore source pointer
	end_timer 4

;; Do FFT level 5
;; On input the 64-byte cache lines hold these data values:
;;	0	+32	...	16	+32	...
;;	...
;;	15	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+32	...	8	+32	...
;;	...
;;	7
;;	16	...
;;	32	...

;; Do 8 four_reals_fft operation
;; Do 56 two_complex_fft operations
;;	distance between fft data elements is 8
;;	do 8 macros each processing 32 data values

	start_timer 5
	bump	rdi, -2*ZMM_SCD7-8*ZMM_SCD1 ;; Load sin/cos pointer
	zloop_init 8			;; 8 iterations
	zr2_four_reals_two_complex_djbfft_preload
b2b:	zr2_four_reals_two_complex_djbfft rsi, 128, 8*128, rdi, ZMM_SCD1, 8
	zloop	8, b2b, -8*128		;; Test loop counter, restore source pointer
	end_timer 5

;; Do FFT levels 6,7,8
;; On input and output the 128-byte cache line pairs hold these data values:
;;	0	+32	...	8	+32	...
;;	...
;;	7
;;	16	...
;;	32	...

;; Do 1 sixteen_reals_fft operation
;; Do 15 eight_complex_fft operations
;;	distance between fft data elements is 1
;;	do 2 macros processing 128 data values

zmid:	start_timer 6
	mov	rcx, 2
	call	zreal_onepass_middle
	end_timer 6

;; Do inverse FFT level 5
;; On input the 64-byte cache lines hold these data values:
;;	0	+32	...	8	+32	...
;;	...
;;	7
;;	16	...
;;	32	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+32	...	16	+32	...
;;	...
;;	15	...

;; Do 8 four_reals_unfft operation
;; Do 56 two_complex_unfft operations
;;	distance between fft data elements is 8
;;	do 8 macros each processing 32 data values

	start_timer 7
	mov	rdi, sincos1		;; Load sin/cos pointer
	zloop_init 8			;; 8 iterations
	zr2_four_reals_two_complex_djbunfft_preload
c2b:	zr2_four_reals_two_complex_djbunfft rsi, 128, 8*128, rdi, ZMM_SCD1, 8
	zloop	8, c2b, -8*128		;; Test loop counter, restore source pointer
	end_timer 7

;; Do inverse FFT levels 1,2,3,4
;; On input the 128-byte cache line pairs hold these data values:
;;	0	+32	...	16	+32	...
;;	...
;;	15	...
;; These are swizzled to:
;;	0	+2	...	16	+2	...
;;	1	...
;;	32	...
;;	64	...
;;	...
;; On output the 128-byte cache line pairs hold these data values:
;;	0	+2	...	128	+2	...
;;	1	...
;;	16	...
;;	32	...
;;	...

;; Do 16 sixteen_reals_unfft operations
;;	distance between fft data elements is 16
;;	do 2 macros processing 128 data values

	start_timer 8
	zloop_init 2			;; 2 iterations
	zr8s_sixteen_reals_last_unfft_preload
c1b:	zr8s_sixteen_reals_last_unfft rsi, 128, 2*128, 4*128, 8*128, rdi, ZMM_SCD7, 2, L1PREFETCH_NONE
	zloop	2, c1b, -2*128		;; Test loop counter, restore source pointer
	end_timer 8

	zfft_3_ret
	ENDM

;; Perform a 256-element negacyclic FFT

zfft256p MACRO type
	LOCAL	b1b, b2b, zmid, c2b, c1b

;; Do a multiply with pre-FFTed inputs

	cmp	ffttype, 4
	je	zmid

;; Do FFT levels 1,2,3,4
;; On input the 128-byte cache line pairs hold these data values:
;;	0	+2	...	128	+2	...
;;	1	...
;;	16	...
;;	32	...
;;	...
;; On output the 128-byte cache line pairs hold these data values:
;;	0	+2	...	16	+2	...
;;	1	...
;;	32	...
;;	64	...
;;	...
;; These are swizzled to:
;;	0	+32	...	16	+32	...
;;	...
;;	15	...

;; Do 16 eight_complex_first_fft operations
;;	distance between fft data elements is 16
;;	do 2 macros processing 128 data values

	start_timer 4
	mov	rdi, sincos1		;; Load sin/cos pointer
	bump	rdi, 8*ZMM_SCD1/8
	mov	r9, norm_col_mults	;; Load weights pointer
	zloop_init 2			;; 2 iterations
	zr8fs_eight_complex_first_fft_preload
b1b:	zr8fs_eight_complex_first_fft rsi, 128, 2*128, 4*128, 8*128, rdi, ZMM_CD8SCD4, r9, 8*128, 2, L1PREFETCH_NONE
	zloop	2, b1b, -2*128		;; Test loop counter, restore source pointer
	end_timer 4

;; Do FFT level 5
;; On input the 64-byte cache lines hold these data values:
;;	0	+32	...	16	+32	...
;;	...
;;	15	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+32	...	8	+32	...
;;	...
;;	7
;;	16	...
;;	32	...

;; Do 64 two_complex_fft operations
;;	distance between fft data elements is 8
;;	do 8 macros each processing 32 data values

	start_timer 5
	bump	rdi, -2*ZMM_CD8SCD4-8*ZMM_SCD1/8 ;; Load sin/cos pointer
	zloop_init 8			;; 8 iterations
	zr2b_two_complex_djbfft_preload
b2b:	zr2b_two_complex_djbfft rsi, 128, 8*128, rdi, ZMM_SCD1/8, 8
	zloop	8, b2b, -8*128		;; Test loop counter, restore source pointer
	end_timer 5

;; Do FFT levels 6,7,8
;; On input and output the 128-byte cache line pairs hold these data values:
;;	0	+32	...	8	+32	...
;;	...
;;	7
;;	16	...
;;	32	...

;; Do 16 eight_complex_fft operations
;;	distance between fft data elements is 1
;;	do 2 macros processing 128 data values

zmid:	start_timer 6
	mov	rcx, 2
	call	zcomplex_onepass_middle
	end_timer 6

;; Do inverse FFT level 5
;; On input the 64-byte cache lines hold these data values:
;;	0	+32	...	8	+32	...
;;	...
;;	7
;;	16	...
;;	32	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+32	...	16	+32	...
;;	...
;;	15	...

;; Do 64 two_complex_unfft operations
;;	distance between fft data elements is 8
;;	do 8 macros each processing 32 data values

	start_timer 7
	mov	rdi, sincos1		;; Load sin/cos pointer
	zloop_init 8			;; 8 iterations
	zr2b_two_complex_djbunfft_preload
c2b:	zr2b_two_complex_djbunfft rsi, 128, 8*128, rdi, ZMM_SCD1/8, 8
	zloop	8, c2b, -8*128		;; Test loop counter, restore source pointer
	end_timer 7

;; Do inverse FFT levels 1,2,3,4
;; On input the 128-byte cache line pairs hold these data values:
;;	0	+32	...	16	+32	...
;;	...
;;	15	...
;; These are swizzled to:
;;	0	+2	...	16	+2	...
;;	1	...
;;	32	...
;;	64	...
;;	...
;; On output the 128-byte cache line pairs hold these data values:
;;	0	+2	...	128	+2	...
;;	1	...
;;	16	...
;;	32	...
;;	...

;; Do 16 eight_complex_last_unfft operations
;;	distance between fft data elements is 16
;;	do 2 macros processing 128 data values

	start_timer 8
	zloop_init 2			;; 2 iterations
	zr8s_eight_complex_last_unfft_preload
c1b:	zr8s_eight_complex_last_unfft rsi, 128, 2*128, 4*128, 8*128, rdi, ZMM_CD8SCD4, 2, L1PREFETCH_NONE
	zloop	2, c1b, -2*128		;; Test loop counter, restore source pointer
	end_timer 8

	zfft_3_ret
	ENDM


;; Perform a 384-element FFT

zfft384 MACRO type
	LOCAL	b1b, b2b, zmid, c2b, c1b

;; Check for a multiply with pre-FFTed inputs

	cmp	ffttype, 4
	je	zmid

;; Do FFT levels 1,2,3,4
;; On input the 128-byte cache line pairs hold these data values:
;;	0	+3	...	192	+3	...
;;	...
;;	2	...
;;	24	...
;;	48	...
;;	...
;; On output the 128-byte cache line pairs hold these data values:
;;	0	+3	...	24	+3	...
;;	...
;;	2	...
;;	48	...
;;	96	...
;;	...
;; These are swizzled to:
;;	0	+48	...	24	+48	...
;;	...
;;	23	...

;; Do 24 sixteen_reals_fft operations
;;	distance between fft data elements is 24
;;	do 3 macros processing 128 data values

	start_timer 4
	mov	rdi, sincos1		;; Load sin/cos data ptr
	bump	rdi, 8*ZMM_SCD2
	mov	r9, norm_col_mults	;; Load weights pointer
	zloop_init 3			;; 3 iterations
	zr8fs_sixteen_reals_first_fft_preload
b1b:	zr8fs_sixteen_reals_first_fft rsi, 128, 3*128, 6*128, 12*128, rdi, ZMM_SCD7, r9, 8*128, 3, L1PREFETCH_NONE
	zloop	3, b1b, -3*128		;; Test loop counter, restore source pointer
	end_timer 4

;; Do FFT levels 5,6
;; On input the 64-byte cache lines hold these data values:
;;	0	+48	...	24	+48	...
;;	...
;;	23	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+48	...	8	+48	...
;;	...
;;	7
;;	16	...
;;	32	...

;; Do 8 six_reals_fft operation
;; Do 56 three_complex_fft operations
;;	distance between fft data elements is 8
;;	do 8 macros each processing 48 data values

	start_timer 5
	bump	rdi, -3*ZMM_SCD7-8*ZMM_SCD2 ;; Load sin/cos pointer
	zloop_init 8			;; 8 iterations
	zr3_six_reals_three_complex_djbfft_preload
b2b:	zr3_six_reals_three_complex_djbfft rsi, 128, 8*128, rdi, ZMM_SCD2, 8
	zloop	8, b2b, -8*128		;; Test loop counter, restore source pointer
	end_timer 5

;; Do FFT levels 7,8,9
;; On input and output the 128-byte cache line pairs hold these data values:
;;	0	+48	...	8	+48	...
;;	...
;;	7
;;	16	...
;;	32	...

;; Do 1 sixteen_reals_fft operation
;; Do 23 eight_complex_fft operations
;;	distance between fft data elements is 1
;;	do 3 macros processing 128 data values

zmid:	start_timer 6
	mov	rcx, 3
	call	zreal_onepass_middle
	end_timer 6

;; Do inverse FFT levels 5,6
;; On input the 64-byte cache lines hold these data values:
;;	0	+48	...	8	+48	...
;;	...
;;	7
;;	16	...
;;	32	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+48	...	24	+48	...
;;	...
;;	23	...

;; Do 8 six_reals_unfft operation
;; Do 56 three_complex_unfft operations
;;	distance between fft data elements is 8
;;	do 8 macros each processing 48 data values

	start_timer 7
	mov	rdi, sincos1		;; Load sin/cos pointer
	zloop_init 8			;; 8 iterations
	zr3_six_reals_three_complex_djbunfft_preload
c2b:	zr3_six_reals_three_complex_djbunfft rsi, 128, 8*128, rdi, ZMM_SCD2, 8
	zloop	8, c2b, -8*128		;; Test loop counter, restore source pointer
	end_timer 7

;; Do inverse FFT levels 1,2,3,4
;; On input the 128-byte cache line pairs hold these data values:
;;	0	+48	...	24	+48	...
;;	...
;;	23	...
;; These are swizzled to:
;;	0	+3	...	24	+3	...
;;	...
;;	2	...
;;	48	...
;;	96	...
;;	...
;; On output the 128-byte cache line pairs hold these data values:
;;	0	+3	...	192	+3	...
;;	...
;;	2	...
;;	24	...
;;	48	...
;;	...

;; Do 24 sixteen_reals_unfft operations
;;	distance between fft data elements is 24
;;	do 3 macros processing 128 data values

	start_timer 8
	zloop_init 3			;; 3 iterations
	zr8s_sixteen_reals_last_unfft_preload
c1b:	zr8s_sixteen_reals_last_unfft rsi, 128, 3*128, 6*128, 12*128, rdi, ZMM_SCD7, 3, L1PREFETCH_NONE
	zloop	3, c1b, -3*128		;; Test loop counter, restore source pointer
	end_timer 8

	zfft_3_ret
	ENDM

;; Perform a 384-element negacyclic FFT

zfft384p MACRO type
	LOCAL	b1b, b2b, zmid, c2b, c1b

;; Do a multiply with pre-FFTed inputs

	cmp	ffttype, 4
	je	zmid

;; Do FFT levels 1,2,3,4
;; On input the 128-byte cache line pairs hold these data values:
;;	0	+3	...	192	+3	...
;;	...
;;	2	...
;;	24	...
;;	48	...
;;	...
;; On output the 128-byte cache line pairs hold these data values:
;;	0	+3	...	24	+3	...
;;	...
;;	2	...
;;	48	...
;;	96	...
;;	...
;; These are swizzled to:
;;	0	+48	...	24	+48	...
;;	...
;;	23	...

;; Do 24 eight_complex_first_fft operations
;;	distance between fft data elements is 24
;;	do 3 macros processing 128 data values

	start_timer 4
	mov	rdi, sincos1		;; Load sin/cos pointer
	bump	rdi, 8*ZMM_SCD1/8
	mov	r9, norm_col_mults	;; Load weights pointer
	zloop_init 3			;; 3 iterations
	zr8fs_eight_complex_first_fft_preload
b1b:	zr8fs_eight_complex_first_fft rsi, 128, 3*128, 6*128, 12*128, rdi, ZMM_CD8SCD4, r9, 8*128, 3, L1PREFETCH_NONE
	zloop	3, b1b, -3*128		;; Test loop counter, restore source pointer
	end_timer 4

;; Do FFT levels 5,6
;; On input the 64-byte cache lines hold these data values:
;;	0	+48	...	24	+48	...
;;	...
;;	23	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+48	...	8	+48	...
;;	...
;;	7
;;	16	...
;;	32	...

;; Do 64 three_complex_fft operations
;;	distance between fft data elements is 8
;;	do 8 macros each processing 48 data values

	start_timer 5
	bump	rdi, -3*ZMM_CD8SCD4-8*ZMM_SCD1/8 ;; Load sin/cos pointer
	zloop_init 8			;; 8 iterations
	zr3b_three_complex_djbfft_preload
b2b:	zr3b_three_complex_djbfft rsi, 128, 8*128, rdi, ZMM_SCD1/8, 8
	zloop	8, b2b, -8*128		;; Test loop counter, restore source pointer
	end_timer 5

;; Do FFT levels 7,8,9
;; On input and output the 128-byte cache line pairs hold these data values:
;;	0	+48	...	8	+48	...
;;	...
;;	7
;;	16	...
;;	32	...

;; Do 24 eight_complex_fft operations
;;	distance between fft data elements is 1
;;	do 3 macros processing 128 data values

zmid:	start_timer 6
	mov	rcx, 3
	call	zcomplex_onepass_middle
	end_timer 6

;; Do inverse FFT levels 5,6
;; On input the 64-byte cache lines hold these data values:
;;	0	+48	...	8	+48	...
;;	...
;;	7
;;	16	...
;;	32	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+48	...	24	+48	...
;;	...
;;	23	...

;; Do 64 three_complex_unfft operations
;;	distance between fft data elements is 8
;;	do 8 macros each processing 48 data values

	start_timer 7
	mov	rdi, sincos1		;; Load sin/cos pointer
	zloop_init 8			;; 8 iterations
	zr3b_three_complex_djbunfft_preload
c2b:	zr3b_three_complex_djbunfft rsi, 128, 8*128, rdi, ZMM_SCD1/8, 8
	zloop	8, c2b, -8*128		;; Test loop counter, restore source pointer
	end_timer 7

;; Do inverse FFT levels 1,2,3,4
;; On input the 128-byte cache line pairs hold these data values:
;;	0	+48	...	24	+48	...
;;	...
;;	23	...
;; These are swizzled to:
;;	0	+3	...	24	+3	...
;;	...
;;	2	...
;;	48	...
;;	96	...
;;	...
;; On output the 128-byte cache line pairs hold these data values:
;;	0	+3	...	192	+3	...
;;	...
;;	2	...
;;	24	...
;;	48	...
;;	...

;; Do 24 eight_complex_last_unfft operations
;;	distance between fft data elements is 24
;;	do 3 macros processing 128 data values

	start_timer 8
	zloop_init 3			;; 3 iterations
	zr8s_eight_complex_last_unfft_preload
c1b:	zr8s_eight_complex_last_unfft rsi, 128, 3*128, 6*128, 12*128, rdi, ZMM_CD8SCD4, 3, L1PREFETCH_NONE
	zloop	3, c1b, -3*128		;; Test loop counter, restore source pointer
	end_timer 8

	zfft_3_ret
	ENDM


;; Perform a 512-element FFT

zfft512 MACRO type
	LOCAL	b1b, b2b, zmid, c2b, c1b

;; Check for a multiply with pre-FFTed inputs

	cmp	ffttype, 4
	je	zmid

;; Do FFT levels 1,2,3,4
;; On input the 128-byte cache line pairs hold these data values:
;;	0	+4	...	256	+4	...
;;	...
;;	3	...
;;	32	...
;;	64	...
;;	...
;; On output the 128-byte cache line pairs hold these data values:
;;	0	+4	...	32	+4	...
;;	...
;;	3	...
;;	64	...
;;	128	...
;;	...
;; These are swizzled to:
;;	0	+64	...	32	+64	...
;;	...
;;	31	...

;; Do 32 sixteen_reals_fft operations
;;	distance between fft data elements is 32
;;	do 4 macros processing 128 data values

	start_timer 4
	mov	rdi, sincos1		;; Load sin/cos data ptr
	bump	rdi, 8*ZMM_SCD3
	mov	r9, norm_col_mults	;; Load weights pointer
	zloop_init 4			;; 4 iterations
	zr8fs_sixteen_reals_first_fft_preload
b1b:	zr8fs_sixteen_reals_first_fft rsi, 128, 4*128, 8*128, 16*128, rdi, ZMM_SCD7, r9, 8*128, 4, L1PREFETCH_NONE
	zloop	4, b1b, -4*128		;; Test loop counter, restore source pointer
	end_timer 4

;; Do FFT levels 5,6
;; On input the 64-byte cache lines hold these data values:
;;	0	+64	...	32	+64	...
;;	...
;;	31	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+64	...	8	+64	...
;;	...
;;	7
;;	16	...
;;	32	...
;;	48	...

;; Do 8 eight_reals_fft operation
;; Do 56 four_complex_fft operations
;;	distance between fft data elements is 8
;;	do 8 macros each processing 64 data values

	start_timer 5
	bump	rdi, -4*ZMM_SCD7-8*ZMM_SCD3 ;; Load sin/cos pointer
	zloop_init 8			;; 8 iterations
	zr4_eight_reals_four_complex_djbfft_preload
b2b:	zr4_eight_reals_four_complex_djbfft rsi, 128, 8*128, 16*128, rdi, ZMM_SCD3, 8
	zloop	8, b2b, -8*128		;; Test loop counter, restore source pointer
	end_timer 5

;; Do FFT levels 7,8,9
;; On input and output the 128-byte cache line pairs hold these data values:
;;	0	+64	...	8	+64	...
;;	...
;;	7
;;	16	...
;;	32	...
;;	48	...

;; Do 1 sixteen_reals_fft operation
;; Do 31 eight_complex_fft operations
;;	distance between fft data elements is 1
;;	do 4 macros processing 128 data values

zmid:	start_timer 6
	mov	rcx, 4
	call	zreal_onepass_middle
	end_timer 6

;; Do inverse FFT levels 5,6
;; On input the 64-byte cache lines hold these data values:
;;	0	+64	...	8	+64	...
;;	...
;;	7
;;	16	...
;;	32	...
;;	48	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+64	...	32	+64	...
;;	...
;;	31	...

;; Do 8 eight_reals_unfft operation
;; Do 56 four_complex_unfft operations
;;	distance between fft data elements is 8
;;	do 8 macros each processing 64 data values

	start_timer 7
	mov	rdi, sincos1		;; Load sin/cos pointer
	zloop_init 8			;; 8 iterations
	zr4_eight_reals_four_complex_djbunfft_preload
c2b:	zr4_eight_reals_four_complex_djbunfft rsi, 128, 8*128, 16*128, rdi, ZMM_SCD3, 8
	zloop	8, c2b, -8*128		;; Test loop counter, restore source pointer
	end_timer 7

;; Do inverse FFT levels 1,2,3,4
;; On input the 128-byte cache line pairs hold these data values:
;;	0	+64	...	32	+64	...
;;	...
;;	63	...
;; These are swizzled to:
;;	0	+4	...	32	+4	...
;;	...
;;	3	...
;;	64	...
;;	128	...
;;	...
;; On output the 128-byte cache line pairs hold these data values:
;;	0	+4	...	256	+4	...
;;	...
;;	3	...
;;	32	...
;;	64	...
;;	...

;; Do 32 sixteen_reals_unfft operations
;;	distance between fft data elements is 32
;;	do 4 macros processing 128 data values

	start_timer 8
	zloop_init 4			;; 4 iterations
	zr8s_sixteen_reals_last_unfft_preload
c1b:	zr8s_sixteen_reals_last_unfft rsi, 128, 4*128, 8*128, 16*128, rdi, ZMM_SCD7, 4, L1PREFETCH_NONE
	zloop	4, c1b, -4*128		;; Test loop counter, restore source pointer
	end_timer 8

	zfft_3_ret
	ENDM

;; Perform a 512-element negacyclic FFT

zfft512p MACRO type
	LOCAL	b1b, b2b, zmid, c2b, c1b

;; Do a multiply with pre-FFTed inputs

	cmp	ffttype, 4
	je	zmid

;; Do FFT levels 1,2,3,4
;; On input the 128-byte cache line pairs hold these data values:
;;	0	+4	...	256	+4	...
;;	...
;;	3	...
;;	32	...
;;	64	...
;;	...
;; On output the 128-byte cache line pairs hold these data values:
;;	0	+4	...	32	+4	...
;;	...
;;	3	...
;;	64	...
;;	128	...
;;	...
;; These are swizzled to:
;;	0	+64	...	32	+64	...
;;	...
;;	31	...

;; Do 32 eight_complex_first_fft operations
;;	distance between fft data elements is 32
;;	do 4 macros processing 128 data values

	start_timer 4
	mov	rdi, sincos1		;; Load sin/cos pointer
	bump	rdi, 8*ZMM_SCD2/8
	mov	r9, norm_col_mults	;; Load weights pointer
	zloop_init 4			;; 4 iterations
	zr8fs_eight_complex_first_fft_preload
b1b:	zr8fs_eight_complex_first_fft rsi, 128, 4*128, 8*128, 16*128, rdi, ZMM_CD8SCD4, r9, 8*128, 4, L1PREFETCH_NONE
	zloop	4, b1b, -4*128		;; Test loop counter, restore source pointer
	end_timer 4

;; Do FFT levels 5,6
;; On input the 64-byte cache lines hold these data values:
;;	0	+64	...	32	+64	...
;;	...
;;	31	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+64	...	8	+64	...
;;	...
;;	7
;;	16	...
;;	32	...
;;	48	...

;; Do 64 four_complex_fft operations
;;	distance between fft data elements is 8
;;	do 8 macros each processing 64 data values

	start_timer 5
	bump	rdi, -4*ZMM_CD8SCD4-8*ZMM_SCD2/8 ;; Load sin/cos pointer
	zloop_init 8			;; 8 iterations
	zr4b_four_complex_djbfft_preload
b2b:	zr4b_four_complex_djbfft rsi, 128, 8*128, 16*128, rdi, ZMM_SCD2/8, 8
	zloop	8, b2b, -8*128		;; Test loop counter, restore source pointer
	end_timer 5

;; Do FFT levels 7,8,9
;; On input and output the 128-byte cache line pairs hold these data values:
;;	0	+64	...	8	+64	...
;;	...
;;	7
;;	16	...
;;	32	...
;;	48	...

;; Do 32 eight_complex_fft operations
;;	distance between fft data elements is 1
;;	do 4 macros processing 128 data values

zmid:	start_timer 6
	mov	rcx, 4
	call	zcomplex_onepass_middle
	end_timer 6

;; Do inverse FFT levels 5,6
;; On input the 64-byte cache lines hold these data values:
;;	0	+64	...	8	+64	...
;;	...
;;	7
;;	16	...
;;	32	...
;;	48	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+64	...	32	+64	...
;;	...
;;	31	...

;; Do 64 four_complex_unfft operations
;;	distance between fft data elements is 8
;;	do 8 macros each processing 64 data values

	start_timer 7
	mov	rdi, sincos1		;; Load sin/cos pointer
	zloop_init 8			;; 8 iterations
	zr4b_four_complex_djbunfft_preload
c2b:	zr4b_four_complex_djbunfft rsi, 128, 8*128, 16*128, rdi, ZMM_SCD2/8, 8
	zloop	8, c2b, -8*128		;; Test loop counter, restore source pointer
	end_timer 7

;; Do inverse FFT levels 1,2,3,4
;; On input the 128-byte cache line pairs hold these data values:
;;	0	+64	...	32	+64	...
;;	...
;;	31	...
;; These are swizzled to:
;;	0	+4	...	32	+4	...
;;	...
;;	3	...
;;	64	...
;;	128	...
;;	...
;; On output the 128-byte cache line pairs hold these data values:
;;	0	+4	...	256	+4	...
;;	...
;;	3	...
;;	32	...
;;	64	...
;;	...

;; Do 32 eight_complex_last_unfft operations
;;	distance between fft data elements is 32
;;	do 4 macros processing 128 data values

	start_timer 8
	zloop_init 4			;; 4 iterations
	zr8s_eight_complex_last_unfft_preload
c1b:	zr8s_eight_complex_last_unfft rsi, 128, 4*128, 8*128, 16*128, rdi, ZMM_CD8SCD4, 4, L1PREFETCH_NONE
	zloop	4, c1b, -4*128		;; Test loop counter, restore source pointer
	end_timer 8

	zfft_3_ret
	ENDM


;; Perform a 640-element FFT

zfft640 MACRO type
	LOCAL	b1b, b2b, zmid, c2b, c1b

;; Check for a multiply with pre-FFTed inputs

	cmp	ffttype, 4
	je	zmid

;; Do FFT levels 1,2,3,4
;; On input the 128-byte cache line pairs hold these data values:
;;	0	+5	...	320	+5	...
;;	...
;;	4	...
;;	40	...
;;	80	...
;;	...
;; On output the 128-byte cache line pairs hold these data values:
;;	0	+5	...	40	+5	...
;;	...
;;	4	...
;;	80	...
;;	160	...
;;	...
;; These are swizzled to:
;;	0	+80	...	40	+80	...
;;	...
;;	39	...

;; Do 40 sixteen_reals_fft operations
;;	distance between fft data elements is 40
;;	do 5 macros processing 128 data values

	start_timer 4
	mov	rdi, sincos1		;; Load sin/cos data ptr
	bump	rdi, 8*ZMM_SCD4
	mov	r9, norm_col_mults	;; Load weights pointer
	zloop_init 5			;; 5 iterations
	zr8fs_sixteen_reals_first_fft_preload
b1b:	zr8fs_sixteen_reals_first_fft rsi, 128, 5*128, 10*128, 20*128, rdi, ZMM_SCD7, r9, 8*128, 5, L1PREFETCH_NONE
	zloop	5, b1b, -5*128		;; Test loop counter, restore source pointer
	end_timer 4

;; Do FFT levels 5,6
;; On input the 64-byte cache lines hold these data values:
;;	0	+80	...	40	+80	...
;;	...
;;	39	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+80	...	8	+80	...
;;	...
;;	7
;;	16	...
;;	32	...

;; Do 8 ten_reals_fft operation
;; Do 56 five_complex_fft operations
;;	distance between fft data elements is 8
;;	do 8 macros each processing 80 data values

	start_timer 5
	bump	rdi, -5*ZMM_SCD7-8*ZMM_SCD4 ;; Load sin/cos pointer
	zloop_init 8			;; 8 iterations
	zr5_ten_reals_five_complex_djbfft_preload
b2b:	zr5_ten_reals_five_complex_djbfft rsi, 128, 8*128, rdi, ZMM_SCD4, 8
	zloop	8, b2b, -8*128		;; Test loop counter, restore source pointer
	end_timer 5

;; Do FFT levels 7,8,9
;; On input and output the 128-byte cache line pairs hold these data values:
;;	0	+80	...	8	+80	...
;;	...
;;	7
;;	16	...
;;	32	...

;; Do 1 sixteen_reals_fft operation
;; Do 39 eight_complex_fft operations
;;	distance between fft data elements is 1
;;	do 5 macros processing 128 data values

zmid:	start_timer 6
	mov	rcx, 5
	call	zreal_onepass_middle
	end_timer 6

;; Do inverse FFT levels 5,6
;; On input the 64-byte cache lines hold these data values:
;;	0	+80	...	8	+80	...
;;	...
;;	7
;;	16	...
;;	32	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+80	...	40	+80	...
;;	...
;;	39	...

;; Do 8 ten_reals_unfft operation
;; Do 56 five_complex_unfft operations
;;	distance between fft data elements is 8
;;	do 8 macros each processing 80 data values

	start_timer 7
	mov	rdi, sincos1		;; Load sin/cos pointer
	zloop_init 8			;; 8 iterations
	zr5_ten_reals_five_complex_djbunfft_preload
c2b:	zr5_ten_reals_five_complex_djbunfft rsi, 128, 8*128, rdi, ZMM_SCD4, 8
	zloop	8, c2b, -8*128		;; Test loop counter, restore source pointer
	end_timer 7

;; Do inverse FFT levels 1,2,3,4
;; On input the 128-byte cache line pairs hold these data values:
;;	0	+80	...	40	+80	...
;;	...
;;	39	...
;; These are swizzled to:
;;	0	+5	...	40	+5	...
;;	...
;;	4	...
;;	80	...
;;	160	...
;;	...
;; On output the 128-byte cache line pairs hold these data values:
;;	0	+5	...	320	+5	...
;;	...
;;	4	...
;;	40	...
;;	80	...
;;	...

;; Do 40 sixteen_reals_unfft operations
;;	distance between fft data elements is 40
;;	do 5 macros processing 128 data values

	start_timer 8
	zloop_init 5			;; 5 iterations
	zr8s_sixteen_reals_last_unfft_preload
c1b:	zr8s_sixteen_reals_last_unfft rsi, 128, 5*128, 10*128, 20*128, rdi, ZMM_SCD7, 5, L1PREFETCH_NONE
	zloop	5, c1b, -5*128		;; Test loop counter, restore source pointer
	end_timer 8

	zfft_3_ret
	ENDM

;; Perform a 640-element negacyclic FFT

zfft640p MACRO type
	LOCAL	b1b, b2b, zmid, c2b, c1b

;; Do a multiply with pre-FFTed inputs

	cmp	ffttype, 4
	je	zmid

;; Do FFT levels 1,2,3,4
;; On input the 128-byte cache line pairs hold these data values:
;;	0	+5	...	320	+5	...
;;	...
;;	4	...
;;	40	...
;;	80	...
;;	...
;; On output the 128-byte cache line pairs hold these data values:
;;	0	+5	...	40	+5	...
;;	...
;;	4	...
;;	80	...
;;	160	...
;;	...
;; These are swizzled to:
;;	0	+80	...	40	+80	...
;;	...
;;	39	...

;; Do 40 eight_complex_first_fft operations
;;	distance between fft data elements is 40
;;	do 5 macros processing 128 data values

	start_timer 4
	mov	rdi, sincos1		;; Load sin/cos pointer
	bump	rdi, 8*ZMM_SCD2/8
	mov	r9, norm_col_mults	;; Load weights pointer
	zloop_init 5			;; 5 iterations
	zr8fs_eight_complex_first_fft_preload
b1b:	zr8fs_eight_complex_first_fft rsi, 128, 5*128, 10*128, 20*128, rdi, ZMM_CD8SCD4, r9, 8*128, 5, L1PREFETCH_NONE
	zloop	5, b1b, -5*128		;; Test loop counter, restore source pointer
	end_timer 4

;; Do FFT levels 5,6
;; On input the 64-byte cache lines hold these data values:
;;	0	+80	...	40	+80	...
;;	...
;;	39	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+80	...	8	+80	...
;;	...
;;	7
;;	16	...
;;	32	...

;; Do 64 five_complex_fft operations
;;	distance between fft data elements is 8
;;	do 8 macros each processing 80 data values

	start_timer 5
	bump	rdi, -5*ZMM_CD8SCD4-8*ZMM_SCD2/8 ;; Load sin/cos pointer
	zloop_init 8			;; 8 iterations
	zr5b_five_complex_djbfft_preload
b2b:	zr5b_five_complex_djbfft rsi, 128, 8*128, rdi, ZMM_SCD2/8, 8
	zloop	8, b2b, -8*128		;; Test loop counter, restore source pointer
	end_timer 5

;; Do FFT levels 7,8,9
;; On input and output the 128-byte cache line pairs hold these data values:
;;	0	+80	...	8	+80	...
;;	...
;;	7
;;	16	...
;;	32	...

;; Do 40 eight_complex_fft operations
;;	distance between fft data elements is 1
;;	do 5 macros processing 128 data values

zmid:	start_timer 6
	mov	rcx, 5
	call	zcomplex_onepass_middle
	end_timer 6

;; Do inverse FFT levels 5,6
;; On input the 64-byte cache lines hold these data values:
;;	0	+80	...	8	+80	...
;;	...
;;	7
;;	16	...
;;	32	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+80	...	40	+80	...
;;	...
;;	39	...

;; Do 64 five_complex_unfft operations
;;	distance between fft data elements is 8
;;	do 8 macros each processing 80 data values

	start_timer 7
	mov	rdi, sincos1		;; Load sin/cos pointer
	zloop_init 8			;; 8 iterations
	zr5b_five_complex_djbunfft_preload
c2b:	zr5b_five_complex_djbunfft rsi, 128, 8*128, rdi, ZMM_SCD2/8, 8
	zloop	8, c2b, -8*128		;; Test loop counter, restore source pointer
	end_timer 7

;; Do inverse FFT levels 1,2,3,4
;; On input the 128-byte cache line pairs hold these data values:
;;	0	+80	...	40	+80	...
;;	...
;;	39	...
;; These are swizzled to:
;;	0	+5	...	40	+5	...
;;	...
;;	4	...
;;	80	...
;;	160	...
;;	...
;; On output the 128-byte cache line pairs hold these data values:
;;	0	+5	...	320	+5	...
;;	...
;;	4	...
;;	40	...
;;	80	...
;;	...

;; Do 40 eight_complex_last_unfft operations
;;	distance between fft data elements is 40
;;	do 5 macros processing 128 data values

	start_timer 8
	zloop_init 5			;; 5 iterations
	zr8s_eight_complex_last_unfft_preload
c1b:	zr8s_eight_complex_last_unfft rsi, 128, 5*128, 10*128, 20*128, rdi, ZMM_CD8SCD4, 5, L1PREFETCH_NONE
	zloop	5, c1b, -5*128		;; Test loop counter, restore source pointer
	end_timer 8

	zfft_3_ret
	ENDM


;; Perform a 768-element FFT

zfft768 MACRO type
	LOCAL	b1b, b2b, zmid, c2b, c1b

;; Check for a multiply with pre-FFTed inputs

	cmp	ffttype, 4
	je	zmid

;; Do FFT levels 1,2,3,4
;; On input the 128-byte cache line pairs hold these data values:
;;	0	+6	...	384	+6	...
;;	...
;;	5	...
;;	48	...
;;	96	...
;;	...
;; On output the 128-byte cache line pairs hold these data values:
;;	0	+6	...	48	+6	...
;;	...
;;	5	...
;;	96	...
;;	192	...
;;	...
;; These are swizzled to:
;;	0	+96	...	48	+96	...
;;	...
;;	47	...

;; Do 48 sixteen_reals_fft operations
;;	distance between fft data elements is 48
;;	do 6 macros processing 128 data values

	start_timer 4
	mov	rdi, sincos1		;; Load sin/cos data ptr
	bump	rdi, 8*ZMM_SCD5
	mov	r9, norm_col_mults	;; Load weights pointer
	zloop_init 6			;; 6 iterations
	zr8fs_sixteen_reals_first_fft_preload
b1b:	zr8fs_sixteen_reals_first_fft rsi, 128, 6*128, 12*128, 24*128, rdi, ZMM_SCD7, r9, 8*128, 6, L1PREFETCH_NONE
	zloop	6, b1b, -6*128		;; Test loop counter, restore source pointer
	end_timer 4

;; Do FFT levels 5,6
;; On input the 64-byte cache lines hold these data values:
;;	0	+96	...	48	+96	...
;;	...
;;	47	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+96	...	8	+96	...
;;	...
;;	7
;;	16	...
;;	32	...

;; Do 8 twelve_reals_fft operation
;; Do 56 six_complex_fft operations
;;	distance between fft data elements is 8
;;	do 8 macros each processing 96 data values

	start_timer 5
	bump	rdi, -6*ZMM_SCD7-8*ZMM_SCD5 ;; Load sin/cos pointer
	zloop_init 8			;; 8 iterations
	zr6_twelve_reals_six_complex_djbfft_preload
b2b:	zr6_twelve_reals_six_complex_djbfft rsi, 128, 8*128, rdi, ZMM_SCD5, 8, L1PREFETCH_NONE
	zloop	8, b2b, -8*128		;; Test loop counter, restore source pointer
	end_timer 5

;; Do FFT levels 7,8,9
;; On input and output the 128-byte cache line pairs hold these data values:
;;	0	+96	...	8	+96	...
;;	...
;;	7
;;	16	...
;;	32	...

;; Do 1 sixteen_reals_fft operation
;; Do 47 eight_complex_fft operations
;;	distance between fft data elements is 1
;;	do 6 macros processing 128 data values

zmid:	start_timer 6
	mov	rcx, 6
	call	zreal_onepass_middle
	end_timer 6

;; Do inverse FFT levels 5,6
;; On input the 64-byte cache lines hold these data values:
;;	0	+96	...	8	+96	...
;;	...
;;	7
;;	16	...
;;	32	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+96	...	48	+96	...
;;	...
;;	47	...

;; Do 8 twelve_reals_unfft operation
;; Do 56 six_complex_unfft operations
;;	distance between fft data elements is 8
;;	do 8 macros each processing 96 data values

	start_timer 7
	mov	rdi, sincos1		;; Load sin/cos pointer
	zloop_init 8			;; 8 iterations
	zr6_twelve_reals_six_complex_djbunfft_preload
c2b:	zr6_twelve_reals_six_complex_djbunfft rsi, 128, 8*128, rdi, ZMM_SCD5, 8, L1PREFETCH_NONE
	zloop	8, c2b, -8*128		;; Test loop counter, restore source pointer
	end_timer 7

;; Do inverse FFT levels 1,2,3,4
;; On input the 128-byte cache line pairs hold these data values:
;;	0	+96	...	48	+96	...
;;	...
;;	47	...
;; These are swizzled to:
;;	0	+6	...	48	+6	...
;;	...
;;	5	...
;;	96	...
;;	192	...
;;	...
;; On output the 128-byte cache line pairs hold these data values:
;;	0	+6	...	384	+6	...
;;	...
;;	5	...
;;	48	...
;;	96	...
;;	...

;; Do 48 sixteen_reals_unfft operations
;;	distance between fft data elements is 48
;;	do 6 macros processing 128 data values

	start_timer 8
	zloop_init 6			;; 6 iterations
	zr8s_sixteen_reals_last_unfft_preload
c1b:	zr8s_sixteen_reals_last_unfft rsi, 128, 6*128, 12*128, 24*128, rdi, ZMM_SCD7, 6, L1PREFETCH_NONE
	zloop	6, c1b, -6*128		;; Test loop counter, restore source pointer
	end_timer 8

	zfft_3_ret
	ENDM

;; Perform a 768-element negacyclic FFT

zfft768p MACRO type
	LOCAL	b1b, b2b, zmid, c2b, c1b

;; Do a multiply with pre-FFTed inputs

	cmp	ffttype, 4
	je	zmid

;; Do FFT levels 1,2,3,4
;; On input the 128-byte cache line pairs hold these data values:
;;	0	+6	...	384	+6	...
;;	...
;;	5	...
;;	48	...
;;	96	...
;;	...
;; On output the 128-byte cache line pairs hold these data values:
;;	0	+6	...	48	+6	...
;;	...
;;	5	...
;;	96	...
;;	192	...
;;	...
;; These are swizzled to:
;;	0	+96	...	48	+96	...
;;	...
;;	47	...

;; Do 48 eight_complex_first_fft operations
;;	distance between fft data elements is 48
;;	do 6 macros processing 128 data values

	start_timer 4
	mov	rdi, sincos1		;; Load sin/cos pointer
	bump	rdi, 8*ZMM_SCD3/8
	mov	r9, norm_col_mults	;; Load weights pointer
	zloop_init 6			;; 5 iterations
	zr8fs_eight_complex_first_fft_preload
b1b:	zr8fs_eight_complex_first_fft rsi, 128, 6*128, 12*128, 24*128, rdi, ZMM_CD8SCD4, r9, 8*128, 6, L1PREFETCH_NONE
	zloop	6, b1b, -6*128		;; Test loop counter, restore source pointer
	end_timer 4

;; Do FFT levels 5,6
;; On input the 64-byte cache lines hold these data values:
;;	0	+96	...	48	+96	...
;;	...
;;	47	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+96	...	8	+96	...
;;	...
;;	7
;;	16	...
;;	32	...

;; Do 64 six_complex_fft operations
;;	distance between fft data elements is 8
;;	do 8 macros each processing 96 data values

	start_timer 5
	bump	rdi, -6*ZMM_CD8SCD4-8*ZMM_SCD3/8 ;; Load sin/cos pointer
	zloop_init 8			;; 8 iterations
	zr6b_six_complex_djbfft_preload
b2b:	zr6b_six_complex_djbfft rsi, 128, 8*128, rdi, ZMM_SCD3/8, 8
	zloop	8, b2b, -8*128		;; Test loop counter, restore source pointer
	end_timer 5

;; Do FFT levels 7,8,9
;; On input and output the 128-byte cache line pairs hold these data values:
;;	0	+96	...	8	+96	...
;;	...
;;	7
;;	16	...
;;	32	...

;; Do 48 eight_complex_fft operations
;;	distance between fft data elements is 1
;;	do 6 macros processing 128 data values

zmid:	start_timer 6
	mov	rcx, 6
	call	zcomplex_onepass_middle
	end_timer 6

;; Do inverse FFT levels 5,6
;; On input the 64-byte cache lines hold these data values:
;;	0	+96	...	8	+96	...
;;	...
;;	7
;;	16	...
;;	32	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+96	...	48	+96	...
;;	...
;;	47	...

;; Do 64 six_complex_unfft operations
;;	distance between fft data elements is 8
;;	do 8 macros each processing 96 data values

	start_timer 7
	mov	rdi, sincos1		;; Load sin/cos pointer
	zloop_init 8			;; 8 iterations
	zr6b_six_complex_djbunfft_preload
c2b:	zr6b_six_complex_djbunfft rsi, 128, 8*128, rdi, ZMM_SCD3/8, 8
	zloop	8, c2b, -8*128		;; Test loop counter, restore source pointer
	end_timer 7

;; Do inverse FFT levels 1,2,3,4
;; On input the 128-byte cache line pairs hold these data values:
;;	0	+96	...	48	+96	...
;;	...
;;	47	...
;; These are swizzled to:
;;	0	+6	...	48	+6	...
;;	...
;;	5	...
;;	96	...
;;	192	...
;;	...
;; On output the 128-byte cache line pairs hold these data values:
;;	0	+6	...	384	+6	...
;;	...
;;	5	...
;;	48	...
;;	96	...
;;	...

;; Do 48 eight_complex_last_unfft operations
;;	distance between fft data elements is 48
;;	do 6 macros processing 128 data values

	start_timer 8
	zloop_init 6			;; 6 iterations
	zr8s_eight_complex_last_unfft_preload
c1b:	zr8s_eight_complex_last_unfft rsi, 128, 6*128, 12*128, 24*128, rdi, ZMM_CD8SCD4, 6, L1PREFETCH_NONE
	zloop	6, c1b, -6*128		;; Test loop counter, restore source pointer
	end_timer 8

	zfft_3_ret
	ENDM


;; Perform a 896-element FFT

zfft896 MACRO type
	LOCAL	b1b, b2b, zmid, c2b, c1b

;; Check for a multiply with pre-FFTed inputs

	cmp	ffttype, 4
	je	zmid

;; Do FFT levels 1,2,3,4
;; On input the 128-byte cache line pairs hold these data values:
;;	0	+7	...	448	+7	...
;;	...
;;	6	...
;;	56	...
;;	112	...
;;	...
;; On output the 128-byte cache line pairs hold these data values:
;;	0	+7	...	56	+7	...
;;	...
;;	6	...
;;	112	...
;;	224	...
;;	...
;; These are swizzled to:
;;	0	+112	...	56	+112	...
;;	...
;;	55	...

;; Do 56 sixteen_reals_fft operations
;;	distance between fft data elements is 56
;;	do 7 macros processing 128 data values

	start_timer 4
	mov	rdi, sincos1		;; Load sin/cos data ptr
	bump	rdi, 8*ZMM_SCD6
	mov	r9, norm_col_mults	;; Load weights pointer
	zloop_init 7			;; 5 iterations
	zr8fs_sixteen_reals_first_fft_preload
b1b:	zr8fs_sixteen_reals_first_fft rsi, 128, 7*128, 14*128, 28*128, rdi, ZMM_SCD7, r9, 8*128, 7, L1PREFETCH_NONE
	zloop	7, b1b, -7*128		;; Test loop counter, restore source pointer
	end_timer 4

;; Do FFT levels 5,6
;; On input the 64-byte cache lines hold these data values:
;;	0	+112	...	56	+112	...
;;	...
;;	55	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+112	...	8	+112	...
;;	...
;;	7
;;	16	...
;;	32	...

;; Do 8 fourteen_reals_fft operation
;; Do 56 seven_complex_fft operations
;;	distance between fft data elements is 8
;;	do 8 macros each processing 112 data values

	start_timer 5
	bump	rdi, -7*ZMM_SCD7-8*ZMM_SCD6 ;; Load sin/cos pointer
	zloop_init 8			;; 8 iterations
	zr7_fourteen_reals_seven_complex_djbfft_preload
b2b:	zr7_fourteen_reals_seven_complex_djbfft rsi, 128, 8*128, rdi, ZMM_SCD6, 8, L1PREFETCH_NONE
	zloop	8, b2b, -8*128		;; Test loop counter, restore source pointer
	end_timer 5

;; Do FFT levels 7,8,9
;; On input and output the 128-byte cache line pairs hold these data values:
;;	0	+112	...	8	+112	...
;;	...
;;	7
;;	16	...
;;	32	...

;; Do 1 sixteen_reals_fft operation
;; Do 55 eight_complex_fft operations
;;	distance between fft data elements is 1
;;	do 7 macros processing 128 data values

zmid:	start_timer 6
	mov	rcx, 7
	call	zreal_onepass_middle
	end_timer 6

;; Do inverse FFT levels 5,6
;; On input the 64-byte cache lines hold these data values:
;;	0	+112	...	8	+112	...
;;	...
;;	7
;;	16	...
;;	32	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+112	...	56	+112	...
;;	...
;;	55	...

;; Do 8 fourteen_reals_unfft operation
;; Do 56 seven_complex_unfft operations
;;	distance between fft data elements is 8
;;	do 8 macros each processing 112 data values

	start_timer 7
	mov	rdi, sincos1		;; Load sin/cos pointer
	zloop_init 8			;; 8 iterations
	zr7_fourteen_reals_seven_complex_djbunfft_preload
c2b:	zr7_fourteen_reals_seven_complex_djbunfft rsi, 128, 8*128, rdi, ZMM_SCD6, 8, L1PREFETCH_NONE
	zloop	8, c2b, -8*128		;; Test loop counter, restore source pointer
	end_timer 7

;; Do inverse FFT levels 1,2,3,4
;; On input the 128-byte cache line pairs hold these data values:
;;	0	+112	...	56	+112	...
;;	...
;;	55	...
;; These are swizzled to:
;;	0	+7	...	56	+7	...
;;	...
;;	6	...
;;	112	...
;;	224	...
;;	...
;; On output the 128-byte cache line pairs hold these data values:
;;	0	+7	...	448	+7	...
;;	...
;;	6	...
;;	56	...
;;	112	...
;;	...

;; Do 56 sixteen_reals_unfft operations
;;	distance between fft data elements is 56
;;	do 7 macros processing 128 data values

	start_timer 8
	zloop_init 7			;; 7 iterations
	zr8s_sixteen_reals_last_unfft_preload
c1b:	zr8s_sixteen_reals_last_unfft rsi, 128, 7*128, 14*128, 28*128, rdi, ZMM_SCD7, 7, L1PREFETCH_NONE
	zloop	7, c1b, -7*128		;; Test loop counter, restore source pointer
	end_timer 8

	zfft_3_ret
	ENDM

;; Perform a 896-element negacyclic FFT

zfft896p MACRO type
	LOCAL	b1b, b2b, zmid, c2b, c1b

;; Do a multiply with pre-FFTed inputs

	cmp	ffttype, 4
	je	zmid

;; Do FFT levels 1,2,3,4
;; On input the 128-byte cache line pairs hold these data values:
;;	0	+7	...	448	+7	...
;;	...
;;	6	...
;;	56	...
;;	112	...
;;	...
;; On output the 128-byte cache line pairs hold these data values:
;;	0	+7	...	56	+7	...
;;	...
;;	6	...
;;	112	...
;;	224	...
;;	...
;; These are swizzled to:
;;	0	+112	...	56	+112	...
;;	...
;;	55	...

;; Do 56 eight_complex_first_fft operations
;;	distance between fft data elements is 56
;;	do 7 macros processing 128 data values

	start_timer 4
	mov	rdi, sincos1		;; Load sin/cos pointer
	bump	rdi, 8*ZMM_SCD3/8
	mov	r9, norm_col_mults	;; Load weights pointer
	zloop_init 7			;; 7 iterations
	zr8fs_eight_complex_first_fft_preload
b1b:	zr8fs_eight_complex_first_fft rsi, 128, 7*128, 14*128, 28*128, rdi, ZMM_CD8SCD4, r9, 8*128, 7, L1PREFETCH_NONE
	zloop	7, b1b, -7*128		;; Test loop counter, restore source pointer
	end_timer 4

;; Do FFT levels 5,6
;; On input the 64-byte cache lines hold these data values:
;;	0	+112	...	56	+112	...
;;	...
;;	55	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+112	...	8	+112	...
;;	...
;;	7
;;	16	...
;;	32	...

;; Do 64 five_complex_fft operations
;;	distance between fft data elements is 8
;;	do 8 macros each processing 112 data values

	start_timer 5
	bump	rdi, -7*ZMM_CD8SCD4-8*ZMM_SCD3/8 ;; Load sin/cos pointer
	zloop_init 8			;; 8 iterations
	zr7b_seven_complex_djbfft_preload
b2b:	zr7b_seven_complex_djbfft rsi, 128, 8*128, rdi, ZMM_SCD3/8, 8, L1PREFETCH_NONE
	zloop	8, b2b, -8*128		;; Test loop counter, restore source pointer
	end_timer 5

;; Do FFT levels 7,8,9
;; On input and output the 128-byte cache line pairs hold these data values:
;;	0	+112	...	8	+112	...
;;	...
;;	7
;;	16	...
;;	32	...

;; Do 56 eight_complex_fft operations
;;	distance between fft data elements is 1
;;	do 7 macros processing 128 data values

zmid:	start_timer 6
	mov	rcx, 7
	call	zcomplex_onepass_middle
	end_timer 6

;; Do inverse FFT levels 5,6
;; On input the 64-byte cache lines hold these data values:
;;	0	+112	...	8	+112	...
;;	...
;;	7
;;	16	...
;;	32	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+112	...	56	+112	...
;;	...
;;	55	...

;; Do 64 seven_complex_unfft operations
;;	distance between fft data elements is 8
;;	do 8 macros each processing 112 data values

	start_timer 7
	mov	rdi, sincos1		;; Load sin/cos pointer
	zloop_init 8			;; 8 iterations
	zr7b_seven_complex_djbunfft_preload
c2b:	zr7b_seven_complex_djbunfft rsi, 128, 8*128, rdi, ZMM_SCD3/8, 8, L1PREFETCH_NONE
	zloop	8, c2b, -8*128		;; Test loop counter, restore source pointer
	end_timer 7

;; Do inverse FFT levels 1,2,3,4
;; On input the 128-byte cache line pairs hold these data values:
;;	0	+112	...	56	+112	...
;;	...
;;	55	...
;; These are swizzled to:
;;	0	+7	...	56	+7	...
;;	...
;;	6	...
;;	112	...
;;	224	...
;;	...
;; On output the 128-byte cache line pairs hold these data values:
;;	0	+7	...	448	+7	...
;;	...
;;	6	...
;;	56	...
;;	112	...
;;	...

;; Do 56 eight_complex_last_unfft operations
;;	distance between fft data elements is 56
;;	do 7 macros processing 128 data values

	start_timer 8
	zloop_init 7			;; 7 iterations
	zr8s_eight_complex_last_unfft_preload
c1b:	zr8s_eight_complex_last_unfft rsi, 128, 7*128, 14*128, 28*128, rdi, ZMM_CD8SCD4, 7, L1PREFETCH_NONE
	zloop	7, c1b, -7*128		;; Test loop counter, restore source pointer
	end_timer 8

	zfft_3_ret
	ENDM


;; Perform a 1K-element FFT

zfft1K MACRO type
	LOCAL	b1b, b2b, zmid, c2b, c1b

;; Check for a multiply with pre-FFTed inputs

	cmp	ffttype, 4
	je	zmid

;; Do FFT levels 1,2,3,4
;; On input the 128-byte cache line pairs hold these data values:
;;	0	+8	...	512	+8	...
;;	...
;;	7	...
;;	64	...
;;	128	...
;;	...
;; On output the 128-byte cache line pairs hold these data values:
;;	0	+8	...	64	+8	...
;;	...
;;	7	...
;;	128	...
;;	256	...
;;	...
;; These are swizzled to:
;;	0	+128	...	64	+128	...
;;	...
;;	63	...

;; Do 64 sixteen_reals_fft operations
;;	distance between fft data elements is 64
;;	do 8 macros processing 128 data values

;; Do FFT levels 5,6,7
;; On input the 64-byte cache lines hold these data values:
;;	0	+128	...	64	+128	...
;;	...
;;	63	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+128	...	8	+128	...
;;	...
;;	7
;;	16	...
;;	32	...
;;	...

;; Do 8 sixteen_reals_fft operation
;; Do 56 eight_complex_fft operations
;;	distance between fft data elements is 8
;;	do 8 macros each processing 128 data values

IFDEF OLDWAY
	start_timer 4
	mov	rdi, sincos1		;; Load sin/cos data ptr
	bump	rdi, 8*ZMM_SCD7
	mov	r9, norm_col_mults	;; Load weights pointer
	zloop_init 8			;; 8 iterations
	zr8fs_sixteen_reals_first_fft_preload
b1b:	zr8fs_sixteen_reals_first_fft rsi, 128, 8*128, 16*128, 32*128, rdi, ZMM_SCD7, r9, 8*128, 8, L1PREFETCH_NONE
	zloop	8, b1b, -8*128		;; Test loop counter, restore source pointer
	end_timer 4

	start_timer 5
	bump	rdi, -8*ZMM_SCD7-8*ZMM_SCD7 ;; Load sin/cos pointer
	zloop_init 8			;; 8 iterations
	zr8_sixteen_reals_eight_complex_djbfft_preload
b2b:	zr8_sixteen_reals_eight_complex_djbfft rsi, 128, 8*128, 16*128, 32*128, rdi, ZMM_SCD7, 8, L1PREFETCH_NONE
	zloop	8, b2b, -8*128		;; Test loop counter, restore source pointer
	end_timer 5
ELSE
	start_timer 4
	mov	rdi, sincos1		;; Load sin/cos data ptr
	lea	r10, [rdi+8*ZMM_SCD7]	;; Second sin/cos pointer
	mov	r9, norm_col_mults	;; Load weights pointer
	zloop_init 8			;; 8 iterations
	zr8fs_hundredtwentyeight_reals_first_fft_preload
b1b:	zr8fs_hundredtwentyeight_reals_first_fft rsi, 128, 8*128, 16*128, 32*128, r10, ZMM_SCD7, rdi, ZMM_SCD7, r9, 8*128, 8, L1PREFETCH_NONE
	zloop	8, b1b, -8*128		;; Test loop counter, restore source pointer
	end_timer 4
ENDIF

;; Do FFT levels 8,9,10
;; On input and output the 128-byte cache line pairs hold these data values:
;;	0	+128	...	8	+128	...
;;	...
;;	7
;;	16	...
;;	32	...
;;	...

;; Do 1 sixteen_reals_fft operation
;; Do 63 eight_complex_fft operations
;;	distance between fft data elements is 1
;;	do 8 macros processing 128 data values

zmid:	start_timer 6
	mov	rcx, 8
	call	zreal_onepass_middle
	end_timer 6

;; Do inverse FFT levels 5,6,7
;; On input the 64-byte cache lines hold these data values:
;;	0	+128	...	8	+128	...
;;	...
;;	7
;;	16	...
;;	32	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+128	...	64	+128	...
;;	...
;;	63	...

;; Do 8 sixteen_reals_unfft operations
;; Do 56 eight_complex_unfft operations
;;	distance between fft data elements is 8
;;	do 8 macros each processing 128 data values

;; Do inverse FFT levels 1,2,3,4
;; On input the 128-byte cache line pairs hold these data values:
;;	0	+128	...	64	+128	...
;;	...
;;	63	...
;; These are swizzled to:
;;	0	+8	...	64	+8	...
;;	...
;;	7	...
;;	128	...
;;	256	...
;;	...
;; On output the 128-byte cache line pairs hold these data values:
;;	0	+8	...	512	+8	...
;;	...
;;	7	...
;;	64	...
;;	128	...
;;	...

;; Do 64 sixteen_reals_unfft operations
;;	distance between fft data elements is 64
;;	do 8 macros processing 128 data values

IFDEF OLDWAY
	start_timer 7
	mov	rdi, sincos1		;; Load sin/cos pointer
	zloop_init 8			;; 8 iterations
	zr8_sixteen_reals_eight_complex_djbunfft_preload
c2b:	zr8_sixteen_reals_eight_complex_djbunfft rsi, 128, 8*128, 16*128, 32*128, rdi, ZMM_SCD7, 8, L1PREFETCH_NONE
	zloop	8, c2b, -8*128		;; Test loop counter, restore source pointer, restore sin/cos pointer
	end_timer 7

	start_timer 8
	zloop_init 8			;; 8 iterations
	zr8s_sixteen_reals_last_unfft_preload
c1b:	zr8s_sixteen_reals_last_unfft rsi, 128, 8*128, 16*128, 32*128, rdi, ZMM_SCD7, 8, L1PREFETCH_NONE
	zloop	8, c1b, -8*128		;; Test loop counter, restore source pointer
	end_timer 8
ELSE
	start_timer 7
	mov	rdi, sincos1		;; Load sin/cos pointer
	lea	r10, [rdi+8*ZMM_SCD7]	;; Second sin/cos pointer
	zloop_init 8			;; 8 iterations
	zr8s_hundredtwentyeight_reals_last_unfft_preload
c1b:	zr8s_hundredtwentyeight_reals_last_unfft rsi, 128, 8*128, 16*128, 32*128, rdi, ZMM_SCD7, r10, ZMM_SCD7, 8, L1PREFETCH_NONE
	zloop	8, c1b, -8*128		;; Test loop counter, restore source pointer
	end_timer 7
ENDIF
	zfft_3_ret
	ENDM

;; Perform a 1K-element negacyclic FFT

zfft1Kp MACRO type
	LOCAL	b1b, b2b, zmid, c2b, c1b

;; Do a multiply with pre-FFTed inputs

	cmp	ffttype, 4
	je	zmid

;; Do FFT levels 1,2,3,4
;; On input the 128-byte cache line pairs hold these data values:
;;	0	+8	...	512	+8	...
;;	...
;;	7	...
;;	64	...
;;	128	...
;;	...
;; On output the 128-byte cache line pairs hold these data values:
;;	0	+8	...	64	+8	...
;;	...
;;	7	...
;;	128	...
;;	256	...
;;	...
;; These are swizzled to:
;;	0	+128	...	64	+128	...
;;	...
;;	63	...

;; Do 64 eight_complex_first_fft operations
;;	distance between fft data elements is 64
;;	do 8 macros processing 128 data values

;; Do FFT levels 5,6,7
;; On input the 64-byte cache lines hold these data values:
;;	0	+128	...	64	+128	...
;;	...
;;	63	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+128	...	8	+128	...
;;	...
;;	7
;;	16	...
;;	32	...
;;	...

;; Do 64 eight_complex_fft operations
;;	distance between fft data elements is 8
;;	do 8 macros each processing 128 data values

IFDEF OLDWAY
	start_timer 4
	mov	rdi, sincos1		;; Load sin/cos pointer
	bump	rdi, 8*ZMM_SCD4/8
	mov	r9, norm_col_mults	;; Load weights pointer
	zloop_init 8			;; 8 iterations
	zr8fs_eight_complex_first_fft_preload
b1b:	zr8fs_eight_complex_first_fft rsi, 128, 8*128, 16*128, 32*128, rdi, ZMM_CD8SCD4, r9, 8*128, 8, L1PREFETCH_NONE
	zloop	8, b1b, -8*128		;; Test loop counter, restore source pointer
	end_timer 4

	start_timer 5
	bump	rdi, -8*ZMM_CD8SCD4-8*ZMM_SCD4/8 ;; Load sin/cos pointer
	zloop_init 8			;; 8 iterations
	zr8b_eight_complex_djbfft_preload
b2b:	zr8b_eight_complex_djbfft rsi, 128, 8*128, 16*128, 32*128, rdi, ZMM_SCD4/8, 8, L1PREFETCH_NONE
	zloop	8, b2b, -8*128		;; Test loop counter, restore source pointer
	end_timer 5
ELSE
	start_timer 4
	mov	rdi, sincos1		;; Load sin/cos pointer
	lea	r10, [rdi+8*ZMM_SCD4/8]	;; Second sin/cos pointer
	mov	r9, norm_col_mults	;; Load weights pointer
	zloop_init 8			;; 8 iterations
	zr8fs_sixtyfour_complex_first_fft_preload
b1b:	zr8fs_sixtyfour_complex_first_fft rsi, 128, 8*128, 16*128, 32*128, r10, ZMM_CD8SCD4, rdi, ZMM_SCD4/8, r9, 8*128, 8, L1PREFETCH_NONE
	zloop	8, b1b, -8*128		;; Test loop counter, restore source pointer
	end_timer 4
ENDIF

;; Do FFT levels 8,9,10
;; On input and output the 128-byte cache line pairs hold these data values:
;;	0	+128	...	8	+128	...
;;	...
;;	7
;;	16	...
;;	32	...
;;	...

;; Do 64 eight_complex_fft operations
;;	distance between fft data elements is 1
;;	do 8 macros processing 128 data values

zmid:	start_timer 6
	mov	rcx, 8
	call	zcomplex_onepass_middle
	end_timer 6

;; Do inverse FFT levels 5,6,7
;; On input the 64-byte cache lines hold these data values:
;;	0	+128	...	8	+128	...
;;	...
;;	7
;;	16	...
;;	32	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+128	...	64	+128	...
;;	...
;;	63	...

;; Do 64 eight_complex_unfft operations
;;	distance between fft data elements is 8
;;	do 8 macros each processing 128 data values

;; Do inverse FFT levels 1,2,3,4
;; On input the 128-byte cache line pairs hold these data values:
;;	0	+128	...	64	+128	...
;;	...
;;	63	...
;; These are swizzled to:
;;	0	+8	...	64	+8	...
;;	...
;;	7	...
;;	128	...
;;	256	...
;;	...
;; On output the 128-byte cache line pairs hold these data values:
;;	0	+8	...	512	+8	...
;;	...
;;	7	...
;;	64	...
;;	128	...
;;	...

;; Do 64 eight_complex_last_unfft operations
;;	distance between fft data elements is 64
;;	do 8 macros processing 128 data values

IFDEF OLDWAY
	start_timer 7
	mov	rdi, sincos1		;; Load sin/cos pointer
	zloop_init 8			;; 8 iterations
	zr8b_eight_complex_djbunfft_preload
c2b:	zr8b_eight_complex_djbunfft rsi, 128, 8*128, 16*128, 32*128, rdi, ZMM_SCD4/8, 8, L1PREFETCH_NONE
	zloop	8, c2b, -8*128		;; Test loop counter, restore source pointer
	end_timer 7

	start_timer 8
	zloop_init 8			;; 8 iterations
	zr8s_eight_complex_last_unfft_preload
c1b:	zr8s_eight_complex_last_unfft rsi, 128, 8*128, 16*128, 32*128, rdi, ZMM_CD8SCD4, 8, L1PREFETCH_NONE
	zloop	8, c1b, -8*128		;; Test loop counter, restore source pointer
	end_timer 8
ELSE
	start_timer 7
	mov	rdi, sincos1		;; Load sin/cos pointer
	lea	r10, [rdi+8*ZMM_SCD4/8]	;; Second sin/cos pointer
	zloop_init 8			;; 8 iterations
	zr8s_sixtyfour_complex_last_unfft_preload
c1b:	zr8s_sixtyfour_complex_last_unfft rsi, 128, 8*128, 16*128, 32*128, rdi, ZMM_SCD4/8, r10, ZMM_CD8SCD4, 8, L1PREFETCH_NONE
	zloop	8, c1b, -8*128		;; Test loop counter, restore source pointer
	end_timer 7
ENDIF
	zfft_3_ret
	ENDM


;; Perform a 1152-element FFT

zfft1152 MACRO type
	LOCAL	b1b, b2b, b3a, b3b, zmid, c3a, c3b, c2b, c1b

;; Check for a multiply with pre-FFTed inputs

	cmp	ffttype, 4
	je	zmid

;; Do FFT levels 1,2,3,4
;; On input the 128-byte cache line pairs hold these data values:
;;	0	+9	...	576	+9	...
;;	...
;;	8	...
;;	72	...
;;	...
;; On output the 128-byte cache line pairs hold these data values:
;;	0	+9	...	72	+9	...
;;	...
;;	9	...
;;	144	...
;;	...
;; These are swizzled to:
;;	0	+144	...	72	+144	...
;;	...
;;	71	...

;; Do 72 sixteen_reals_fft operations
;;	distance between fft data elements is 72
;;	do 9 macros processing 128 data values

	start_timer 4
	mov	rdi, sincos1		;; Load sin/cos data ptr
	bump	rdi, 24*ZMM_SCD2
	mov	r9, norm_col_mults	;; Load weights pointer
	zloop_init 9			;; 9 iterations
	zr8fs_sixteen_reals_first_fft_preload
b1b:	zr8fs_sixteen_reals_first_fft rsi, 128, 9*128, 18*128, 36*128, rdi, ZMM_SCD7, r9, 8*128, 9, L1PREFETCH_NONE
	zloop	9, b1b, -9*128		;; Test loop counter, restore source pointer
	end_timer 4

;; Do FFT levels 5,6
;; On input the 64-byte cache lines hold these data values:
;;	0	+144	...	72	+144	...
;;	...
;;	71	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+144	...	24	+144	...
;;	...
;;	23
;;	48	...
;;	...

;; Do 24 six_reals_fft operation
;; Do 168 three_complex_fft operations
;;	distance between fft data elements is 24
;;	do 24 macros each processing 48 data values

;; Do FFT levels 7
;; On input the 64-byte cache lines hold these data values:
;;	0	+144	...	24	+144	...
;;	...
;;	23
;;	48	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+144	...	8	+144	...
;;	...
;;	7
;;	16	...
;;	...

;; Do 8 six_reals_fft operation
;; Do 184 two_complex_fft operations
;;	distance between fft data elements is 8
;;	do 24 macros each processing 48 data values

	start_timer 5
	lea	r10, [rdi-9*ZMM_SCD7-24*ZMM_SCD2] ;; Load sin/cos pointers
	mov	rdi, r10
	zloop_init 8			;; 8 iterations
	zr33_eighteen_reals_nine_complex_djbfft_preload
b2b:	zr33_eighteen_reals_nine_complex_djbfft rsi, 128, 8*128, r10, 8*ZMM_SCD2, ZMM_SCD2, rdi, 0*3*ZMM_SCD2, 3*ZMM_SCD2, 8
	zloop	8, b2b, -8*128		;; Test loop counter, restore source pointer
	end_timer 5

;; Do FFT levels 8,9,10
;; On input and output the 128-byte cache line pairs hold these data values:
;;	0	+144	...	8	+144	...
;;	...
;;	7
;;	16	...
;;	...

;; Do 1 sixteen_reals_fft operation
;; Do 71 eight_complex_fft operations
;;	distance between fft data elements is 1
;;	do 9 macros processing 128 data values

zmid:	start_timer 7
	mov	rcx, 9
	call	zreal_onepass_middle
	end_timer 7

;; Do inverse FFT levels 7
;; On input the 64-byte cache lines hold these data values:
;;	0	+144	...	8	+144	...
;;	...
;;	7
;;	16	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+144	...	24	+144	...
;;	...
;;	23
;;	48	...
;;	...

;; Do 8 six_reals_unfft operations
;; Do 184 three_complex_unfft operations
;;	distance between fft data elements is 8
;;	do 24 macros each processing 48 data values

;; Do inverse FFT levels 5,6
;; On input the 64-byte cache lines hold these data values:
;;	0	+144	...	24	+144	...
;;	...
;;	23
;;	48	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+144	...	72	+144	...
;;	...
;;	71	...

;; Do 24 six_reals_unfft operations
;; Do 169 three_complex_unfft operations
;;	distance between fft data elements is 24
;;	do 24 macros each processing 48 data values

	start_timer 8
	mov	rdi, sincos1		;; Load sin/cos pointers
	mov	r10, rdi
	zloop_init 8			;; 8 iterations
	zr33_eighteen_reals_nine_complex_djbunfft_preload
c3b:	zr33_eighteen_reals_nine_complex_djbunfft rsi, 128, 8*128, rdi, 0*3*ZMM_SCD2, 3*ZMM_SCD2, r10, 8*ZMM_SCD2, ZMM_SCD2, 8
	zloop	8, c3b, -8*128		;; Test loop counter, restore source pointer
	end_timer 8

;; Do inverse FFT levels 1,2,3,4
;; On input the 128-byte cache line pairs hold these data values:
;;	0	+144	...	72	+144	...
;;	...
;;	71	...
;; These are swizzled to:
;;	0	+9	...	72	+9	...
;;	...
;;	9	...
;;	144	...
;;	288	...
;;	...
;; On output the 128-byte cache line pairs hold these data values:
;;	0	+9	...	576	+9	...
;;	...
;;	8	...
;;	72	...
;;	144	...
;;	...

;; Do 72 sixteen_reals_unfft operations
;;	distance between fft data elements is 72
;;	do 9 macros processing 128 data values

	start_timer 10
	zloop_init 9			;; 9 iterations
	zr8s_sixteen_reals_last_unfft_preload
c1b:	zr8s_sixteen_reals_last_unfft rsi, 128, 9*128, 18*128, 36*128, rdi, ZMM_SCD7, 9, L1PREFETCH_NONE
	zloop	9, c1b, -9*128		;; Test loop counter, restore source pointer
	end_timer 10

	zfft_3_ret
	ENDM

;; Perform a 1152-element negacyclic FFT

zfft1152p MACRO type
	LOCAL	b1b, b2b, b3b, zmid, c3b, c2b, c1b

;; Do a multiply with pre-FFTed inputs

	cmp	ffttype, 4
	je	zmid

;; Do FFT levels 1,2,3,4
;; On input the 128-byte cache line pairs hold these data values:
;;	0	+9	...	576	+9	...
;;	...
;;	8	...
;;	72	...
;;	...
;; On output the 128-byte cache line pairs hold these data values:
;;	0	+9	...	72	+9	...
;;	...
;;	9	...
;;	144	...
;;	...
;; These are swizzled to:
;;	0	+144	...	72	+144	...
;;	...
;;	71	...

;; Do 72 eight_complex_first_fft operations
;;	distance between fft data elements is 72
;;	do 9 macros processing 128 data values

	start_timer 4
	mov	rdi, sincos1		;; Load sin/cos pointer
	bump	rdi, 24*ZMM_SCD1/8
	mov	r9, norm_col_mults	;; Load weights pointer
	zloop_init 9			;; 9 iterations
	zr8fs_eight_complex_first_fft_preload
b1b:	zr8fs_eight_complex_first_fft rsi, 128, 9*128, 18*128, 36*128, rdi, ZMM_CD8SCD4, r9, 8*128, 9, L1PREFETCH_NONE
	zloop	9, b1b, -9*128		;; Test loop counter, restore source pointer
	end_timer 4

;; Do FFT levels 5,6
;; On input the 64-byte cache lines hold these data values:
;;	0	+144	...	72	+144	...
;;	...
;;	71	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+144	...	24	+144	...
;;	...
;;	23
;;	48	...
;;	...

;; Do 192 three_complex_fft operations
;;	distance between fft data elements is 24
;;	do 24 macros each processing 48 data values

;; Do FFT levels 7
;; On input the 64-byte cache lines hold these data values:
;;	0	+144	...	24	+144	...
;;	...
;;	23
;;	48	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+144	...	8	+144	...
;;	...
;;	7
;;	16	...
;;	...

;; Do 192 three_complex_fft operations
;;	distance between fft data elements is 8
;;	do 24 macros each processing 48 data values

	start_timer 5
	lea	r10, [rdi-9*ZMM_CD8SCD4-24*ZMM_SCD1/8] ;; Load sin/cos pointers
	mov	rdi, r10
	zloop_init 8			;; 8 iterations
	zr33b_nine_complex_djbfft_preload
b2b:	zr33b_nine_complex_djbfft rsi, 128, 8*128, r10, 8*ZMM_SCD1/8, ZMM_SCD1/8, rdi, 0*3*ZMM_SCD1/8, 3*ZMM_SCD1/8, 8
	zloop	8, b2b, -8*128		;; Test loop counter, restore source pointer
	end_timer 5

;; Do FFT levels 8,9,10
;; On input and output the 128-byte cache line pairs hold these data values:
;;	0	+144	...	8	+144	...
;;	...
;;	7
;;	16	...
;;	...

;; Do 72 eight_complex_fft operations
;;	distance between fft data elements is 1
;;	do 9 macros processing 128 data values

zmid:	start_timer 7
	mov	rcx, 9
	call	zcomplex_onepass_middle
	end_timer 7

;; Do inverse FFT levels 7
;; On input the 64-byte cache lines hold these data values:
;;	0	+144	...	8	+144	...
;;	...
;;	7
;;	16	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+144	...	24	+144	...
;;	...
;;	23
;;	48	...
;;	...

;; Do 192 three_complex_unfft operations
;;	distance between fft data elements is 8
;;	do 24 macros each processing 48 data values

;; Do inverse FFT levels 5,6
;; On input the 64-byte cache lines hold these data values:
;;	0	+144	...	24	+144	...
;;	...
;;	23
;;	48	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+144	...	72	+144	...
;;	...
;;	71	...

;; Do 192 three_complex_unfft operations
;;	distance between fft data elements is 24
;;	do 24 macros each processing 48 data values

	start_timer 8
	mov	rdi, sincos1		;; Load sin/cos pointers
	mov	r10, rdi
	zloop_init 8			;; 8 iterations
	zr33b_nine_complex_djbunfft_preload
c3b:	zr33b_nine_complex_djbunfft rsi, 128, 8*128, rdi, 0*3*ZMM_SCD1/8, 3*ZMM_SCD1/8, r10, 8*ZMM_SCD1/8, ZMM_SCD1/8, 8
	zloop	8, c3b, -8*128		;; Test loop counter, restore source pointer
	end_timer 8

;; Do inverse FFT levels 1,2,3,4
;; On input the 128-byte cache line pairs hold these data values:
;;	0	+144	...	72	+144	...
;;	...
;;	71	...
;; These are swizzled to:
;;	0	+9	...	72	+9	...
;;	...
;;	9	...
;;	144	...
;;	288	...
;;	...
;; On output the 128-byte cache line pairs hold these data values:
;;	0	+9	...	576	+9	...
;;	...
;;	8	...
;;	72	...
;;	144	...
;;	...

;; Do 72 eight_complex_last_unfft operations
;;	distance between fft data elements is 72
;;	do 9 macros processing 128 data values

	start_timer 10
	zloop_init 9			;; 9 iterations
	zr8s_eight_complex_last_unfft_preload
c1b:	zr8s_eight_complex_last_unfft rsi, 128, 9*128, 18*128, 36*128, rdi, ZMM_CD8SCD4, 9, L1PREFETCH_NONE
	zloop	9, c1b, -9*128		;; Test loop counter, restore source pointer
	end_timer 10

	zfft_3_ret
	ENDM


;; Perform a 1280-element FFT

zfft1280 MACRO type
	LOCAL	b1b, b2b, b3a, b3b, zmid, c3a, c3b, c2b, c1b

;; Check for a multiply with pre-FFTed inputs

	cmp	ffttype, 4
	je	zmid

;; Do FFT levels 1,2,3,4
;; On input the 128-byte cache line pairs hold these data values:
;;	0	+10	...	640	+10	...
;;	...
;;	9	...
;;	80	...
;;	160	...
;;	...
;; On output the 128-byte cache line pairs hold these data values:
;;	0	+10	...	80	+10	...
;;	...
;;	9	...
;;	160	...
;;	320	...
;;	...
;; These are swizzled to:
;;	0	+160	...	80	+160	...
;;	...
;;	79	...

;; Do 80 sixteen_reals_fft operations
;;	distance between fft data elements is 80
;;	do 10 macros processing 128 data values

	start_timer 4
	mov	rdi, sincos1		;; Load sin/cos data ptr
	bump	rdi, 16*ZMM_SCD4+8*ZMM_SCD1
	mov	r9, norm_col_mults	;; Load weights pointer
	zloop_init 10			;; 10 iterations
	zr8fs_sixteen_reals_first_fft_preload
b1b:	zr8fs_sixteen_reals_first_fft rsi, 128, 10*128, 20*128, 40*128, rdi, ZMM_SCD7, r9, 8*128, 10, L1PREFETCH_NONE
	zloop	10, b1b, -10*128	;; Test loop counter, restore source pointer
	end_timer 4

;; Do FFT levels 5,6
;; On input the 64-byte cache lines hold these data values:
;;	0	+160	...	80	+160	...
;;	...
;;	79	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+160	...	16	+160	...
;;	...
;;	15
;;	32	...
;;	64	...
;;	...

;; Do 16 ten_reals_fft operation
;; Do 112 five_complex_fft operations
;;	distance between fft data elements is 16
;;	do 16 macros each processing 80 data values

;; Do FFT levels 7
;; On input the 64-byte cache lines hold these data values:
;;	0	+160	...	16	+160	...
;;	...
;;	15
;;	32	...
;;	64	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+160	...	8	+160	...
;;	...
;;	7
;;	16	...
;;	32	...
;;	...

;; Do 8 four_reals_fft operation
;; Do 312 two_complex_fft operations
;;	distance between fft data elements is 8
;;	do 40 macros each processing 32 data values

IFDEF OLDWAY
	start_timer 5
	bump	rdi, -10*ZMM_SCD7-16*ZMM_SCD4 ;; Load sin/cos pointer
	zloop_init 16			;; 16 iterations
	zr5_ten_reals_five_complex_djbfft_preload
b2b:	zr5_ten_reals_five_complex_djbfft rsi, 128, 16*128, rdi, ZMM_SCD4, 16
	zloop	16, b2b, -16*128	;; Test loop counter, restore source pointer
	end_timer 5

	start_timer 6
	bump	rdi, -16*ZMM_SCD4-8*ZMM_SCD1 ;; Load sin/cos ptr
	zloop_init 40			;; 5 iterations of 8
	zr2_four_reals_two_complex_djbfft_preload
b3a:	zr2_four_reals_two_complex_djbfft rsi, 128, 8*128, rdi, ZMM_SCD1, 8
	zloop	8, b3a, -8*128+16*128, rdi, -8*ZMM_SCD1 ;; Test loop counter, next source pointer, restore sin/cos pointer

	loops_reset			;; 4 more iters of 8
	zr2rb_two_complex_djbfft_preload
b3b:	zr2rb_two_complex_djbfft rsi, 128, 8*128, rdi, ZMM_SCD1, 8
	zloop	8, b3b, -8*128+16*128, rdi, -8*ZMM_SCD1 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	5, b3b, -5*16*128	;; Test loop counter, restore source pointer
	end_timer 6
ELSE
	start_timer 5
	lea	r10, [rdi-10*ZMM_SCD7-16*ZMM_SCD4] ;; Load sin/cos pointers
	lea	rdi, [r10-8*ZMM_SCD1]
	zloop_init 8			;; 8 iterations
	zr25_twenty_reals_ten_complex_djbfft_preload
b2b:	zr25_twenty_reals_ten_complex_djbfft rsi, 128, 8*128, r10, 8*ZMM_SCD4, ZMM_SCD4, rdi, 0*ZMM_SCD1, ZMM_SCD1, 8
	zloop	8, b2b, -8*128		;; Test loop counter, restore source pointer
	end_timer 5
ENDIF

;; Do FFT levels 8,9,10
;; On input and output the 128-byte cache line pairs hold these data values:
;;	0	+160	...	8	+160	...
;;	...
;;	7
;;	16	...
;;	32	...
;;	...

;; Do 1 sixteen_reals_fft operation
;; Do 79 eight_complex_fft operations
;;	distance between fft data elements is 1
;;	do 10 macros processing 128 data values

zmid:	start_timer 7
	mov	rcx, 10
	call	zreal_onepass_middle
	end_timer 7

;; Do inverse FFT levels 7
;; On input the 64-byte cache lines hold these data values:
;;	0	+160	...	8	+160	...
;;	...
;;	7
;;	16	...
;;	32	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+160	...	16	+160	...
;;	...
;;	15
;;	32	...
;;	64	...
;;	...

;; Do 8 four_reals_unfft operations
;; Do 312 two_complex_unfft operations
;;	distance between fft data elements is 8
;;	do 40 macros each processing 32 data values

;; Do inverse FFT levels 5,6
;; On input the 64-byte cache lines hold these data values:
;;	0	+160	...	16	+160	...
;;	...
;;	15
;;	32	...
;;	64	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+160	...	80	+160	...
;;	...
;;	79	...

;; Do 16 ten_reals_unfft operations
;; Do 112 five_complex_unfft operations
;;	distance between fft data elements is 16
;;	do 16 macros each processing 80 data values

IFDEF OLDWAY
	start_timer 8
	mov	rdi, sincos1		;; Load sin/cos pointer
	zloop_init 40			;; 5 iterations of 8
	zr2_four_reals_two_complex_djbunfft_preload
c3a:	zr2_four_reals_two_complex_djbunfft rsi, 128, 8*128, rdi, ZMM_SCD1, 8
	zloop	8, c3a, -8*128+16*128, rdi, -8*ZMM_SCD1 ;; Test loop counter, restore source pointer, restore sin/cos pointer

	loops_reset			;; 4 more iters of 8
	zr2rb_two_complex_djbunfft_preload
c3b:	zr2rb_two_complex_djbunfft rsi, 128, 8*128, rdi, ZMM_SCD1, 8
	zloop	8, c3b, -8*128+16*128, rdi, -8*ZMM_SCD1 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	5, c3b, -5*16*128, rdi, 8*ZMM_SCD1 ;; Test loop counter, restore source pointer, next sin/cos pointer
	end_timer 8

	start_timer 9
	zloop_init 16			;; 16 iterations
	zr5_ten_reals_five_complex_djbunfft_preload
c2b:	zr5_ten_reals_five_complex_djbunfft rsi, 128, 16*128, rdi, ZMM_SCD4, 16
	zloop	16, c2b, -16*128	;; Test loop counter, restore source pointer
	end_timer 9
ELSE
	start_timer 8
	mov	rdi, sincos1		;; Load sin/cos pointers
	lea	r10, [rdi+8*ZMM_SCD1]
	zloop_init 8			;; 8 iterations
	zr25_twenty_reals_ten_complex_djbunfft_preload
c3b:	zr25_twenty_reals_ten_complex_djbunfft rsi, 128, 8*128, rdi, 0*ZMM_SCD1, ZMM_SCD1, r10, 8*ZMM_SCD4, ZMM_SCD4, 8
	zloop	8, c3b, -8*128, rdi, 16*ZMM_SCD4 ;; Test loop counter, restore source pointer, next sin/cos pointer
	end_timer 8
ENDIF

;; Do inverse FFT levels 1,2,3,4
;; On input the 128-byte cache line pairs hold these data values:
;;	0	+160	...	80	+160	...
;;	...
;;	79	...
;; These are swizzled to:
;;	0	+10	...	80	+10	...
;;	...
;;	9	...
;;	160	...
;;	320	...
;;	...
;; On output the 128-byte cache line pairs hold these data values:
;;	0	+10	...	640	+10	...
;;	...
;;	9	...
;;	80	...
;;	160	...
;;	...

;; Do 80 sixteen_reals_unfft operations
;;	distance between fft data elements is 80
;;	do 10 macros processing 128 data values

	start_timer 10
	zloop_init 10			;; 10 iterations
	zr8s_sixteen_reals_last_unfft_preload
c1b:	zr8s_sixteen_reals_last_unfft rsi, 128, 10*128, 20*128, 40*128, rdi, ZMM_SCD7, 10, L1PREFETCH_NONE
	zloop	10, c1b, -10*128	;; Test loop counter, restore source pointer
	end_timer 10

	zfft_3_ret
	ENDM

;; Perform a 1280-element negacyclic FFT

zfft1280p MACRO type
	LOCAL	b1b, b2b, b3b, zmid, c3b, c2b, c1b

;; Do a multiply with pre-FFTed inputs

	cmp	ffttype, 4
	je	zmid

;; Do FFT levels 1,2,3,4
;; On input the 128-byte cache line pairs hold these data values:
;;	0	+10	...	640	+10	...
;;	...
;;	9	...
;;	80	...
;;	160	...
;;	...
;; On output the 128-byte cache line pairs hold these data values:
;;	0	+10	...	80	+10	...
;;	...
;;	9	...
;;	160	...
;;	320	...
;;	...
;; These are swizzled to:
;;	0	+160	...	80	+160	...
;;	...
;;	79	...

;; Do 80 eight_complex_first_fft operations
;;	distance between fft data elements is 80
;;	do 10 macros processing 128 data values

	start_timer 4
	mov	rdi, sincos1		;; Load sin/cos pointer
	bump	rdi, 16*ZMM_SCD2/8+8*ZMM_SCD1/8
	mov	r9, norm_col_mults	;; Load weights pointer
	zloop_init 10			;; 10 iterations
	zr8fs_eight_complex_first_fft_preload
b1b:	zr8fs_eight_complex_first_fft rsi, 128, 10*128, 20*128, 40*128, rdi, ZMM_CD8SCD4, r9, 8*128, 10, L1PREFETCH_NONE
	zloop	10, b1b, -10*128	;; Test loop counter, restore source pointer
	end_timer 4

;; Do FFT levels 5,6
;; On input the 64-byte cache lines hold these data values:
;;	0	+160	...	80	+160	...
;;	...
;;	79	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+160	...	16	+160	...
;;	...
;;	15
;;	32	...
;;	64	...
;;	...

;; Do 128 five_complex_fft operations
;;	distance between fft data elements is 16
;;	do 16 macros each processing 80 data values

;; Do FFT levels 7
;; On input the 64-byte cache lines hold these data values:
;;	0	+160	...	16	+160	...
;;	...
;;	15
;;	32	...
;;	64	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+160	...	8	+160	...
;;	...
;;	7
;;	16	...
;;	32	...
;;	...

;; Do 320 two_complex_fft operations
;;	distance between fft data elements is 8
;;	do 40 macros each processing 32 data values

IFDEF OLDWAY
	start_timer 5
	bump	rdi, -10*ZMM_CD8SCD4-16*ZMM_SCD2/8 ;; Load sin/cos pointer
	zloop_init 16			;; 16 iterations
	zr5b_five_complex_djbfft_preload
b2b:	zr5b_five_complex_djbfft rsi, 128, 16*128, rdi, ZMM_SCD2/8, 16
	zloop	16, b2b, -16*128	;; Test loop counter, restore source pointer
	end_timer 5

	start_timer 6
	bump	rdi, -16*ZMM_SCD2/8-8*ZMM_SCD1/8 ;; Load sin/cos pointer
	zloop_init 40			;; 5 iterations of 8
	zr2b_two_complex_djbfft_preload
b3b:	zr2b_two_complex_djbfft rsi, 128, 8*128, rdi, ZMM_SCD1/8, 8
	zloop	8, b3b, -8*128+16*128, rdi, -8*ZMM_SCD1/8 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	5, b3b, -5*16*128	;; Test loop counter, restore source pointer
	end_timer 6
ELSE
	start_timer 5
	lea	r10, [rdi-10*ZMM_CD8SCD4-16*ZMM_SCD2/8] ;; Load sin/cos pointers
	lea	rdi, [r10-8*ZMM_SCD1/8]
	zloop_init 8			;; 8 iterations
	zr25b_ten_complex_djbfft_preload
b2b:	zr25b_ten_complex_djbfft rsi, 128, 8*128, r10, 8*ZMM_SCD2/8, ZMM_SCD2/8, rdi, 0*ZMM_SCD1/8, ZMM_SCD1/8, 8
	zloop	8, b2b, -8*128		;; Test loop counter, restore source pointer
	end_timer 5
ENDIF

;; Do FFT levels 8,9,10
;; On input and output the 128-byte cache line pairs hold these data values:
;;	0	+160	...	8	+160	...
;;	...
;;	7
;;	16	...
;;	32	...
;;	...

;; Do 80 eight_complex_fft operations
;;	distance between fft data elements is 1
;;	do 10 macros processing 128 data values

zmid:	start_timer 7
	mov	rcx, 10
	call	zcomplex_onepass_middle
	end_timer 7

;; Do inverse FFT levels 7
;; On input the 64-byte cache lines hold these data values:
;;	0	+160	...	8	+160	...
;;	...
;;	7
;;	16	...
;;	32	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+160	...	16	+160	...
;;	...
;;	15
;;	32	...
;;	64	...
;;	...

;; Do 320 two_complex_unfft operations
;;	distance between fft data elements is 8
;;	do 40 macros each processing 32 data values

;; Do inverse FFT levels 5,6
;; On input the 64-byte cache lines hold these data values:
;;	0	+160	...	16	+160	...
;;	...
;;	15
;;	32	...
;;	64	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+160	...	80	+160	...
;;	...
;;	79	...

;; Do 128 five_complex_unfft operations
;;	distance between fft data elements is 16
;;	do 16 macros each processing 80 data values

IFDEF OLDWAY
	start_timer 8
	mov	rdi, sincos1		;; Load sin/cos pointer
	zloop_init 40			;; 5 iterations of 8
	zr2b_two_complex_djbunfft_preload
c3b:	zr2b_two_complex_djbunfft rsi, 128, 8*128, rdi, ZMM_SCD1/8, 8
	zloop	8, c3b, -8*128+16*128, rdi, -8*ZMM_SCD1/8 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	5, c3b, -5*16*128, rdi, 8*ZMM_SCD1/8 ;; Test loop counter, restore source pointer, next sin/cos pointer
	end_timer 8

	start_timer 9
	zloop_init 16			;; 16 iterations
	zr5b_five_complex_djbunfft_preload
c2b:	zr5b_five_complex_djbunfft rsi, 128, 16*128, rdi, ZMM_SCD2/8, 16
	zloop	16, c2b, -16*128	;; Test loop counter, restore source pointer
	end_timer 9
ELSE
	start_timer 8
	mov	rdi, sincos1		;; Load sin/cos pointers
	lea	r10, [rdi+8*ZMM_SCD1/8]
	zloop_init 8			;; 8 iterations
	zr25b_ten_complex_djbunfft_preload
c3b:	zr25b_ten_complex_djbunfft rsi, 128, 8*128, rdi, 0*ZMM_SCD1/8, ZMM_SCD1/8, r10, 8*ZMM_SCD2/8, ZMM_SCD2/8, 8
	zloop	8, c3b, -8*128, rdi, 16*ZMM_SCD2/8 ;; Test loop counter, restore source pointer, next sin/cos pointer
	end_timer 8
ENDIF

;; Do inverse FFT levels 1,2,3,4
;; On input the 128-byte cache line pairs hold these data values:
;;	0	+160	...	80	+160	...
;;	...
;;	79	...
;; These are swizzled to:
;;	0	+10	...	80	+10	...
;;	...
;;	9	...
;;	160	...
;;	320	...
;;	...
;; On output the 128-byte cache line pairs hold these data values:
;;	0	+10	...	640	+10	...
;;	...
;;	9	...
;;	80	...
;;	160	...
;;	...

;; Do 80 eight_complex_last_unfft operations
;;	distance between fft data elements is 80
;;	do 10 macros processing 128 data values

	start_timer 10
	zloop_init 10			;; 10 iterations
	zr8s_eight_complex_last_unfft_preload
c1b:	zr8s_eight_complex_last_unfft rsi, 128, 10*128, 20*128, 40*128, rdi, ZMM_CD8SCD4, 10, L1PREFETCH_NONE
	zloop	10, c1b, -10*128	;; Test loop counter, restore source pointer
	end_timer 10

	zfft_3_ret
	ENDM


;; Perform a 1536-element FFT

zfft1536 MACRO type
	LOCAL	b1b, b2b, b3a, b3b, zmid, c3a, c3b, c2b, c1b

;; Check for a multiply with pre-FFTed inputs

	cmp	ffttype, 4
	je	zmid

;; Do FFT levels 1,2,3,4
;; On input the 128-byte cache line pairs hold these data values:
;;	0	+12	...	768	+12	...
;;	...
;;	11	...
;;	96	...
;;	192	...
;;	...
;; On output the 128-byte cache line pairs hold these data values:
;;	0	+12	...	96	+12	...
;;	...
;;	11	...
;;	192	...
;;	384	...
;;	...
;; These are swizzled to:
;;	0	+192	...	96	+192	...
;;	...
;;	95	...

;; Do 96 sixteen_reals_fft operations
;;	distance between fft data elements is 96
;;	do 12 macros processing 128 data values

	start_timer 4
	mov	rdi, sincos1		;; Load sin/cos data ptr
	bump	rdi, 32*ZMM_SCD2+8*ZMM_SCD3
	mov	r9, norm_col_mults	;; Load weights pointer
	zloop_init 12			;; 12 iterations
	zr8fs_sixteen_reals_first_fft_preload
b1b:	zr8fs_sixteen_reals_first_fft rsi, 128, 12*128, 24*128, 48*128, rdi, ZMM_SCD7, r9, 8*128, 12, L1PREFETCH_ALL
	zloop	12, b1b, -12*128	;; Test loop counter, restore source pointer
	end_timer 4

;; Do FFT levels 5,6
;; On input the 64-byte cache lines hold these data values:
;;	0	+192	...	96	+192	...
;;	...
;;	95	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+192	...	32	+192	...
;;	...
;;	31
;;	64	...
;;	128	...
;;	...

;; Do 16 six_reals_fft operation
;; Do 240 three_complex_fft operations
;;	distance between fft data elements is 32
;;	do 32 macros each processing 48 data values

;; Do FFT levels 7,8
;; On input the 64-byte cache lines hold these data values:
;;	0	+192	...	32	+192	...
;;	...
;;	31
;;	64	...
;;	128	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+192	...	8	+192	...
;;	...
;;	7
;;	16	...
;;	32	...
;;	...

;; Do 8 eight_reals_fft operation
;; Do 184 four_complex_fft operations
;;	distance between fft data elements is 8
;;	do 24 macros each processing 64 data values

IFDEF OLDWAY
	start_timer 5
	bump	rdi, -12*ZMM_SCD7-32*ZMM_SCD2 ;; Load sin/cos pointer
	zloop_init 32			;; 32 iterations
	zr3_six_reals_three_complex_djbfft_preload
b2b:	zr3_six_reals_three_complex_djbfft rsi, 128, 32*128, rdi, ZMM_SCD2, 32
	zloop	32, b2b, -32*128	;; Test loop counter, restore source pointer
	end_timer 5

	start_timer 6
	bump	rdi, -32*ZMM_SCD2-8*ZMM_SCD3 ;; Load sin/cos ptr
	zloop_init 24			;; 3 iterations of 8
	zr4_eight_reals_four_complex_djbfft_preload
b3a:	zr4_eight_reals_four_complex_djbfft rsi, 128, 8*128, 16*128, rdi, ZMM_SCD3, 8
	zloop	8, b3a, -8*128+32*128, rdi, -8*ZMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer

	loops_reset			;; 2 more iters of 8
	zr4rb_four_complex_djbfft_preload
b3b:	zr4rb_four_complex_djbfft rsi, 128, 8*128, 16*128, rdi, ZMM_SCD3, 8
	zloop	8, b3b, -8*128+32*128, rdi, -8*ZMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	3, b3b, -3*32*128	;; Test loop counter, restore source pointer
	end_timer 6
ELSE
	start_timer 5
	lea	r10, [rdi-12*ZMM_SCD7-32*ZMM_SCD2] ;; Load sin/cos pointers
	lea	rdi, [r10-8*ZMM_SCD3]
	zloop_init 8			;; 8 iterations
	zr34_twentyfour_reals_twelve_complex_djbfft_preload
b2b:	zr34_twentyfour_reals_twelve_complex_djbfft rsi, 128, 8*128, r10, 8*ZMM_SCD2, ZMM_SCD2, rdi, 0*ZMM_SCD3, ZMM_SCD3, 8
	zloop	8, b2b, -8*128		;; Test loop counter, restore source pointer
	end_timer 5
ENDIF

;; Do FFT levels 9,10,11
;; On input and output the 128-byte cache line pairs hold these data values:
;;	0	+192	...	8	+192	...
;;	...
;;	7
;;	16	...
;;	32	...
;;	...

;; Do 1 sixteen_reals_fft operation
;; Do 95 eight_complex_fft operations
;;	distance between fft data elements is 1
;;	do 12 macros processing 128 data values

zmid:	start_timer 7
	mov	rcx, 12
	call	zreal_onepass_middle
	end_timer 7

;; Do inverse FFT levels 7,8
;; On input the 64-byte cache lines hold these data values:
;;	0	+192	...	8	+192	...
;;	...
;;	7
;;	16	...
;;	32	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+192	...	32	+192	...
;;	...
;;	31
;;	64	...
;;	128	...
;;	...

;; Do 8 eight_reals_unfft operations
;; Do 184 four_complex_unfft operations
;;	distance between fft data elements is 8
;;	do 24 macros each processing 64 data values

;; Do inverse FFT levels 5,6
;; On input the 64-byte cache lines hold these data values:
;;	0	+192	...	32	+192	...
;;	...
;;	31
;;	64	...
;;	128	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+192	...	96	+192	...
;;	...
;;	95	...

;; Do 16 six_reals_unfft operations
;; Do 240 three_complex_unfft operations
;;	distance between fft data elements is 32
;;	do 32 macros each processing 48 data values

IFDEF OLDWAY
	start_timer 8
	mov	rdi, sincos1		;; Load sin/cos pointer
	zloop_init 24			;; 3 iterations of 8
	zr4_eight_reals_four_complex_djbunfft_preload
c3a:	zr4_eight_reals_four_complex_djbunfft rsi, 128, 8*128, 16*128, rdi, ZMM_SCD3, 8
	zloop	8, c3a, -8*128+32*128, rdi, -8*ZMM_SCD3 ;; Test loop counter, restore source pointer, restore sin/cos pointer

	loops_reset			;; 2 more iters of 8
	zr4rb_four_complex_djbunfft_preload
c3b:	zr4rb_four_complex_djbunfft rsi, 128, 8*128, 16*128, rdi, ZMM_SCD3, 8
	zloop	8, c3b, -8*128+32*128, rdi, -8*ZMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	3, c3b, -3*32*128, rdi, 8*ZMM_SCD3 ;; Test loop counter, restore source pointer, next sin/cos pointer
	end_timer 8

	start_timer 9
	zloop_init 32			;; 32 iterations
	zr3_six_reals_three_complex_djbunfft_preload
c2b:	zr3_six_reals_three_complex_djbunfft rsi, 128, 32*128, rdi, ZMM_SCD2, 32
	zloop	32, c2b, -32*128	;; Test loop counter, restore source pointer
	end_timer 9
ELSE
	start_timer 8
	mov	rdi, sincos1		;; Load sin/cos pointers
	lea	r10, [rdi+8*ZMM_SCD3]
	zloop_init 8			;; 8 iterations
	zr34_twentyfour_reals_twelve_complex_djbunfft_preload
c2b:	zr34_twentyfour_reals_twelve_complex_djbunfft rsi, 128, 8*128, rdi, 0*ZMM_SCD3, ZMM_SCD3, r10, 8*ZMM_SCD2, ZMM_SCD2, 8
	zloop	8, c2b, -8*128, rdi, 32*ZMM_SCD2 ;; Test loop counter, restore source pointer, next sin/cos pointer
	end_timer 8
ENDIF

;; Do inverse FFT levels 1,2,3,4
;; On input the 128-byte cache line pairs hold these data values:
;;	0	+192	...	96	+192	...
;;	...
;;	95	...
;; These are swizzled to:
;;	0	+12	...	96	+12	...
;;	...
;;	11	...
;;	192	...
;;	384	...
;;	...
;; On output the 128-byte cache line pairs hold these data values:
;;	0	+12	...	768	+12	...
;;	...
;;	11	...
;;	96	...
;;	192	...
;;	...

;; Do 96 sixteen_reals_unfft operations
;;	distance between fft data elements is 96
;;	do 12 macros processing 128 data values

	start_timer 10
	zloop_init 12			;; 12 iterations
	zr8s_sixteen_reals_last_unfft_preload
c1b:	zr8s_sixteen_reals_last_unfft rsi, 128, 12*128, 24*128, 48*128, rdi, ZMM_SCD7, 12, L1PREFETCH_ALL
	zloop	12, c1b, -12*128	;; Test loop counter, restore source pointer
	end_timer 10

	zfft_3_ret
	ENDM

;; Perform a 1536-element negacyclic FFT

zfft1536p MACRO type
	LOCAL	b1b, b2b, b3b, zmid, c3b, c2b, c1b

;; Do a multiply with pre-FFTed inputs

	cmp	ffttype, 4
	je	zmid

;; Do FFT levels 1,2,3,4
;; On input the 128-byte cache line pairs hold these data values:
;;	0	+12	...	768	+12	...
;;	...
;;	11	...
;;	96	...
;;	192	...
;;	...
;; On output the 128-byte cache line pairs hold these data values:
;;	0	+12	...	96	+12	...
;;	...
;;	11	...
;;	192	...
;;	384	...
;;	...
;; These are swizzled to:
;;	0	+192	...	96	+192	...
;;	...
;;	95	...

;; Do 96 eight_complex_first_fft operations
;;	distance between fft data elements is 96
;;	do 12 macros processing 128 data values

	start_timer 4
	mov	rdi, sincos1		;; Load sin/cos pointer
	bump	rdi, 32*ZMM_SCD1/8+8*ZMM_SCD2/8
	mov	r9, norm_col_mults	;; Load weights pointer
	zloop_init 12			;; 12 iterations
	zr8fs_eight_complex_first_fft_preload
b1b:	zr8fs_eight_complex_first_fft rsi, 128, 12*128, 24*128, 48*128, rdi, ZMM_CD8SCD4, r9, 8*128, 12, L1PREFETCH_ALL
	zloop	12, b1b, -12*128	;; Test loop counter, restore source pointer
	end_timer 4

;; Do FFT levels 5,6
;; On input the 64-byte cache lines hold these data values:
;;	0	+192	...	96	+192	...
;;	...
;;	95	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+192	...	32	+192	...
;;	...
;;	31
;;	64	...
;;	128	...
;;	...

;; Do 256 three_complex_fft operations
;;	distance between fft data elements is 32
;;	do 32 macros each processing 48 data values

;; Do FFT levels 7,8
;; On input the 64-byte cache lines hold these data values:
;;	0	+192	...	32	+192	...
;;	...
;;	31
;;	64	...
;;	128	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+192	...	8	+192	...
;;	...
;;	7
;;	16	...
;;	32	...
;;	...

;; Do 192 four_complex_fft operations
;;	distance between fft data elements is 8
;;	do 24 macros each processing 64 data values

IFDEF OLDWAY
	start_timer 5
	bump	rdi, -12*ZMM_CD8SCD4-32*ZMM_SCD1/8 ;; Load sin/cos pointer
	zloop_init 32			;; 32 iterations
	zr3b_three_complex_djbfft_preload
b2b:	zr3b_three_complex_djbfft rsi, 128, 32*128, rdi, ZMM_SCD1/8, 32
	zloop	32, b2b, -32*128	;; Test loop counter, restore source pointer
	end_timer 5

	start_timer 6
	bump	rdi, -32*ZMM_SCD1/8-8*ZMM_SCD2/8 ;; Load sin/cos pointer
	zloop_init 24			;; 3 iterations of 8
	zr4b_four_complex_djbfft_preload
b3b:	zr4b_four_complex_djbfft rsi, 128, 8*128, 16*128, rdi, ZMM_SCD2/8, 8
	zloop	8, b3b, -8*128+32*128, rdi, -8*ZMM_SCD2/8 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	3, b3b, -3*32*128	;; Test loop counter, restore source pointer
	end_timer 6
ELSE
	start_timer 5
	lea	r10, [rdi-12*ZMM_CD8SCD4-32*ZMM_SCD1/8] ;; Load sin/cos pointers
	lea	rdi, [r10-8*ZMM_SCD2/8]
	zloop_init 8			;; 8 iterations
	zr34b_twelve_complex_djbfft_preload
b2b:	zr34b_twelve_complex_djbfft rsi, 128, 8*128, r10, 8*ZMM_SCD1/8, ZMM_SCD1/8, rdi, 0*ZMM_SCD2/8, ZMM_SCD2/8, 8
	zloop	8, b2b, -8*128		;; Test loop counter, restore source pointer
	end_timer 5
ENDIF

;; Do FFT levels 9,10,11
;; On input and output the 128-byte cache line pairs hold these data values:
;;	0	+192	...	8	+192	...
;;	...
;;	7
;;	16	...
;;	32	...
;;	...

;; Do 96 eight_complex_fft operations
;;	distance between fft data elements is 1
;;	do 12 macros processing 128 data values

zmid:	start_timer 7
	mov	rcx, 12
	call	zcomplex_onepass_middle
	end_timer 7

;; Do inverse FFT levels 7,8
;; On input the 64-byte cache lines hold these data values:
;;	0	+192	...	8	+192	...
;;	...
;;	7
;;	16	...
;;	32	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+192	...	32	+192	...
;;	...
;;	31
;;	64	...
;;	128	...
;;	...

;; Do 192 four_complex_unfft operations
;;	distance between fft data elements is 8
;;	do 24 macros each processing 64 data values

;; Do inverse FFT levels 5,6
;; On input the 64-byte cache lines hold these data values:
;;	0	+192	...	32	+192	...
;;	...
;;	31
;;	64	...
;;	128	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+192	...	96	+192	...
;;	...
;;	95	...

;; Do 256 three_complex_unfft operations
;;	distance between fft data elements is 32
;;	do 32 macros each processing 48 data values

IFDEF OLDWAY
	start_timer 8
	mov	rdi, sincos1		;; Load sin/cos pointer
	zloop_init 24			;; 3 iterations of 8
	zr4b_four_complex_djbunfft_preload
c3b:	zr4b_four_complex_djbunfft rsi, 128, 8*128, 16*128, rdi, ZMM_SCD2/8, 8
	zloop	8, c3b, -8*128+32*128, rdi, -8*ZMM_SCD2/8 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	3, c3b, -3*32*128, rdi, 8*ZMM_SCD2/8 ;; Test loop counter, restore source pointer, next sin/cos pointer
	end_timer 8

	start_timer 9
	zloop_init 32			;; 32 iterations
	zr3b_three_complex_djbunfft_preload
c2b:	zr3b_three_complex_djbunfft rsi, 128, 32*128, rdi, ZMM_SCD1/8, 32
	zloop	32, c2b, -32*128	;; Test loop counter, restore source pointer
	end_timer 9
ELSE
	start_timer 8
	mov	rdi, sincos1		;; Load sin/cos pointer
	lea	r10, [rdi+8*ZMM_SCD2/8]
	zloop_init 8			;; 8 iterations
	zr34b_twelve_complex_djbunfft_preload
c3b:	zr34b_twelve_complex_djbunfft rsi, 128, 8*128, rdi, 0*ZMM_SCD2/8, ZMM_SCD2/8, r10, 8*ZMM_SCD1/8, ZMM_SCD1/8, 8
	zloop	8, c3b, -8*128, rdi, 32*ZMM_SCD1/8 ;; Test loop counter, restore source pointer, next sin/cos pointer
	end_timer 8
ENDIF

;; Do inverse FFT levels 1,2,3,4
;; On input the 128-byte cache line pairs hold these data values:
;;	0	+192	...	96	+192	...
;;	...
;;	95	...
;; These are swizzled to:
;;	0	+12	...	96	+12	...
;;	...
;;	11	...
;;	192	...
;;	384	...
;;	...
;; On output the 128-byte cache line pairs hold these data values:
;;	0	+12	...	768	+12	...
;;	...
;;	11	...
;;	96	...
;;	192	...
;;	...

;; Do 96 eight_complex_last_unfft operations
;;	distance between fft data elements is 96
;;	do 12 macros processing 128 data values

	start_timer 10
	zloop_init 12			;; 12 iterations
	zr8s_eight_complex_last_unfft_preload
c1b:	zr8s_eight_complex_last_unfft rsi, 128, 12*128, 24*128, 48*128, rdi, ZMM_CD8SCD4, 12, L1PREFETCH_NONE
	zloop	12, c1b, -12*128	;; Test loop counter, restore source pointer
	end_timer 10

	zfft_3_ret
	ENDM


;; Perform a 1792-element FFT

zfft1792 MACRO type
	LOCAL	b1b, b2b, b3a, b3b, zmid, c3a, c3b, c2b, c1b

;; Check for a multiply with pre-FFTed inputs

	cmp	ffttype, 4
	je	zmid

;; Do FFT levels 1,2,3,4
;; On input the 128-byte cache line pairs hold these data values:
;;	0	+14	...	896	+14	...
;;	...
;;	13	...
;;	112	...
;;	224	...
;;	...
;; On output the 128-byte cache line pairs hold these data values:
;;	0	+14	...	112	+14	...
;;	...
;;	13	...
;;	224	...
;;	448	...
;;	...
;; These are swizzled to:
;;	0	+224	...	112	+224	...
;;	...
;;	111	...

;; Do 112 sixteen_reals_fft operations
;;	distance between fft data elements is 112
;;	do 14 macros processing 128 data values

	start_timer 4
	mov	rdi, sincos1		;; Load sin/cos data ptr
	bump	rdi, 16*ZMM_SCD6+8*ZMM_SCD1
	mov	r9, norm_col_mults	;; Load weights pointer
	zloop_init 14			;; 14 iterations
	zr8fs_sixteen_reals_first_fft_preload
b1b:	zr8fs_sixteen_reals_first_fft rsi, 128, 14*128, 28*128, 56*128, rdi, ZMM_SCD7, r9, 8*128, 14, L1PREFETCH_ALL
	zloop	14, b1b, -14*128	;; Test loop counter, restore source pointer
	end_timer 4

;; Do FFT levels 5,6
;; On input the 64-byte cache lines hold these data values:
;;	0	+224	...	112	+224	...
;;	...
;;	111	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+224	...	316	+224	...
;;	...
;;	15
;;	32	...
;;	64	...
;;	...

;; Do 16 fourteen_reals_fft operation
;; Do 112 seven_complex_fft operations
;;	distance between fft data elements is 16
;;	do 16 macros each processing 112 data values

	start_timer 5
	bump	rdi, -14*ZMM_SCD7-16*ZMM_SCD6 ;; Load sin/cos pointer
	zloop_init 16			;; 16 iterations
	zr7_fourteen_reals_seven_complex_djbfft_preload
b2b:	zr7_fourteen_reals_seven_complex_djbfft rsi, 128, 16*128, rdi, ZMM_SCD6, 16, L1PREFETCH_NONE
	zloop	16, b2b, -16*128	;; Test loop counter, restore source pointer
	end_timer 5

;; Do FFT levels 7
;; On input the 64-byte cache lines hold these data values:
;;	0	+224	...	316	+224	...
;;	...
;;	15
;;	32	...
;;	64	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+224	...	8	+224	...
;;	...
;;	7
;;	16	...
;;	32	...
;;	...

;; Do 8 four_reals_fft operation
;; Do 440 two_complex_fft operations
;;	distance between fft data elements is 8
;;	do 56 macros each processing 32 data values

	start_timer 6
	bump	rdi, -16*ZMM_SCD6-8*ZMM_SCD1 ;; Load sin/cos ptr
	zloop_init 56			;; 7 iterations of 8
	zr2_four_reals_two_complex_djbfft_preload
b3a:	zr2_four_reals_two_complex_djbfft rsi, 128, 8*128, rdi, ZMM_SCD1, 8
	zloop	8, b3a, -8*128+16*128, rdi, -8*ZMM_SCD1 ;; Test loop counter, next source pointer, restore sin/cos pointer

	loops_reset			;; 6 more iters of 8
	zr2rb_two_complex_djbfft_preload
b3b:	zr2rb_two_complex_djbfft rsi, 128, 8*128, rdi, ZMM_SCD1, 8
	zloop	8, b3b, -8*128+16*128, rdi, -8*ZMM_SCD1 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	7, b3b, -7*16*128	;; Test loop counter, restore source pointer
	end_timer 6

;; Do FFT levels 8,9,10
;; On input and output the 128-byte cache line pairs hold these data values:
;;	0	+224	...	8	+224	...
;;	...
;;	7
;;	16	...
;;	32	...
;;	...

;; Do 1 sixteen_reals_fft operation
;; Do 111 eight_complex_fft operations
;;	distance between fft data elements is 1
;;	do 14 macros processing 128 data values

zmid:	start_timer 7
	mov	rcx, 14
	call	zreal_onepass_middle
	end_timer 7

;; Do inverse FFT levels 7
;; On input the 64-byte cache lines hold these data values:
;;	0	+224	...	8	+224	...
;;	...
;;	7
;;	16	...
;;	32	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+224	...	316	+224	...
;;	...
;;	15
;;	32	...
;;	64	...
;;	...

;; Do 8 four_reals_unfft operations
;; Do 440 two_complex_unfft operations
;;	distance between fft data elements is 8
;;	do 56 macros each processing 32 data values

	start_timer 8
	mov	rdi, sincos1		;; Load sin/cos pointer
	zloop_init 56			;; 7 iterations of 8
	zr2_four_reals_two_complex_djbunfft_preload
c3a:	zr2_four_reals_two_complex_djbunfft rsi, 128, 8*128, rdi, ZMM_SCD1, 8
	zloop	8, c3a, -8*128+16*128, rdi, -8*ZMM_SCD1 ;; Test loop counter, restore source pointer, restore sin/cos pointer

	loops_reset			;; 6 more iters of 8
	zr2rb_two_complex_djbunfft_preload
c3b:	zr2rb_two_complex_djbunfft rsi, 128, 8*128, rdi, ZMM_SCD1, 8
	zloop	8, c3b, -8*128+16*128, rdi, -8*ZMM_SCD1 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	7, c3b, -7*16*128, rdi, 8*ZMM_SCD1 ;; Test loop counter, restore source pointer, next sin/cos pointer
	end_timer 8

;; Do inverse FFT levels 5,6
;; On input the 64-byte cache lines hold these data values:
;;	0	+224	...	316	+224	...
;;	...
;;	15
;;	32	...
;;	64	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+224	...	112	+224	...
;;	...
;;	111	...

;; Do 16 fourteen_reals_unfft operations
;; Do 112 seven_complex_unfft operations
;;	distance between fft data elements is 16
;;	do 16 macros each processing 112 data values

	start_timer 9
	zloop_init 16			;; 16 iterations
	zr7_fourteen_reals_seven_complex_djbunfft_preload
c2b:	zr7_fourteen_reals_seven_complex_djbunfft rsi, 128, 16*128, rdi, ZMM_SCD6, 16, L1PREFETCH_NONE
	zloop	16, c2b, -16*128	;; Test loop counter, restore source pointer
	end_timer 9

;; Do inverse FFT levels 1,2,3,4
;; On input the 128-byte cache line pairs hold these data values:
;;	0	+224	...	112	+224	...
;;	...
;;	111	...
;; These are swizzled to:
;;	0	+14	...	112	+14	...
;;	...
;;	13	...
;;	224	...
;;	448	...
;;	...
;; On output the 128-byte cache line pairs hold these data values:
;;	0	+14	...	896	+14	...
;;	...
;;	13	...
;;	112	...
;;	224	...
;;	...

;; Do 112 sixteen_reals_unfft operations
;;	distance between fft data elements is 112
;;	do 14 macros processing 128 data values

	start_timer 10
	zloop_init 14			;; 14 iterations
	zr8s_sixteen_reals_last_unfft_preload
c1b:	zr8s_sixteen_reals_last_unfft rsi, 128, 14*128, 28*128, 56*128, rdi, ZMM_SCD7, 14, L1PREFETCH_ALL
	zloop	14, c1b, -14*128	;; Test loop counter, restore source pointer
	end_timer 10

	zfft_3_ret
	ENDM

;; Perform a 1792-element negacyclic FFT

zfft1792p MACRO type
	LOCAL	b1b, b2b, b3b, zmid, c3b, c2b, c1b

;; Do a multiply with pre-FFTed inputs

	cmp	ffttype, 4
	je	zmid

;; Do FFT levels 1,2,3,4
;; On input the 128-byte cache line pairs hold these data values:
;;	0	+14	...	896	+14	...
;;	...
;;	13	...
;;	112	...
;;	224	...
;;	...
;; On output the 128-byte cache line pairs hold these data values:
;;	0	+14	...	112	+14	...
;;	...
;;	13	...
;;	224	...
;;	448	...
;;	...
;; These are swizzled to:
;;	0	+224	...	112	+224	...
;;	...
;;	111	...

;; Do 112 eight_complex_first_fft operations
;;	distance between fft data elements is 112
;;	do 14 macros processing 128 data values

	start_timer 4
	mov	rdi, sincos1		;; Load sin/cos pointer
	bump	rdi, 16*ZMM_SCD3/8+8*ZMM_SCD1/8
	mov	r9, norm_col_mults	;; Load weights pointer
	zloop_init 14			;; 14 iterations
	zr8fs_eight_complex_first_fft_preload
b1b:	zr8fs_eight_complex_first_fft rsi, 128, 14*128, 28*128, 56*128, rdi, ZMM_CD8SCD4, r9, 8*128, 14, L1PREFETCH_ALL
	zloop	14, b1b, -14*128	;; Test loop counter, restore source pointer
	end_timer 4

;; Do FFT levels 5,6
;; On input the 64-byte cache lines hold these data values:
;;	0	+224	...	112	+224	...
;;	...
;;	111	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+224	...	316	+224	...
;;	...
;;	15
;;	32	...
;;	64	...
;;	...

;; Do 128 seven_complex_fft operations
;;	distance between fft data elements is 16
;;	do 16 macros each processing 112 data values

	start_timer 5
	bump	rdi, -14*ZMM_CD8SCD4-16*ZMM_SCD3/8 ;; Load sin/cos pointer
	zloop_init 16			;; 16 iterations
	zr7b_seven_complex_djbfft_preload
b2b:	zr7b_seven_complex_djbfft rsi, 128, 16*128, rdi, ZMM_SCD3/8, 16, L1PREFETCH_NONE
	zloop	16, b2b, -16*128	;; Test loop counter, restore source pointer
	end_timer 5

;; Do FFT levels 7
;; On input the 64-byte cache lines hold these data values:
;;	0	+224	...	316	+224	...
;;	...
;;	15
;;	32	...
;;	64	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+224	...	8	+224	...
;;	...
;;	7
;;	16	...
;;	32	...
;;	...

;; Do 448 two_complex_fft operations
;;	distance between fft data elements is 8
;;	do 56 macros each processing 32 data values

	start_timer 6
	bump	rdi, -16*ZMM_SCD3/8-8*ZMM_SCD1/8 ;; Load sin/cos pointer
	zloop_init 56			;; 7 iterations of 8
	zr2b_two_complex_djbfft_preload
b3b:	zr2b_two_complex_djbfft rsi, 128, 8*128, rdi, ZMM_SCD1/8, 8
	zloop	8, b3b, -8*128+16*128, rdi, -8*ZMM_SCD1/8 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	7, b3b, -7*16*128	;; Test loop counter, restore source pointer
	end_timer 6

;; Do FFT levels 8,9,10
;; On input and output the 128-byte cache line pairs hold these data values:
;;	0	+224	...	8	+224	...
;;	...
;;	7
;;	16	...
;;	32	...
;;	...

;; Do 112 eight_complex_fft operations
;;	distance between fft data elements is 1
;;	do 14 macros processing 128 data values

zmid:	start_timer 7
	mov	rcx, 14
	call	zcomplex_onepass_middle
	end_timer 7

;; Do inverse FFT levels 7
;; On input the 64-byte cache lines hold these data values:
;;	0	+224	...	8	+224	...
;;	...
;;	7
;;	16	...
;;	32	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+224	...	316	+224	...
;;	...
;;	15
;;	32	...
;;	64	...
;;	...

;; Do 448 two_complex_unfft operations
;;	distance between fft data elements is 8
;;	do 56 macros each processing 32 data values

	start_timer 8
	mov	rdi, sincos1		;; Load sin/cos pointer
	zloop_init 56			;; 7 iterations of 8
	zr2b_two_complex_djbunfft_preload
c3b:	zr2b_two_complex_djbunfft rsi, 128, 8*128, rdi, ZMM_SCD1/8, 8
	zloop	8, c3b, -8*128+16*128, rdi, -8*ZMM_SCD1/8 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	7, c3b, -7*16*128, rdi, 8*ZMM_SCD1/8 ;; Test loop counter, restore source pointer, next sin/cos pointer
	end_timer 8

;; Do inverse FFT levels 5,6
;; On input the 64-byte cache lines hold these data values:
;;	0	+224	...	316	+224	...
;;	...
;;	15
;;	32	...
;;	64	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+224	...	112	+224	...
;;	...
;;	111	...

;; Do 128 seven_complex_unfft operations
;;	distance between fft data elements is 16
;;	do 16 macros each processing 112 data values

	start_timer 9
	zloop_init 16			;; 16 iterations
	zr7b_seven_complex_djbunfft_preload
c2b:	zr7b_seven_complex_djbunfft rsi, 128, 16*128, rdi, ZMM_SCD3/8, 16, L1PREFETCH_NONE
	zloop	16, c2b, -16*128	;; Test loop counter, restore source pointer
	end_timer 9

;; Do inverse FFT levels 1,2,3,4
;; On input the 128-byte cache line pairs hold these data values:
;;	0	+224	...	112	+224	...
;;	...
;;	111	...
;; These are swizzled to:
;;	0	+14	...	112	+14	...
;;	...
;;	13	...
;;	224	...
;;	448	...
;;	...
;; On output the 128-byte cache line pairs hold these data values:
;;	0	+14	...	896	+14	...
;;	...
;;	13	...
;;	112	...
;;	224	...
;;	...

;; Do 112 eight_complex_last_unfft operations
;;	distance between fft data elements is 112
;;	do 14 macros processing 128 data values

	start_timer 10
	zloop_init 14			;; 14 iterations
	zr8s_eight_complex_last_unfft_preload
c1b:	zr8s_eight_complex_last_unfft rsi, 128, 14*128, 28*128, 56*128, rdi, ZMM_CD8SCD4, 14, L1PREFETCH_NONE
	zloop	14, c1b, -14*128	;; Test loop counter, restore source pointer
	end_timer 10

	zfft_3_ret
	ENDM


;; Perform a 2K-element FFT

zfft2K MACRO type
	LOCAL	b1b, b2b, b3a, b3b, zmid, c3a, c3b, c2b, c1b

;; Check for a multiply with pre-FFTed inputs

	cmp	ffttype, 4
	je	zmid

;; Do FFT levels 1,2,3,4
;; On input the 128-byte cache line pairs hold these data values:
;;	0	+16	...	1024	+16	...
;;	...
;;	15	...
;;	128	...
;;	256	...
;;	...
;; On output the 128-byte cache line pairs hold these data values:
;;	0	+16	...	128	+16	...
;;	...
;;	15	...
;;	256	...
;;	512	...
;;	...
;; These are swizzled to:
;;	0	+256	...	128	+256	...
;;	...
;;	127	...

;; Do 128 sixteen_reals_fft operations
;;	distance between fft data elements is 128
;;	do 16 macros processing 128 data values

;; Do FFT level 5,6,7
;; On input the 64-byte cache lines hold these data values:
;;	0	+256	...	128	+256	...
;;	...
;;	127	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+256	...	16	+256	...
;;	...
;;	15
;;	32	...
;;	64	...
;;	...

;; Do 16 sixteen_reals_fft operation
;; Do 112 eight_complex_fft operations
;;	distance between fft data elements is 16
;;	do 16 macros each processing 128 data values

IFDEF OLDWAY
	start_timer 4
	mov	rdi, sincos1		;; Load sin/cos data ptr
	bump	rdi, 16*ZMM_SCD7+8*ZMM_SCD1
	mov	r9, norm_col_mults	;; Load weights pointer
	zloop_init 16			;; 16 iterations
	zr8fs_sixteen_reals_first_fft_preload
b1b:	zr8fs_sixteen_reals_first_fft rsi, 128, 16*128, 32*128, 64*128, rdi, ZMM_SCD7, r9, 8*128, 16, L1PREFETCH_ALL
	zloop	16, b1b, -16*128	;; Test loop counter, restore source pointer
	end_timer 4

	start_timer 5
	bump	rdi, -16*ZMM_SCD7-16*ZMM_SCD7 ;; Load sin/cos pointer
	zloop_init 16			;; 16 iterations
	zr8_sixteen_reals_eight_complex_djbfft_preload
b2b:	zr8_sixteen_reals_eight_complex_djbfft rsi, 128, 16*128, 32*128, 64*128, rdi, ZMM_SCD7, 16, L1PREFETCH_NONE
	zloop	16, b2b, -16*128	;; Test loop counter, next source pointer
	end_timer 5
ELSE
	start_timer 4
	mov	rdi, sincos1		;; Load sin/cos data ptr
	bump	rdi, 8*ZMM_SCD1
	lea	r10, [rdi+16*ZMM_SCD7]
	mov	r9, norm_col_mults	;; Load weights pointer
	zloop_init 16			;; 16 iterations
	zr8fs_hundredtwentyeight_reals_first_fft_preload
b1b:	zr8fs_hundredtwentyeight_reals_first_fft rsi, 128, 16*128, 32*128, 64*128, r10, ZMM_SCD7, rdi, ZMM_SCD7, r9, 8*128, 16, L1PREFETCH_ALL
	zloop	16, b1b, -16*128	;; Test loop counter, restore source pointer
	end_timer 4
ENDIF

;; Do FFT level 8
;; On input the 64-byte cache lines hold these data values:
;;	0	+256	...	16	+256	...
;;	...
;;	15
;;	32	...
;;	64	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+256	...	8	+256	...
;;	...
;;	7
;;	16	...
;;	32	...
;;	...

;; Do 8 four_reals_fft operation
;; Do 504 two_complex_fft operations
;;	distance between fft data elements is 8
;;	do 64 macros each processing 32 data values

	start_timer 6
	bump	rdi, -16*ZMM_SCD7-8*ZMM_SCD1 ;; Load sin/cos ptr
	zloop_init 64			;; 8 iterations of 8
	zr2_four_reals_two_complex_djbfft_preload
b3a:	zr2_four_reals_two_complex_djbfft rsi, 128, 8*128, rdi, ZMM_SCD1, 8
	zloop	8, b3a, -8*128+16*128, rdi, -8*ZMM_SCD1 ;; Test loop counter, next source pointer, restore sin/cos pointer

	loops_reset			;; 7 more iters of 8
	zr2rb_two_complex_djbfft_preload
b3b:	zr2rb_two_complex_djbfft rsi, 128, 8*128, rdi, ZMM_SCD1, 8
	zloop	8, b3b, -8*128+16*128, rdi, -8*ZMM_SCD1 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	8, b3b, -8*16*128	;; Test loop counter, restore source pointer
	end_timer 6

;; Do FFT levels 9,10,11
;; On input and output the 128-byte cache line pairs hold these data values:
;;	0	+256	...	8	+256	...
;;	...
;;	7
;;	16	...
;;	32	...
;;	...

;; Do 1 sixteen_reals_fft operation
;; Do 127 eight_complex_fft operations
;;	distance between fft data elements is 1
;;	do 16 macros processing 128 data values

zmid:	start_timer 7
	mov	rcx, 16
	call	zreal_onepass_middle
	end_timer 7

;; Do inverse FFT level 8
;; On input the 64-byte cache lines hold these data values:
;;	0	+256	...	8	+256	...
;;	...
;;	7
;;	16	...
;;	32	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+256	...	16	+256	...
;;	...
;;	15
;;	32	...
;;	64	...
;;	...

;; Do 8 four_reals_unfft operations
;; Do 504 two_complex_unfft operations
;;	distance between fft data elements is 8
;;	do 64 macros each processing 32 data values

	start_timer 8
	mov	rdi, sincos1		;; Load sin/cos pointer
	zloop_init 64			;; 8 iterations of 8
	zr2_four_reals_two_complex_djbunfft_preload
c3a:	zr2_four_reals_two_complex_djbunfft rsi, 128, 8*128, rdi, ZMM_SCD1, 8
	zloop	8, c3a, -8*128+16*128, rdi, -8*ZMM_SCD1 ;; Test loop counter, next source pointer, restore sin/cos pointer

	loops_reset			;; 7 more iters of 8
	zr2rb_two_complex_djbunfft_preload
c3b:	zr2rb_two_complex_djbunfft rsi, 128, 8*128, rdi, ZMM_SCD1, 8
	zloop	8, c3b, -8*128+16*128, rdi, -8*ZMM_SCD1 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	8, c3b, -8*16*128, rdi, 8*ZMM_SCD1 ;; Test loop counter, restore source pointer, next sin/cos pointer
	end_timer 8

;; Do inverse FFT level 5,6,7
;; On input the 64-byte cache lines hold these data values:
;;	0	+256	...	16	+256	...
;;	...
;;	15
;;	32	...
;;	64	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+256	...	128	+256	...
;;	...
;;	127	...

;; Do 16 sixteen_reals_unfft operations
;; Do 112 eight_complex_unfft operations
;;	distance between fft data elements is 16
;;	do 16 macros each processing 128 data values

;; Do inverse FFT levels 1,2,3,4
;; On input the 128-byte cache line pairs hold these data values:
;;	0	+256	...	128	+256	...
;;	...
;;	127	...
;; These are swizzled to:
;;	0	+16	...	128	+16	...
;;	...
;;	15	...
;;	256	...
;;	512	...
;;	...
;; On output the 128-byte cache line pairs hold these data values:
;;	0	+16	...	1024	+16	...
;;	...
;;	15	...
;;	128	...
;;	256	...
;;	...

;; Do 128 sixteen_reals_unfft operations
;;	distance between fft data elements is 128
;;	do 16 macros processing 128 data values

IFDEF OLDWAY
	start_timer 9
	zloop_init 16			;; 16 iterations
	zr8_sixteen_reals_eight_complex_djbunfft_preload
c2b:	zr8_sixteen_reals_eight_complex_djbunfft rsi, 128, 16*128, 32*128, 64*128, rdi, ZMM_SCD7, 16, L1PREFETCH_NONE
	zloop	16, c2b, -16*128	;; Test loop counter, restore source pointer
	end_timer 9

	start_timer 10
	zloop_init 16			;; 16 iterations
	zr8s_sixteen_reals_last_unfft_preload
c1b:	zr8s_sixteen_reals_last_unfft rsi, 128, 16*128, 32*128, 64*128, rdi, ZMM_SCD7, 16, L1PREFETCH_ALL
	zloop	16, c1b, -16*128	;; Test loop counter, restore source pointer
	end_timer 10
ELSE
	start_timer 10
	lea	r10, [rdi+16*ZMM_SCD7]
	zloop_init 16			;; 16 iterations
	zr8s_hundredtwentyeight_reals_last_unfft_preload
c1b:	zr8s_hundredtwentyeight_reals_last_unfft rsi, 128, 16*128, 32*128, 64*128, rdi, ZMM_SCD7, r10, ZMM_SCD7, 16, L1PREFETCH_ALL
	zloop	16, c1b, -16*128	;; Test loop counter, restore source pointer
	end_timer 10
ENDIF

	zfft_3_ret
	ENDM

;; Perform a 2K-element negacyclic FFT

zfft2Kp MACRO type
	LOCAL	b1b, b2b, b3b, zmid, c3b, c2b, c1b

;; Do a multiply with pre-FFTed inputs

	cmp	ffttype, 4
	je	zmid

;; Do FFT levels 1,2,3,4
;; On input the 128-byte cache line pairs hold these data values:
;;	0	+16	...	1024	+16	...
;;	...
;;	15	...
;;	128	...
;;	256	...
;;	...
;; On output the 128-byte cache line pairs hold these data values:
;;	0	+16	...	128	+16	...
;;	...
;;	15	...
;;	256	...
;;	512	...
;;	...
;; These are swizzled to:
;;	0	+256	...	128	+256	...
;;	...
;;	127	...

;; Do 128 eight_complex_first_fft operations
;;	distance between fft data elements is 128
;;	do 16 macros processing 128 data values

;; Do FFT level 5,6,7
;; On input the 64-byte cache lines hold these data values:
;;	0	+256	...	128	+256	...
;;	...
;;	127	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+256	...	16	+256	...
;;	...
;;	15
;;	32	...
;;	64	...
;;	...

;; Do 128 eight_complex_fft operations
;;	distance between fft data elements is 16
;;	do 16 macros each processing 128 data values

IFDEF OLDWAY
	start_timer 4
	mov	rdi, sincos1		;; Load sin/cos pointer
	bump	rdi, 16*ZMM_SCD4/8+8*ZMM_SCD1/8
	mov	r9, norm_col_mults	;; Load weights pointer
	zloop_init 16			;; 16 iterations
	zr8fs_eight_complex_first_fft_preload
b1b:	zr8fs_eight_complex_first_fft rsi, 128, 16*128, 32*128, 64*128, rdi, ZMM_CD8SCD4, r9, 8*128, 16, L1PREFETCH_ALL
	zloop	16, b1b, -16*128	;; Test loop counter, restore source pointer
	end_timer 4

	start_timer 5
	bump	rdi, -16*ZMM_CD8SCD4-16*ZMM_SCD4/8 ;; Load sin/cos pointer
	zloop_init 16			;; 16 iterations
	zr8b_eight_complex_djbfft_preload
b2b:	zr8b_eight_complex_djbfft rsi, 128, 16*128, 32*128, 64*128, rdi, ZMM_SCD4/8, 16, L1PREFETCH_NONE
	zloop	16, b2b, -16*128	;; Test loop counter, restore source pointer
	end_timer 5
ELSE
	start_timer 4
	mov	rdi, sincos1		;; Load sin/cos pointer
	bump	rdi, 8*ZMM_SCD1/8
	lea	r10, [rdi+16*ZMM_SCD4/8]
	mov	r9, norm_col_mults	;; Load weights pointer
	zloop_init 16			;; 16 iterations
	zr8fs_sixtyfour_complex_first_fft_preload
b1b:	zr8fs_sixtyfour_complex_first_fft rsi, 128, 16*128, 32*128, 64*128, r10, ZMM_CD8SCD4, rdi, ZMM_SCD4/8, r9, 8*128, 16, L1PREFETCH_ALL
	zloop	16, b1b, -16*128	;; Test loop counter, restore source pointer
	end_timer 4
ENDIF

;; Do FFT level 8
;; On input the 64-byte cache lines hold these data values:
;;	0	+256	...	16	+256	...
;;	...
;;	15
;;	32	...
;;	64	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+256	...	8	+256	...
;;	...
;;	7
;;	16	...
;;	32	...
;;	...

;; Do 512 two_complex_fft operations
;;	distance between fft data elements is 8
;;	do 64 macros each processing 32 data values

	start_timer 6
	bump	rdi, -16*ZMM_SCD4/8-8*ZMM_SCD1/8 ;; Load sin/cos pointer
	zloop_init 64			;; 8 iterations of 8
	zr2b_two_complex_djbfft_preload
b3b:	zr2b_two_complex_djbfft rsi, 128, 8*128, rdi, ZMM_SCD1/8, 8
	zloop	8, b3b, -8*128+16*128, rdi, -8*ZMM_SCD1/8 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	8, b3b, -8*16*128	;; Test loop counter, restore source pointer
	end_timer 6

;; Do FFT levels 9,10,11
;; On input and output the 128-byte cache line pairs hold these data values:
;;	0	+256	...	8	+256	...
;;	...
;;	7
;;	16	...
;;	32	...
;;	...

;; Do 128 eight_complex_fft operations
;;	distance between fft data elements is 1
;;	do 16 macros processing 128 data values

zmid:	start_timer 7
	mov	rcx, 16
	call	zcomplex_onepass_middle
	end_timer 7

;; Do inverse FFT level 8
;; On input the 64-byte cache lines hold these data values:
;;	0	+256	...	8	+256	...
;;	...
;;	7
;;	16	...
;;	32	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+256	...	16	+256	...
;;	...
;;	15
;;	32	...
;;	64	...
;;	...

;; Do 512 two_complex_unfft operations
;;	distance between fft data elements is 8
;;	do 64 macros each processing 32 data values

	start_timer 8
	mov	rdi, sincos1		;; Load sin/cos pointer
	zloop_init 64			;; 8 iterations of 8
	zr2b_two_complex_djbunfft_preload
c3b:	zr2b_two_complex_djbunfft rsi, 128, 8*128, rdi, ZMM_SCD1/8, 8
	zloop	8, c3b, -8*128+16*128, rdi, -8*ZMM_SCD1/8 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	8, c3b, -8*16*128, rdi, 8*ZMM_SCD1/8 ;; Test loop counter, restore source pointer, next sin/cos pointer
	end_timer 8

;; Do inverse FFT level 5,6,7
;; On input the 64-byte cache lines hold these data values:
;;	0	+256	...	16	+256	...
;;	...
;;	15
;;	32	...
;;	64	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+256	...	128	+256	...
;;	...
;;	127	...

;; Do 128 eight_complex_unfft operations
;;	distance between fft data elements is 16
;;	do 16 macros each processing 128 data values

;; Do inverse FFT levels 1,2,3,4
;; On input the 128-byte cache line pairs hold these data values:
;;	0	+256	...	128	+256	...
;;	...
;;	127	...
;; These are swizzled to:
;;	0	+16	...	128	+16	...
;;	...
;;	15	...
;;	256	...
;;	512	...
;;	...
;; On output the 128-byte cache line pairs hold these data values:
;;	0	+16	...	1024	+16	...
;;	...
;;	15	...
;;	128	...
;;	256	...
;;	...

;; Do 128 eight_complex_last_unfft operations
;;	distance between fft data elements is 128
;;	do 16 macros processing 128 data values

IFDEF OLDWAY
	start_timer 9
	zloop_init 16			;; 16 iterations
	zr8b_eight_complex_djbunfft_preload
c2b:	zr8b_eight_complex_djbunfft rsi, 128, 16*128, 32*128, 64*128, rdi, ZMM_SCD4/8, 16, L1PREFETCH_NONE
	zloop	16, c2b, -16*128	;; Test loop counter, restore source pointer
	end_timer 9

	start_timer 10
	zloop_init 16			;; 16 iterations
	zr8s_eight_complex_last_unfft_preload
c1b:	zr8s_eight_complex_last_unfft rsi, 128, 16*128, 32*128, 64*128, rdi, ZMM_CD8SCD4, 16, L1PREFETCH_NONE
	zloop	16, c1b, -16*128	;; Test loop counter, restore source pointer
	end_timer 10
ELSE
	start_timer 10
	lea	r10, [rdi+16*ZMM_SCD4/8]
	zloop_init 16			;; 16 iterations
	zr8s_sixtyfour_complex_last_unfft_preload
c1b:	zr8s_sixtyfour_complex_last_unfft rsi, 128, 16*128, 32*128, 64*128, rdi, ZMM_SCD4/8, r10, ZMM_CD8SCD4, 16, L1PREFETCH_NONE
	zloop	16, c1b, -16*128	;; Test loop counter, restore source pointer
	end_timer 10
ENDIF

	zfft_3_ret
	ENDM


;; Perform a 2304-element FFT

zfft2304 MACRO type
	LOCAL	b1b, b2b, b3a, b3b, zmid, c3a, c3b, c2b, c1b

;; Check for a multiply with pre-FFTed inputs

	cmp	ffttype, 4
	je	zmid

;; Do FFT levels 1,2,3,4
;; On input the 128-byte cache line pairs hold these data values:
;;	0	+18	...	1152	+18	...
;;	...
;;	17	...
;;	144	...
;;	288	...
;;	...
;; On output the 128-byte cache line pairs hold these data values:
;;	0	+18	...	144	+18	...
;;	...
;;	17	...
;;	288	...
;;	576	...
;;	...
;; These are swizzled to:
;;	0	+288	...	144	+288	...
;;	...
;;	143	...

;; Do 144 sixteen_reals_fft operations
;;	distance between fft data elements is 144
;;	do 18 macros processing 128 data values

	start_timer 4
	mov	rdi, sincos1		;; Load sin/cos data ptr
	bump	rdi, 48*ZMM_SCD2+8*ZMM_SCD5
	mov	r9, norm_col_mults	;; Load weights pointer
	zloop_init 18			;; 18 iterations
	zr8fs_sixteen_reals_first_fft_preload
b1b:	zr8fs_sixteen_reals_first_fft rsi, 128, 18*128, 36*128, 72*128, rdi, ZMM_SCD7, r9, 8*128, 18, L1PREFETCH_ALL
	zloop	18, b1b, -18*128	;; Test loop counter, restore source pointer
	end_timer 4

;; Do FFT levels 5,6
;; On input the 64-byte cache lines hold these data values:
;;	0	+288	...	144	+288	...
;;	...
;;	143	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+288	...	48	+288	...
;;	...
;;	47
;;	96	...
;;	192	...
;;	...

;; Do 24 six_reals_fft operation
;; Do 360 three_complex_fft operations
;;	distance between fft data elements is 48
;;	do 48 macros each processing 48 data values

	start_timer 5
	bump	rdi, -18*ZMM_SCD7-48*ZMM_SCD2 ;; Load sin/cos pointer
	zloop_init 48			;; 48 iterations
	zr3_six_reals_three_complex_djbfft_preload
b2b:	zr3_six_reals_three_complex_djbfft rsi, 128, 48*128, rdi, ZMM_SCD2, 48
	zloop	48, b2b, -48*128	;; Test loop counter, restore source pointer
	end_timer 5

;; Do FFT levels 7,8,9
;; On input the 64-byte cache lines hold these data values:
;;	0	+288	...	48	+288	...
;;	...
;;	47
;;	96	...
;;	192	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+288	...	8	+288	...
;;	...
;;	7
;;	16	...
;;	32	...
;;	...

;; Do 8 twelve_reals_fft operation
;; Do 184 six_complex_fft operations
;;	distance between fft data elements is 8
;;	do 24 macros each processing 96 data values

	start_timer 6
	bump	rdi, -48*ZMM_SCD2-8*ZMM_SCD5	;; Load sin/cos ptr
	zloop_init 24			;; 3 iterations of 8
	zr6_twelve_reals_six_complex_djbfft_preload
b3a:	zr6_twelve_reals_six_complex_djbfft rsi, 128, 8*128, rdi, ZMM_SCD5, 8, L1PREFETCH_NONE
	zloop	8, b3a, -8*128+48*128, rdi, -8*ZMM_SCD5 ;; Test loop counter, next source pointer, restore sin/cos pointer

	loops_reset			;; 2 more iters of 8
	zr6rb_six_complex_djbfft_preload
b3b:	zr6rb_six_complex_djbfft rsi, 128, 8*128, rdi, ZMM_SCD5, 8
	zloop	8, b3b, -8*128+48*128, rdi, -8*ZMM_SCD5 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	3, b3b, -3*48*128	;; Test loop counter, restore source pointer
	end_timer 6

;; Do FFT levels 10,11,12
;; On input and output the 128-byte cache line pairs hold these data values:
;;	0	+288	...	8	+288	...
;;	...
;;	7
;;	16	...
;;	32	...
;;	...

;; Do 1 sixteen_reals_fft operation
;; Do 143 eight_complex_fft operations
;;	distance between fft data elements is 1
;;	do 18 macros processing 128 data values

zmid:	start_timer 7
	mov	rcx, 18
	call	zreal_onepass_middle
	end_timer 7

;; Do inverse FFT levels 7,8,9
;; On input the 64-byte cache lines hold these data values:
;;	0	+288	...	8	+288	...
;;	...
;;	7
;;	16	...
;;	32	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+288	...	48	+288	...
;;	...
;;	47
;;	96	...
;;	192	...
;;	...

;; Do 8 twelve_reals_unfft operations
;; Do 184 six_complex_unfft operations
;;	distance between fft data elements is 8
;;	do 24 macros each processing 96 data values

	start_timer 8
	mov	rdi, sincos1		;; Load sin/cos pointer
	zloop_init 24			;; 3 iterations of 8
	zr6_twelve_reals_six_complex_djbunfft_preload
c3a:	zr6_twelve_reals_six_complex_djbunfft rsi, 128, 8*128, rdi, ZMM_SCD5, 8, L1PREFETCH_NONE
	zloop	8, c3a, -8*128+48*128, rdi, -8*ZMM_SCD5 ;; Test loop counter, restore source pointer, restore sin/cos pointer

	loops_reset			;; 2 more iters of 8
	zr6rb_six_complex_djbunfft_preload
c3b:	zr6rb_six_complex_djbunfft rsi, 128, 8*128, rdi, ZMM_SCD5, 8
	zloop	8, c3b, -8*128+48*128, rdi, -8*ZMM_SCD5 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	3, c3b, -3*48*128, rdi, 8*ZMM_SCD5 ;; Test loop counter, restore source pointer, next sin/cos pointer
	end_timer 8

;; Do inverse FFT levels 5,6
;; On input the 64-byte cache lines hold these data values:
;;	0	+288	...	48	+288	...
;;	...
;;	47
;;	96	...
;;	192	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+288	...	144	+288	...
;;	...
;;	143	...

;; Do 24 six_reals_unfft operations
;; Do 360 three_complex_unfft operations
;;	distance between fft data elements is 48
;;	do 48 macros each processing 48 data values

	start_timer 9
	zloop_init 48			;; 48 iterations
	zr3_six_reals_three_complex_djbunfft_preload
c2b:	zr3_six_reals_three_complex_djbunfft rsi, 128, 48*128, rdi, ZMM_SCD2, 48
	zloop	48, c2b, -48*128	;; Test loop counter, restore source pointer
	end_timer 9

;; Do inverse FFT levels 1,2,3,4
;; On input the 128-byte cache line pairs hold these data values:
;;	0	+288	...	144	+288	...
;;	...
;;	143	...
;; These are swizzled to:
;;	0	+18	...	144	+18	...
;;	...
;;	17	...
;;	288	...
;;	576	...
;;	...
;; On output the 128-byte cache line pairs hold these data values:
;;	0	+18	...	1152	+18	...
;;	...
;;	17	...
;;	144	...
;;	288	...
;;	...

;; Do 144 sixteen_reals_unfft operations
;;	distance between fft data elements is 144
;;	do 18 macros processing 128 data values

	start_timer 10
	zloop_init 18			;; 18 iterations
	zr8s_sixteen_reals_last_unfft_preload
c1b:	zr8s_sixteen_reals_last_unfft rsi, 128, 18*128, 36*128, 72*128, rdi, ZMM_SCD7, 18, L1PREFETCH_ALL
	zloop	18, c1b, -18*128	;; Test loop counter, restore source pointer
	end_timer 10

	zfft_3_ret
	ENDM

;; Perform a 2304-element negacyclic FFT

zfft2304p MACRO type
	LOCAL	b1b, b2b, b3b, zmid, c3b, c2b, c1b

;; Do a multiply with pre-FFTed inputs

	cmp	ffttype, 4
	je	zmid

;; Do FFT levels 1,2,3,4
;; On input the 128-byte cache line pairs hold these data values:
;;	0	+18	...	1152	+18	...
;;	...
;;	17	...
;;	144	...
;;	288	...
;;	...
;; On output the 128-byte cache line pairs hold these data values:
;;	0	+18	...	144	+18	...
;;	...
;;	17	...
;;	288	...
;;	576	...
;;	...
;; These are swizzled to:
;;	0	+288	...	144	+288	...
;;	...
;;	143	...

;; Do 144 eight_complex_first_fft operations
;;	distance between fft data elements is 144
;;	do 18 macros processing 128 data values

	start_timer 4
	mov	rdi, sincos1		;; Load sin/cos pointer
	bump	rdi, 48*ZMM_SCD1/8+8*ZMM_SCD3/8
	mov	r9, norm_col_mults	;; Load weights pointer
	zloop_init 18			;; 18 iterations
	zr8fs_eight_complex_first_fft_preload
b1b:	zr8fs_eight_complex_first_fft rsi, 128, 18*128, 36*128, 72*128, rdi, ZMM_CD8SCD4, r9, 8*128, 18, L1PREFETCH_ALL
	zloop	18, b1b, -18*128	;; Test loop counter, restore source pointer
	end_timer 4

;; Do FFT levels 5,6
;; On input the 64-byte cache lines hold these data values:
;;	0	+288	...	144	+288	...
;;	...
;;	143	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+288	...	48	+288	...
;;	...
;;	47
;;	96	...
;;	192	...
;;	...

;; Do 384 three_complex_fft operations
;;	distance between fft data elements is 48
;;	do 48 macros each processing 48 data values

	start_timer 5
	bump	rdi, -18*ZMM_CD8SCD4-48*ZMM_SCD1/8 ;; Load sin/cos pointer
	zloop_init 48			;; 48 iterations
	zr3b_three_complex_djbfft_preload
b2b:	zr3b_three_complex_djbfft rsi, 128, 48*128, rdi, ZMM_SCD1/8, 48
	zloop	48, b2b, -48*128	;; Test loop counter, restore source pointer
	end_timer 5

;; Do FFT levels 7,8,9
;; On input the 64-byte cache lines hold these data values:
;;	0	+288	...	48	+288	...
;;	...
;;	47
;;	96	...
;;	192	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+288	...	8	+288	...
;;	...
;;	7
;;	16	...
;;	32	...
;;	...

;; Do 192 six_complex_fft operations
;;	distance between fft data elements is 8
;;	do 24 macros each processing 96 data values

	start_timer 6
	bump	rdi, -48*ZMM_SCD1/8-8*ZMM_SCD3/8 ;; Load sin/cos pointer
	zloop_init 24			;; 3 iterations of 8
	zr6b_six_complex_djbfft_preload
b3b:	zr6b_six_complex_djbfft rsi, 128, 8*128, rdi, ZMM_SCD3/8, 8
	zloop	8, b3b, -8*128+48*128, rdi, -8*ZMM_SCD3/8 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	3, b3b, -3*48*128	;; Test loop counter, restore source pointer
	end_timer 6

;; Do FFT levels 10,11,12
;; On input and output the 128-byte cache line pairs hold these data values:
;;	0	+288	...	8	+288	...
;;	...
;;	7
;;	16	...
;;	32	...
;;	...

;; Do 144 eight_complex_fft operations
;;	distance between fft data elements is 1
;;	do 18 macros processing 128 data values

zmid:	start_timer 7
	mov	rcx, 18
	call	zcomplex_onepass_middle
	end_timer 7

;; Do inverse FFT levels 7,8,9
;; On input the 64-byte cache lines hold these data values:
;;	0	+288	...	8	+288	...
;;	...
;;	7
;;	16	...
;;	32	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+288	...	48	+288	...
;;	...
;;	47
;;	96	...
;;	192	...
;;	...

;; Do 192 six_complex_unfft operations
;;	distance between fft data elements is 8
;;	do 24 macros each processing 96 data values

	start_timer 8
	mov	rdi, sincos1		;; Load sin/cos pointer
	zloop_init 24			;; 3 iterations of 8
	zr6b_six_complex_djbunfft_preload
c3b:	zr6b_six_complex_djbunfft rsi, 128, 8*128, rdi, ZMM_SCD3/8, 8
	zloop	8, c3b, -8*128+48*128, rdi, -8*ZMM_SCD3/8 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	3, c3b, -3*48*128, rdi, 8*ZMM_SCD3/8	 ;; Test loop counter, restore source pointer, next sin/cos pointer
	end_timer 8

;; Do inverse FFT levels 5,6
;; On input the 64-byte cache lines hold these data values:
;;	0	+288	...	48	+288	...
;;	...
;;	47
;;	96	...
;;	192	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+288	...	144	+288	...
;;	...
;;	143	...

;; Do 384 three_complex_unfft operations
;;	distance between fft data elements is 48
;;	do 48 macros each processing 48 data values

	start_timer 9
	zloop_init 48			;; 48 iterations
	zr3b_three_complex_djbunfft_preload
c2b:	zr3b_three_complex_djbunfft rsi, 128, 48*128, rdi, ZMM_SCD1/8, 48
	zloop	48, c2b, -48*128	;; Test loop counter, restore source pointer
	end_timer 9

;; Do inverse FFT levels 1,2,3,4
;; On input the 128-byte cache line pairs hold these data values:
;;	0	+288	...	144	+288	...
;;	...
;;	143	...
;; These are swizzled to:
;;	0	+18	...	144	+18	...
;;	...
;;	17	...
;;	288	...
;;	576	...
;;	...
;; On output the 128-byte cache line pairs hold these data values:
;;	0	+18	...	1152	+18	...
;;	...
;;	17	...
;;	144	...
;;	288	...
;;	...

;; Do 144 eight_complex_last_unfft operations
;;	distance between fft data elements is 144
;;	do 18 macros processing 128 data values

	start_timer 10
	zloop_init 18			;; 18 iterations
	zr8s_eight_complex_last_unfft_preload
c1b:	zr8s_eight_complex_last_unfft rsi, 128, 18*128, 36*128, 72*128, rdi, ZMM_CD8SCD4, 18, L1PREFETCH_ALL
	zloop	18, c1b, -18*128	;; Test loop counter, restore source pointer
	end_timer 10

	zfft_3_ret
	ENDM


;; Perform a 2560-element FFT

zfft2560 MACRO type
	LOCAL	b1b, b2b, b3a, b3b, zmid, c3a, c3b, c2b, c1b

;; Check for a multiply with pre-FFTed inputs

	cmp	ffttype, 4
	je	zmid

;; Do FFT levels 1,2,3,4
;; On input the 128-byte cache line pairs hold these data values:
;;	0	+20	...	1280	+20	...
;;	...
;;	19	...
;;	160	...
;;	240	...
;;	...
;; On output the 128-byte cache line pairs hold these data values:
;;	0	+20	...	160	+20	...
;;	...
;;	19	...
;;	320	...
;;	640	...
;;	...
;; These are swizzled to:
;;	0	+320	...	160	+320	...
;;	...
;;	159	...

;; Do 160 sixteen_reals_fft operations
;;	distance between fft data elements is 160
;;	do 20 macros processing 128 data values

	start_timer 4
	mov	rdi, sincos1		;; Load sin/cos data ptr
	bump	rdi, 32*ZMM_SCD4+8*ZMM_SCD3
	mov	r9, norm_col_mults	;; Load weights pointer
	zloop_init 20			;; 20 iterations
	zr8fs_sixteen_reals_first_fft_preload
b1b:	zr8fs_sixteen_reals_first_fft rsi, 128, 20*128, 40*128, 80*128, rdi, ZMM_SCD7, r9, 8*128, 20, L1PREFETCH_ALL
	zloop	20, b1b, -20*128	;; Test loop counter, restore source pointer
	end_timer 4

;; Do FFT levels 5,6
;; On input the 64-byte cache lines hold these data values:
;;	0	+320	...	160	+320	...
;;	...
;;	159	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+320	...	32	+320	...
;;	...
;;	31
;;	64	...
;;	128	...
;;	...

;; Do 16 ten_reals_fft operation
;; Do 240 five_complex_fft operations
;;	distance between fft data elements is 32
;;	do 32 macros each processing 80 data values

	start_timer 5
	bump	rdi, -20*ZMM_SCD7-32*ZMM_SCD4 ;; Load sin/cos pointer
	zloop_init 32			;; 32 iterations
	zr5_ten_reals_five_complex_djbfft_preload
b2b:	zr5_ten_reals_five_complex_djbfft rsi, 128, 32*128, rdi, ZMM_SCD4, 32
	zloop	32, b2b, -32*128	;; Test loop counter, restore source pointer
	end_timer 5

;; Do FFT levels 7,8
;; On input the 64-byte cache lines hold these data values:
;;	0	+320	...	32	+320	...
;;	...
;;	31
;;	64	...
;;	128	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+320	...	8	+320	...
;;	...
;;	7
;;	16	...
;;	32	...
;;	...

;; Do 8 eight_reals_fft operation
;; Do 312 four_complex_fft operations
;;	distance between fft data elements is 8
;;	do 40 macros each processing 64 data values

	start_timer 6
	bump	rdi, -32*ZMM_SCD4-8*ZMM_SCD3 ;; Load sin/cos ptr
	zloop_init 40			;; 5 iterations of 8
	zr4_eight_reals_four_complex_djbfft_preload
b3a:	zr4_eight_reals_four_complex_djbfft rsi, 128, 8*128, 16*128, rdi, ZMM_SCD3, 8
	zloop	8, b3a, -8*128+32*128, rdi, -8*ZMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer

	loops_reset			;; 4 more iters of 8
	zr4rb_four_complex_djbfft_preload
b3b:	zr4rb_four_complex_djbfft rsi, 128, 8*128, 16*128, rdi, ZMM_SCD3, 8
	zloop	8, b3b, -8*128+32*128, rdi, -8*ZMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	5, b3b, -5*32*128	;; Test loop counter, restore source pointer
	end_timer 6

;; Do FFT levels 9,10,11
;; On input and output the 128-byte cache line pairs hold these data values:
;;	0	+320	...	8	+320	...
;;	...
;;	7
;;	16	...
;;	32	...
;;	...

;; Do 1 sixteen_reals_fft operation
;; Do 159 eight_complex_fft operations
;;	distance between fft data elements is 1
;;	do 20 macros processing 128 data values

zmid:	start_timer 7
	mov	rcx, 20
	call	zreal_onepass_middle
	end_timer 7

;; Do inverse FFT levels 7,8
;; On input the 64-byte cache lines hold these data values:
;;	0	+320	...	8	+320	...
;;	...
;;	7
;;	16	...
;;	32	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+320	...	32	+320	...
;;	...
;;	31
;;	64	...
;;	128	...
;;	...

;; Do 8 eight_reals_unfft operations
;; Do 312 four_complex_unfft operations
;;	distance between fft data elements is 8
;;	do 40 macros each processing 64 data values

	start_timer 8
	mov	rdi, sincos1		;; Load sin/cos pointer
	zloop_init 40			;; 5 iterations of 8
	zr4_eight_reals_four_complex_djbunfft_preload
c3a:	zr4_eight_reals_four_complex_djbunfft rsi, 128, 8*128, 16*128, rdi, ZMM_SCD3, 8
	zloop	8, c3a, -8*128+32*128, rdi, -8*ZMM_SCD3 ;; Test loop counter, restore source pointer, restore sin/cos pointer

	loops_reset			;; 4 more iters of 8
	zr4rb_four_complex_djbunfft_preload
c3b:	zr4rb_four_complex_djbunfft rsi, 128, 8*128, 16*128, rdi, ZMM_SCD3, 8
	zloop	8, c3b, -8*128+32*128, rdi, -8*ZMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	5, c3b, -5*32*128, rdi, 8*ZMM_SCD3 ;; Test loop counter, restore source pointer, next sin/cos pointer
	end_timer 8

;; Do inverse FFT levels 5,6
;; On input the 64-byte cache lines hold these data values:
;;	0	+320	...	32	+320	...
;;	...
;;	31
;;	64	...
;;	128	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+320	...	160	+320	...
;;	...
;;	159	...

;; Do 16 ten_reals_unfft operations
;; Do 240 five_complex_unfft operations
;;	distance between fft data elements is 32
;;	do 32 macros each processing 80 data values

	start_timer 9
	zloop_init 32			;; 32 iterations
	zr5_ten_reals_five_complex_djbunfft_preload
c2b:	zr5_ten_reals_five_complex_djbunfft rsi, 128, 32*128, rdi, ZMM_SCD4, 32
	zloop	32, c2b, -32*128	;; Test loop counter, restore source pointer
	end_timer 9

;; Do inverse FFT levels 1,2,3,4
;; On input the 128-byte cache line pairs hold these data values:
;;	0	+320	...	160	+320	...
;;	...
;;	159	...
;; These are swizzled to:
;;	0	+20	...	160	+20	...
;;	...
;;	19	...
;;	320	...
;;	640	...
;;	...
;; On output the 128-byte cache line pairs hold these data values:
;;	0	+20	...	1280	+20	...
;;	...
;;	19	...
;;	160	...
;;	240	...
;;	...

;; Do 160 sixteen_reals_unfft operations
;;	distance between fft data elements is 160
;;	do 20 macros processing 128 data values

	start_timer 10
	zloop_init 20			;; 20 iterations
	zr8s_sixteen_reals_last_unfft_preload
c1b:	zr8s_sixteen_reals_last_unfft rsi, 128, 20*128, 40*128, 80*128, rdi, ZMM_SCD7, 20, L1PREFETCH_ALL
	zloop	20, c1b, -20*128	;; Test loop counter, restore source pointer
	end_timer 10

	zfft_3_ret
	ENDM

;; Perform a 2560-element negacyclic FFT

zfft2560p MACRO type
	LOCAL	b1b, b2b, b3b, zmid, c3b, c2b, c1b

;; Do a multiply with pre-FFTed inputs

	cmp	ffttype, 4
	je	zmid

;; Do FFT levels 1,2,3,4
;; On input the 128-byte cache line pairs hold these data values:
;;	0	+20	...	1280	+20	...
;;	...
;;	19	...
;;	160	...
;;	240	...
;;	...
;; On output the 128-byte cache line pairs hold these data values:
;;	0	+20	...	160	+20	...
;;	...
;;	19	...
;;	320	...
;;	640	...
;;	...
;; These are swizzled to:
;;	0	+320	...	160	+320	...
;;	...
;;	159	...

;; Do 160 eight_complex_first_fft operations
;;	distance between fft data elements is 160
;;	do 20 macros processing 128 data values

	start_timer 4
	mov	rdi, sincos1		;; Load sin/cos pointer
	bump	rdi, 32*ZMM_SCD2/8+8*ZMM_SCD2/8
	mov	r9, norm_col_mults	;; Load weights pointer
	zloop_init 20			;; 20 iterations
	zr8fs_eight_complex_first_fft_preload
b1b:	zr8fs_eight_complex_first_fft rsi, 128, 20*128, 40*128, 80*128, rdi, ZMM_CD8SCD4, r9, 8*128, 20, L1PREFETCH_ALL
	zloop	20, b1b, -20*128	;; Test loop counter, restore source pointer
	end_timer 4

;; Do FFT levels 5,6
;; On input the 64-byte cache lines hold these data values:
;;	0	+320	...	160	+320	...
;;	...
;;	159	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+320	...	32	+320	...
;;	...
;;	31
;;	64	...
;;	128	...
;;	...

;; Do 256 five_complex_fft operations
;;	distance between fft data elements is 32
;;	do 32 macros each processing 80 data values

	start_timer 5
	bump	rdi, -20*ZMM_CD8SCD4-32*ZMM_SCD2/8 ;; Load sin/cos pointer
	zloop_init 32			;; 32 iterations
	zr5b_five_complex_djbfft_preload
b2b:	zr5b_five_complex_djbfft rsi, 128, 32*128, rdi, ZMM_SCD2/8, 32
	zloop	32, b2b, -32*128	;; Test loop counter, restore source pointer
	end_timer 5

;; Do FFT levels 7,8
;; On input the 64-byte cache lines hold these data values:
;;	0	+320	...	32	+320	...
;;	...
;;	31
;;	64	...
;;	128	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+320	...	8	+320	...
;;	...
;;	7
;;	16	...
;;	32	...
;;	...

;; Do 320 four_complex_fft operations
;;	distance between fft data elements is 8
;;	do 40 macros each processing 64 data values

	start_timer 6
	bump	rdi, -32*ZMM_SCD2/8-8*ZMM_SCD2/8 ;; Load sin/cos pointer
	zloop_init 40			;; 5 iterations of 8
	zr4b_four_complex_djbfft_preload
b3b:	zr4b_four_complex_djbfft rsi, 128, 8*128, 16*128, rdi, ZMM_SCD2/8, 8
	zloop	8, b3b, -8*128+32*128, rdi, -8*ZMM_SCD2/8 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	5, b3b, -5*32*128	;; Test loop counter, restore source pointer
	end_timer 6

;; Do FFT levels 9,10,11
;; On input and output the 128-byte cache line pairs hold these data values:
;;	0	+320	...	8	+320	...
;;	...
;;	7
;;	16	...
;;	32	...
;;	...

;; Do 160 eight_complex_fft operations
;;	distance between fft data elements is 1
;;	do 20 macros processing 128 data values

zmid:	start_timer 7
	mov	rcx, 20
	call	zcomplex_onepass_middle
	end_timer 7

;; Do inverse FFT levels 7,8
;; On input the 64-byte cache lines hold these data values:
;;	0	+320	...	8	+320	...
;;	...
;;	7
;;	16	...
;;	32	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+320	...	32	+320	...
;;	...
;;	31
;;	64	...
;;	128	...
;;	...

;; Do 320 four_complex_unfft operations
;;	distance between fft data elements is 8
;;	do 40 macros each processing 64 data values

	start_timer 8
	mov	rdi, sincos1		;; Load sin/cos pointer
	zloop_init 40			;; 5 iterations of 8
	zr4b_four_complex_djbunfft_preload
c3b:	zr4b_four_complex_djbunfft rsi, 128, 8*128, 16*128, rdi, ZMM_SCD2/8, 8
	zloop	8, c3b, -8*128+32*128, rdi, -8*ZMM_SCD2/8 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	5, c3b, -5*32*128, rdi, 8*ZMM_SCD2/8 ;; Test loop counter, restore source pointer, next sin/cos pointer
	end_timer 8

;; Do inverse FFT levels 5,6
;; On input the 64-byte cache lines hold these data values:
;;	0	+320	...	32	+320	...
;;	...
;;	31
;;	64	...
;;	128	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+320	...	160	+320	...
;;	...
;;	159	...

;; Do 256 five_complex_unfft operations
;;	distance between fft data elements is 32
;;	do 32 macros each processing 80 data values

	start_timer 9
	zloop_init 32			;; 32 iterations
	zr5b_five_complex_djbunfft_preload
c2b:	zr5b_five_complex_djbunfft rsi, 128, 32*128, rdi, ZMM_SCD2/8, 32
	zloop	32, c2b, -32*128	;; Test loop counter, restore source pointer
	end_timer 9

;; Do inverse FFT levels 1,2,3,4
;; On input the 128-byte cache line pairs hold these data values:
;;	0	+320	...	160	+320	...
;;	...
;;	159	...
;; These are swizzled to:
;;	0	+20	...	160	+20	...
;;	...
;;	19	...
;;	320	...
;;	640	...
;;	...
;; On output the 128-byte cache line pairs hold these data values:
;;	0	+20	...	1280	+20	...
;;	...
;;	19	...
;;	160	...
;;	240	...
;;	...

;; Do 160 eight_complex_last_unfft operations
;;	distance between fft data elements is 160
;;	do 20 macros processing 128 data values

	start_timer 10
	zloop_init 20			;; 20 iterations
	zr8s_eight_complex_last_unfft_preload
c1b:	zr8s_eight_complex_last_unfft rsi, 128, 20*128, 40*128, 80*128, rdi, ZMM_CD8SCD4, 20, L1PREFETCH_ALL
	zloop	20, c1b, -20*128	;; Test loop counter, restore source pointer
	end_timer 10

	zfft_3_ret
	ENDM


;; Perform a 3K-element FFT

zfft3K MACRO type
	LOCAL	b1b, b2b, b3a, b3b, zmid, c3a, c3b, c2b, c1b

;; Check for a multiply with pre-FFTed inputs

	cmp	ffttype, 4
	je	zmid

;; Do FFT levels 1,2,3,4
;; On input the 128-byte cache line pairs hold these data values:
;;	0	+24	...	1536	+24	...
;;	...
;;	23	...
;;	192	...
;;	384	...
;;	...
;; On output the 128-byte cache line pairs hold these data values:
;;	0	+24	...	192	+24	...
;;	...
;;	23	...
;;	384	...
;;	768	...
;;	...
;; These are swizzled to:
;;	0	+384	...	192	+384	...
;;	...
;;	191	...

;; Do 192 sixteen_reals_fft operations
;;	distance between fft data elements is 192
;;	do 24 macros processing 128 data values

;; Do FFT levels 5,6,7
;; On input the 64-byte cache lines hold these data values:
;;	0	+384	...	192	+384	...
;;	...
;;	191	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+384	...	24	+384	...
;;	...
;;	23
;;	48	...
;;	96	...
;;	...

;; Do 24 sixteen_reals_fft operation
;; Do 168 eight_complex_fft operations
;;	distance between fft data elements is 24
;;	do 24 macros each processing 128 data values

IFDEF OLDWAY
	start_timer 4
	mov	rdi, sincos1		;; Load sin/cos data ptr
	bump	rdi, 24*ZMM_SCD7+8*ZMM_SCD2
	mov	r9, norm_col_mults	;; Load weights pointer
	zloop_init 24			;; 24 iterations
	zr8fs_sixteen_reals_first_fft_preload
b1b:	zr8fs_sixteen_reals_first_fft rsi, 128, 24*128, 48*128, 96*128, rdi, ZMM_SCD7, r9, 8*128, 24, L1PREFETCH_ALL
	zloop	24, b1b, -24*128	;; Test loop counter, restore source pointer
	end_timer 4

	start_timer 5
	bump	rdi, -24*ZMM_SCD7-24*ZMM_SCD7 ;; Load sin/cos pointer
	zloop_init 24			;; 24 iterations
	zr8_sixteen_reals_eight_complex_djbfft_preload
b2a:	zr8_sixteen_reals_eight_complex_djbfft rsi, 128, 24*128, 48*128, 96*128, rdi, ZMM_SCD7, 24, L1PREFETCH_ALL
	zloop	24, b2a, -24*128		;; Test loop counter, next source pointer
	end_timer 5
ELSE
	start_timer 4
	mov	rdi, sincos1		;; Load sin/cos data ptr
	bump	rdi, 8*ZMM_SCD2
	lea	r10, [rdi+24*ZMM_SCD7]
	mov	r9, norm_col_mults	;; Load weights pointer
	zloop_init 24			;; 24 iterations
	zr8fs_hundredtwentyeight_reals_first_fft_preload
b1b:	zr8fs_hundredtwentyeight_reals_first_fft rsi, 128, 24*128, 48*128, 96*128, r10, ZMM_SCD7, rdi, ZMM_SCD7, r9, 8*128, 24, L1PREFETCH_ALL
	zloop	24, b1b, -24*128	;; Test loop counter, restore source pointer
	end_timer 4
ENDIF

;; Do FFT levels 8,9
;; On input the 64-byte cache lines hold these data values:
;;	0	+384	...	24	+384	...
;;	...
;;	23
;;	48	...
;;	96	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+384	...	8	+384	...
;;	...
;;	7
;;	16	...
;;	32	...
;;	...

;; Do 8 six_reals_fft operation
;; Do 504 three_complex_fft operations
;;	distance between fft data elements is 8
;;	do 64 macros each processing 48 data values

	start_timer 6
	bump	rdi, -24*ZMM_SCD7-8*ZMM_SCD2 ;; Load sin/cos ptr
	zloop_init 64			;; 8 iterations of 8
	zr3_six_reals_three_complex_djbfft_preload
b3a:	zr3_six_reals_three_complex_djbfft rsi, 128, 8*128, rdi, ZMM_SCD2, 8
	zloop	8, b3a, -8*128+24*128, rdi, -8*ZMM_SCD2 ;; Test loop counter, next source pointer, restore sin/cos pointer

	loops_reset			;; 7 more iters of 8
	zr3rb_three_complex_djbfft_preload
b3b:	zr3rb_three_complex_djbfft rsi, 128, 8*128, rdi, ZMM_SCD2, 8
	zloop	8, b3b, -8*128+24*128, rdi, -8*ZMM_SCD2 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	8, b3b, -8*24*128	;; Test loop counter, restore source pointer
	end_timer 6

;; Do FFT levels 10,11,12
;; On input and output the 128-byte cache line pairs hold these data values:
;;	0	+384	...	8	+384	...
;;	...
;;	7
;;	16	...
;;	32	...
;;	...

;; Do 1 sixteen_reals_fft operation
;; Do 191 eight_complex_fft operations
;;	distance between fft data elements is 1
;;	do 24 macros processing 128 data values

zmid:	start_timer 7
	mov	rcx, 24
	call	zreal_onepass_middle
	end_timer 7

;; Do inverse FFT levels 8,9
;; On input the 64-byte cache lines hold these data values:
;;	0	+384	...	8	+384	...
;;	...
;;	7
;;	16	...
;;	32	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+384	...	24	+384	...
;;	...
;;	23
;;	48	...
;;	96	...
;;	...

;; Do 8 six_reals_unfft operations
;; Do 504 three_complex_unfft operations
;;	distance between fft data elements is 8
;;	do 64 macros each processing 48 data values

	start_timer 8
	mov	rdi, sincos1		;; Load sin/cos pointer
	zloop_init 64			;; 8 iterations of 8
	zr3_six_reals_three_complex_djbunfft_preload
c3a:	zr3_six_reals_three_complex_djbunfft rsi, 128, 8*128, rdi, ZMM_SCD2, 8
	zloop	8, c3a, -8*128+24*128, rdi, -8*ZMM_SCD2 ;; Test loop counter, next source pointer, restore sin/cos pointer

	loops_reset			;; 7 more iters of 8
	zr3rb_three_complex_djbunfft_preload
c3b:	zr3rb_three_complex_djbunfft rsi, 128, 8*128, rdi, ZMM_SCD2, 8
	zloop	8, c3b, -8*128+24*128, rdi, -8*ZMM_SCD2 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	8, c3b, -8*24*128, rdi, 8*ZMM_SCD2 ;; Test loop counter, restore source pointer, next sin/cos pointer
	end_timer 8

;; Do inverse FFT levels 5,6,7
;; On input the 64-byte cache lines hold these data values:
;;	0	+384	...	24	+384	...
;;	...
;;	23
;;	48	...
;;	96	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+384	...	192	+384	...
;;	...
;;	191	...

;; Do 24 sixteen_reals_unfft operations
;; Do 168 eight_complex_unfft operations
;;	distance between fft data elements is 24
;;	do 24 macros each processing 128 data values

;; Do inverse FFT levels 1,2,3,4
;; On input the 128-byte cache line pairs hold these data values:
;;	0	+384	...	192	+384	...
;;	...
;;	191	...
;; These are swizzled to:
;;	0	+24	...	192	+24	...
;;	...
;;	23	...
;;	384	...
;;	768	...
;;	...
;; On output the 128-byte cache line pairs hold these data values:
;;	0	+24	...	1536	+24	...
;;	...
;;	23	...
;;	192	...
;;	384	...
;;	...

;; Do 192 sixteen_reals_unfft operations
;;	distance between fft data elements is 192
;;	do 24 macros processing 128 data values

IFDEF OLDWAY
	start_timer 9
	zloop_init 24			;; 24 iterations
	zr8_sixteen_reals_eight_complex_djbunfft_preload
c2b:	zr8_sixteen_reals_eight_complex_djbunfft rsi, 128, 24*128, 48*128, 96*128, rdi, ZMM_SCD7, 24, L1PREFETCH_NONE
	zloop	24, c2b, -24*128	;; Test loop counter, restore source pointer
	end_timer 9

	start_timer 10
	zloop_init 24			;; 24 iterations
	zr8s_sixteen_reals_last_unfft_preload
c1b:	zr8s_sixteen_reals_last_unfft rsi, 128, 24*128, 48*128, 96*128, rdi, ZMM_SCD7, 24, L1PREFETCH_ALL
	zloop	24, c1b, -24*128	;; Test loop counter, restore source pointer
	end_timer 10
ELSE
	start_timer 10
	lea	r10, [rdi+24*ZMM_SCD7]
	zloop_init 24			;; 24 iterations
	zr8s_hundredtwentyeight_reals_last_unfft_preload
c1b:	zr8s_hundredtwentyeight_reals_last_unfft rsi, 128, 24*128, 48*128, 96*128, rdi, ZMM_SCD7, r10, ZMM_SCD7, 24, L1PREFETCH_ALL
	zloop	24, c1b, -24*128	;; Test loop counter, restore source pointer
	end_timer 10
ENDIF

	zfft_3_ret
	ENDM

;; Perform a 3K-element negacyclic FFT

zfft3Kp MACRO type
	LOCAL	b1b, b2b, b3b, zmid, c3b, c2b, c1b

;; Do a multiply with pre-FFTed inputs

	cmp	ffttype, 4
	je	zmid

;; Do FFT levels 1,2,3,4
;; On input the 128-byte cache line pairs hold these data values:
;;	0	+24	...	1536	+24	...
;;	...
;;	23	...
;;	192	...
;;	384	...
;;	...
;; On output the 128-byte cache line pairs hold these data values:
;;	0	+24	...	192	+24	...
;;	...
;;	23	...
;;	384	...
;;	768	...
;;	...
;; These are swizzled to:
;;	0	+384	...	192	+384	...
;;	...
;;	191	...

;; Do 192 eight_complex_first_fft operations
;;	distance between fft data elements is 192
;;	do 24 macros processing 128 data values

;; Do FFT levels 5,6,7
;; On input the 64-byte cache lines hold these data values:
;;	0	+384	...	192	+384	...
;;	...
;;	191	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+384	...	24	+384	...
;;	...
;;	23
;;	48	...
;;	96	...
;;	...

;; Do 192 eight_complex_fft operations
;;	distance between fft data elements is 24
;;	do 24 macros each processing 128 data values

IFDEF OLDWAY
	start_timer 4
	mov	rdi, sincos1		;; Load sin/cos pointer
	bump	rdi, 24*ZMM_SCD4/8+8*ZMM_SCD1/8
	mov	r9, norm_col_mults	;; Load weights pointer
	zloop_init 24			;; 24 iterations
	zr8fs_eight_complex_first_fft_preload
b1b:	zr8fs_eight_complex_first_fft rsi, 128, 24*128, 48*128, 96*128, rdi, ZMM_CD8SCD4, r9, 8*128, 24, L1PREFETCH_ALL
	zloop	24, b1b, -24*128	;; Test loop counter, restore source pointer
	end_timer 4

	start_timer 5
	bump	rdi, -24*ZMM_CD8SCD4-24*ZMM_SCD4/8 ;; Load sin/cos pointer
	zloop_init 24			;; 24 iterations
	zr8b_eight_complex_djbfft_preload
b2b:	zr8b_eight_complex_djbfft rsi, 128, 24*128, 48*128, 96*128, rdi, ZMM_SCD4/8, 24, L1PREFETCH_NONE
	zloop	24, b2b, -24*128	;; Test loop counter, restore source pointer
	end_timer 5
ELSE
	start_timer 4
	mov	rdi, sincos1		;; Load sin/cos pointer
	bump	rdi, 8*ZMM_SCD1/8
	lea	r10, [rdi+24*ZMM_SCD4/8]
	mov	r9, norm_col_mults	;; Load weights pointer
	zloop_init 24			;; 24 iterations
	zr8fs_sixtyfour_complex_first_fft_preload
b1b:	zr8fs_sixtyfour_complex_first_fft rsi, 128, 24*128, 48*128, 96*128, r10, ZMM_CD8SCD4, rdi, ZMM_SCD4/8, r9, 8*128, 24, L1PREFETCH_ALL
	zloop	24, b1b, -24*128	;; Test loop counter, restore source pointer
	end_timer 4
ENDIF

;; Do FFT levels 8,9
;; On input the 64-byte cache lines hold these data values:
;;	0	+384	...	24	+384	...
;;	...
;;	23
;;	48	...
;;	96	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+384	...	8	+384	...
;;	...
;;	7
;;	16	...
;;	32	...
;;	...

;; Do 512 three_complex_fft operations
;;	distance between fft data elements is 8
;;	do 64 macros each processing 48 data values

	start_timer 6
	bump	rdi, -24*ZMM_SCD4/8-8*ZMM_SCD1/8 ;; Load sin/cos pointer
	zloop_init 64			;; 8 iterations of 8
	zr3b_three_complex_djbfft_preload
b3b:	zr3b_three_complex_djbfft rsi, 128, 8*128, rdi, ZMM_SCD1/8, 8
	zloop	8, b3b, -8*128+24*128, rdi, -8*ZMM_SCD1/8 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	8, b3b, -8*24*128	;; Test loop counter, restore source pointer
	end_timer 6

;; Do FFT levels 10,11,12
;; On input and output the 128-byte cache line pairs hold these data values:
;;	0	+384	...	8	+384	...
;;	...
;;	7
;;	16	...
;;	32	...
;;	...

;; Do 192 eight_complex_fft operations
;;	distance between fft data elements is 1
;;	do 24 macros processing 128 data values

zmid:	start_timer 7
	mov	rcx, 24
	call	zcomplex_onepass_middle
	end_timer 7

;; Do inverse FFT levels 8,9
;; On input the 64-byte cache lines hold these data values:
;;	0	+384	...	8	+384	...
;;	...
;;	7
;;	16	...
;;	32	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+384	...	24	+384	...
;;	...
;;	23
;;	48	...
;;	96	...
;;	...

;; Do 512 three_complex_unfft operations
;;	distance between fft data elements is 8
;;	do 64 macros each processing 48 data values

	start_timer 8
	mov	rdi, sincos1		;; Load sin/cos pointer
	zloop_init 64			;; 8 iterations of 8
	zr3b_three_complex_djbunfft_preload
c3b:	zr3b_three_complex_djbunfft rsi, 128, 8*128, rdi, ZMM_SCD1/8, 8
	zloop	8, c3b, -8*128+24*128, rdi, -8*ZMM_SCD1/8 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	8, c3b, -8*24*128, rdi, 8*ZMM_SCD1/8 ;; Test loop counter, restore source pointer, next sin/cos pointer
	end_timer 8

;; Do inverse FFT levels 5,6,7
;; On input the 64-byte cache lines hold these data values:
;;	0	+384	...	24	+384	...
;;	...
;;	23
;;	48	...
;;	96	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+384	...	192	+384	...
;;	...
;;	191	...

;; Do 192 eight_complex_unfft operations
;;	distance between fft data elements is 24
;;	do 24 macros each processing 128 data values

;; Do inverse FFT levels 1,2,3,4
;; On input the 128-byte cache line pairs hold these data values:
;;	0	+384	...	192	+384	...
;;	...
;;	191	...
;; These are swizzled to:
;;	0	+24	...	192	+24	...
;;	...
;;	23	...
;;	384	...
;;	768	...
;;	...
;; On output the 128-byte cache line pairs hold these data values:
;;	0	+24	...	1536	+24	...
;;	...
;;	23	...
;;	192	...
;;	384	...
;;	...

;; Do 192 eight_complex_last_unfft operations
;;	distance between fft data elements is 192
;;	do 24 macros processing 128 data values

IFDEF OLDWAY
	start_timer 9
	zloop_init 24			;; 24 iterations
	zr8b_eight_complex_djbunfft_preload
c2b:	zr8b_eight_complex_djbunfft rsi, 128, 24*128, 48*128, 96*128, rdi, ZMM_SCD4/8, 24, L1PREFETCH_NONE
	zloop	24, c2b, -24*128	;; Test loop counter, restore source pointer
	end_timer 9

	start_timer 10
	zloop_init 24			;; 24 iterations
	zr8s_eight_complex_last_unfft_preload
c1b:	zr8s_eight_complex_last_unfft rsi, 128, 24*128, 48*128, 96*128, rdi, ZMM_CD8SCD4, 24, L1PREFETCH_ALL
	zloop	24, c1b, -24*128	;; Test loop counter, restore source pointer
	end_timer 10
ELSE
	start_timer 10
	lea	r10, [rdi+24*ZMM_SCD4/8]
	zloop_init 24			;; 24 iterations
	zr8s_sixtyfour_complex_last_unfft_preload
c1b:	zr8s_sixtyfour_complex_last_unfft rsi, 128, 24*128, 48*128, 96*128, rdi, ZMM_SCD4/8, r10, ZMM_CD8SCD4, 24, L1PREFETCH_ALL
	zloop	24, c1b, -24*128	;; Test loop counter, restore source pointer
	end_timer 10
ENDIF

	zfft_3_ret
	ENDM


;; Perform a 3584-element FFT

zfft3584 MACRO type
	LOCAL	b1b, b2b, b3a, b3b, zmid, c3a, c3b, c2b, c1b

;; Check for a multiply with pre-FFTed inputs

	cmp	ffttype, 4
	je	zmid

;; Do FFT levels 1,2,3,4
;; On input the 128-byte cache line pairs hold these data values:
;;	0	+28	...	1792	+28	...
;;	...
;;	27	...
;;	224	...
;;	448	...
;;	...
;; On output the 128-byte cache line pairs hold these data values:
;;	0	+28	...	224	+28	...
;;	...
;;	27	...
;;	448	...
;;	896	...
;;	...
;; These are swizzled to:
;;	0	+448	...	224	+448	...
;;	...
;;	223	...

;; Do 224 sixteen_reals_fft operations
;;	distance between fft data elements is 224
;;	do 28 macros processing 128 data values

	start_timer 4
	mov	rdi, sincos1		;; Load sin/cos data ptr
	bump	rdi, 32*ZMM_SCD6+8*ZMM_SCD3
	mov	r9, norm_col_mults	;; Load weights pointer
	zloop_init 28			;; 28 iterations
	zr8fs_sixteen_reals_first_fft_preload
b1b:	zr8fs_sixteen_reals_first_fft rsi, 128, 28*128, 56*128, 112*128, rdi, ZMM_SCD7, r9, 8*128, 28, L1PREFETCH_ALL
	zloop	28, b1b, -28*128	;; Test loop counter, restore source pointer
	end_timer 4

;; Do FFT levels 5,6
;; On input the 64-byte cache lines hold these data values:
;;	0	+448	...	224	+448	...
;;	...
;;	223	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+448	...	32	+448	...
;;	...
;;	31
;;	64	...
;;	128	...
;;	...

;; Do 16 fourteen_reals_fft operation
;; Do 240 seven_complex_fft operations
;;	distance between fft data elements is 32
;;	do 32 macros each processing 112 data values

	start_timer 5
	bump	rdi, -28*ZMM_SCD7-32*ZMM_SCD6 ;; Load sin/cos pointer
	zloop_init 32			;; 32 iterations
	zr7_fourteen_reals_seven_complex_djbfft_preload
b2b:	zr7_fourteen_reals_seven_complex_djbfft rsi, 128, 32*128, rdi, ZMM_SCD6, 32, L1PREFETCH_ALL
	zloop	32, b2b, -32*128	;; Test loop counter, restore source pointer
	end_timer 5

;; Do FFT levels 7,8
;; On input the 64-byte cache lines hold these data values:
;;	0	+448	...	32	+448	...
;;	...
;;	31
;;	64	...
;;	128	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+448	...	8	+448	...
;;	...
;;	7
;;	16	...
;;	32	...
;;	...

;; Do 8 eight_reals_fft operation
;; Do 440 four_complex_fft operations
;;	distance between fft data elements is 8
;;	do 56 macros each processing 64 data values

	start_timer 6
	bump	rdi, -32*ZMM_SCD6-8*ZMM_SCD3 ;; Load sin/cos ptr
	zloop_init 56			;; 7 iterations of 8
	zr4_eight_reals_four_complex_djbfft_preload
b3a:	zr4_eight_reals_four_complex_djbfft rsi, 128, 8*128, 16*128, rdi, ZMM_SCD3, 8
	zloop	8, b3a, -8*128+32*128, rdi, -8*ZMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer

	loops_reset			;; 6 more iters of 8
	zr4rb_four_complex_djbfft_preload
b3b:	zr4rb_four_complex_djbfft rsi, 128, 8*128, 16*128, rdi, ZMM_SCD3, 8
	zloop	8, b3b, -8*128+32*128, rdi, -8*ZMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	7, b3b, -7*32*128	;; Test loop counter, restore source pointer
	end_timer 6

;; Do FFT levels 9,10,11
;; On input and output the 128-byte cache line pairs hold these data values:
;;	0	+448	...	8	+448	...
;;	...
;;	7
;;	16	...
;;	32	...
;;	...

;; Do 1 sixteen_reals_fft operation
;; Do 223 eight_complex_fft operations
;;	distance between fft data elements is 1
;;	do 28 macros processing 128 data values

zmid:	start_timer 7
	mov	rcx, 28
	call	zreal_onepass_middle
	end_timer 7

;; Do inverse FFT levels 7,8
;; On input the 64-byte cache lines hold these data values:
;;	0	+448	...	8	+448	...
;;	...
;;	7
;;	16	...
;;	32	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+448	...	32	+448	...
;;	...
;;	31
;;	64	...
;;	128	...
;;	...

;; Do 8 eight_reals_unfft operations
;; Do 440 four_complex_unfft operations
;;	distance between fft data elements is 8
;;	do 56 macros each processing 64 data values

	start_timer 8
	mov	rdi, sincos1		;; Load sin/cos pointer
	zloop_init 56			;; 7 iterations of 8
	zr4_eight_reals_four_complex_djbunfft_preload
c3a:	zr4_eight_reals_four_complex_djbunfft rsi, 128, 8*128, 16*128, rdi, ZMM_SCD3, 8
	zloop	8, c3a, -8*128+32*128, rdi, -8*ZMM_SCD3 ;; Test loop counter, restore source pointer, restore sin/cos pointer

	loops_reset			;; 6 more iters of 8
	zr4rb_four_complex_djbunfft_preload
c3b:	zr4rb_four_complex_djbunfft rsi, 128, 8*128, 16*128, rdi, ZMM_SCD3, 8
	zloop	8, c3b, -8*128+32*128, rdi, -8*ZMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	7, c3b, -7*32*128, rdi, 8*ZMM_SCD3 ;; Test loop counter, restore source pointer, next sin/cos pointer
	end_timer 8

;; Do inverse FFT levels 5,6
;; On input the 64-byte cache lines hold these data values:
;;	0	+448	...	32	+448	...
;;	...
;;	31
;;	64	...
;;	128	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+448	...	224	+448	...
;;	...
;;	223	...

;; Do 16 fourteen_reals_unfft operations
;; Do 240 seven_complex_unfft operations
;;	distance between fft data elements is 32
;;	do 32 macros each processing 112 data values

	start_timer 9
	zloop_init 32			;; 32 iterations
	zr7_fourteen_reals_seven_complex_djbunfft_preload
c2b:	zr7_fourteen_reals_seven_complex_djbunfft rsi, 128, 32*128, rdi, ZMM_SCD6, 32, L1PREFETCH_ALL
	zloop	32, c2b, -32*128	;; Test loop counter, restore source pointer
	end_timer 9

;; Do inverse FFT levels 1,2,3,4
;; On input the 128-byte cache line pairs hold these data values:
;;	0	+448	...	224	+448	...
;;	...
;;	223	...
;; These are swizzled to:
;;	0	+28	...	224	+28	...
;;	...
;;	27	...
;;	448	...
;;	896	...
;;	...
;; On output the 128-byte cache line pairs hold these data values:
;;	0	+28	...	1792	+28	...
;;	...
;;	27	...
;;	224	...
;;	448	...
;;	...

;; Do 224 sixteen_reals_unfft operations
;;	distance between fft data elements is 224
;;	do 28 macros processing 128 data values

	start_timer 10
	zloop_init 28			;; 28 iterations
	zr8s_sixteen_reals_last_unfft_preload
c1b:	zr8s_sixteen_reals_last_unfft rsi, 128, 28*128, 56*128, 112*128, rdi, ZMM_SCD7, 28, L1PREFETCH_ALL
	zloop	28, c1b, -28*128	;; Test loop counter, restore source pointer
	end_timer 10

	zfft_3_ret
	ENDM

;; Perform a 3584-element negacyclic FFT

zfft3584p MACRO type
	LOCAL	b1b, b2b, b3b, zmid, c3b, c2b, c1b

;; Do a multiply with pre-FFTed inputs

	cmp	ffttype, 4
	je	zmid

;; Do FFT levels 1,2,3,4
;; On input the 128-byte cache line pairs hold these data values:
;;	0	+28	...	1792	+28	...
;;	...
;;	27	...
;;	224	...
;;	448	...
;;	...
;; On output the 128-byte cache line pairs hold these data values:
;;	0	+28	...	224	+28	...
;;	...
;;	27	...
;;	448	...
;;	896	...
;;	...
;; These are swizzled to:
;;	0	+448	...	224	+448	...
;;	...
;;	223	...

;; Do 224 eight_complex_first_fft operations
;;	distance between fft data elements is 224
;;	do 28 macros processing 128 data values

	start_timer 4
	mov	rdi, sincos1		;; Load sin/cos pointer
	bump	rdi, 32*ZMM_SCD3/8+8*ZMM_SCD2/8
	mov	r9, norm_col_mults	;; Load weights pointer
	zloop_init 28			;; 28 iterations
	zr8fs_eight_complex_first_fft_preload
b1b:	zr8fs_eight_complex_first_fft rsi, 128, 28*128, 56*128, 112*128, rdi, ZMM_CD8SCD4, r9, 8*128, 28, L1PREFETCH_ALL
	zloop	28, b1b, -28*128	;; Test loop counter, restore source pointer
	end_timer 4

;; Do FFT levels 5,6
;; On input the 64-byte cache lines hold these data values:
;;	0	+448	...	224	+448	...
;;	...
;;	223	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+448	...	32	+448	...
;;	...
;;	31
;;	64	...
;;	128	...
;;	...

;; Do 256 seven_complex_fft operations
;;	distance between fft data elements is 32
;;	do 32 macros each processing 112 data values

	start_timer 5
	bump	rdi, -28*ZMM_CD8SCD4-32*ZMM_SCD3/8 ;; Load sin/cos pointer
	zloop_init 32			;; 32 iterations
	zr7b_seven_complex_djbfft_preload
b2b:	zr7b_seven_complex_djbfft rsi, 128, 32*128, rdi, ZMM_SCD3/8, 32, L1PREFETCH_ALL, 128
	zloop	32, b2b, -32*128	;; Test loop counter, restore source pointer
	end_timer 5

;; Do FFT levels 7,8
;; On input the 64-byte cache lines hold these data values:
;;	0	+448	...	32	+448	...
;;	...
;;	31
;;	64	...
;;	128	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+448	...	8	+448	...
;;	...
;;	7
;;	16	...
;;	32	...
;;	...

;; Do 448 four_complex_fft operations
;;	distance between fft data elements is 8
;;	do 56 macros each processing 64 data values

	start_timer 6
	bump	rdi, -32*ZMM_SCD3/8-8*ZMM_SCD2/8 ;; Load sin/cos pointer
	zloop_init 56			;; 7 iterations of 8
	zr4b_four_complex_djbfft_preload
b3b:	zr4b_four_complex_djbfft rsi, 128, 8*128, 16*128, rdi, ZMM_SCD2/8, 8
	zloop	8, b3b, -8*128+32*128, rdi, -8*ZMM_SCD2/8 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	7, b3b, -7*32*128	;; Test loop counter, restore source pointer
	end_timer 6

;; Do FFT levels 9,10,11
;; On input and output the 128-byte cache line pairs hold these data values:
;;	0	+448	...	8	+448	...
;;	...
;;	7
;;	16	...
;;	32	...
;;	...

;; Do 224 eight_complex_fft operations
;;	distance between fft data elements is 1
;;	do 28 macros processing 128 data values

zmid:	start_timer 7
	mov	rcx, 28
	call	zcomplex_onepass_middle
	end_timer 7

;; Do inverse FFT levels 7,8
;; On input the 64-byte cache lines hold these data values:
;;	0	+448	...	8	+448	...
;;	...
;;	7
;;	16	...
;;	32	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+448	...	32	+448	...
;;	...
;;	31
;;	64	...
;;	128	...
;;	...

;; Do 448 four_complex_unfft operations
;;	distance between fft data elements is 8
;;	do 56 macros each processing 64 data values

	start_timer 8
	mov	rdi, sincos1		;; Load sin/cos pointer
	zloop_init 56			;; 7 iterations of 8
	zr4b_four_complex_djbunfft_preload
c3b:	zr4b_four_complex_djbunfft rsi, 128, 8*128, 16*128, rdi, ZMM_SCD2/8, 8
	zloop	8, c3b, -8*128+32*128, rdi, -8*ZMM_SCD2/8 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	7, c3b, -7*32*128, rdi, 8*ZMM_SCD2/8 ;; Test loop counter, restore source pointer, next sin/cos pointer
	end_timer 8

;; Do inverse FFT levels 5,6
;; On input the 64-byte cache lines hold these data values:
;;	0	+448	...	32	+448	...
;;	...
;;	31
;;	64	...
;;	128	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+448	...	224	+448	...
;;	...
;;	223	...

;; Do 256 seven_complex_unfft operations
;;	distance between fft data elements is 32
;;	do 32 macros each processing 112 data values

	start_timer 9
	zloop_init 32			;; 32 iterations
	zr7b_seven_complex_djbunfft_preload
c2b:	zr7b_seven_complex_djbunfft rsi, 128, 32*128, rdi, ZMM_SCD3/8, 32, L1PREFETCH_NONE
	zloop	32, c2b, -32*128	;; Test loop counter, restore source pointer
	end_timer 9

;; Do inverse FFT levels 1,2,3,4
;; On input the 128-byte cache line pairs hold these data values:
;;	0	+448	...	224	+448	...
;;	...
;;	223	...
;; These are swizzled to:
;;	0	+28	...	224	+28	...
;;	...
;;	27	...
;;	448	...
;;	896	...
;;	...
;; On output the 128-byte cache line pairs hold these data values:
;;	0	+28	...	1792	+28	...
;;	...
;;	27	...
;;	224	...
;;	448	...
;;	...

;; Do 224 eight_complex_last_unfft operations
;;	distance between fft data elements is 224
;;	do 28 macros processing 128 data values

	start_timer 10
	zloop_init 28			;; 28 iterations
	zr8s_eight_complex_last_unfft_preload
c1b:	zr8s_eight_complex_last_unfft rsi, 128, 28*128, 56*128, 112*128, rdi, ZMM_CD8SCD4, 28, L1PREFETCH_ALL
	zloop	28, c1b, -28*128	;; Test loop counter, restore source pointer
	end_timer 10

	zfft_3_ret
	ENDM


;; Perform a 3840-element FFT

zfft3840 MACRO type
	LOCAL	b1b, b2b, b3a, b3b, zmid, c3a, c3b, c2b, c1b

;; Check for a multiply with pre-FFTed inputs

	cmp	ffttype, 4
	je	zmid

;; Do FFT levels 1,2,3,4
;; On input the 128-byte cache line pairs hold these data values:
;;	0	+30	...	1920	+30	...
;;	...
;;	29	...
;;	240	...
;;	480	...
;;	...
;; On output the 128-byte cache line pairs hold these data values:
;;	0	+30	...	240	+30	...
;;	...
;;	29	...
;;	480	...
;;	960	...
;;	...
;; These are swizzled to:
;;	0	+480	...	240	+480	...
;;	...
;;	239	...

;; Do 240 sixteen_reals_fft operations
;;	distance between fft data elements is 240
;;	do 30 macros processing 128 data values

	start_timer 4
	mov	rdi, sincos1		;; Load sin/cos data ptr
	bump	rdi, 48*ZMM_SCD4+8*ZMM_SCD5
	mov	r9, norm_col_mults	;; Load weights pointer
	zloop_init 30			;; 30 iterations
	zr8fs_sixteen_reals_first_fft_preload
b1b:	zr8fs_sixteen_reals_first_fft rsi, 128, 30*128, 60*128, 120*128, rdi, ZMM_SCD7, r9, 8*128, 30, L1PREFETCH_ALL
	zloop	30, b1b, -30*128	;; Test loop counter, restore source pointer
	end_timer 4

;; Do FFT levels 5,6
;; On input the 64-byte cache lines hold these data values:
;;	0	+480	...	240	+480	...
;;	...
;;	239	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+480	...	48	+480	...
;;	...
;;	47
;;	96	...
;;	192	...
;;	...

;; Do 24 ten_reals_fft operation
;; Do 360 five_complex_fft operations
;;	distance between fft data elements is 48
;;	do 48 macros each processing 80 data values

	start_timer 5
	bump	rdi, -30*ZMM_SCD7-48*ZMM_SCD4 ;; Load sin/cos pointer
	zloop_init 48			;; 48 iterations
	zr5_ten_reals_five_complex_djbfft_preload
b2b:	zr5_ten_reals_five_complex_djbfft rsi, 128, 48*128, rdi, ZMM_SCD4, 48
	zloop	48, b2b, -48*128	;; Test loop counter, restore source pointer
	end_timer 5

;; Do FFT levels 7,8,9
;; On input the 64-byte cache lines hold these data values:
;;	0	+480	...	48	+480	...
;;	...
;;	47
;;	96	...
;;	192	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+480	...	8	+480	...
;;	...
;;	7
;;	16	...
;;	32	...
;;	...

;; Do 8 twelve_reals_fft operation
;; Do 312 six_complex_fft operations
;;	distance between fft data elements is 8
;;	do 40 macros each processing 96 data values

	start_timer 6
	bump	rdi, -48*ZMM_SCD4-8*ZMM_SCD5	;; Load sin/cos ptr
	zloop_init 40			;; 5 iterations of 8
	zr6_twelve_reals_six_complex_djbfft_preload
b3a:	zr6_twelve_reals_six_complex_djbfft rsi, 128, 8*128, rdi, ZMM_SCD5, 8, L1PREFETCH_NONE
	zloop	8, b3a, -8*128+48*128, rdi, -8*ZMM_SCD5 ;; Test loop counter, next source pointer, restore sin/cos pointer

	loops_reset			;; 4 more iters of 8
	zr6rb_six_complex_djbfft_preload
b3b:	zr6rb_six_complex_djbfft rsi, 128, 8*128, rdi, ZMM_SCD5, 8
	zloop	8, b3b, -8*128+48*128, rdi, -8*ZMM_SCD5 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	5, b3b, -5*48*128	;; Test loop counter, restore source pointer
	end_timer 6

;; Do FFT levels 10,11,12
;; On input and output the 128-byte cache line pairs hold these data values:
;;	0	+480	...	8	+480	...
;;	...
;;	7
;;	16	...
;;	32	...
;;	...

;; Do 1 sixteen_reals_fft operation
;; Do 239 eight_complex_fft operations
;;	distance between fft data elements is 1
;;	do 30 macros processing 128 data values

zmid:	start_timer 7
	mov	rcx, 30
	call	zreal_onepass_middle
	end_timer 7

;; Do inverse FFT levels 7,8,9
;; On input the 64-byte cache lines hold these data values:
;;	0	+480	...	8	+480	...
;;	...
;;	7
;;	16	...
;;	32	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+480	...	48	+480	...
;;	...
;;	47
;;	96	...
;;	192	...
;;	...

;; Do 8 twelve_reals_unfft operations
;; Do 312 six_complex_unfft operations
;;	distance between fft data elements is 8
;;	do 40 macros each processing 96 data values

	start_timer 8
	mov	rdi, sincos1		;; Load sin/cos pointer
	zloop_init 40			;; 5 iterations of 8
	zr6_twelve_reals_six_complex_djbunfft_preload
c3a:	zr6_twelve_reals_six_complex_djbunfft rsi, 128, 8*128, rdi, ZMM_SCD5, 8, L1PREFETCH_NONE
	zloop	8, c3a, -8*128+48*128, rdi, -8*ZMM_SCD5 ;; Test loop counter, restore source pointer, restore sin/cos pointer

	loops_reset			;; 4 more iters of 8
	zr6rb_six_complex_djbunfft_preload
c3b:	zr6rb_six_complex_djbunfft rsi, 128, 8*128, rdi, ZMM_SCD5, 8
	zloop	8, c3b, -8*128+48*128, rdi, -8*ZMM_SCD5 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	5, c3b, -5*48*128, rdi, 8*ZMM_SCD5 ;; Test loop counter, restore source pointer, next sin/cos pointer
	end_timer 8

;; Do inverse FFT levels 5,6
;; On input the 64-byte cache lines hold these data values:
;;	0	+480	...	48	+480	...
;;	...
;;	47
;;	96	...
;;	192	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+480	...	240	+480	...
;;	...
;;	239	...

;; Do 24 ten_reals_unfft operations
;; Do 360 five_complex_unfft operations
;;	distance between fft data elements is 48
;;	do 48 macros each processing 80 data values

	start_timer 9
	zloop_init 48			;; 48 iterations
	zr5_ten_reals_five_complex_djbunfft_preload
c2b:	zr5_ten_reals_five_complex_djbunfft rsi, 128, 48*128, rdi, ZMM_SCD4, 48
	zloop	48, c2b, -48*128	;; Test loop counter, restore source pointer
	end_timer 9

;; Do inverse FFT levels 1,2,3,4
;; On input the 128-byte cache line pairs hold these data values:
;;	0	+480	...	240	+480	...
;;	...
;;	239	...
;; These are swizzled to:
;;	0	+30	...	240	+30	...
;;	...
;;	29	...
;;	480	...
;;	960	...
;;	...
;; On output the 128-byte cache line pairs hold these data values:
;;	0	+30	...	1920	+30	...
;;	...
;;	29	...
;;	240	...
;;	480	...
;;	...

;; Do 240 sixteen_reals_unfft operations
;;	distance between fft data elements is 240
;;	do 30 macros processing 128 data values

	start_timer 10
	zloop_init 30			;; 30 iterations
	zr8s_sixteen_reals_last_unfft_preload
c1b:	zr8s_sixteen_reals_last_unfft rsi, 128, 30*128, 60*128, 120*128, rdi, ZMM_SCD7, 30, L1PREFETCH_ALL
	zloop	30, c1b, -30*128	;; Test loop counter, restore source pointer
	end_timer 10

	zfft_3_ret
	ENDM

;; Perform a 3840-element negacyclic FFT

zfft3840p MACRO type
	LOCAL	b1b, b2b, b3b, zmid, c3b, c2b, c1b

;; Do a multiply with pre-FFTed inputs

	cmp	ffttype, 4
	je	zmid

;; Do FFT levels 1,2,3,4
;; On input the 128-byte cache line pairs hold these data values:
;;	0	+30	...	1920	+30	...
;;	...
;;	29	...
;;	240	...
;;	480	...
;;	...
;; On output the 128-byte cache line pairs hold these data values:
;;	0	+30	...	240	+30	...
;;	...
;;	29	...
;;	480	...
;;	960	...
;;	...
;; These are swizzled to:
;;	0	+480	...	240	+480	...
;;	...
;;	239	...

;; Do 240 eight_complex_first_fft operations
;;	distance between fft data elements is 240
;;	do 30 macros processing 128 data values

	start_timer 4
	mov	rdi, sincos1		;; Load sin/cos pointer
	bump	rdi, 48*ZMM_SCD2/8+8*ZMM_SCD3/8
	mov	r9, norm_col_mults	;; Load weights pointer
	zloop_init 30			;; 30 iterations
	zr8fs_eight_complex_first_fft_preload
b1b:	zr8fs_eight_complex_first_fft rsi, 128, 30*128, 60*128, 120*128, rdi, ZMM_CD8SCD4, r9, 8*128, 30, L1PREFETCH_ALL
	zloop	30, b1b, -30*128	;; Test loop counter, restore source pointer
	end_timer 4

;; Do FFT levels 5,6
;; On input the 64-byte cache lines hold these data values:
;;	0	+480	...	240	+480	...
;;	...
;;	239	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+480	...	48	+480	...
;;	...
;;	47
;;	96	...
;;	192	...
;;	...

;; Do 384 five_complex_fft operations
;;	distance between fft data elements is 48
;;	do 48 macros each processing 80 data values

	start_timer 5
	bump	rdi, -30*ZMM_CD8SCD4-48*ZMM_SCD2/8 ;; Load sin/cos pointer
	zloop_init 48			;; 48 iterations
	zr5b_five_complex_djbfft_preload
b2b:	zr5b_five_complex_djbfft rsi, 128, 48*128, rdi, ZMM_SCD2/8, 48
	zloop	48, b2b, -48*128	;; Test loop counter, restore source pointer
	end_timer 5

;; Do FFT levels 7,8,9
;; On input the 64-byte cache lines hold these data values:
;;	0	+480	...	48	+480	...
;;	...
;;	47
;;	96	...
;;	192	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+480	...	8	+480	...
;;	...
;;	7
;;	16	...
;;	32	...
;;	...

;; Do 320 six_complex_fft operations
;;	distance between fft data elements is 8
;;	do 40 macros each processing 96 data values

	start_timer 6
	bump	rdi, -48*ZMM_SCD2/8-8*ZMM_SCD3/8 ;; Load sin/cos pointer
	zloop_init 40			;; 5 iterations of 8
	zr6b_six_complex_djbfft_preload
b3b:	zr6b_six_complex_djbfft rsi, 128, 8*128, rdi, ZMM_SCD3/8, 8
	zloop	8, b3b, -8*128+48*128, rdi, -8*ZMM_SCD3/8 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	5, b3b, -5*48*128	;; Test loop counter, restore source pointer
	end_timer 6

;; Do FFT levels 10,11,12
;; On input and output the 128-byte cache line pairs hold these data values:
;;	0	+480	...	8	+480	...
;;	...
;;	7
;;	16	...
;;	32	...
;;	...

;; Do 240 eight_complex_fft operations
;;	distance between fft data elements is 1
;;	do 30 macros processing 128 data values

zmid:	start_timer 7
	mov	rcx, 30
	call	zcomplex_onepass_middle
	end_timer 7

;; Do inverse FFT levels 7,8,9
;; On input the 64-byte cache lines hold these data values:
;;	0	+480	...	8	+480	...
;;	...
;;	7
;;	16	...
;;	32	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+480	...	48	+480	...
;;	...
;;	47
;;	96	...
;;	192	...
;;	...

;; Do 320 six_complex_unfft operations
;;	distance between fft data elements is 8
;;	do 40 macros each processing 96 data values

	start_timer 8
	mov	rdi, sincos1		;; Load sin/cos pointer
	zloop_init 40			;; 5 iterations of 8
	zr6b_six_complex_djbunfft_preload
c3b:	zr6b_six_complex_djbunfft rsi, 128, 8*128, rdi, ZMM_SCD3/8, 8
	zloop	8, c3b, -8*128+48*128, rdi, -8*ZMM_SCD3/8 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	5, c3b, -5*48*128, rdi, 8*ZMM_SCD3/8	 ;; Test loop counter, restore source pointer, next sin/cos pointer
	end_timer 8

;; Do inverse FFT levels 5,6
;; On input the 64-byte cache lines hold these data values:
;;	0	+480	...	48	+480	...
;;	...
;;	47
;;	96	...
;;	192	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+480	...	240	+480	...
;;	...
;;	239	...

;; Do 384 five_complex_unfft operations
;;	distance between fft data elements is 48
;;	do 48 macros each processing 80 data values

	start_timer 9
	zloop_init 48			;; 48 iterations
	zr5b_five_complex_djbunfft_preload
c2b:	zr5b_five_complex_djbunfft rsi, 128, 48*128, rdi, ZMM_SCD2/8, 48
	zloop	48, c2b, -48*128	;; Test loop counter, restore source pointer
	end_timer 9

;; Do inverse FFT levels 1,2,3,4
;; On input the 128-byte cache line pairs hold these data values:
;;	0	+480	...	240	+480	...
;;	...
;;	239	...
;; These are swizzled to:
;;	0	+30	...	240	+30	...
;;	...
;;	29	...
;;	480	...
;;	960	...
;;	...
;; On output the 128-byte cache line pairs hold these data values:
;;	0	+30	...	1920	+30	...
;;	...
;;	29	...
;;	240	...
;;	480	...
;;	...

;; Do 240 eight_complex_last_unfft operations
;;	distance between fft data elements is 240
;;	do 30 macros processing 128 data values

	start_timer 10
	zloop_init 30			;; 30 iterations
	zr8s_eight_complex_last_unfft_preload
c1b:	zr8s_eight_complex_last_unfft rsi, 128, 30*128, 60*128, 120*128, rdi, ZMM_CD8SCD4, 30, L1PREFETCH_ALL
	zloop	30, c1b, -30*128	;; Test loop counter, restore source pointer
	end_timer 10

	zfft_3_ret
	ENDM


;; Perform a 4K-element FFT

zfft4K MACRO type
	LOCAL	b1b, b2b, b3a, b3b, zmid, c3a, c3b, c2b, c1b

;; Check for a multiply with pre-FFTed inputs

	cmp	ffttype, 4
	je	zmid

;; Do FFT levels 1,2,3,4
;; On input the 128-byte cache line pairs hold these data values:
;;	0	+32	...	2K	+32	...
;;	...
;;	31	...
;;	256	...
;;	512	...
;;	...
;; On output the 128-byte cache line pairs hold these data values:
;;	0	+32	...	256	+32	...
;;	...
;;	31	...
;;	512	...
;;	1K	...
;;	...
;; These are swizzled to:
;;	0	+512	...	256	+512	...
;;	...
;;	255	...

;; Do 256 sixteen_reals_fft operations
;;	distance between fft data elements is 256
;;	do 32 macros processing 128 data values

;; Do FFT levels 5,6,7
;; On input the 64-byte cache lines hold these data values:
;;	0	+512	...	256	+512	...
;;	...
;;	255	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+512	...	32	+512	...
;;	...
;;	31
;;	64	...
;;	128	...
;;	...

;; Do 32 sixteen_reals_fft operation
;; Do 224 eight_complex_fft operations
;;	distance between fft data elements is 32
;;	do 32 macros each processing 128 data values

IFDEF OLDWAY
	start_timer 4
	mov	rdi, sincos1		;; Load sin/cos data ptr
	bump	rdi, 32*ZMM_SCD7+8*ZMM_SCD3
	mov	r9, norm_col_mults	;; Load weights pointer
	zloop_init 32			;; 32 iterations
	zr8fs_sixteen_reals_first_fft_preload
b1b:	zr8fs_sixteen_reals_first_fft rsi, 128, 32*128, 64*128, 128*128, rdi, ZMM_SCD7, r9, 8*128, 32, L1PREFETCH_ALL
	zloop	32, b1b, -32*128	;; Test loop counter, restore source pointer
	end_timer 4

	start_timer 5
	bump	rdi, -32*ZMM_SCD7-32*ZMM_SCD7 ;; Load sin/cos pointer
	zloop_init 32			;; 32 iterations
	zr8_sixteen_reals_eight_complex_djbfft_preload
b2b:	zr8_sixteen_reals_eight_complex_djbfft rsi, 128, 32*128, 64*128, 128*128, rdi, ZMM_SCD7, 32, L1PREFETCH_ALL
	zloop	32, b2b, -32*128	;; Test loop counter, next source pointer, restore sin/cos pointer
	end_timer 5
ELSE
	start_timer 4
	mov	rdi, sincos1		;; Load sin/cos data ptr
	bump	rdi, 8*ZMM_SCD3
	lea	r10, [rdi+32*ZMM_SCD7]
	mov	r9, norm_col_mults	;; Load weights pointer
	zloop_init 32			;; 32 iterations
	zr8fs_hundredtwentyeight_reals_first_fft_preload
b1b:	zr8fs_hundredtwentyeight_reals_first_fft rsi, 128, 32*128, 64*128, 128*128, r10, ZMM_SCD7, rdi, ZMM_SCD7, r9, 8*128, 32, L1PREFETCH_ALL
	zloop	32, b1b, -32*128	;; Test loop counter, restore source pointer
	end_timer 4
ENDIF

;; Do FFT levels 8,9
;; On input the 64-byte cache lines hold these data values:
;;	0	+512	...	32	+512	...
;;	...
;;	31
;;	64	...
;;	128	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+512	...	8	+512	...
;;	...
;;	7
;;	16	...
;;	32	...
;;	...

;; Do 8 eight_reals_fft operation
;; Do 504 four_complex_fft operations
;;	distance between fft data elements is 8
;;	do 64 macros each processing 64 data values

	start_timer 6
	bump	rdi, -32*ZMM_SCD7-8*ZMM_SCD3 ;; Load sin/cos pointer
	zloop_init 64			;; 8 iterations of 8
	zr4_eight_reals_four_complex_djbfft_preload
b3a:	zr4_eight_reals_four_complex_djbfft rsi, 128, 8*128, 16*128, rdi, ZMM_SCD3, 8
	zloop	8, b3a, -8*128+32*128, rdi, -8*ZMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer

	loops_reset			;; 7 more iters of 8
	zr4rb_four_complex_djbfft_preload
b3b:	zr4rb_four_complex_djbfft rsi, 128, 8*128, 16*128, rdi, ZMM_SCD3, 8, L1PREFETCH_NONE
	zloop	8, b3b, -8*128+32*128, rdi, -8*ZMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	8, b3b, -8*32*128	;; Test loop counter, restore source pointer
	end_timer 6

;; Do FFT levels 10,11,12
;; On input and output the 128-byte cache line pairs hold these data values:
;;	0	+512	...	8	+512	...
;;	...
;;	7
;;	16	...
;;	32	...
;;	...

;; Do 1 sixteen_reals_fft operation
;; Do 255 eight_complex_fft operations
;;	distance between fft data elements is 1
;;	do 32 macros processing 128 data values

zmid:	start_timer 7
	mov	rcx, 32
	call	zreal_onepass_middle
	end_timer 7

;; Do inverse FFT levels 8,9
;; On input the 64-byte cache lines hold these data values:
;;	0	+512	...	8	+512	...
;;	...
;;	7
;;	16	...
;;	32	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+512	...	32	+512	...
;;	...
;;	31
;;	64	...
;;	128	...
;;	...

;; Do 8 eight_reals_unfft operations
;; Do 504 four_complex_unfft operations
;;	distance between fft data elements is 8
;;	do 64 macros each processing 64 data values

	start_timer 8
	mov	rdi, sincos1		;; Load sin/cos pointer
	zloop_init 64			;; 8 iterations of 8
	zr4_eight_reals_four_complex_djbunfft_preload
c3a:	zr4_eight_reals_four_complex_djbunfft rsi, 128, 8*128, 16*128, rdi, ZMM_SCD3, 8
	zloop	8, c3a, -8*128+32*128, rdi, -8*ZMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer

	loops_reset			;; 7 more iters of 8
	zr4rb_four_complex_djbunfft_preload
c3b:	zr4rb_four_complex_djbunfft rsi, 128, 8*128, 16*128, rdi, ZMM_SCD3, 8
	zloop	8, c3b, -8*128+32*128, rdi, -8*ZMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	8, c3b, -8*32*128, rdi, 8*ZMM_SCD3 ;; Test loop counter, restore source pointer, next sin/cos pointer
	end_timer 8

;; Do inverse FFT levels 5,6,7
;; On input the 64-byte cache lines hold these data values:
;;	0	+512	...	32	+512	...
;;	...
;;	31
;;	64	...
;;	128	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+512	...	256	+512	...
;;	...
;;	255	...

;; Do 32 sixteen_reals_unfft operations
;; Do 224 eight_complex_unfft operations
;;	distance between fft data elements is 32
;;	do 32 macros each processing 128 data values

;; Do inverse FFT levels 1,2,3,4
;; On input the 128-byte cache line pairs hold these data values:
;;	0	+512	...	256	+512	...
;;	...
;;	255	...
;; These are swizzled to:
;;	0	+32	...	256	+32	...
;;	...
;;	31	...
;;	512	...
;;	1K	...
;;	...
;; On output the 128-byte cache line pairs hold these data values:
;;	0	+32	...	2K	+32	...
;;	...
;;	31	...
;;	256	...
;;	512	...
;;	...

;; Do 256 sixteen_reals_unfft operations
;;	distance between fft data elements is 256
;;	do 32 macros processing 128 data values

IFDEF OLDWAY
	start_timer 9
	zloop_init 32			;; 32 iterations
	zr8_sixteen_reals_eight_complex_djbunfft_preload
c2b:	zr8_sixteen_reals_eight_complex_djbunfft rsi, 128, 32*128, 64*128, 128*128, rdi, ZMM_SCD7, 32, L1PREFETCH_NONE
	zloop	32, c2b, -32*128	;; Test loop counter, restore source pointer
	end_timer 9

	start_timer 10
	zloop_init 32			;; 32 iterations
	zr8s_sixteen_reals_last_unfft_preload
c1b:	zr8s_sixteen_reals_last_unfft rsi, 128, 32*128, 64*128, 128*128, rdi, ZMM_SCD7, 32, L1PREFETCH_ALL
	zloop	32, c1b, -32*128	;; Test loop counter, restore source pointer
	end_timer 10
ELSE
	start_timer 10
	lea	r10, [rdi+32*ZMM_SCD7]
	zloop_init 32			;; 32 iterations
	zr8s_hundredtwentyeight_reals_last_unfft_preload
c1b:	zr8s_hundredtwentyeight_reals_last_unfft rsi, 128, 32*128, 64*128, 128*128, rdi, ZMM_SCD7, r10, ZMM_SCD7, 32, L1PREFETCH_ALL
	zloop	32, c1b, -32*128	;; Test loop counter, restore source pointer
	end_timer 10
ENDIF

	zfft_3_ret
	ENDM

;; Perform a 4K-element negacyclic FFT

zfft4Kp MACRO type
	LOCAL	b1b, b2b, b3b, zmid, c3b, c2b, c1b

;; Do a multiply with pre-FFTed inputs

	cmp	ffttype, 4
	je	zmid

;; Do FFT levels 1,2,3,4
;; On input the 128-byte cache line pairs hold these data values:
;;	0	+32	...	2K	+32	...
;;	...
;;	31	...
;;	256	...
;;	512	...
;;	...
;; On output the 128-byte cache line pairs hold these data values:
;;	0	+32	...	256	+32	...
;;	...
;;	31	...
;;	512	...
;;	1K	...
;;	...
;; These are swizzled to:
;;	0	+512	...	256	+512	...
;;	...
;;	255	...

;; Do 256 eight_complex_first_fft operations
;;	distance between fft data elements is 256
;;	do 32 macros processing 128 data values

;; Do FFT levels 5,6,7
;; On input the 64-byte cache lines hold these data values:
;;	0	+512	...	256	+512	...
;;	...
;;	255	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+512	...	32	+512	...
;;	...
;;	31
;;	64	...
;;	128	...
;;	...

;; Do 256 eight_complex_fft operations
;;	distance between fft data elements is 32
;;	do 32 macros each processing 128 data values

IFNDEF NEWWAY				;; No better despite better L1 cache locality
IFDEF OLDWAY
	start_timer 4
	mov	rdi, sincos1		;; Load sin/cos pointer
	bump	rdi, 32*ZMM_SCD4/8+8*ZMM_SCD2/8
	mov	r9, norm_col_mults	;; Load weights pointer
	zloop_init 32			;; 32 iterations
	zr8fs_eight_complex_first_fft_preload
b1b:	zr8fs_eight_complex_first_fft rsi, 128, 32*128, 64*128, 128*128, rdi, ZMM_CD8SCD4, r9, 8*128, 32, L1PREFETCH_ALL
	zloop	32, b1b, -32*128	;; Test loop counter, restore source pointer
	end_timer 4

	start_timer 5
	bump	rdi, -32*ZMM_CD8SCD4-32*ZMM_SCD4/8 ;; Load sin/cos pointer
	zloop_init 32			;; 32 iterations
	zr8b_eight_complex_djbfft_preload
b2b:	zr8b_eight_complex_djbfft rsi, 128, 32*128, 64*128, 128*128, rdi, ZMM_SCD4/8, 32, L1PREFETCH_NONE
	zloop	32, b2b, -32*128	;; Test loop counter, restore source pointer
	end_timer 5
ELSE
	start_timer 4
	mov	rdi, sincos1		;; Load sin/cos pointer
	bump	rdi, 8*ZMM_SCD2/8
	lea	r10, [rdi+32*ZMM_SCD4/8]
	mov	r9, norm_col_mults	;; Load weights pointer
	zloop_init 32			;; 32 iterations
	zr8fs_sixtyfour_complex_first_fft_preload
b1b:	zr8fs_sixtyfour_complex_first_fft rsi, 128, 32*128, 64*128, 128*128, r10, ZMM_CD8SCD4, rdi, ZMM_SCD4/8, r9, 8*128, 32, L1PREFETCH_ALL
	zloop	32, b1b, -32*128	;; Test loop counter, restore source pointer
	end_timer 4
ENDIF

;; Do FFT levels 8,9
;; On input the 64-byte cache lines hold these data values:
;;	0	+512	...	32	+512	...
;;	...
;;	31
;;	64	...
;;	128	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+512	...	8	+512	...
;;	...
;;	7
;;	16	...
;;	32	...
;;	...

;; Do 512 four_complex_fft operations
;;	distance between fft data elements is 8
;;	do 64 macros each processing 64 data values

	start_timer 6
	bump	rdi, -32*ZMM_SCD4/8-8*ZMM_SCD2/8 ;; Load sin/cos pointer
	zloop_init 64			;; 8 iterations of 8
	zr4b_four_complex_djbfft_preload
b3b:	zr4b_four_complex_djbfft rsi, 128, 8*128, 16*128, rdi, ZMM_SCD2/8, 8
	zloop	8, b3b, -8*128+32*128, rdi, -8*ZMM_SCD2/8 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	8, b3b, -8*32*128	;; Test loop counter, restore source pointer
	end_timer 6

ELSE
	start_timer 4
	mov	rdi, sincos1		;; Load sin/cos pointer
	lea	r12, [rdi+8*ZMM_SCD2/8]
	lea	r10, [r12+32*ZMM_SCD4/8]
	mov	r9, norm_col_mults	;; Load weights pointer
	zloop_init 64			;; 4 or 8 iterations of 8
b1a:	zr8fs_sixtyfour_complex_first_fft_preload
b1b:	zr8fs_sixtyfour_complex_first_fft rsi, 8*128, 32*128, 64*128, 128*128, r10, 8*ZMM_CD8SCD4, r12, 8*ZMM_SCD4/8, r9, 8*8*128, 4, L1PREFETCH_ALL
	zloop	4, b1b, -4*8*128	;; Test loop counter, restore source pointer
	bump	r12, -4*8*ZMM_SCD4/8+ZMM_SCD4/8	;; Next sin/cos and weights pointers
	bump	r10, -4*8*ZMM_CD8SCD4+ZMM_CD8SCD4
	bump	r9, -4*8*8*128+8*128
	loops_undo 4
	zr4b_four_complex_djbfft_preload
b3b:	zr4b_four_complex_djbfft rsi, 32*128, 8*128, 16*128, rdi, 0, 8
	zloop	8, b3b, -8*32*128+128	;; Test loop counter, next outer loop source pointer
	bump	rdi, ZMM_SCD2/8		;; Next sin/cos pointer
	zloop	8, b1a, -8*128		;; Test outer loop counter, restore source pointer
	end_timer 4
ENDIF

;; Do FFT levels 10,11,12
;; On input and output the 128-byte cache line pairs hold these data values:
;;	0	+512	...	8	+512	...
;;	...
;;	7
;;	16	...
;;	32	...
;;	...

;; Do 256 eight_complex_fft operations
;;	distance between fft data elements is 1
;;	do 32 macros processing 128 data values

zmid:	start_timer 7
	mov	rcx, 32
	call	zcomplex_onepass_middle
	end_timer 7

;; Do inverse FFT levels 8,9
;; On input the 64-byte cache lines hold these data values:
;;	0	+512	...	8	+512	...
;;	...
;;	7
;;	16	...
;;	32	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+512	...	32	+512	...
;;	...
;;	31
;;	64	...
;;	128	...
;;	...

;; Do 512 four_complex_unfft operations
;;	distance between fft data elements is 8
;;	do 64 macros each processing 64 data values

IFNDEF NEWWAY				;; No better despite better L1 cache locality
	start_timer 8
	mov	rdi, sincos1		;; Load sin/cos pointer
	zloop_init 64			;; 8 iterations of 8
	zr4b_four_complex_djbunfft_preload
c3b:	zr4b_four_complex_djbunfft rsi, 128, 8*128, 16*128, rdi, ZMM_SCD2/8, 8
	zloop	8, c3b, -8*128+32*128, rdi, -8*ZMM_SCD2/8 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	8, c3b, -8*32*128, rdi, 8*ZMM_SCD2/8 ;; Test loop counter, restore source pointer, next sin/cos pointer
	end_timer 8

;; Do inverse FFT levels 5,6,7
;; On input the 64-byte cache lines hold these data values:
;;	0	+512	...	32	+512	...
;;	...
;;	31
;;	64	...
;;	128	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+512	...	256	+512	...
;;	...
;;	255	...

;; Do 256 eight_complex_unfft operations
;;	distance between fft data elements is 8
;;	do 32 macros each processing 128 data values

;; Do inverse FFT levels 1,2,3,4
;; On input the 128-byte cache line pairs hold these data values:
;;	0	+512	...	256	+512	...
;;	...
;;	255	...
;; These are swizzled to:
;;	0	+32	...	256	+32	...
;;	...
;;	31	...
;;	512	...
;;	1K	...
;;	...
;; On output the 128-byte cache line pairs hold these data values:
;;	0	+32	...	2K	+32	...
;;	...
;;	31	...
;;	256	...
;;	512	...
;;	...

;; Do 256 eight_complex_last_unfft operations
;;	distance between fft data elements is 256
;;	do 32 macros processing 128 data values

IFDEF OLDWAY
	start_timer 9
	zloop_init 32			;; 32 iterations
	zr8b_eight_complex_djbunfft_preload
c2b:	zr8b_eight_complex_djbunfft rsi, 128, 32*128, 64*128, 128*128, rdi, ZMM_SCD4/8, 32, L1PREFETCH_NONE
	zloop	32, c2b, -32*128	;; Test loop counter, restore source pointer
	end_timer 9

	start_timer 10
	zloop_init 32			;; 32 iterations
	zr8s_eight_complex_last_unfft_preload
c1b:	zr8s_eight_complex_last_unfft rsi, 128, 32*128, 64*128, 128*128, rdi, ZMM_CD8SCD4, 32, L1PREFETCH_ALL
	zloop	32, c1b, -32*128	;; Test loop counter, restore source pointer
	end_timer 10
ELSE
	start_timer 10
	lea	r10, [rdi+32*ZMM_SCD4/8]
	zloop_init 32			;; 32 iterations
	zr8s_sixtyfour_complex_last_unfft_preload
c1b:	zr8s_sixtyfour_complex_last_unfft rsi, 128, 32*128, 64*128, 128*128, rdi, ZMM_SCD4/8, r10, ZMM_CD8SCD4, 32, L1PREFETCH_ALL
	zloop	32, c1b, -32*128	;; Test loop counter, restore source pointer
	end_timer 10
ENDIF

ELSE
	start_timer 10
	mov	rdi, sincos1		;; Load sin/cos pointer
	lea	r12, [rdi+8*ZMM_SCD2/8]
	lea	r10, [r12+32*ZMM_SCD4/8]
	zloop_init 64 			;; 8 or 4 iterations of 8
c3a:	zr4b_four_complex_djbunfft_preload
c3b:	zr4b_four_complex_djbunfft rsi, 32*128, 8*128, 16*128, rdi, 0, 8, L1PREFETCH_ALL
	zloop	8, c3b, -8*32*128, rdi, ZMM_SCD2/8 ;; Test loop counter, restore source pointer, next sin/cos pointer
	loops_undo 8
	zloop_adjust 4			;; Do 4 rather than 8 iterations in this loop
	zr8s_sixtyfour_complex_last_unfft_preload
c1b:	zr8s_sixtyfour_complex_last_unfft rsi, 8*128, 32*128, 64*128, 128*128, r12, 8*ZMM_SCD4/8, r10, 8*ZMM_CD8SCD4, 4, L1PREFETCH_NONE
	zloop	8, c1b, -4*8*128+128	;; 4 iterations!!! Test loop counter, next outer loop source pointer
	bump	r12, -4*8*ZMM_SCD4/8+ZMM_SCD4/8	;; Next sin/cos pointers
	bump	r10, -4*8*ZMM_CD8SCD4+ZMM_CD8SCD4
	zloop	8, c3a, -8*128		;; Test outer loop counter, restore source pointer
	end_timer 10
ENDIF

	zfft_3_ret
	ENDM


;; Perform a 4608-element FFT

zfft4608 MACRO type
	LOCAL	b1b, b2b, b3a, b3b, zmid, c3a, c3b, c2b, c1b

;; Check for a multiply with pre-FFTed inputs

	cmp	ffttype, 4
	je	zmid

;; Do FFT levels 1,2,3,4
;; On input the 128-byte cache line pairs hold these data values:
;;	0	+36	...	2304	+36	...
;;	...
;;	35	...
;;	288	...
;;	576	...
;;	...
;; On output the 128-byte cache line pairs hold these data values:
;;	0	+36	...	288	+36	...
;;	...
;;	35	...
;;	576	...
;;	1152	...
;;	...
;; These are swizzled to:
;;	0	+576	...	288	+576	...
;;	...
;;	287	...

;; Do 288 sixteen_reals_fft operations
;;	distance between fft data elements is 288
;;	do 36 macros processing 128 data values

	start_timer 4
	mov	rdi, sincos1		;; Load sin/cos data ptr
	bump	rdi, 48*ZMM_SCD5
	mov	r9, norm_col_mults	;; Load weights pointer
	zloop_init 36			;; 36 iterations
	zr8fs_sixteen_reals_first_fft_preload
b1b:	zr8fs_sixteen_reals_first_fft rsi, 128, 36*128, 72*128, 144*128, rdi, ZMM_SCD7, r9, 8*128, 36, L1PREFETCH_ALL
	zloop	36, b1b, -36*128	;; Test loop counter, restore source pointer
	end_timer 4

;; Do FFT levels 5,6
;; On input the 64-byte cache lines hold these data values:
;;	0	+576	...	288	+576	...
;;	...
;;	287	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+576	...	48	+576	...
;;	...
;;	47
;;	96	...
;;	192	...
;;	...

;; Do 24 twelve_reals_fft operation
;; Do 360 six_complex_fft operations
;;	distance between fft data elements is 48
;;	do 48 macros each processing 96 data values

	start_timer 5
	bump	rdi, -36*ZMM_SCD7-48*ZMM_SCD5 ;; Load sin/cos pointer
	zloop_init 48			;; 48 iterations
	zr6_twelve_reals_six_complex_djbfft_preload
b2b:	zr6_twelve_reals_six_complex_djbfft rsi, 128, 48*128, rdi, ZMM_SCD5, 48, L1PREFETCH_NONE
	zloop	48, b2b, -48*128	;; Test loop counter, restore source pointer
	end_timer 5

;; Do FFT levels 7,8,9
;; On input the 64-byte cache lines hold these data values:
;;	0	+576	...	48	+576	...
;;	...
;;	47
;;	96	...
;;	192	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+576	...	8	+576	...
;;	...
;;	7
;;	16	...
;;	32	...
;;	...

;; Do 8 twelve_reals_fft operation
;; Do 376 six_complex_fft operations
;;	distance between fft data elements is 8
;;	do 48 macros each processing 96 data values

	start_timer 6
	bump	rdi, -48*ZMM_SCD5	;; Load sin/cos ptr
	zloop_init 48			;; 6 iterations of 8
	zr6_twelve_reals_six_complex_djbfft_preload
b3a:	zr6_twelve_reals_six_complex_djbfft rsi, 128, 8*128, rdi, 6*ZMM_SCD5, 8, L1PREFETCH_NONE
	zloop	8, b3a, -8*128+48*128, rdi, -8*6*ZMM_SCD5 ;; Test loop counter, next source pointer, restore sin/cos pointer

	loops_reset			;; 5 more iters of 8
	zr6rb_six_complex_djbfft_preload
b3b:	zr6rb_six_complex_djbfft rsi, 128, 8*128, rdi, 6*ZMM_SCD5, 8
	zloop	8, b3b, -8*128+48*128, rdi, -8*6*ZMM_SCD5 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	6, b3b, -6*48*128	;; Test loop counter, restore source pointer
	end_timer 6

;; Do FFT levels 10,11,12
;; On input and output the 128-byte cache line pairs hold these data values:
;;	0	+576	...	8	+576	...
;;	...
;;	7
;;	16	...
;;	32	...
;;	...

;; Do 1 sixteen_reals_fft operation
;; Do 287 eight_complex_fft operations
;;	distance between fft data elements is 1
;;	do 36 macros processing 128 data values

zmid:	start_timer 7
	mov	rcx, 36
	call	zreal_onepass_middle
	end_timer 7

;; Do inverse FFT levels 7,8,9
;; On input the 64-byte cache lines hold these data values:
;;	0	+576	...	8	+576	...
;;	...
;;	7
;;	16	...
;;	32	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+576	...	48	+576	...
;;	...
;;	47
;;	96	...
;;	192	...
;;	...

;; Do 8 twelve_reals_unfft operations
;; Do 376 six_complex_unfft operations
;;	distance between fft data elements is 8
;;	do 48 macros each processing 96 data values

	start_timer 8
	mov	rdi, sincos1		;; Load sin/cos pointer
	zloop_init 48			;; 6 iterations of 8
	zr6_twelve_reals_six_complex_djbunfft_preload
c3a:	zr6_twelve_reals_six_complex_djbunfft rsi, 128, 8*128, rdi, 6*ZMM_SCD5, 8, L1PREFETCH_NONE
	zloop	8, c3a, -8*128+48*128, rdi, -8*6*ZMM_SCD5 ;; Test loop counter, restore source pointer, restore sin/cos pointer

	loops_reset			;; 5 more iters of 8
	zr6rb_six_complex_djbunfft_preload
c3b:	zr6rb_six_complex_djbunfft rsi, 128, 8*128, rdi, 6*ZMM_SCD5, 8
	zloop	8, c3b, -8*128+48*128, rdi, -8*6*ZMM_SCD5 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	6, c3b, -6*48*128	 ;; Test loop counter, restore source pointer
	end_timer 8

;; Do inverse FFT levels 5,6
;; On input the 64-byte cache lines hold these data values:
;;	0	+576	...	48	+576	...
;;	...
;;	47
;;	96	...
;;	192	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+576	...	288	+576	...
;;	...
;;	287	...

;; Do 24 twelve_reals_unfft operations
;; Do 360 six_complex_unfft operations
;;	distance between fft data elements is 48
;;	do 48 macros each processing 96 data values

	start_timer 9
	zloop_init 48			;; 48 iterations
	zr6_twelve_reals_six_complex_djbunfft_preload
c2b:	zr6_twelve_reals_six_complex_djbunfft rsi, 128, 48*128, rdi, ZMM_SCD5, 48, L1PREFETCH_NONE
	zloop	48, c2b, -48*128	;; Test loop counter, restore source pointer
	end_timer 9

;; Do inverse FFT levels 1,2,3,4
;; On input the 128-byte cache line pairs hold these data values:
;;	0	+576	...	288	+576	...
;;	...
;;	287	...
;; These are swizzled to:
;;	0	+36	...	288	+36	...
;;	...
;;	35	...
;;	576	...
;;	1152	...
;;	...
;; On output the 128-byte cache line pairs hold these data values:
;;	0	+36	...	2304	+36	...
;;	...
;;	35	...
;;	288	...
;;	576	...
;;	...

;; Do 288 sixteen_reals_unfft operations
;;	distance between fft data elements is 288
;;	do 36 macros processing 128 data values

	start_timer 10
	zloop_init 36			;; 36 iterations
	zr8s_sixteen_reals_last_unfft_preload
c1b:	zr8s_sixteen_reals_last_unfft rsi, 128, 36*128, 72*128, 144*128, rdi, ZMM_SCD7, 36, L1PREFETCH_ALL
	zloop	36, c1b, -36*128	;; Test loop counter, restore source pointer
	end_timer 10

	zfft_3_ret
	ENDM

;; Perform a 4608-element negacyclic FFT

zfft4608p MACRO type
	LOCAL	b1b, b2b, b3b, zmid, c3b, c2b, c1b

;; Do a multiply with pre-FFTed inputs

	cmp	ffttype, 4
	je	zmid

;; Do FFT levels 1,2,3,4
;; On input the 128-byte cache line pairs hold these data values:
;;	0	+36	...	2304	+36	...
;;	...
;;	35	...
;;	288	...
;;	576	...
;;	...
;; On output the 128-byte cache line pairs hold these data values:
;;	0	+36	...	288	+36	...
;;	...
;;	35	...
;;	576	...
;;	1152	...
;;	...
;; These are swizzled to:
;;	0	+576	...	288	+576	...
;;	...
;;	287	...

;; Do 288 eight_complex_first_fft operations
;;	distance between fft data elements is 288
;;	do 36 macros processing 128 data values

	start_timer 4
	mov	rdi, sincos1		;; Load sin/cos pointer
	bump	rdi, 48*ZMM_SCD3/8
	mov	r9, norm_col_mults	;; Load weights pointer
	zloop_init 36			;; 36 iterations
	zr8fs_eight_complex_first_fft_preload
b1b:	zr8fs_eight_complex_first_fft rsi, 128, 36*128, 72*128, 144*128, rdi, ZMM_CD8SCD4, r9, 8*128, 36, L1PREFETCH_ALL
	zloop	36, b1b, -36*128	;; Test loop counter, restore source pointer
	end_timer 4

;; Do FFT levels 5,6
;; On input the 64-byte cache lines hold these data values:
;;	0	+576	...	288	+576	...
;;	...
;;	287	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+576	...	48	+576	...
;;	...
;;	47
;;	96	...
;;	192	...
;;	...

;; Do 384 six_complex_fft operations
;;	distance between fft data elements is 48
;;	do 48 macros each processing 96 data values

	start_timer 5
	bump	rdi, -36*ZMM_CD8SCD4-48*ZMM_SCD3/8 ;; Load sin/cos pointer
	zloop_init 48			;; 48 iterations
	zr6b_six_complex_djbfft_preload
b2b:	zr6b_six_complex_djbfft rsi, 128, 48*128, rdi, ZMM_SCD3/8, 48
	zloop	48, b2b, -48*128	;; Test loop counter, restore source pointer
	end_timer 5

;; Do FFT levels 7,8,9
;; On input the 64-byte cache lines hold these data values:
;;	0	+576	...	48	+576	...
;;	...
;;	47
;;	96	...
;;	192	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+576	...	8	+576	...
;;	...
;;	7
;;	16	...
;;	32	...
;;	...

;; Do 384 six_complex_fft operations
;;	distance between fft data elements is 8
;;	do 48 macros each processing 96 data values

	start_timer 6
	bump	rdi, -48*ZMM_SCD3/8	;; Load sin/cos pointer
	zloop_init 48			;; 6 iterations of 8
	zr6b_six_complex_djbfft_preload
b3b:	zr6b_six_complex_djbfft rsi, 128, 8*128, rdi, 6*ZMM_SCD3/8, 8
	zloop	8, b3b, -8*128+48*128, rdi, -8*6*ZMM_SCD3/8 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	6, b3b, -6*48*128	;; Test loop counter, restore source pointer
	end_timer 6

;; Do FFT levels 10,11,12
;; On input and output the 128-byte cache line pairs hold these data values:
;;	0	+576	...	8	+576	...
;;	...
;;	7
;;	16	...
;;	32	...
;;	...

;; Do 288 eight_complex_fft operations
;;	distance between fft data elements is 1
;;	do 36 macros processing 128 data values

zmid:	start_timer 7
	mov	rcx, 36
	call	zcomplex_onepass_middle
	end_timer 7

;; Do inverse FFT levels 7,8,9
;; On input the 64-byte cache lines hold these data values:
;;	0	+576	...	8	+576	...
;;	...
;;	7
;;	16	...
;;	32	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+576	...	48	+576	...
;;	...
;;	47
;;	96	...
;;	192	...
;;	...

;; Do 384 six_complex_unfft operations
;;	distance between fft data elements is 8
;;	do 48 macros each processing 96 data values

	start_timer 8
	mov	rdi, sincos1		;; Load sin/cos pointer
	zloop_init 48			;; 6 iterations of 8
	zr6b_six_complex_djbunfft_preload
c3b:	zr6b_six_complex_djbunfft rsi, 128, 8*128, rdi, 6*ZMM_SCD3/8, 8
	zloop	8, c3b, -8*128+48*128, rdi, -8*6*ZMM_SCD3/8 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	6, c3b, -6*48*128	 ;; Test loop counter, restore source pointer
	end_timer 8

;; Do inverse FFT levels 5,6
;; On input the 64-byte cache lines hold these data values:
;;	0	+576	...	48	+576	...
;;	...
;;	47
;;	96	...
;;	192	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+576	...	288	+576	...
;;	...
;;	287	...

;; Do 384 six_complex_unfft operations
;;	distance between fft data elements is 48
;;	do 48 macros each processing 96 data values

	start_timer 9
	zloop_init 48			;; 48 iterations
	zr6b_six_complex_djbunfft_preload
c2b:	zr6b_six_complex_djbunfft rsi, 128, 48*128, rdi, ZMM_SCD3/8, 48
	zloop	48, c2b, -48*128	;; Test loop counter, restore source pointer
	end_timer 9

;; Do inverse FFT levels 1,2,3,4
;; On input the 128-byte cache line pairs hold these data values:
;;	0	+576	...	288	+576	...
;;	...
;;	287	...
;; These are swizzled to:
;;	0	+36	...	288	+36	...
;;	...
;;	35	...
;;	576	...
;;	1152	...
;;	...
;; On output the 128-byte cache line pairs hold these data values:
;;	0	+36	...	2304	+36	...
;;	...
;;	35	...
;;	288	...
;;	576	...
;;	...

;; Do 288 eight_complex_last_unfft operations
;;	distance between fft data elements is 288
;;	do 36 macros processing 128 data values

	start_timer 10
	zloop_init 36			;; 36 iterations
	zr8s_eight_complex_last_unfft_preload
c1b:	zr8s_eight_complex_last_unfft rsi, 128, 36*128, 72*128, 144*128, rdi, ZMM_CD8SCD4, 36, L1PREFETCH_ALL
	zloop	36, c1b, -36*128	;; Test loop counter, restore source pointer
	end_timer 10

	zfft_3_ret
	ENDM


;; Perform a 5K-element FFT

zfft5K MACRO type
	LOCAL	b1b, b2b, b3a, b3b, zmid, c3a, c3b, c2b, c1b

;; Check for a multiply with pre-FFTed inputs

	cmp	ffttype, 4
	je	zmid

;; Do FFT levels 1,2,3,4
;; On input the 128-byte cache line pairs hold these data values:
;;	0	+40	...	2560	+40	...
;;	...
;;	39	...
;;	320	...
;;	640	...
;;	...
;; On output the 128-byte cache line pairs hold these data values:
;;	0	+40	...	320	+40	...
;;	...
;;	39	...
;;	640	...
;;	1280	...
;;	...
;; These are swizzled to:
;;	0	+640	...	320	+640	...
;;	...
;;	319	...

;; Do 320 sixteen_reals_fft operations
;;	distance between fft data elements is 320
;;	do 40 macros processing 128 data values

;; Do FFT levels 5,6,7
;; On input the 64-byte cache lines hold these data values:
;;	0	+640	...	320	+640	...
;;	...
;;	319	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+640	...	40	+640	...
;;	...
;;	39
;;	80	...
;;	160	...
;;	...

;; Do 40 sixteen_reals_fft operation
;; Do 280 eight_complex_fft operations
;;	distance between fft data elements is 40
;;	do 40 macros each processing 128 data values

IFDEF OLDWAY
	start_timer 4
	mov	rdi, sincos1		;; Load sin/cos data ptr
	bump	rdi, 40*ZMM_SCD7+8*ZMM_SCD4
	mov	r9, norm_col_mults	;; Load weights pointer
	zloop_init 40			;; 40 iterations
	zr8fs_sixteen_reals_first_fft_preload
b1b:	zr8fs_sixteen_reals_first_fft rsi, 128, 40*128, 80*128, 160*128, rdi, ZMM_SCD7, r9, 8*128, 40, L1PREFETCH_ALL
	zloop	40, b1b, -40*128	;; Test loop counter, restore source pointer
	end_timer 4

	start_timer 5
	bump	rdi, -40*ZMM_SCD7-40*ZMM_SCD7 ;; Load sin/cos pointer
	zloop_init 40			;; 40 iterations
	zr8_sixteen_reals_eight_complex_djbfft_preload
b2b:	zr8_sixteen_reals_eight_complex_djbfft rsi, 128, 40*128, 80*128, 160*128, rdi, ZMM_SCD7, 40, L1PREFETCH_ALL
	zloop	40, b2b, -40*128	;; Test loop counter, restore source pointer
	end_timer 5
ELSE
	start_timer 4
	mov	rdi, sincos1		;; Load sin/cos data ptr
	bump	rdi, 8*ZMM_SCD4
	lea	r10, [rdi+40*ZMM_SCD7]
	mov	r9, norm_col_mults	;; Load weights pointer
	zloop_init 40			;; 40 iterations
	zr8fs_hundredtwentyeight_reals_first_fft_preload
b1b:	zr8fs_hundredtwentyeight_reals_first_fft rsi, 128, 40*128, 80*128, 160*128, r10, ZMM_SCD7, rdi, ZMM_SCD7, r9, 8*128, 40, L1PREFETCH_ALL
	zloop	40, b1b, -40*128	;; Test loop counter, restore source pointer
	end_timer 4
ENDIF

;; Do FFT levels 8,9
;; On input the 64-byte cache lines hold these data values:
;;	0	+640	...	40	+640	...
;;	...
;;	39
;;	80	...
;;	160	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+640	...	8	+640	...
;;	...
;;	7
;;	16	...
;;	32	...
;;	...

;; Do 8 ten_reals_fft operation
;; Do 504 five_complex_fft operations
;;	distance between fft data elements is 8
;;	do 64 macros each processing 80 data values

	start_timer 6
	bump	rdi, -40*ZMM_SCD7-8*ZMM_SCD4 ;; Load sin/cos ptr
	zloop_init 64			;; 8 iterations of 8
	zr5_ten_reals_five_complex_djbfft_preload
b3a:	zr5_ten_reals_five_complex_djbfft rsi, 128, 8*128, rdi, ZMM_SCD4, 8
	zloop	8, b3a, -8*128+40*128, rdi, -8*ZMM_SCD4 ;; Test loop counter, next source pointer, restore sin/cos pointer

	loops_reset			;; 7 more iters of 8
	zr5rb_five_complex_djbfft_preload
b3b:	zr5rb_five_complex_djbfft rsi, 128, 8*128, rdi, ZMM_SCD4, 8
	zloop	8, b3b, -8*128+40*128, rdi, -8*ZMM_SCD4 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	8, b3b, -8*40*128	;; Test loop counter, restore source pointer
	end_timer 6

;; Do FFT levels 10,11,12
;; On input and output the 128-byte cache line pairs hold these data values:
;;	0	+640	...	8	+640	...
;;	...
;;	7
;;	16	...
;;	32	...
;;	...

;; Do 1 sixteen_reals_fft operation
;; Do 319 eight_complex_fft operations
;;	distance between fft data elements is 1
;;	do 40 macros processing 128 data values

zmid:	start_timer 7
	mov	rcx, 40
	call	zreal_onepass_middle
	end_timer 7

;; Do inverse FFT levels 8,9
;; On input the 64-byte cache lines hold these data values:
;;	0	+640	...	8	+640	...
;;	...
;;	7
;;	16	...
;;	32	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+640	...	40	+640	...
;;	...
;;	39
;;	80	...
;;	160	...
;;	...

;; Do 8 ten_reals_unfft operations
;; Do 504 five_complex_unfft operations
;;	distance between fft data elements is 8
;;	do 64 macros each processing 48 data values

	start_timer 8
	mov	rdi, sincos1		;; Load sin/cos pointer
	zloop_init 64			;; 8 iterations of 8
	zr5_ten_reals_five_complex_djbunfft_preload
c3a:	zr5_ten_reals_five_complex_djbunfft rsi, 128, 8*128, rdi, ZMM_SCD4, 8
	zloop	8, c3a, -8*128+40*128, rdi, -8*ZMM_SCD4 ;; Test loop counter, restore source pointer, restore sin/cos pointer

	loops_reset			;; 7 more iters of 8
	zr5rb_five_complex_djbunfft_preload
c3b:	zr5rb_five_complex_djbunfft rsi, 128, 8*128, rdi, ZMM_SCD4, 8
	zloop	8, c3b, -8*128+40*128, rdi, -8*ZMM_SCD4 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	8, c3b, -8*40*128, rdi, 8*ZMM_SCD4 ;; Test loop counter, restore source pointer, next sin/cos pointer
	end_timer 8

;; Do inverse FFT levels 5,6,7
;; On input the 64-byte cache lines hold these data values:
;;	0	+640	...	40	+640	...
;;	...
;;	39
;;	80	...
;;	160	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+640	...	320	+640	...
;;	...
;;	319	...

;; Do 40 sixteen_reals_unfft operations
;; Do 280 eight_complex_unfft operations
;;	distance between fft data elements is 40
;;	do 40 macros each processing 128 data values

;; Do inverse FFT levels 1,2,3,4
;; On input the 128-byte cache line pairs hold these data values:
;;	0	+640	...	320	+640	...
;;	...
;;	319	...
;; These are swizzled to:
;;	0	+40	...	320	+40	...
;;	...
;;	39	...
;;	640	...
;;	1280	...
;;	...
;; On output the 128-byte cache line pairs hold these data values:
;;	0	+40	...	2560	+40	...
;;	...
;;	39	...
;;	320	...
;;	640	...
;;	...

;; Do 320 sixteen_reals_unfft operations
;;	distance between fft data elements is 320
;;	do 40 macros processing 128 data values

IFDEF OLDWAY
	start_timer 9
	zloop_init 40			;; 40 iterations
	zr8_sixteen_reals_eight_complex_djbunfft_preload
c2b:	zr8_sixteen_reals_eight_complex_djbunfft rsi, 128, 40*128, 80*128, 160*128, rdi, ZMM_SCD7, 40, L1PREFETCH_ALL
	zloop	40, c2b, -40*128	;; Test loop counter, restore source pointer
	end_timer 9

	start_timer 10
	zloop_init 40			;; 40 iterations
	zr8s_sixteen_reals_last_unfft_preload
c1b:	zr8s_sixteen_reals_last_unfft rsi, 128, 40*128, 80*128, 160*128, rdi, ZMM_SCD7, 40, L1PREFETCH_ALL
	zloop	40, c1b, -40*128	;; Test loop counter, restore source pointer
	end_timer 10
ELSE
	start_timer 10
	lea	r10, [rdi+40*ZMM_SCD7]
	zloop_init 40			;; 40 iterations
	zr8s_hundredtwentyeight_reals_last_unfft_preload
c1b:	zr8s_hundredtwentyeight_reals_last_unfft rsi, 128, 40*128, 80*128, 160*128, rdi, ZMM_SCD7, r10, ZMM_SCD7, 40, L1PREFETCH_ALL
	zloop	40, c1b, -40*128	;; Test loop counter, restore source pointer
	end_timer 10
ENDIF

	zfft_3_ret
	ENDM

;; Perform a 5K-element negacyclic FFT

zfft5Kp MACRO type
	LOCAL	b1b, b2b, b3b, zmid, c3b, c2b, c1b

;; Do a multiply with pre-FFTed inputs

	cmp	ffttype, 4
	je	zmid

;; Do FFT levels 1,2,3,4
;; On input the 128-byte cache line pairs hold these data values:
;;	0	+40	...	2560	+40	...
;;	...
;;	39	...
;;	320	...
;;	640	...
;;	...
;; On output the 128-byte cache line pairs hold these data values:
;;	0	+40	...	320	+40	...
;;	...
;;	39	...
;;	640	...
;;	1280	...
;;	...
;; These are swizzled to:
;;	0	+640	...	320	+640	...
;;	...
;;	319	...

;; Do 320 eight_complex_first_fft operations
;;	distance between fft data elements is 320
;;	do 40 macros processing 128 data values

;; Do FFT levels 5,6,7
;; On input the 64-byte cache lines hold these data values:
;;	0	+640	...	320	+640	...
;;	...
;;	319	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+640	...	40	+640	...
;;	...
;;	39
;;	80	...
;;	160	...
;;	...

;; Do 320 eight_complex_fft operations
;;	distance between fft data elements is 40
;;	do 40 macros each processing 128 data values

IFDEF OLDWAY
	start_timer 4
	mov	rdi, sincos1		;; Load sin/cos pointer
	bump	rdi, 40*ZMM_SCD4/8+8*ZMM_SCD2/8
	mov	r9, norm_col_mults	;; Load weights pointer
	zloop_init 40			;; 40 iterations
	zr8fs_eight_complex_first_fft_preload
b1b:	zr8fs_eight_complex_first_fft rsi, 128, 40*128, 80*128, 160*128, rdi, ZMM_CD8SCD4, r9, 8*128, 40, L1PREFETCH_ALL
	zloop	40, b1b, -40*128	;; Test loop counter, restore source pointer
	end_timer 4

	start_timer 5
	bump	rdi, -40*ZMM_CD8SCD4-40*ZMM_SCD4/8 ;; Load sin/cos pointer
	zloop_init 40			;; 40 iterations
	zr8b_eight_complex_djbfft_preload
b2b:	zr8b_eight_complex_djbfft rsi, 128, 40*128, 80*128, 160*128, rdi, ZMM_SCD4/8, 40, L1PREFETCH_NONE
	zloop	40, b2b, -40*128	;; Test loop counter, restore source pointer
	end_timer 5
ELSE
	start_timer 4
	mov	rdi, sincos1		;; Load sin/cos pointer
	bump	rdi, 8*ZMM_SCD2/8
	lea	r10, [rdi+40*ZMM_SCD4/8]
	mov	r9, norm_col_mults	;; Load weights pointer
	zloop_init 40			;; 40 iterations
	zr8fs_sixtyfour_complex_first_fft_preload
b1b:	zr8fs_sixtyfour_complex_first_fft rsi, 128, 40*128, 80*128, 160*128, r10, ZMM_CD8SCD4, rdi, ZMM_SCD4/8, r9, 8*128, 40, L1PREFETCH_ALL
	zloop	40, b1b, -40*128	;; Test loop counter, restore source pointer
	end_timer 4
ENDIF

;; Do FFT levels 8,9
;; On input the 64-byte cache lines hold these data values:
;;	0	+640	...	40	+640	...
;;	...
;;	39
;;	80	...
;;	160	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+640	...	8	+640	...
;;	...
;;	7
;;	16	...
;;	32	...
;;	...

;; Do 512 five_complex_fft operations
;;	distance between fft data elements is 8
;;	do 64 macros each processing 80 data values

	start_timer 5
	bump	rdi, -40*ZMM_SCD4/8-8*ZMM_SCD2/8 ;; Load sin/cos pointer
	zloop_init 64			;; 8 iterations of 8
	zr5b_five_complex_djbfft_preload
b3b:	zr5b_five_complex_djbfft rsi, 128, 8*128, rdi, ZMM_SCD2/8, 8
	zloop	8, b3b, -8*128+40*128, rdi, -8*ZMM_SCD2/8 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	8, b3b, -8*40*128	;; Test loop counter, restore source pointer
	end_timer 5

;; Do FFT levels 10,11,12
;; On input and output the 128-byte cache line pairs hold these data values:
;;	0	+640	...	8	+640	...
;;	...
;;	7
;;	16	...
;;	32	...
;;	...

;; Do 320 eight_complex_fft operations
;;	distance between fft data elements is 1
;;	do 40 macros processing 128 data values

zmid:	start_timer 7
	mov	rcx, 40
	call	zcomplex_onepass_middle
	end_timer 7

;; Do inverse FFT levels 8,9
;; On input the 64-byte cache lines hold these data values:
;;	0	+640	...	8	+640	...
;;	...
;;	7
;;	16	...
;;	32	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+640	...	40	+640	...
;;	...
;;	39
;;	80	...
;;	160	...
;;	...

;; Do 512 five_complex_unfft operations
;;	distance between fft data elements is 8
;;	do 64 macros each processing 80 data values

	start_timer 8
	mov	rdi, sincos1		;; Load sin/cos pointer
	zloop_init 64			;; 8 iterations of 8
	zr5b_five_complex_djbunfft_preload
c3b:	zr5b_five_complex_djbunfft rsi, 128, 8*128, rdi, ZMM_SCD2/8, 8
	zloop	8, c3b, -8*128+40*128, rdi, -8*ZMM_SCD2/8 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	8, c3b, -8*40*128, rdi, 8*ZMM_SCD2/8 ;; Test loop counter, restore source pointer, next sin/cos pointer
	end_timer 8

;; Do inverse FFT levels 5,6,7
;; On input the 64-byte cache lines hold these data values:
;;	0	+640	...	40	+640	...
;;	...
;;	39
;;	80	...
;;	160	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+640	...	320	+640	...
;;	...
;;	319	...

;; Do 320 eight_complex_unfft operations
;;	distance between fft data elements is 40
;;	do 40 macros each processing 128 data values

;; Do inverse FFT levels 1,2,3,4
;; On input the 128-byte cache line pairs hold these data values:
;;	0	+640	...	320	+640	...
;;	...
;;	319	...
;; These are swizzled to:
;;	0	+40	...	320	+40	...
;;	...
;;	39	...
;;	640	...
;;	1280	...
;;	...
;; On output the 128-byte cache line pairs hold these data values:
;;	0	+40	...	2560	+40	...
;;	...
;;	39	...
;;	320	...
;;	640	...
;;	...

;; Do 320 eight_complex_last_unfft operations
;;	distance between fft data elements is 320
;;	do 40 macros processing 128 data values

IFDEF OLDWAY
	start_timer 9
	zloop_init 40			;; 40 iterations
	zr8b_eight_complex_djbunfft_preload
c2b:	zr8b_eight_complex_djbunfft rsi, 128, 40*128, 80*128, 160*128, rdi, ZMM_SCD4/8, 40, L1PREFETCH_ALL, 128
	zloop	40, c2b, -40*128	;; Test loop counter, restore source pointer
	end_timer 9

	start_timer 10
	zloop_init 40			;; 40 iterations
	zr8s_eight_complex_last_unfft_preload
c1b:	zr8s_eight_complex_last_unfft rsi, 128, 40*128, 80*128, 160*128, rdi, ZMM_CD8SCD4, 40, L1PREFETCH_ALL
	zloop	40, c1b, -40*128	;; Test loop counter, restore source pointer
	end_timer 10
ELSE
	start_timer 10
	lea	r10, [rdi+40*ZMM_SCD4/8]
	zloop_init 40			;; 40 iterations
	zr8s_sixtyfour_complex_last_unfft_preload
c1b:	zr8s_sixtyfour_complex_last_unfft rsi, 128, 40*128, 80*128, 160*128, rdi, ZMM_SCD4/8, r10, ZMM_CD8SCD4, 40, L1PREFETCH_ALL
	zloop	40, c1b, -40*128	;; Test loop counter, restore source pointer
	end_timer 10
ENDIF

	zfft_3_ret
	ENDM


;; Perform a 5376-element FFT

zfft5376 MACRO type
	LOCAL	b1b, b2b, b3a, b3b, zmid, c3a, c3b, c2b, c1b

;; Check for a multiply with pre-FFTed inputs

	cmp	ffttype, 4
	je	zmid

;; Do FFT levels 1,2,3,4
;; On input the 128-byte cache line pairs hold these data values:
;;	0	+42	...	2688	+42	...
;;	...
;;	41	...
;;	336	...
;;	672	...
;;	...
;; On output the 128-byte cache line pairs hold these data values:
;;	0	+42	...	336	+42	...
;;	...
;;	41	...
;;	672	...
;;	1344	...
;;	...
;; These are swizzled to:
;;	0	+672	...	336	+672	...
;;	...
;;	335	...

;; Do 336 sixteen_reals_fft operations
;;	distance between fft data elements is 336
;;	do 42 macros processing 128 data values

	start_timer 4
	mov	rdi, sincos1		;; Load sin/cos data ptr
	bump	rdi, 48*ZMM_SCD6+8*ZMM_SCD5
	mov	r9, norm_col_mults	;; Load weights pointer
	zloop_init 42			;; 42 iterations
	zr8fs_sixteen_reals_first_fft_preload
b1b:	zr8fs_sixteen_reals_first_fft rsi, 128, 42*128, 84*128, 168*128, rdi, ZMM_SCD7, r9, 8*128, 42, L1PREFETCH_ALL
	zloop	42, b1b, -42*128	;; Test loop counter, restore source pointer
	end_timer 4

;; Do FFT levels 5,6
;; On input the 64-byte cache lines hold these data values:
;;	0	+672	...	336	+672	...
;;	...
;;	335	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+672	...	48	+672	...
;;	...
;;	47
;;	96	...
;;	192	...
;;	...

;; Do 24 fourteen_reals_fft operation
;; Do 360 seven_complex_fft operations
;;	distance between fft data elements is 48
;;	do 48 macros each processing 112 data values

	start_timer 5
	bump	rdi, -42*ZMM_SCD7-48*ZMM_SCD6 ;; Load sin/cos pointer
	zloop_init 48			;; 48 iterations
	zr7_fourteen_reals_seven_complex_djbfft_preload
b2b:	zr7_fourteen_reals_seven_complex_djbfft rsi, 128, 48*128, rdi, ZMM_SCD6, 48, L1PREFETCH_ALL
	zloop	48, b2b, -48*128	;; Test loop counter, restore source pointer
	end_timer 5

;; Do FFT levels 7,8,9
;; On input the 64-byte cache lines hold these data values:
;;	0	+672	...	48	+672	...
;;	...
;;	47
;;	96	...
;;	192	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+672	...	8	+672	...
;;	...
;;	7
;;	16	...
;;	32	...
;;	...

;; Do 8 twelve_reals_fft operation
;; Do 440 six_complex_fft operations
;;	distance between fft data elements is 8
;;	do 56 macros each processing 96 data values

	start_timer 6
	bump	rdi, -48*ZMM_SCD6-8*ZMM_SCD5	;; Load sin/cos ptr
	zloop_init 56			;; 7 iterations of 8
	zr6_twelve_reals_six_complex_djbfft_preload
b3a:	zr6_twelve_reals_six_complex_djbfft rsi, 128, 8*128, rdi, ZMM_SCD5, 8, L1PREFETCH_NONE
	zloop	8, b3a, -8*128+48*128, rdi, -8*ZMM_SCD5 ;; Test loop counter, next source pointer, restore sin/cos pointer

	loops_reset			;; 6 more iters of 8
	zr6rb_six_complex_djbfft_preload
b3b:	zr6rb_six_complex_djbfft rsi, 128, 8*128, rdi, ZMM_SCD5, 8
	zloop	8, b3b, -8*128+48*128, rdi, -8*ZMM_SCD5 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	7, b3b, -7*48*128	;; Test loop counter, restore source pointer
	end_timer 6

;; Do FFT levels 10,11,12
;; On input and output the 128-byte cache line pairs hold these data values:
;;	0	+672	...	8	+672	...
;;	...
;;	7
;;	16	...
;;	32	...
;;	...

;; Do 1 sixteen_reals_fft operation
;; Do 335 eight_complex_fft operations
;;	distance between fft data elements is 1
;;	do 42 macros processing 128 data values

zmid:	start_timer 7
	mov	rcx, 42
	call	zreal_onepass_middle
	end_timer 7

;; Do inverse FFT levels 7,8,9
;; On input the 64-byte cache lines hold these data values:
;;	0	+672	...	8	+672	...
;;	...
;;	7
;;	16	...
;;	32	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+672	...	48	+672	...
;;	...
;;	47
;;	96	...
;;	192	...
;;	...

;; Do 8 twelve_reals_unfft operations
;; Do 440 six_complex_unfft operations
;;	distance between fft data elements is 8
;;	do 56 macros each processing 96 data values

	start_timer 8
	mov	rdi, sincos1		;; Load sin/cos pointer
	zloop_init 56			;; 7 iterations of 8
	zr6_twelve_reals_six_complex_djbunfft_preload
c3a:	zr6_twelve_reals_six_complex_djbunfft rsi, 128, 8*128, rdi, ZMM_SCD5, 8, L1PREFETCH_NONE
	zloop	8, c3a, -8*128+48*128, rdi, -8*ZMM_SCD5 ;; Test loop counter, restore source pointer, restore sin/cos pointer

	loops_reset			;; 6 more iters of 8
	zr6rb_six_complex_djbunfft_preload
c3b:	zr6rb_six_complex_djbunfft rsi, 128, 8*128, rdi, ZMM_SCD5, 8
	zloop	8, c3b, -8*128+48*128, rdi, -8*ZMM_SCD5 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	7, c3b, -7*48*128, rdi, 8*ZMM_SCD5 ;; Test loop counter, restore source pointer, next sin/cos pointer
	end_timer 8

;; Do inverse FFT levels 5,6
;; On input the 64-byte cache lines hold these data values:
;;	0	+672	...	48	+672	...
;;	...
;;	47
;;	96	...
;;	192	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+672	...	336	+672	...
;;	...
;;	335	...

;; Do 24 fourteen_reals_unfft operations
;; Do 360 seven_complex_unfft operations
;;	distance between fft data elements is 48
;;	do 48 macros each processing 112 data values

	start_timer 9
	zloop_init 48			;; 48 iterations
	zr7_fourteen_reals_seven_complex_djbunfft_preload
c2b:	zr7_fourteen_reals_seven_complex_djbunfft rsi, 128, 48*128, rdi, ZMM_SCD6, 48, L1PREFETCH_ALL
	zloop	48, c2b, -48*128	;; Test loop counter, restore source pointer
	end_timer 9

;; Do inverse FFT levels 1,2,3,4
;; On input the 128-byte cache line pairs hold these data values:
;;	0	+672	...	336	+672	...
;;	...
;;	335	...
;; These are swizzled to:
;;	0	+42	...	336	+42	...
;;	...
;;	41	...
;;	672	...
;;	1344	...
;;	...
;; On output the 128-byte cache line pairs hold these data values:
;;	0	+42	...	2688	+42	...
;;	...
;;	41	...
;;	336	...
;;	672	...
;;	...

;; Do 336 sixteen_reals_unfft operations
;;	distance between fft data elements is 336
;;	do 42 macros processing 128 data values

	start_timer 10
	zloop_init 42			;; 42 iterations
	zr8s_sixteen_reals_last_unfft_preload
c1b:	zr8s_sixteen_reals_last_unfft rsi, 128, 42*128, 84*128, 168*128, rdi, ZMM_SCD7, 42, L1PREFETCH_ALL
	zloop	42, c1b, -42*128	;; Test loop counter, restore source pointer
	end_timer 10

	zfft_3_ret
	ENDM

;; Perform a 5376-element negacyclic FFT

zfft5376p MACRO type
	LOCAL	b1b, b2b, b3b, zmid, c3b, c2b, c1b

;; Do a multiply with pre-FFTed inputs

	cmp	ffttype, 4
	je	zmid

;; Do FFT levels 1,2,3,4
;; On input the 128-byte cache line pairs hold these data values:
;;	0	+42	...	2688	+42	...
;;	...
;;	41	...
;;	336	...
;;	672	...
;;	...
;; On output the 128-byte cache line pairs hold these data values:
;;	0	+42	...	336	+42	...
;;	...
;;	41	...
;;	672	...
;;	1344	...
;;	...
;; These are swizzled to:
;;	0	+672	...	336	+672	...
;;	...
;;	335	...

;; Do 336 eight_complex_first_fft operations
;;	distance between fft data elements is 336
;;	do 42 macros processing 128 data values

	start_timer 4
	mov	rdi, sincos1		;; Load sin/cos pointer
	bump	rdi, 48*ZMM_SCD3/8+8*ZMM_SCD3/8
	mov	r9, norm_col_mults	;; Load weights pointer
	zloop_init 42			;; 42 iterations
	zr8fs_eight_complex_first_fft_preload
b1b:	zr8fs_eight_complex_first_fft rsi, 128, 42*128, 84*128, 168*128, rdi, ZMM_CD8SCD4, r9, 8*128, 42, L1PREFETCH_ALL
	zloop	42, b1b, -42*128	;; Test loop counter, restore source pointer
	end_timer 4

;; Do FFT levels 5,6
;; On input the 64-byte cache lines hold these data values:
;;	0	+672	...	336	+672	...
;;	...
;;	335	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+672	...	48	+672	...
;;	...
;;	47
;;	96	...
;;	192	...
;;	...

;; Do 384 seven_complex_fft operations
;;	distance between fft data elements is 48
;;	do 48 macros each processing 80 data values

	start_timer 5
	bump	rdi, -42*ZMM_CD8SCD4-48*ZMM_SCD3/8 ;; Load sin/cos pointer
	zloop_init 48			;; 48 iterations
	zr7b_seven_complex_djbfft_preload
b2b:	zr7b_seven_complex_djbfft rsi, 128, 48*128, rdi, ZMM_SCD3/8, 48, L1PREFETCH_ALL, 128
	zloop	48, b2b, -48*128	;; Test loop counter, restore source pointer
	end_timer 5

;; Do FFT levels 7,8,9
;; On input the 64-byte cache lines hold these data values:
;;	0	+672	...	48	+672	...
;;	...
;;	47
;;	96	...
;;	192	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+672	...	8	+672	...
;;	...
;;	7
;;	16	...
;;	32	...
;;	...

;; Do 448 six_complex_fft operations
;;	distance between fft data elements is 8
;;	do 56 macros each processing 96 data values

	start_timer 6
	bump	rdi, -48*ZMM_SCD3/8-8*ZMM_SCD3/8 ;; Load sin/cos pointer
	zloop_init 56			;; 7 iterations of 8
	zr6b_six_complex_djbfft_preload
b3b:	zr6b_six_complex_djbfft rsi, 128, 8*128, rdi, ZMM_SCD3/8, 8
	zloop	8, b3b, -8*128+48*128, rdi, -8*ZMM_SCD3/8 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	7, b3b, -7*48*128	;; Test loop counter, restore source pointer
	end_timer 6

;; Do FFT levels 10,11,12
;; On input and output the 128-byte cache line pairs hold these data values:
;;	0	+672	...	8	+672	...
;;	...
;;	7
;;	16	...
;;	32	...
;;	...

;; Do 336 eight_complex_fft operations
;;	distance between fft data elements is 1
;;	do 42 macros processing 128 data values

zmid:	start_timer 7
	mov	rcx, 42
	call	zcomplex_onepass_middle
	end_timer 7

;; Do inverse FFT levels 7,8,9
;; On input the 64-byte cache lines hold these data values:
;;	0	+672	...	8	+672	...
;;	...
;;	7
;;	16	...
;;	32	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+672	...	48	+672	...
;;	...
;;	47
;;	96	...
;;	192	...
;;	...

;; Do 448 six_complex_unfft operations
;;	distance between fft data elements is 8
;;	do 56 macros each processing 96 data values

	start_timer 8
	mov	rdi, sincos1		;; Load sin/cos pointer
	zloop_init 56			;; 7 iterations of 8
	zr6b_six_complex_djbunfft_preload
c3b:	zr6b_six_complex_djbunfft rsi, 128, 8*128, rdi, ZMM_SCD3/8, 8
	zloop	8, c3b, -8*128+48*128, rdi, -8*ZMM_SCD3/8 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	7, c3b, -7*48*128, rdi, 8*ZMM_SCD3/8	 ;; Test loop counter, restore source pointer, next sin/cos pointer
	end_timer 8

;; Do inverse FFT levels 5,6
;; On input the 64-byte cache lines hold these data values:
;;	0	+672	...	48	+672	...
;;	...
;;	47
;;	96	...
;;	192	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+672	...	336	+672	...
;;	...
;;	335	...

;; Do 384 seven_complex_unfft operations
;;	distance between fft data elements is 48
;;	do 48 macros each processing 112 data values

	start_timer 9
	zloop_init 48			;; 48 iterations
	zr7b_seven_complex_djbunfft_preload
c2b:	zr7b_seven_complex_djbunfft rsi, 128, 48*128, rdi, ZMM_SCD3/8, 48, L1PREFETCH_NONE
	zloop	48, c2b, -48*128	;; Test loop counter, restore source pointer
	end_timer 9

;; Do inverse FFT levels 1,2,3,4
;; On input the 128-byte cache line pairs hold these data values:
;;	0	+672	...	336	+672	...
;;	...
;;	335	...
;; These are swizzled to:
;;	0	+42	...	336	+42	...
;;	...
;;	41	...
;;	672	...
;;	1344	...
;;	...
;; On output the 128-byte cache line pairs hold these data values:
;;	0	+42	...	2688	+42	...
;;	...
;;	41	...
;;	336	...
;;	672	...
;;	...

;; Do 336 eight_complex_last_unfft operations
;;	distance between fft data elements is 336
;;	do 42 macros processing 128 data values

	start_timer 10
	zloop_init 42			;; 42 iterations
	zr8s_eight_complex_last_unfft_preload
c1b:	zr8s_eight_complex_last_unfft rsi, 128, 42*128, 84*128, 168*128, rdi, ZMM_CD8SCD4, 42, L1PREFETCH_ALL
	zloop	42, c1b, -42*128	;; Test loop counter, restore source pointer
	end_timer 10

	zfft_3_ret
	ENDM


;; Perform a 6K-element FFT

zfft6K MACRO type
	LOCAL	b1b, b2b, b3a, b3b, zmid, c3a, c3b, c2b, c1b

;; Check for a multiply with pre-FFTed inputs

	cmp	ffttype, 4
	je	zmid

;; Do FFT levels 1,2,3,4
;; On input the 128-byte cache line pairs hold these data values:
;;	0	+48	...	3K	+48	...
;;	...
;;	48	...
;;	384	...
;;	768	...
;;	...
;; On output the 128-byte cache line pairs hold these data values:
;;	0	+48	...	384	+48	...
;;	...
;;	47	...
;;	768	...
;;	1536	...
;;	...
;; These are swizzled to:
;;	0	+768	...	384	+768	...
;;	...
;;	383	...

;; Do 384 sixteen_reals_fft operations
;;	distance between fft data elements is 384
;;	do 48 macros processing 128 data values

;; Do FFT levels 5,6,7
;; On input the 64-byte cache lines hold these data values:
;;	0	+768	...	384	+768	...
;;	...
;;	383	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+768	...	48	+768	...
;;	...
;;	47
;;	96	...
;;	192	...
;;	...

;; Do 48 sixteen_reals_fft operation
;; Do 336 eight_complex_fft operations
;;	distance between fft data elements is 48
;;	do 48 macros each processing 128 data values

IFDEF OLDWAY
	start_timer 4
	mov	rdi, sincos1		;; Load sin/cos data ptr
	bump	rdi, 48*ZMM_SCD7+8*ZMM_SCD5
	mov	r9, norm_col_mults	;; Load weights pointer
	zloop_init 48			;; 48 iterations
	zr8fs_sixteen_reals_first_fft_preload
b1b:	zr8fs_sixteen_reals_first_fft rsi, 128, 48*128, 96*128, 192*128, rdi, ZMM_SCD7, r9, 8*128, 48, L1PREFETCH_ALL
	zloop	48, b1b, -48*128	;; Test loop counter, restore source pointer
	end_timer 4

	start_timer 5
	bump	rdi, -48*ZMM_SCD7-48*ZMM_SCD7 ;; Load sin/cos ptr
	zloop_init 48			;; 48 iterations
	zr8_sixteen_reals_eight_complex_djbfft_preload
b2a:	zr8_sixteen_reals_eight_complex_djbfft rsi, 128, 48*128, 96*128, 192*128, rdi, ZMM_SCD7, 48, L1PREFETCH_ALL
	zloop	48, b2a, -48*128	;; Test loop counter, restore source pointer
	end_timer 5
ELSE
	start_timer 4
	mov	rdi, sincos1		;; Load sin/cos data ptrs
	bump	rdi, 8*ZMM_SCD5
	lea	r10, [rdi+48*ZMM_SCD7]
	mov	r9, norm_col_mults	;; Load weights pointer
	zloop_init 48			;; 48 iterations
	zr8fs_hundredtwentyeight_reals_first_fft_preload
b1b:	zr8fs_hundredtwentyeight_reals_first_fft rsi, 128, 48*128, 96*128, 192*128, r10, ZMM_SCD7, rdi, ZMM_SCD7, r9, 8*128, 48, L1PREFETCH_ALL
	zloop	48, b1b, -48*128	;; Test loop counter, restore source pointer
	end_timer 4
ENDIF

;; Do FFT levels 8,9
;; On input the 64-byte cache lines hold these data values:
;;	0	+768	...	48	+768	...
;;	...
;;	47
;;	96	...
;;	192	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+768	...	8	+768	...
;;	...
;;	7
;;	16	...
;;	32	...
;;	...

;; Do 8 twelve_reals_fft operation
;; Do 504 six_complex_fft operations
;;	distance between fft data elements is 8
;;	do 64 macros each processing 96 data values

	start_timer 6
	bump	rdi, -48*ZMM_SCD7-8*ZMM_SCD5 ;; Load sin/cos pointer
	zloop_init 64			;; 8 iterations of 8
	zr6_twelve_reals_six_complex_djbfft_preload
b3a:	zr6_twelve_reals_six_complex_djbfft rsi, 128, 8*128, rdi, ZMM_SCD5, 8, L1PREFETCH_ALL
	zloop	8, b3a, -8*128+48*128, rdi, -8*ZMM_SCD5 ;; Test loop counter, next source pointer, restore sin/cos pointer

	loops_reset			;; 7 more iters of 8
	zr6rb_six_complex_djbfft_preload
b3b:	zr6rb_six_complex_djbfft rsi, 128, 8*128, rdi, ZMM_SCD5, 8
	zloop	8, b3b, -8*128+48*128, rdi, -8*ZMM_SCD5 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	8, b3b, -8*48*128	;; Test loop counter, restore source pointer
	end_timer 6

;; Do FFT levels 10,11,12
;; On input and output the 128-byte cache line pairs hold these data values:
;;	0	+768	...	8	+768	...
;;	...
;;	7
;;	16	...
;;	32	...
;;	...

;; Do 1 sixteen_reals_fft operation
;; Do 383 eight_complex_fft operations
;;	distance between fft data elements is 1
;;	do 48 macros processing 128 data values

zmid:	start_timer 7
	mov	rcx, 48
	call	zreal_onepass_middle
	end_timer 7

;; Do inverse FFT levels 8,9
;; On input the 64-byte cache lines hold these data values:
;;	0	+768	...	8	+768	...
;;	...
;;	7
;;	16	...
;;	32	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+768	...	8	+768	...
;;	...
;;	7
;;	16	...
;;	32	...
;;	...

;; Do 8 twelve_reals_unfft operations
;; Do 504 six_complex_unfft operations
;;	distance between fft data elements is 8
;;	do 64 macros each processing 96 data values

	start_timer 8
	mov	rdi, sincos1		;; Load sin/cos pointer
	zloop_init 64			;; 64 iterations
	zr6_twelve_reals_six_complex_djbunfft_preload
c3a:	zr6_twelve_reals_six_complex_djbunfft rsi, 128, 8*128, rdi, ZMM_SCD5, 8, L1PREFETCH_ALL
	zloop	8, c3a, -8*128+48*128, rdi, -8*ZMM_SCD5 ;; Test loop counter, next source pointer, restore sin/cos pointer

	loops_reset			;; 7 more iters of 8
	zr6rb_six_complex_djbunfft_preload
c3b:	zr6rb_six_complex_djbunfft rsi, 128, 8*128, rdi, ZMM_SCD5, 8
	zloop	8, c3b, -8*128+48*128, rdi, -8*ZMM_SCD5 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	8, c3b, -8*48*128, rdi, 8*ZMM_SCD5 ;; Test loop counter, restore source pointer, next sin/cos ptr
	end_timer 9

;; Do inverse FFT levels 5,6,7
;; On input the 64-byte cache lines hold these data values:
;;	0	+768	...	8	+768	...
;;	...
;;	7
;;	16	...
;;	32	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+768	...	384	+768	...
;;	...
;;	383	...

;; Do 48 sixteen_reals_unfft operations
;; Do 336 eight_complex_unfft operations
;;	distance between fft data elements is 48
;;	do 48 macros each processing 128 data values

;; Do inverse FFT levels 1,2,3,4
;; On input the 128-byte cache line pairs hold these data values:
;;	0	+768	...	384	+768	...
;;	...
;;	383	...
;; These are swizzled to:
;;	0	+48	...	384	+48	...
;;	...
;;	47	...
;;	768	...
;;	1536	...
;;	...
;; On output the 128-byte cache line pairs hold these data values:
;;	0	+48	...	3K	+48	...
;;	...
;;	48	...
;;	384	...
;;	768	...
;;	...

;; Do 384 sixteen_reals_unfft operations
;;	distance between fft data elements is 384
;;	do 48 macros processing 128 data values

IFDEF OLDWAY
	zloop_init 48			;; 48 iterations
	zr8_sixteen_reals_eight_complex_djbunfft_preload
c2a:	zr8_sixteen_reals_eight_complex_djbunfft rsi, 128, 48*128, 96*128, 192*128, rdi, ZMM_SCD7, 48, L1PREFETCH_NONE
	zloop	48, c2a, -48*128	;; Test loop counter, restore source pointer, restore sin/cos pointer
	end_timer 8

	start_timer 10
	zloop_init 48			;; 48 iterations
	zr8s_sixteen_reals_last_unfft_preload
c1b:	zr8s_sixteen_reals_last_unfft rsi, 128, 48*128, 96*128, 192*128, rdi, ZMM_SCD7, 48, L1PREFETCH_ALL
	zloop	48, c1b, -56*128	;; Test loop counter, restore source pointer
	end_timer 10
ELSE
	start_timer 10
	lea	r10, [rdi+48*ZMM_SCD7]
	zloop_init 48			;; 48 iterations
	zr8s_hundredtwentyeight_reals_last_unfft_preload
c1b:	zr8s_hundredtwentyeight_reals_last_unfft rsi, 128, 48*128, 96*128, 192*128, rdi, ZMM_SCD7, r10, ZMM_SCD7, 48, L1PREFETCH_ALL
	zloop	48, c1b, -56*128	;; Test loop counter, restore source pointer
	end_timer 10
ENDIF

	zfft_3_ret
	ENDM

;; Perform a 6K-element negacyclic FFT

zfft6Kp MACRO type
	LOCAL	b1b, b2b, b3b, zmid, c3b, c2b, c1b

;; Do a multiply with pre-FFTed inputs

	cmp	ffttype, 4
	je	zmid

;; Do FFT levels 1,2,3,4
;; On input the 128-byte cache line pairs hold these data values:
;;	0	+48	...	3K	+48	...
;;	...
;;	48	...
;;	384	...
;;	768	...
;;	...
;; On output the 128-byte cache line pairs hold these data values:
;;	0	+48	...	384	+48	...
;;	...
;;	47	...
;;	768	...
;;	1536	...
;;	...
;; These are swizzled to:
;;	0	+768	...	384	+768	...
;;	...
;;	383	...

;; Do 384 eight_complex_first_fft operations
;;	distance between fft data elements is 384
;;	do 48 macros processing 128 data values

;; Do FFT levels 5,6,7
;; On input the 64-byte cache lines hold these data values:
;;	0	+768	...	384	+768	...
;;	...
;;	383	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+768	...	48	+768	...
;;	...
;;	47
;;	96	...
;;	192	...
;;	...

;; Do 384 eight_complex_fft operations
;;	distance between fft data elements is 48
;;	do 48 macros each processing 128 data values

IFDEF OLDWAY
	start_timer 4
	mov	rdi, sincos1		;; Load sin/cos pointer
	bump	rdi, 48*ZMM_SCD4/8+8*ZMM_SCD3/8
	mov	r9, norm_col_mults	;; Load weights pointer
	zloop_init 48			;; 48 iterations
	zr8fs_eight_complex_first_fft_preload
b1b:	zr8fs_eight_complex_first_fft rsi, 128, 48*128, 96*128, 192*128, rdi, ZMM_CD8SCD4, r9, 8*128, 48, L1PREFETCH_ALL
	zloop	48, b1b, -48*128	;; Test loop counter, restore source pointer
	end_timer 4

	start_timer 5
	bump	rdi, -48*ZMM_CD8SCD4-48*ZMM_SCD4/8 ;; Load sin/cos pointer
	zloop_init 48			;; 48 iterations
	zr8b_eight_complex_djbfft_preload
b2b:	zr8b_eight_complex_djbfft rsi, 128, 48*128, 96*128, 192*128, rdi, ZMM_SCD4/8, 48, L1PREFETCH_ALL, 128
	zloop	48, b2b, -48*128	;; Test loop counter, restore source pointer
	end_timer 5
ELSE
	start_timer 4
	mov	rdi, sincos1		;; Load sin/cos pointers
	bump	rdi, 8*ZMM_SCD3/8
	lea	r10, [rdi+48*ZMM_SCD4/8]
	mov	r9, norm_col_mults	;; Load weights pointer
	zloop_init 48			;; 48 iterations
	zr8fs_sixtyfour_complex_first_fft_preload
b1b:	zr8fs_sixtyfour_complex_first_fft rsi, 128, 48*128, 96*128, 192*128, r10, ZMM_CD8SCD4, rdi, ZMM_SCD4/8, r9, 8*128, 48, L1PREFETCH_ALL
	zloop	48, b1b, -48*128	;; Test loop counter, restore source pointer
	end_timer 4
ENDIF

;; Do FFT levels 8,9
;; On input the 64-byte cache lines hold these data values:
;;	0	+768	...	48	+768	...
;;	...
;;	47
;;	96	...
;;	192	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+768	...	8	+768	...
;;	...
;;	7
;;	16	...
;;	32	...
;;	...

;; Do 512 six_complex_fft operations
;;	distance between fft data elements is 8
;;	do 64 macros each processing 96 data values

	start_timer 6
	bump	rdi, -48*ZMM_SCD4/8-8*ZMM_SCD3/8 ;; Load sin/cos pointer
	zloop_init 64			;; 8 iterations of 8
	zr6b_six_complex_djbfft_preload
b3b:	zr6b_six_complex_djbfft rsi, 128, 8*128, rdi, ZMM_SCD3/8, 8
	zloop	8, b3b, -8*128+48*128, rdi, -8*ZMM_SCD3/8 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	8, b3b, -8*48*128	;; Test loop counter, restore source pointer
	end_timer 6

;; Do FFT levels 10,11,12
;; On input and output the 128-byte cache line pairs hold these data values:
;;	0	+768	...	8	+768	...
;;	...
;;	7
;;	16	...
;;	32	...
;;	...

;; Do 384 eight_complex_fft operations
;;	distance between fft data elements is 1
;;	do 48 macros processing 128 data values

zmid:	start_timer 7
	mov	rcx, 48
	call	zcomplex_onepass_middle
	end_timer 7

;; Do inverse FFT levels 8,9
;; On input the 64-byte cache lines hold these data values:
;;	0	+768	...	8	+768	...
;;	...
;;	7
;;	16	...
;;	32	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+768	...	8	+768	...
;;	...
;;	7
;;	16	...
;;	32	...
;;	...

;; Do 512 six_complex_unfft operations
;;	distance between fft data elements is 8
;;	do 64 macros each processing 96 data values

	start_timer 8
	mov	rdi, sincos1		;; Load sin/cos pointer
	zloop_init 64			;; 8 iterations of 8
	zr6b_six_complex_djbunfft_preload
c3b:	zr6b_six_complex_djbunfft rsi, 128, 8*128, rdi, ZMM_SCD3/8, 8
	zloop	8, c3b, -8*128+48*128, rdi, -8*ZMM_SCD3/8 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	8, c3b, -8*48*128, rdi, 8*ZMM_SCD3/8 ;; Test loop counter, restore source pointer, next sin/cos ptr
	end_timer 8

;; Do inverse FFT levels 5,6,7
;; On input the 64-byte cache lines hold these data values:
;;	0	+768	...	8	+768	...
;;	...
;;	7
;;	16	...
;;	32	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+768	...	384	+768	...
;;	...
;;	383	...

;; Do 384 eight_complex_unfft operations
;;	distance between fft data elements is 48
;;	do 48 macros each processing 128 data values

;; Do inverse FFT levels 1,2,3,4
;; On input the 128-byte cache line pairs hold these data values:
;;	0	+768	...	384	+768	...
;;	...
;;	383	...
;; These are swizzled to:
;;	0	+48	...	384	+48	...
;;	...
;;	47	...
;;	768	...
;;	1536	...
;;	...
;; On output the 128-byte cache line pairs hold these data values:
;;	0	+48	...	3K	+48	...
;;	...
;;	48	...
;;	384	...
;;	768	...
;;	...

;; Do 384 eight_complex_last_unfft operations
;;	distance between fft data elements is 384
;;	do 48 macros processing 128 data values

IFDEF OLDWAY
	start_timer 9
	zloop_init 48			;; 48 iterations
	zr8b_eight_complex_djbunfft_preload
c2b:	zr8b_eight_complex_djbunfft rsi, 128, 48*128, 96*128, 192*128, rdi, ZMM_SCD4/8, 48, L1PREFETCH_ALL, 128
	zloop	48, c2b, -48*128	;; Test loop counter, restore source pointer
	end_timer 9

	start_timer 10
	zloop_init 48			;; 48 iterations
	zr8s_eight_complex_last_unfft_preload
c1b:	zr8s_eight_complex_last_unfft rsi, 128, 48*128, 96*128, 192*128, rdi, ZMM_CD8SCD4, 48, L1PREFETCH_ALL
	zloop	48, c1b, -48*128	;; Test loop counter, restore source pointer
	end_timer 10
ELSE
	start_timer 10
	lea	r10, [rdi+48*ZMM_SCD4/8]
	zloop_init 48			;; 48 iterations
	zr8s_sixtyfour_complex_last_unfft_preload
c1b:	zr8s_sixtyfour_complex_last_unfft rsi, 128, 48*128, 96*128, 192*128, rdi, ZMM_SCD4/8, r10, ZMM_CD8SCD4, 48, L1PREFETCH_ALL
	zloop	48, c1b, -48*128	;; Test loop counter, restore source pointer
	end_timer 10
ENDIF

	zfft_3_ret
	ENDM


;; Perform a 7K-element FFT

zfft7K MACRO type
	LOCAL	b1b, b2b, b3a, b3b, zmid, c3a, c3b, c2b, c1b

;; Check for a multiply with pre-FFTed inputs

	cmp	ffttype, 4
	je	zmid

;; Do FFT levels 1,2,3,4
;; On input the 128-byte cache line pairs hold these data values:
;;	0	+56	...	3584	+56	...
;;	...
;;	55	...
;;	448	...
;;	896	...
;;	...
;; On output the 128-byte cache line pairs hold these data values:
;;	0	+56	...	448	+56	...
;;	...
;;	55	...
;;	896	...
;;	1792	...
;;	...
;; These are swizzled to:
;;	0	+896	...	448	+896	...
;;	...
;;	447	...

;; Do 448 sixteen_reals_fft operations
;;	distance between fft data elements is 448
;;	do 56 macros processing 128 data values

;; Do FFT levels 5,6,7
;; On input the 64-byte cache lines hold these data values:
;;	0	+896	...	448	+896	...
;;	...
;;	447	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+896	...	56	+896	...
;;	...
;;	55
;;	112	...
;;	224	...
;;	...

;; Do 56 sixteen_reals_fft operation
;; Do 392 eight_complex_fft operations
;;	distance between fft data elements is 56
;;	do 56 macros each processing 128 data values

IFDEF OLDWAY
	start_timer 4
	mov	rdi, sincos1		;; Load sin/cos data ptr
	bump	rdi, 56*ZMM_SCD7+8*ZMM_SCD6
	mov	r9, norm_col_mults	;; Load weights pointer
	zloop_init 56			;; 56 iterations
	zr8fs_sixteen_reals_first_fft_preload
b1b:	zr8fs_sixteen_reals_first_fft rsi, 128, 56*128, 112*128, 224*128, rdi, ZMM_SCD7, r9, 8*128, 56, L1PREFETCH_ALL
	zloop	56, b1b, -56*128	;; Test loop counter, restore source pointer
	end_timer 4

	start_timer 5
	bump	rdi, -56*ZMM_SCD7-56*ZMM_SCD7 ;; Load sin/cos ptr
	zloop_init 56			;; 56 iterations
	zr8_sixteen_reals_eight_complex_djbfft_preload
b2b:	zr8_sixteen_reals_eight_complex_djbfft rsi, 128, 56*128, 112*128, 224*128, rdi, ZMM_SCD7, 56, L1PREFETCH_ALL
	zloop	56, b2b, -56*128	;; Test loop counter, restore source pointer
	end_timer 5
ELSE
	start_timer 4
	mov	rdi, sincos1		;; Load sin/cos data ptrs
	bump	rdi, 8*ZMM_SCD6
	lea	r10, [rdi+56*ZMM_SCD7]
	mov	r9, norm_col_mults	;; Load weights pointer
	zloop_init 56			;; 56 iterations
	zr8fs_hundredtwentyeight_reals_first_fft_preload
b1b:	zr8fs_hundredtwentyeight_reals_first_fft rsi, 128, 56*128, 112*128, 224*128, r10, ZMM_SCD7, rdi, ZMM_SCD7, r9, 8*128, 56, L1PREFETCH_ALL
	zloop	56, b1b, -56*128	;; Test loop counter, restore source pointer
	end_timer 4
ENDIF

;; Do FFT levels 8,9
;; On input the 64-byte cache lines hold these data values:
;;	0	+896	...	56	+896	...
;;	...
;;	55
;;	112	...
;;	224	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+896	...	8	+896	...
;;	...
;;	7
;;	16	...
;;	32	...
;;	...

;; Do 8 fourteen_reals_fft operation
;; Do 504 seven_complex_fft operations
;;	distance between fft data elements is 8
;;	do 64 macros each processing 112 data values

	start_timer 6
	bump	rdi, -56*ZMM_SCD7-8*ZMM_SCD6 ;; Load sin/cos pointer
	zloop_init 64			;; 8 iterations of 8
	zr7_fourteen_reals_seven_complex_djbfft_preload
b3a:	zr7_fourteen_reals_seven_complex_djbfft rsi, 128, 8*128, rdi, ZMM_SCD6, 8, L1PREFETCH_NONE
	zloop	8, b3a, -8*128+56*128, rdi, -8*ZMM_SCD6	;; Test loop counter, next source pointer, restore sin/cos ptr

	loops_reset			;; 7 more iters of 8
	zr7rb_seven_complex_djbfft_preload
b3b:	zr7rb_seven_complex_djbfft rsi, 128, 8*128, rdi, ZMM_SCD6, 8, L1PREFETCH_NONE
	zloop	8, b3b, -8*128+56*128, rdi, -8*ZMM_SCD6 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	8, b3b, -8*56*128	;; Test loop counter, restore source pointer
	end_timer 6

;; Do FFT levels 10,11,12
;; On input and output the 128-byte cache line pairs hold these data values:
;;	0	+896	...	8	+896	...
;;	...
;;	7
;;	16	...
;;	32	...
;;	...

;; Do 1 sixteen_reals_fft operation
;; Do 447 eight_complex_fft operations
;;	distance between fft data elements is 1
;;	do 56 macros processing 128 data values

zmid:	start_timer 7
	mov	rcx, 56
	call	zreal_onepass_middle
	end_timer 7

;; Do inverse FFT levels 8,9
;; On input the 64-byte cache lines hold these data values:
;;	0	+896	...	8	+896	...
;;	...
;;	7
;;	16	...
;;	32	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+896	...	56	+896	...
;;	...
;;	55
;;	112	...
;;	224	...
;;	...

;; Do 8 fourteen_reals_unfft operation
;; Do 504 seven_complex_unfft operations
;;	distance between fft data elements is 8
;;	do 64 macros each processing 112 data values

	start_timer 8
	mov	rdi, sincos1		;; Load sin/cos pointer
	zloop_init 64			;; 8 iterations of 8
	zr7_fourteen_reals_seven_complex_djbunfft_preload
c3a:	zr7_fourteen_reals_seven_complex_djbunfft rsi, 128, 8*128, rdi, ZMM_SCD6, 8, L1PREFETCH_NONE
	zloop	8, c3a, -8*128+56*128, rdi, -8*ZMM_SCD6 ;; Test loop counter, restore source pointer, restore sin/cos pointer

	loops_reset			;; 7 more iters of 8
	zr7rb_seven_complex_djbunfft_preload
c3b:	zr7rb_seven_complex_djbunfft rsi, 128, 8*128, rdi, ZMM_SCD6, 8, L1PREFETCH_NONE
	zloop	8, c3b, -8*128+56*128, rdi, -8*ZMM_SCD6 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	8, c3b, -8*56*128, rdi, 8*ZMM_SCD6 ;; Test loop counter, restore source pointer, next sin/cos pointer
	end_timer 8

;; Do inverse FFT levels 1,2,3,4
;; On input the 128-byte cache line pairs hold these data values:
;;	0	+896	...	448	+896	...
;;	...
;;	447	...
;; These are swizzled to:
;;	0	+56	...	448	+56	...
;;	...
;;	55	...
;;	896	...
;;	1792	...
;;	...
;; On output the 128-byte cache line pairs hold these data values:
;;	0	+56	...	3584	+56	...
;;	...
;;	55	...
;;	448	...
;;	896	...
;;	...

;; Do 448 sixteen_reals_unfft operations
;;	distance between fft data elements is 448
;;	do 56 macros processing 128 data values

;; Do inverse FFT levels 5,6,7
;; On input the 64-byte cache lines hold these data values:
;;	0	+896	...	56	+896	...
;;	...
;;	55
;;	112	...
;;	224	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+896	...	448	+896	...
;;	...
;;	447	...

;; Do 56 sixteen_reals_unfft operations
;; Do 392 eight_complex_unfft operations
;;	distance between fft data elements is 56
;;	do 56 macros each processing 128 data values

IFDEF OLDWAY
	start_timer 9
	zloop_init 56			;; 56 iterations
	zr8_sixteen_reals_eight_complex_djbunfft_preload
c2b:	zr8_sixteen_reals_eight_complex_djbunfft rsi, 128, 56*128, 112*128, 224*128, rdi, ZMM_SCD7, 8, L1PREFETCH_ALL
	zloop	56, c2b, -56*128	;; Test loop counter, restore source pointer
	end_timer 9

	start_timer 10
	zloop_init 56			;; 56 iterations
	zr8s_sixteen_reals_last_unfft_preload
c1b:	zr8s_sixteen_reals_last_unfft rsi, 128, 56*128, 112*128, 224*128, rdi, ZMM_SCD7, 56, L1PREFETCH_ALL
	zloop	56, c1b, -56*128	;; Test loop counter, restore source pointer
	end_timer 10
ELSE
	start_timer 10
	lea	r10, [rdi+56*ZMM_SCD7]	;; Second sin/cos ptr
	zloop_init 56			;; 56 iterations
	zr8s_hundredtwentyeight_reals_last_unfft_preload
c1b:	zr8s_hundredtwentyeight_reals_last_unfft rsi, 128, 56*128, 112*128, 224*128, rdi, ZMM_SCD7, r10, ZMM_SCD7, 56, L1PREFETCH_ALL
	zloop	56, c1b, -56*128	;; Test loop counter, restore source pointer
	end_timer 10
ENDIF

	zfft_3_ret
	ENDM

;; Perform a 7K-element negacyclic FFT

zfft7Kp MACRO type
	LOCAL	b1b, b2b, b3b, zmid, c3b, c2b, c1b

;; Do a multiply with pre-FFTed inputs

	cmp	ffttype, 4
	je	zmid

;; Do FFT levels 1,2,3,4
;; On input the 128-byte cache line pairs hold these data values:
;;	0	+56	...	3584	+56	...
;;	...
;;	55	...
;;	448	...
;;	896	...
;;	...
;; On output the 128-byte cache line pairs hold these data values:
;;	0	+56	...	448	+56	...
;;	...
;;	55	...
;;	896	...
;;	1792	...
;;	...
;; These are swizzled to:
;;	0	+896	...	448	+896	...
;;	...
;;	447	...

;; Do 448 eight_complex_first_fft operations
;;	distance between fft data elements is 448
;;	do 56 macros processing 128 data values

;; Do FFT levels 5,6,7
;; On input the 64-byte cache lines hold these data values:
;;	0	+896	...	448	+896	...
;;	...
;;	447	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+896	...	56	+896	...
;;	...
;;	55
;;	112	...
;;	224	...
;;	...

;; Do 448 eight_complex_fft operations
;;	distance between fft data elements is 56
;;	do 56 macros each processing 128 data values

IFDEF OLDWAY
	start_timer 4
	mov	rdi, sincos1		;; Load sin/cos pointer
	bump	rdi, 56*ZMM_SCD4/8+8*ZMM_SCD3/8
	mov	r9, norm_col_mults	;; Load weights pointer
	zloop_init 56			;; 56 iterations
	zr8fs_eight_complex_first_fft_preload
b1b:	zr8fs_eight_complex_first_fft rsi, 128, 56*128, 112*128, 224*128, rdi, ZMM_CD8SCD4, r9, 8*128, 56, L1PREFETCH_ALL
	zloop	56, b1b, -56*128	;; Test loop counter, restore source pointer
	end_timer 4

	start_timer 5
	bump	rdi, -56*ZMM_CD8SCD4-56*ZMM_SCD4/8 ;; Load sin/cos pointer
	zloop_init 56			;; 56 iterations
	zr8b_eight_complex_djbfft_preload
b2b:	zr8b_eight_complex_djbfft rsi, 128, 56*128, 112*128, 224*128, rdi, ZMM_SCD4/8, 56, L1PREFETCH_ALL, 128
	zloop	56, b2b, -56*128	;; Test loop counter, next source pointer, restore sin/cos pointer
	end_timer 5
ELSE
	start_timer 4
	mov	rdi, sincos1		;; Load sin/cos pointer
	bump	rdi, 8*ZMM_SCD3/8
	lea	r10, [rdi+56*ZMM_SCD4/8]
	mov	r9, norm_col_mults	;; Load weights pointer
	zloop_init 56			;; 56 iterations
	zr8fs_sixtyfour_complex_first_fft_preload
b1b:	zr8fs_sixtyfour_complex_first_fft rsi, 128, 56*128, 112*128, 224*128, r10, ZMM_CD8SCD4, rdi, ZMM_SCD4/8, r9, 8*128, 56, L1PREFETCH_ALL
	zloop	56, b1b, -56*128	;; Test loop counter, restore source pointer
	end_timer 4
ENDIF

;; Do FFT levels 8,9
;; On input the 64-byte cache lines hold these data values:
;;	0	+896	...	56	+896	...
;;	...
;;	55
;;	112	...
;;	224	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+896	...	8	+896	...
;;	...
;;	7
;;	16	...
;;	32	...
;;	...

;; Do 512 seven_complex_fft operations
;;	distance between fft data elements is 8
;;	do 64 macros each processing 112 data values

	start_timer 5
	bump	rdi, -56*ZMM_SCD4/8-8*ZMM_SCD3/8 ;; Load sin/cos pointer
	zloop_init 64			;; 8 iterations of 8
	zr7b_seven_complex_djbfft_preload
b3b:	zr7b_seven_complex_djbfft rsi, 128, 8*128, rdi, ZMM_SCD3/8, 8, L1PREFETCH_ALL, 128
	zloop	8, b3b, -8*128+56*128, rdi, -8*ZMM_SCD3/8 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	8, b3b, -8*56*128	;; Test loop counter, restore source pointer
	end_timer 5

;; Do FFT levels 10,11,12
;; On input and output the 128-byte cache line pairs hold these data values:
;;	0	+896	...	8	+896	...
;;	...
;;	7
;;	16	...
;;	32	...
;;	...

;; Do 448 eight_complex_fft operations
;;	distance between fft data elements is 1
;;	do 56 macros processing 128 data values

zmid:	start_timer 7
	mov	rcx, 56
	call	zcomplex_onepass_middle
	end_timer 7

;; Do inverse FFT levels 8,9
;; On input the 64-byte cache lines hold these data values:
;;	0	+896	...	8	+896	...
;;	...
;;	7
;;	16	...
;;	32	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+896	...	56	+896	...
;;	...
;;	55
;;	112	...
;;	224	...
;;	...

;; Do 512 seven_complex_unfft operations
;;	distance between fft data elements is 8
;;	do 64 macros each processing 112 data values

	start_timer 8
	mov	rdi, sincos1		;; Load sin/cos pointer
	zloop_init 64			;; 8 iterations of 8
	zr7b_seven_complex_djbunfft_preload
c3b:	zr7b_seven_complex_djbunfft rsi, 128, 8*128, rdi, ZMM_SCD3/8, 64, L1PREFETCH_NONE
	zloop	8, c3b, -8*128+56*128, rdi, -8*ZMM_SCD3/8 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	8, c3b, -8*56*128, rdi, 8*ZMM_SCD3/8 ;; Test loop counter, restore source pointer, next sin/cos pointer
	end_timer 8

;; Do inverse FFT levels 5,6,7
;; On input the 64-byte cache lines hold these data values:
;;	0	+896	...	56	+896	...
;;	...
;;	55
;;	112	...
;;	224	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+896	...	448	+896	...
;;	...
;;	447	...

;; Do 448 eight_complex_unfft operations
;;	distance between fft data elements is 56
;;	do 56 macros each processing 128 data values

;; Do inverse FFT levels 1,2,3,4
;; On input the 128-byte cache line pairs hold these data values:
;;	0	+896	...	448	+896	...
;;	...
;;	447	...
;; These are swizzled to:
;;	0	+56	...	448	+56	...
;;	...
;;	55	...
;;	896	...
;;	1792	...
;;	...
;; On output the 128-byte cache line pairs hold these data values:
;;	0	+56	...	3584	+56	...
;;	...
;;	55	...
;;	448	...
;;	896	...
;;	...

;; Do 448 eight_complex_last_unfft operations
;;	distance between fft data elements is 448
;;	do 56 macros processing 128 data values

IFDEF OLDWAY
	start_timer 9
	zloop_init 56			;; 56 iterations
	zr8b_eight_complex_djbunfft_preload
c2b:	zr8b_eight_complex_djbunfft rsi, 128, 56*128, 112*128, 224*128, rdi, ZMM_SCD4/8, 56, L1PREFETCH_ALL, 128
	zloop	56, c2b, -56*128	;; Test loop counter, restore source pointer
	end_timer 9

	start_timer 10
	zloop_init 56			;; 56 iterations
	zr8s_eight_complex_last_unfft_preload
c1b:	zr8s_eight_complex_last_unfft rsi, 128, 56*128, 112*128, 224*128, rdi, ZMM_CD8SCD4, 56, L1PREFETCH_ALL
	zloop	56, c1b, -56*128	;; Test loop counter, restore source pointer
	end_timer 10
ELSE
	start_timer 9
	zloop_init 56			;; 56 iterations
	lea	r10, [rdi+56*ZMM_SCD4/8]
	zr8s_sixtyfour_complex_last_unfft_preload
c1b:	zr8s_sixtyfour_complex_last_unfft rsi, 128, 56*128, 112*128, 224*128, rdi, ZMM_SCD4/8, r10, ZMM_CD8SCD4, 56, L1PREFETCH_ALL
	zloop	56, c1b, -56*128	;; Test loop counter, restore source pointer
	end_timer 9
ENDIF

	zfft_3_ret
	ENDM


;; Perform a 8K-element FFT

zfft8K MACRO type
	LOCAL	b1b, b2b, b3a, b3b, zmid, c3a, c3b, c2b, c1b

;; Check for a multiply with pre-FFTed inputs

	cmp	ffttype, 4
	je	zmid

;; Do FFT levels 1,2,3,4
;; On input the 128-byte cache line pairs hold these data values:
;;	0	+64	...	4K	+64	...
;;	...
;;	63	...
;;	512	...
;;	1K	...
;;	...
;; On output the 128-byte cache line pairs hold these data values:
;;	0	+64	...	512	+64	...
;;	...
;;	63	...
;;	1K	...
;;	2K	...
;;	...
;; These are swizzled to:
;;	0	+1K	...	512	+1K	...
;;	...
;;	511	...

;; Do 512 sixteen_reals_fft operations
;;	distance between fft data elements is 512
;;	do 64 macros processing 128 data values

;; Do FFT levels 5,6,7
;; On input the 64-byte cache lines hold these data values:
;;	0	+1K	...	512	+1K	...
;;	...
;;	511	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+1K	...	64	+1K	...
;;	...
;;	63
;;	128	...
;;	256	...
;;	...

;; Do 64 sixteen_reals_fft operation
;; Do 448 eight_complex_fft operations
;;	distance between fft data elements is 64
;;	do 64 macros each processing 128 data values

IFDEF OLDWAY
	start_timer 4
	mov	rdi, sincos1		;; Load sin/cos data ptr
	bump	rdi, 64*ZMM_SCD7
	mov	r9, norm_col_mults	;; Load weights pointer
	zloop_init 64			;; 64 iterations
	zr8fs_sixteen_reals_first_fft_preload
b1b:	zr8fs_sixteen_reals_first_fft rsi, 128, 64*128, 128*128, 256*128, rdi, ZMM_SCD7, r9, 8*128, 64, L1PREFETCH_ALL
	zloop	64, b1b, -64*128	;; Test loop counter, restore source pointer
	end_timer 4

	start_timer 5
	bump	rdi, -64*ZMM_SCD7-64*ZMM_SCD7 ;; Load sin/cos pointer
	zloop_init 64			;; 64 iterations
	zr8_sixteen_reals_eight_complex_djbfft_preload
b2b:	zr8_sixteen_reals_eight_complex_djbfft rsi, 128, 64*128, 128*128, 256*128, rdi, ZMM_SCD7, 64, L1PREFETCH_ALL
	zloop	64, b2b, -64*128, rdi, -64*ZMM_SCD7 ;; Test loop counter, restore source pointer, restore sin/cos pointer
	end_timer 5
ELSE
	start_timer 4
	mov	rdi, sincos1		;; Load sin/cos data ptr
	lea	r10, [rdi+64*ZMM_SCD7]	;; Second sin/cos pointer
	mov	r9, norm_col_mults	;; Load weights pointer
	zloop_init 64			;; 64 iterations
	zr8fs_hundredtwentyeight_reals_first_fft_preload
b1b:	zr8fs_hundredtwentyeight_reals_first_fft rsi, 128, 64*128, 128*128, 256*128, r10, ZMM_SCD7, rdi, ZMM_SCD7, r9, 8*128, 64, L1PREFETCH_ALL
	zloop	64, b1b, -64*128, rdi, -64*ZMM_SCD7 ;; Test loop counter, restore source pointer, restore sin/cos pointer
	end_timer 4
ENDIF

;; Do FFT levels 8,9,10
;; On input the 64-byte cache lines hold these data values:
;;	0	+1K	...	64	+1K	...
;;	...
;;	63
;;	128	...
;;	256	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+1K	...	8	+1K	...
;;	...
;;	7
;;	16	...
;;	32	...
;;	...

;; Do 8 sixteen_reals_fft operation
;; Do 504 eight_complex_fft operations
;;	distance between fft data elements is 8
;;	do 64 macros each processing 128 data values

	start_timer 6
	zloop_init 64			;; 8 iterations of 8
	zr8_sixteen_reals_eight_complex_djbfft_preload
b3a:	zr8_sixteen_reals_eight_complex_djbfft rsi, 128, 8*128, 16*128, 32*128, rdi, 8*ZMM_SCD7, 8, L1PREFETCH_ALL
	zloop	8, b3a, -8*128+64*128, rdi, -8*8*ZMM_SCD7 ;; Test loop counter, next source pointer, restore sin/cos pointer

	loops_reset			;; 7 more iters of 8
	zr8rb_eight_complex_djbfft_preload
b3b:	zr8rb_eight_complex_djbfft rsi, 128, 8*128, 16*128, 32*128, rdi, 8*ZMM_SCD7, 8, L1PREFETCH_ALL, 128
	zloop	8, b3b, -8*128+64*128, rdi, -8*8*ZMM_SCD7 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	8, b3b, -8*64*128	;; Test loop counter, restore source pointer
	end_timer 6

;; Do FFT levels 11,12,13
;; On input and output the 128-byte cache line pairs hold these data values:
;;	0	+1K	...	8	+1K	...
;;	...
;;	7
;;	16	...
;;	32	...
;;	...

;; Do 1 sixteen_reals_fft operation
;; Do 511 eight_complex_fft operations
;;	distance between fft data elements is 1
;;	do 64 macros processing 128 data values

zmid:	start_timer 7
	mov	rcx, 64
	call	zreal_onepass_middle
	end_timer 7

;; Do inverse FFT levels 8,9,10
;; On input the 64-byte cache lines hold these data values:
;;	0	+1K	...	8	+1K	...
;;	...
;;	7
;;	16	...
;;	32	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+1K	...	64	+1K	...
;;	...
;;	63
;;	128	...
;;	256	...
;;	...

;; Do 8 sixteen_reals_unfft operations
;; Do 504 eight_complex_unfft operations
;;	distance between fft data elements is 8
;;	do 64 macros each processing 128 data values

	start_timer 8
	mov	rdi, sincos1		;; Load sin/cos pointer
	zloop_init 64			;; 8 iterations of 8
	zr8_sixteen_reals_eight_complex_djbunfft_preload
c3a:	zr8_sixteen_reals_eight_complex_djbunfft rsi, 128, 8*128, 16*128, 32*128, rdi, 8*ZMM_SCD7, 8, L1PREFETCH_NONE
	zloop	8, c3a, -8*128+64*128, rdi, -8*8*ZMM_SCD7 ;; Test loop counter, restore source pointer, restore sin/cos pointer

	loops_reset			;; 7 more iters of 8
	zr8rb_eight_complex_djbunfft_preload
c3b:	zr8rb_eight_complex_djbunfft rsi, 128, 8*128, 16*128, 32*128, rdi, 8*ZMM_SCD7, 8, L1PREFETCH_NONE
	zloop	8, c3b, -8*128+64*128, rdi, -8*8*ZMM_SCD7 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	8, c3b, -8*64*128	;; Test loop counter, restore source pointer
	end_timer 8

;; Do inverse FFT levels 5,6,7
;; On input the 64-byte cache lines hold these data values:
;;	0	+1K	...	64	+1K	...
;;	...
;;	63
;;	128	...
;;	256	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+1K	...	512	+1K	...
;;	...
;;	511	...

;; Do 64 sixteen_reals_unfft operations
;; Do 448 eight_complex_unfft operations
;;	distance between fft data elements is 64
;;	do 64 macros each processing 128 data values

;; Do inverse FFT levels 1,2,3,4
;; On input the 128-byte cache line pairs hold these data values:
;;	0	+1K	...	512	+1K	...
;;	...
;;	511	...
;; These are swizzled to:
;;	0	+64	...	512	+64	...
;;	...
;;	63	...
;;	1K	...
;;	2K	...
;;	...
;; On output the 128-byte cache line pairs hold these data values:
;;	0	+64	...	4K	+64	...
;;	...
;;	63	...
;;	512	...
;;	1K	...
;;	...

;; Do 512 sixteen_reals_unfft operations
;;	distance between fft data elements is 512
;;	do 64 macros processing 128 data values

IFDEF OLDWAY
	start_timer 9
	zloop_init 64			;; 64 iterations
	zr8_sixteen_reals_eight_complex_djbunfft_preload
c2b:	zr8_sixteen_reals_eight_complex_djbunfft rsi, 128, 64*128, 128*128, 256*128, rdi, ZMM_SCD7, 64, L1PREFETCH_NONE
	zloop	64, c2b, -64*128	;; Test loop counter, restore source pointer, restore sin/cos pointer
	end_timer 9

	start_timer 10
	zloop_init 64			;; 64 iterations
	zr8s_sixteen_reals_last_unfft_preload
c1b:	zr8s_sixteen_reals_last_unfft rsi, 128, 64*128, 128*128, 256*128, rdi, ZMM_SCD7, 64, L1PREFETCH_ALL
	zloop	64, c1b, -64*128	;; Test loop counter, restore source pointer
	end_timer 10
ELSE
	start_timer 9
	lea	r10, [rdi+64*ZMM_SCD7]	;; Second sin/cos pointer
	zloop_init 64			;; 64 iterations
	zr8s_hundredtwentyeight_reals_last_unfft_preload
c1b:	zr8s_hundredtwentyeight_reals_last_unfft rsi, 128, 64*128, 128*128, 256*128, rdi, ZMM_SCD7, r10, ZMM_SCD7, 64, L1PREFETCH_ALL
	zloop	64, c1b, -64*128	;; Test loop counter, restore source pointer
	end_timer 9
ENDIF

	zfft_3_ret
	ENDM

;; Perform a 8K-element negacyclic FFT

zfft8Kp MACRO type
	LOCAL	b1b, b2b, b3b, zmid, c3b, c2b, c1b

;; Do a multiply with pre-FFTed inputs

	cmp	ffttype, 4
	je	zmid

;; Do FFT levels 1,2,3,4
;; On input the 128-byte cache line pairs hold these data values:
;;	0	+64	...	4K	+64	...
;;	...
;;	63	...
;;	512	...
;;	1K	...
;;	...
;; On output the 128-byte cache line pairs hold these data values:
;;	0	+64	...	512	+64	...
;;	...
;;	63	...
;;	1K	...
;;	2K	...
;;	...
;; These are swizzled to:
;;	0	+1K	...	512	+1K	...
;;	...
;;	511	...

;; Do 512 eight_complex_first_fft operations
;;	distance between fft data elements is 512
;;	do 64 macros processing 128 data values

;; Do FFT levels 5,6,7
;; On input the 64-byte cache lines hold these data values:
;;	0	+1K	...	512	+1K	...
;;	...
;;	511	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+1K	...	64	+1K	...
;;	...
;;	63
;;	128	...
;;	256	...
;;	...

;; Do 512 eight_complex_fft operations
;;	distance between fft data elements is 64
;;	do 64 macros each processing 128 data values

IFDEF OLDWAY
	start_timer 4
	mov	rdi, sincos1		;; Load sin/cos pointer
	bump	rdi, 64*ZMM_SCD4/8
	mov	r9, norm_col_mults	;; Load weights pointer
	zloop_init 64			;; 64 iterations
	zr8fs_eight_complex_first_fft_preload
b1b:	zr8fs_eight_complex_first_fft rsi, 128, 64*128, 128*128, 256*128, rdi, ZMM_CD8SCD4, r9, 8*128, 64, L1PREFETCH_ALL
	zloop	64, b1b, -64*128	;; Test loop counter, restore source pointer
	end_timer 4

	start_timer 5
	bump	rdi, -64*ZMM_CD8SCD4-64*ZMM_SCD4/8 ;; Load sin/cos pointer
	zloop_init 64			;; 64 iterations
	zr8b_eight_complex_djbfft_preload
b2b:	zr8b_eight_complex_djbfft rsi, 128, 64*128, 128*128, 256*128, rdi, ZMM_SCD4/8, 64, L1PREFETCH_ALL, 128
	zloop	64, b2b, -64*128	;; Test loop counter, restore source pointer
	end_timer 5
ELSE
	start_timer 4
	mov	rdi, sincos1		;; Load sin/cos pointer
	lea	r10, [rdi+64*ZMM_SCD4/8] ;; Second sin/cos pointer
	mov	r9, norm_col_mults	;; Load weights pointer
	zloop_init 64			;; 64 iterations
	zr8fs_sixtyfour_complex_first_fft_preload
b1b:	zr8fs_sixtyfour_complex_first_fft rsi, 128, 64*128, 128*128, 256*128, r10, ZMM_CD8SCD4, rdi, ZMM_SCD4/8, r9, 8*128, 64, L1PREFETCH_ALL
	zloop	64, b1b, -64*128	;; Test loop counter, restore source pointer
	end_timer 4
ENDIF

;; Do FFT levels 8,9,10
;; On input the 64-byte cache lines hold these data values:
;;	0	+1K	...	64	+1K	...
;;	...
;;	63
;;	128	...
;;	256	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+1K	...	8	+1K	...
;;	...
;;	7
;;	16	...
;;	32	...
;;	...

;; Do 512 eight_complex_fft operations
;;	distance between fft data elements is 8
;;	do 64 macros each processing 128 data values

	start_timer 6
	bump	rdi, -64*ZMM_SCD4/8	;; Load sin/cos pointer
	zloop_init 64			;; 8 iterations of 8
	zr8b_eight_complex_djbfft_preload
b3b:	zr8b_eight_complex_djbfft rsi, 128, 8*128, 16*128, 32*128, rdi, 8*ZMM_SCD4/8, 8, L1PREFETCH_ALL, 128
	zloop	8, b3b, -8*128+64*128, rdi, -8*8*ZMM_SCD4/8 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	8, b3b, -8*64*128	;; Test loop counter, restore source pointer
	end_timer 6

;; Do FFT levels 11,12,13
;; On input and output the 128-byte cache line pairs hold these data values:
;;	0	+1K	...	8	+1K	...
;;	...
;;	7
;;	16	...
;;	32	...
;;	...

;; Do 512 eight_complex_fft operations
;;	distance between fft data elements is 1
;;	do 64 macros processing 128 data values

zmid:	start_timer 7
	mov	rcx, 64
	call	zcomplex_onepass_middle
	end_timer 7

;; Do inverse FFT levels 8,9,10
;; On input the 64-byte cache lines hold these data values:
;;	0	+1K	...	8	+1K	...
;;	...
;;	7
;;	16	...
;;	32	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+1K	...	64	+1K	...
;;	...
;;	63
;;	128	...
;;	256	...
;;	...

;; Do 512 eight_complex_unfft operations
;;	distance between fft data elements is 8
;;	do 64 macros each processing 128 data values

	start_timer 8
	mov	rdi, sincos1		;; Load sin/cos pointer
	zloop_init 64			;; 8 iterations of 8
	zr8b_eight_complex_djbunfft_preload
c3b:	zr8b_eight_complex_djbunfft rsi, 128, 8*128, 16*128, 32*128, rdi, 8*ZMM_SCD4/8, 8, L1PREFETCH_ALL, 128
	zloop	8, c3b, -8*128+64*128, rdi, -8*8*ZMM_SCD4/8 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	8, c3b, -8*64*128	;; Test loop counter, restore source pointer
	end_timer 8

;; Do inverse FFT levels 5,6,7
;; On input the 64-byte cache lines hold these data values:
;;	0	+1K	...	64	+1K	...
;;	...
;;	63
;;	128	...
;;	256	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+1K	...	512	+1K	...
;;	...
;;	511	...

;; Do 512 eight_complex_unfft operations
;;	distance between fft data elements is 64
;;	do 64 macros each processing 128 data values

;; Do inverse FFT levels 1,2,3,4
;; On input the 128-byte cache line pairs hold these data values:
;;	0	+1K	...	512	+1K	...
;;	...
;;	511	...
;; These are swizzled to:
;;	0	+64	...	512	+64	...
;;	...
;;	63	...
;;	1K	...
;;	2K	...
;;	...
;; On output the 128-byte cache line pairs hold these data values:
;;	0	+64	...	4K	+64	...
;;	...
;;	63	...
;;	512	...
;;	1K	...
;;	...

;; Do 512 eight_complex_last_unfft operations
;;	distance between fft data elements is 512
;;	do 64 macros processing 128 data values

IFDEF OLDWAY
	start_timer 9
	zloop_init 64			;; 64 iterations
	zr8b_eight_complex_djbunfft_preload
c2b:	zr8b_eight_complex_djbunfft rsi, 128, 64*128, 128*128, 256*128, rdi, ZMM_SCD4/8, 64, L1PREFETCH_ALL, 128
	zloop	64, c2b, -64*128	;; Test loop counter, restore source pointer
	end_timer 9

	start_timer 10
	zloop_init 64			;; 64 iterations
	zr8s_eight_complex_last_unfft_preload
c1b:	zr8s_eight_complex_last_unfft rsi, 128, 64*128, 128*128, 256*128, rdi, ZMM_CD8SCD4, 64, L1PREFETCH_ALL
	zloop	64, c1b, -64*128	;; Test loop counter, restore source pointer
	end_timer 10
ELSE
	start_timer 9
	lea	r10, [rdi+64*ZMM_SCD4/8] ;; Second sin/cos pointer
	zloop_init 64			;; 64 iterations
	zr8s_sixtyfour_complex_last_unfft_preload
c1b:	zr8s_sixtyfour_complex_last_unfft rsi, 128, 64*128, 128*128, 256*128, rdi, ZMM_SCD4/8, r10, ZMM_CD8SCD4, 64, L1PREFETCH_ALL
	zloop	64, c1b, -64*128	;; Test loop counter, restore source pointer
	end_timer 9
ENDIF

	zfft_3_ret
	ENDM


;; Perform a 9K-element FFT

zfft9K MACRO type
	LOCAL	b1b, b2b, b3a, b3b, b4a, b4b, zmid, c4a, c4b, c3a, c3b, c2b, c1b

;; Check for a multiply with pre-FFTed inputs

	cmp	ffttype, 4
	je	zmid

;; Do FFT levels 1,2,3,4
;; On input the 128-byte cache line pairs hold these data values:
;;	0	+72	...	4.5K	+72	...
;;	...
;;	71	...
;;	576	...
;;	1152	...
;;	...
;; On output the 128-byte cache line pairs hold these data values:
;;	0	+72	...	576	+72	...
;;	...
;;	71	...
;;	1152	...
;;	2304	...
;;	...
;; These are swizzled to:
;;	0	+1152	...	576	+1152	...
;;	...
;;	575	...

;; Do 576 sixteen_reals_fft operations
;;	distance between fft data elements is 576
;;	do 72 macros processing 128 data values

;; Do FFT levels 5,6,7
;; On input the 64-byte cache lines hold these data values:
;;	0	+1152	...	576	+1152	...
;;	...
;;	575	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+1152	...	72	+1152	...
;;	...
;;	71
;;	144	...
;;	288	...
;;	...

;; Do 72 sixteen_reals_fft operation
;; Do 504 eight_complex_fft operations
;;	distance between fft data elements is 72
;;	do 72 macros each processing 128 data values

IFDEF OLDWAY
	start_timer 4
	mov	rdi, sincos1		;; Load sin/cos data ptr
	bump	rdi, 72*ZMM_SCD7+72*ZMM_SCD2+24*ZMM_SCD2
	mov	r9, norm_col_mults	;; Load weights pointer
	zloop_init 72			;; 72 iterations
	zr8fs_sixteen_reals_first_fft_preload
b1b:	zr8fs_sixteen_reals_first_fft rsi, 128, 72*128, 144*128, 288*128, rdi, ZMM_SCD7, r9, 8*128, 72, L1PREFETCH_ALL
	zloop	72, b1b, -72*128	;; Test loop counter, restore source pointer
	end_timer 4

	start_timer 5
	bump	rdi, -72*ZMM_SCD7-72*ZMM_SCD7 ;; Load sin/cos pointer
	zloop_init 72			;; 72 iterations
	zr8_sixteen_reals_eight_complex_djbfft_preload
b2b:	zr8_sixteen_reals_eight_complex_djbfft rsi, 128, 72*128, 144*128, 288*128, rdi, ZMM_SCD7, 72, L1PREFETCH_ALL
	zloop	72, b2b, -72*128	;; Test loop counter, restore source pointer
	end_timer 5
ELSE
	start_timer 4
	mov	rdi, sincos1		;; Load sin/cos data ptr
	bump	rdi, 24*ZMM_SCD2
	lea	r10, [rdi+72*ZMM_SCD7]	;; Second sin/cos pointer
	mov	r9, norm_col_mults	;; Load weights pointer
	zloop_init 72			;; 72 iterations
	zr8fs_hundredtwentyeight_reals_first_fft_preload
b1b:	zr8fs_hundredtwentyeight_reals_first_fft rsi, 128, 72*128, 144*128, 288*128, r10, ZMM_SCD7, rdi, ZMM_SCD7, r9, 8*128, 72, L1PREFETCH_ALL
	zloop	72, b1b, -72*128	;; Test loop counter, restore source pointer
	end_timer 4
ENDIF

;; Do FFT levels 8,9
;; On input the 64-byte cache lines hold these data values:
;;	0	+1152	...	72	+1152	...
;;	...
;;	71
;;	144	...
;;	288	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+1152	...	24	+1152	...
;;	...
;;	23
;;	48	...
;;	...

;; Do 24 six_reals_fft operation
;; Do 1512 three_complex_fft operations
;;	distance between fft data elements is 24
;;	do 192 macros each processing 48 data values

;; Do FFT levels 10,11
;; On input the 64-byte cache lines hold these data values:
;;	0	+1152	...	24	+1152	...
;;	...
;;	23
;;	48	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+1152	...	8	+1152	...
;;	...
;;	7
;;	16	...
;;	32	...
;;	...

;; Do 8 six_reals_fft operation
;; Do 1528 three_complex_fft operations
;;	distance between fft data elements is 8
;;	do 192 macros each processing 48 data values

IFDEF OLDWAY
	start_timer 6
	bump	rdi, -72*ZMM_SCD7-24*ZMM_SCD2
	zloop_init 192			;; 8 iterations of 24
	zr3_six_reals_three_complex_djbfft_preload
b3a:	zr3_six_reals_three_complex_djbfft rsi, 128, 24*128, rdi, ZMM_SCD2, 24
	zloop	24, b3a, -24*128+72*128, rdi, -24*ZMM_SCD2 ;; Test loop counter, next source pointer, restore sin/cos pointer

	loops_reset			;; 7 more iters of 24
	zr3rb_three_complex_djbfft_preload
b3b:	zr3rb_three_complex_djbfft rsi, 128, 24*128, rdi, ZMM_SCD2, 24
	zloop	24, b3b, -24*128+72*128, rdi, -24*ZMM_SCD2 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	8, b3b, -8*72*128	;; Test loop counter, restore source pointer
	end_timer 6

	start_timer 7
	zloop_init 192			;; 24 iterations of 8
	zr3_six_reals_three_complex_djbfft_preload
b4a:	zr3_six_reals_three_complex_djbfft rsi, 128, 8*128, rdi, 3*ZMM_SCD2, 8
	zloop	8, b4a, -8*128+24*128, rdi, -8*3*ZMM_SCD2 ;; Test loop counter, next source pointer, restore sin/cos pointer

	loops_reset			;; 23 more iters of 8
	zr3rb_three_complex_djbfft_preload
b4b:	zr3rb_three_complex_djbfft rsi, 128, 8*128, rdi, 3*ZMM_SCD2, 8
	zloop	8, b4b, -8*128+24*128, rdi, -8*3*ZMM_SCD2 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	24, b4b, -24*24*128	;; Test loop counter, restore source pointer
	end_timer 7
ELSE
	start_timer 6
	bump	rdi, -72*ZMM_SCD7-24*ZMM_SCD2
	mov	r10, rdi
	zloop_init 64			;; 8 iterations of 8
	zr33_eighteen_reals_nine_complex_djbfft_preload
b3a:	zr33_eighteen_reals_nine_complex_djbfft rsi, 128, 8*128, r10, 8*ZMM_SCD2, ZMM_SCD2, rdi, 0*3*ZMM_SCD2, 3*ZMM_SCD2, 8
	zloop	8, b3a, -8*128+72*128, r10, -8*ZMM_SCD2, rdi, -8*3*ZMM_SCD2 ;; Test loop counter, next source pointer, restore sin/cos pointers
	loops_reset			;; 7 more iterations of 8
	zr33rb_nine_complex_djbfft_preload
b3b:	zr33rb_nine_complex_djbfft rsi, 128, 8*128, r10, 8*ZMM_SCD2, ZMM_SCD2, rdi, 0*3*ZMM_SCD2, 3*ZMM_SCD2, 8
	zloop	8, b3b, -8*128+72*128, r10, -8*ZMM_SCD2, rdi, -8*3*ZMM_SCD2 ;; Test loop counter, next source pointer, restore sin/cos pointers
	zloop	8, b3b, -8*72*128	;; Test loop counter, restore source pointer
	end_timer 6
ENDIF

;; Do FFT levels 12,13,14
;; On input and output the 128-byte cache line pairs hold these data values:
;;	0	+1152	...	8	+1152	...
;;	...
;;	7
;;	16	...
;;	32	...
;;	...

;; Do 1 sixteen_reals_fft operation
;; Do 575 eight_complex_fft operations
;;	distance between fft data elements is 1
;;	do 72 macros processing 128 data values

zmid:	start_timer 8
	mov	rcx, 72
	call	zreal_onepass_middle
	end_timer 8

;; Do inverse FFT levels 10,11
;; On input the 64-byte cache lines hold these data values:
;;	0	+1152	...	8	+1152	...
;;	...
;;	7
;;	16	...
;;	32	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+1152	...	24	+1152	...
;;	...
;;	23
;;	48	...
;;	...

;; Do 8 six_reals_unfft operations
;; Do 1528 three_complex_unfft operations
;;	distance between fft data elements is 8
;;	do 192 macros each processing 48 data values

;; Do inverse FFT levels 8,9
;; On input the 64-byte cache lines hold these data values:
;;	0	+1152	...	24	+1152	...
;;	...
;;	23
;;	48	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+1152	...	72	+1152	...
;;	...
;;	71
;;	144	...
;;	288	...
;;	...

;; Do 24 six_reals_unfft operations
;; Do 1512 three_complex_unfft operations
;;	distance between fft data elements is 24
;;	do 192 macros each processing 48 data values

IFDEF OLDWAY
	start_timer 9
	mov	rdi, sincos1		;; Load sin/cos pointer
	zloop_init 192			;; 24 iterations of 8
	zr3_six_reals_three_complex_djbunfft_preload
c4a:	zr3_six_reals_three_complex_djbunfft rsi, 128, 8*128, rdi, 3*ZMM_SCD2, 8
	zloop	8, c4a, -8*128+24*128, rdi, -8*3*ZMM_SCD2 ;; Test loop counter, restore source pointer, restore sin/cos pointer

	loops_reset			;; 23 more iters of 8
	zr3rb_three_complex_djbunfft_preload
c4b:	zr3rb_three_complex_djbunfft rsi, 128, 8*128, rdi, 3*ZMM_SCD2, 8
	zloop	8, c4b, -8*128+24*128, rdi, -8*3*ZMM_SCD2 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	24, c4b, -24*24*128	;; Test loop counter, restore source pointer
	end_timer 9

	start_timer 10
	zloop_init 192			;; 8 iterations of 24
	zr3_six_reals_three_complex_djbunfft_preload
c3a:	zr3_six_reals_three_complex_djbunfft rsi, 128, 24*128, rdi, ZMM_SCD2, 24
	zloop	24, c3a, -24*128+72*128, rdi, -24*ZMM_SCD2 ;; Test loop counter, restore source pointer, restore sin/cos pointer

	loops_reset			;; 7 more iters of 24
	zr3rb_three_complex_djbunfft_preload
c3b:	zr3rb_three_complex_djbunfft rsi, 128, 24*128, rdi, ZMM_SCD2, 24
	zloop	24, c3b, -24*128+72*128, rdi, -24*ZMM_SCD2 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	8, c3b, -8*72*128, rdi, 24*ZMM_SCD2 ;; Test loop counter, restore source pointer, next sin/cos ptr
	end_timer 10
ELSE
	start_timer 9
	mov	rdi, sincos1		;; Load sin/cos pointers
	mov	r10,rdi
	zloop_init 64			;; 8 iterations of 8
	zr33_eighteen_reals_nine_complex_djbunfft_preload
c3a:	zr33_eighteen_reals_nine_complex_djbunfft rsi, 128, 8*128, rdi, 0*3*ZMM_SCD2, 3*ZMM_SCD2, r10, 8*ZMM_SCD2, ZMM_SCD2, 8
	zloop	8, c3a, -8*128+72*128, rdi, -8*3*ZMM_SCD2, r10, -8*ZMM_SCD2 ;; Test loop counter, next source pointer, restore sin/cos pointers
	loops_reset			;; 7 more iters of 8
	zr33rb_nine_complex_djbunfft_preload
c3b:	zr33rb_nine_complex_djbunfft rsi, 128, 8*128, rdi, 0*3*ZMM_SCD2, 3*ZMM_SCD2, r10, 8*ZMM_SCD2, ZMM_SCD2, 8
	zloop	8, c3b, -8*128+72*128, rdi, -8*3*ZMM_SCD2, r10, -8*ZMM_SCD2 ;; Test loop counter, next source pointer, restore sin/cos pointers
	zloop	8, c3b, -8*72*128, rdi, 24*ZMM_SCD2 ;; Test loop counter, restore source pointer, next sin/cos ptr
	end_timer 9
ENDIF

;; Do inverse FFT levels 5,6,7
;; On input the 64-byte cache lines hold these data values:
;;	0	+1152	...	72	+1152	...
;;	...
;;	71
;;	144	...
;;	288	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+1152	...	576	+1152	...
;;	...
;;	575	...

;; Do 72 sixteen_reals_unfft operations
;; Do 504 eight_complex_unfft operations
;;	distance between fft data elements is 72
;;	do 72 macros each processing 128 data values

;; Do inverse FFT levels 1,2,3,4
;; On input the 128-byte cache line pairs hold these data values:
;;	0	+1152	...	576	+1152	...
;;	...
;;	575	...
;; These are swizzled to:
;;	0	+72	...	576	+72	...
;;	...
;;	71	...
;;	1152	...
;;	2304	...
;;	...
;; On output the 128-byte cache line pairs hold these data values:
;;	0	+72	...	4.5K	+72	...
;;	...
;;	71	...
;;	576	...
;;	1152	...
;;	...

;; Do 576 sixteen_reals_unfft operations
;;	distance between fft data elements is 576
;;	do 72 macros processing 128 data values

IFDEF OLDWAY
	start_timer 11
	zloop_init 72			;; 72 iterations
	zr8_sixteen_reals_eight_complex_djbunfft_preload
c2b:	zr8_sixteen_reals_eight_complex_djbunfft rsi, 128, 72*128, 144*128, 288*128, rdi, ZMM_SCD7, 72
	zloop	72, c2b, -72*128	;; Test loop counter, restore source pointer, restore sin/cos pointer
	end_timer 11

	start_timer 12
	zloop_init 72			;; 72 iterations
	zr8s_sixteen_reals_last_unfft_preload
c1b:	zr8s_sixteen_reals_last_unfft rsi, 128, 72*128, 144*128, 288*128, rdi, ZMM_SCD7, 72, L1PREFETCH_ALL
	zloop	72, c1b, -72*128	;; Test loop counter, restore source pointer
	end_timer 12
ELSE
	start_timer 12
	lea	r10, [rdi+72*ZMM_SCD7]	;; Second sin/cos pointer
	zloop_init 72			;; 72 iterations
	zr8s_hundredtwentyeight_reals_last_unfft_preload
c1b:	zr8s_hundredtwentyeight_reals_last_unfft rsi, 128, 72*128, 144*128, 288*128, rdi, ZMM_SCD7, r10, ZMM_SCD7, 72, L1PREFETCH_ALL
	zloop	72, c1b, -72*128	;; Test loop counter, restore source pointer
	end_timer 12
ENDIF

	zfft_3_ret
	ENDM

;; Perform a 9K-element negacyclic FFT

zfft9Kp MACRO type
	LOCAL	b1b, b2b, b3b, b4b, zmid, c4b, c3b, c2b, c1b

;; Do a multiply with pre-FFTed inputs

	cmp	ffttype, 4
	je	zmid

;; Do FFT levels 1,2,3,4
;; On input the 128-byte cache line pairs hold these data values:
;;	0	+72	...	4.5K	+72	...
;;	...
;;	71	...
;;	576	...
;;	1152	...
;;	...
;; On output the 128-byte cache line pairs hold these data values:
;;	0	+72	...	576	+72	...
;;	...
;;	71	...
;;	1152	...
;;	2304	...
;;	...
;; These are swizzled to:
;;	0	+1152	...	576	+1152	...
;;	...
;;	575	...

;; Do 576 eight_complex_first_fft operations
;;	distance between fft data elements is 576
;;	do 72 macros processing 128 data values

;; Do FFT levels 5,6,7
;; On input the 64-byte cache lines hold these data values:
;;	0	+1152	...	576	+1152	...
;;	...
;;	575	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+1152	...	72	+1152	...
;;	...
;;	71
;;	144	...
;;	288	...
;;	...

;; Do 576 eight_complex_fft operations
;;	distance between fft data elements is 72
;;	do 72 macros each processing 128 data values

IFDEF OLDWAY
	start_timer 4
	mov	rdi, sincos1		;; Load sin/cos pointer
	bump	rdi, 72*ZMM_SCD4/8+24*ZMM_SCD1/8
	mov	r9, norm_col_mults	;; Load weights pointer
	zloop_init 72			;; 96 iterations
	zr8fs_eight_complex_first_fft_preload
b1b:	zr8fs_eight_complex_first_fft rsi, 128, 72*128, 144*128, 288*128, rdi, ZMM_CD8SCD4, r9, 8*128, 72, L1PREFETCH_ALL
	zloop	72, b1b, -72*128	;; Test loop counter, restore source pointer
	end_timer 4

	start_timer 5
	bump	rdi, -72*ZMM_CD8SCD4-72*ZMM_SCD4/8 ;; Load sin/cos pointer
	zloop_init 72			;; 72 iterations
	zr8b_eight_complex_djbfft_preload
b2b:	zr8b_eight_complex_djbfft rsi, 128, 72*128, 144*128, 288*128, rdi, ZMM_SCD4/8, 72, L1PREFETCH_ALL, 128
	zloop	72, b2b, -72*128	;; Test loop counter, restore source pointer
	end_timer 5
ELSE
	start_timer 4
	mov	rdi, sincos1		;; Load sin/cos pointer
	bump	rdi, 24*ZMM_SCD1/8
	lea	r10, [rdi+72*ZMM_SCD4/8] ;; Second sin/cos pointer
	mov	r9, norm_col_mults	;; Load weights pointer
	zloop_init 72			;; 72 iterations
	zr8fs_sixtyfour_complex_first_fft_preload
b1b:	zr8fs_sixtyfour_complex_first_fft rsi, 128, 72*128, 144*128, 288*128, r10, ZMM_CD8SCD4, rdi, ZMM_SCD4/8, r9, 8*128, 72, L1PREFETCH_ALL
	zloop	72, b1b, -72*128	;; Test loop counter, restore source pointer
	end_timer 4
ENDIF

;; Do FFT levels 8,9
;; On input the 64-byte cache lines hold these data values:
;;	0	+1152	...	72	+1152	...
;;	...
;;	71
;;	144	...
;;	288	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+1152	...	24	+1152	...
;;	...
;;	23
;;	48	...
;;	...

;; Do 1536 three_complex_fft operations
;;	distance between fft data elements is 24
;;	do 192 macros each processing 48 data values

;; Do FFT levels 10,11
;; On input the 64-byte cache lines hold these data values:
;;	0	+1152	...	24	+1152	...
;;	...
;;	23
;;	48	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+1152	...	8	+1152	...
;;	...
;;	7
;;	16	...
;;	32	...
;;	...

;; Do 1536 three_complex_fft operations
;;	distance between fft data elements is 8
;;	do 192 macros each processing 48 data values

IFDEF OLDWAY
	start_timer 6
	bump	rdi, -72*ZMM_SCD4/8-24*ZMM_SCD1/8 ;; Load sin/cos pointer
	zloop_init 192			;; 8 iterations of 24
	zr3b_three_complex_djbfft_preload
b3b:	zr3b_three_complex_djbfft rsi, 128, 24*128, rdi, ZMM_SCD1/8, 24
	zloop	24, b3b, -24*128+72*128, rdi, -24*ZMM_SCD1/8 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	8, b3b, -8*72*128	;; Test loop counter, restore source pointer
	end_timer 6

	start_timer 7
	zloop_init 192			;; 24 iterations of 8
	zr3b_three_complex_djbfft_preload
b4b:	zr3b_three_complex_djbfft rsi, 128, 8*128, rdi, 3*ZMM_SCD1/8, 8
	zloop	8, b4b, -8*128+24*128, rdi, -8*3*ZMM_SCD1/8 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	24, b4b, -24*24*128	;; Test loop counter, restore source pointer
	end_timer 7
ELSE
	start_timer 6
	bump	rdi, -72*ZMM_SCD4/8-24*ZMM_SCD1/8 ;; Load sin/cos pointers
	mov	r10, rdi
	zloop_init 64			;; 8 iterations of 8
	zr33b_nine_complex_djbfft_preload
b3b:	zr33b_nine_complex_djbfft rsi, 128, 8*128, r10, 8*ZMM_SCD1/8, ZMM_SCD1/8, rdi, 0*3*ZMM_SCD1/8, 3*ZMM_SCD1/8, 8
	zloop	8, b3b, -8*128+72*128, r10, -8*ZMM_SCD1/8, rdi, -8*3*ZMM_SCD1/8 ;; Test loop counter, next source pointer, restore sin/cos pointers
	zloop	8, b3b, -8*72*128	;; Test loop counter, restore source pointer
	end_timer 6
ENDIF

;; Do FFT levels 12,13,14
;; On input and output the 128-byte cache line pairs hold these data values:
;;	0	+1152	...	8	+1152	...
;;	...
;;	7
;;	16	...
;;	32	...
;;	...

;; Do 576 eight_complex_fft operations
;;	distance between fft data elements is 1
;;	do 72 macros processing 128 data values

zmid:	start_timer 8
	mov	rcx, 72
	call	zcomplex_onepass_middle
	end_timer 8

;; Do inverse FFT levels 10,11
;; On input the 64-byte cache lines hold these data values:
;;	0	+1152	...	8	+1152	...
;;	...
;;	7
;;	16	...
;;	32	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+1152	...	24	+1152	...
;;	...
;;	23
;;	48	...
;;	...

;; Do 1536 three_complex_unfft operations
;;	distance between fft data elements is 8
;;	do 192 macros each processing 48 data values

;; Do inverse FFT levels 8,9
;; On input the 64-byte cache lines hold these data values:
;;	0	+1152	...	24	+1152	...
;;	...
;;	23
;;	48	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+1152	...	72	+1152	...
;;	...
;;	71
;;	144	...
;;	288	...
;;	...

;; Do 1536 three_complex_unfft operations
;;	distance between fft data elements is 24
;;	do 192 macros each processing 48 data values

IFDEF OLDWAY
	start_timer 9
	mov	rdi, sincos1		;; Load sin/cos pointer
	zloop_init 192			;; 24 iterations of 8
	zr3b_three_complex_djbunfft_preload
c4b:	zr3b_three_complex_djbunfft rsi, 128, 8*128, rdi, 3*ZMM_SCD1/8, 8
	zloop	8, c4b, -8*128+24*128, rdi, -8*3*ZMM_SCD1/8 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	24, c4b, -24*24*128	;; Test loop counter, restore source pointer
	end_timer 9

	start_timer 10
	zloop_init 192			;; 8 iterations of 24
	zr3b_three_complex_djbunfft_preload
c3b:	zr3b_three_complex_djbunfft rsi, 128, 24*128, rdi, ZMM_SCD1/8, 24
	zloop	24, c3b, -24*128+72*128, rdi, -24*ZMM_SCD1/8 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	8, c3b, -8*72*128, rdi, 24*ZMM_SCD1/8 ;; Test loop counter, restore source pointer, next sin/cos ptr
	end_timer 10
ELSE
	start_timer 9
	mov	rdi, sincos1		;; Load sin/cos pointers
	mov	r10, rdi
	zloop_init 64			;; 8 iterations of 8
	zr33b_nine_complex_djbunfft_preload
c4b:	zr33b_nine_complex_djbunfft rsi, 128, 8*128, rdi, 0*3*ZMM_SCD1/8, 3*ZMM_SCD1/8, r10, 8*ZMM_SCD1/8, ZMM_SCD1/8, 8
	zloop	8, c4b, -8*128+72*128, rdi, -8*3*ZMM_SCD1/8, r10, -8*ZMM_SCD1/8 ;; Test loop counter, next source pointer, restore sin/cos pointers
	zloop	8, c4b, -8*72*128, rdi, 24*ZMM_SCD1/8 ;; Test loop counter, restore source pointer, next sin/cos ptr
	end_timer 9
ENDIF

;; Do inverse FFT levels 5,6,7
;; On input the 64-byte cache lines hold these data values:
;;	0	+1152	...	72	+1152	...
;;	...
;;	71
;;	144	...
;;	288	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+1152	...	576	+1152	...
;;	...
;;	575	...

;; Do 576 eight_complex_unfft operations
;;	distance between fft data elements is 72
;;	do 72 macros each processing 128 data values

;; Do inverse FFT levels 1,2,3,4
;; On input the 128-byte cache line pairs hold these data values:
;;	0	+1152	...	576	+1152	...
;;	...
;;	575	...
;; These are swizzled to:
;;	0	+72	...	576	+72	...
;;	...
;;	71	...
;;	1152	...
;;	2304	...
;;	...
;; On output the 128-byte cache line pairs hold these data values:
;;	0	+72	...	4.5K	+72	...
;;	...
;;	71	...
;;	576	...
;;	1152	...
;;	...

;; Do 576 eight_complex_last_unfft operations
;;	distance between fft data elements is 576
;;	do 72 macros processing 128 data values

IFDEF OLDWAY
	start_timer 11
	zloop_init 72			;; 72 iterations
	zr8b_eight_complex_djbunfft_preload
c2b:	zr8b_eight_complex_djbunfft rsi, 128, 72*128, 144*128, 288*128, rdi, ZMM_SCD4/8, 72, L1PREFETCH_ALL, 128
	zloop	72, c2b, -72*128	;; Test loop counter, restore source pointer
	end_timer 11

	start_timer 12
	zloop_init 72			;; 72 iterations
	zr8s_eight_complex_last_unfft_preload
c1b:	zr8s_eight_complex_last_unfft rsi, 128, 72*128, 144*128, 288*128, rdi, ZMM_CD8SCD4, 72, L1PREFETCH_ALL
	zloop	72, c1b, -72*128	;; Test loop counter, restore source pointer
	end_timer 12
ELSE
	start_timer 12
	lea	r10, [rdi+72*ZMM_SCD4/8] ;; Second sin/cos pointer
	zloop_init 72			;; 72 iterations
	zr8s_sixtyfour_complex_last_unfft_preload
c1b:	zr8s_sixtyfour_complex_last_unfft rsi, 128, 72*128, 144*128, 288*128, rdi, ZMM_SCD4/8, r10, ZMM_CD8SCD4, 72, L1PREFETCH_ALL
	zloop	72, c1b, -72*128	;; Test loop counter, restore source pointer
	end_timer 12
ENDIF

	zfft_3_ret
	ENDM


;; Perform a 10K-element FFT

zfft10K MACRO type
	LOCAL	b1b, b2b, b3a, b3b, b4a, b4b, zmid, c4a, c4b, c3a, c3b, c2b, c1b

;; Check for a multiply with pre-FFTed inputs

	cmp	ffttype, 4
	je	zmid

;; Do FFT levels 1,2,3,4
;; On input the 128-byte cache line pairs hold these data values:
;;	0	+80	...	5K	+80	...
;;	...
;;	79	...
;;	640	...
;;	1280	...
;;	...
;; On output the 128-byte cache line pairs hold these data values:
;;	0	+80	...	640	+80	...
;;	...
;;	79	...
;;	1280	...
;;	2560	...
;;	...
;; These are swizzled to:
;;	0	+1280	...	640	+1280	...
;;	...
;;	639	...

;; Do 640 sixteen_reals_fft operations
;;	distance between fft data elements is 640
;;	do 80 macros processing 128 data values

;; Do FFT levels 5,6,7
;; On input the 64-byte cache lines hold these data values:
;;	0	+1280	...	640	+1280	...
;;	...
;;	639	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+1280	...	80	+1280	...
;;	...
;;	79
;;	160	...
;;	320	...
;;	...

;; Do 80 sixteen_reals_fft operation
;; Do 560 eight_complex_fft operations
;;	distance between fft data elements is 80
;;	do 80 macros each processing 128 data values

IFDEF OLDWAY
	start_timer 4
	mov	rdi, sincos1		;; Load sin/cos data ptr
	bump	rdi, 80*ZMM_SCD7+16*ZMM_SCD4+8*ZMM_SCD1
	mov	r9, norm_col_mults	;; Load weights pointer
	zloop_init 80			;; 80 iterations
	zr8fs_sixteen_reals_first_fft_preload
b1b:	zr8fs_sixteen_reals_first_fft rsi, 128, 80*128, 160*128, 320*128, rdi, ZMM_SCD7, r9, 8*128, 80, L1PREFETCH_ALL
	zloop	80, b1b, -80*128	;; Test loop counter, restore source pointer
	end_timer 4

	start_timer 5
	bump	rdi, -80*ZMM_SCD7-80*ZMM_SCD7 ;; Load sin/cos pointer
	zloop_init 80			;; 80 iterations
	zr8_sixteen_reals_eight_complex_djbfft_preload
b2b:	zr8_sixteen_reals_eight_complex_djbfft rsi, 128, 80*128, 160*128, 320*128, rdi, ZMM_SCD7, 80, L1PREFETCH_ALL
	zloop	80, b2b, -80*128	;; Test loop counter, restore source pointer
	end_timer 5
ELSE
	start_timer 4
	mov	rdi, sincos1		;; Load sin/cos data ptr
	bump	rdi, 16*ZMM_SCD4+8*ZMM_SCD1
	lea	r10, [rdi+80*ZMM_SCD7]	;; Second sin/cos pointer
	mov	r9, norm_col_mults	;; Load weights pointer
	zloop_init 80			;; 80 iterations
	zr8fs_hundredtwentyeight_reals_first_fft_preload
b1b:	zr8fs_hundredtwentyeight_reals_first_fft rsi, 128, 80*128, 160*128, 320*128, r10, ZMM_SCD7, rdi, ZMM_SCD7, r9, 8*128, 80, L1PREFETCH_ALL
	zloop	80, b1b, -80*128	;; Test loop counter, restore source pointer
	end_timer 4
ENDIF

;; Do FFT levels 8,9,10
;; On input the 64-byte cache lines hold these data values:
;;	0	+1280	...	80	+1280	...
;;	...
;;	79
;;	160	...
;;	320	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+1280	...	16	+1280	...
;;	...
;;	15
;;	32	...
;;	...

;; Do 16 ten_reals_fft operation
;; Do 1008 five_complex_fft operations
;;	distance between fft data elements is 16
;;	do 128 macros each processing 80 data values

;; Do FFT levels 11
;; On input the 64-byte cache lines hold these data values:
;;	0	+1280	...	16	+1280	...
;;	...
;;	15
;;	32	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+1280	...	8	+1280	...
;;	...
;;	7
;;	16	...
;;	32	...
;;	...

;; Do 8 four_reals_fft operation
;; Do 2552 two_complex_fft operations
;;	distance between fft data elements is 8
;;	do 320 macros each processing 32 data values

IFDEF OLDWAY
	start_timer 6
	bump	rdi, -80*ZMM_SCD7-16*ZMM_SCD4
	zloop_init 128			;; 8 iterations of 16
	zr5_ten_reals_five_complex_djbfft_preload
b3a:	zr5_ten_reals_five_complex_djbfft rsi, 128, 16*128, rdi, ZMM_SCD4, 16
	zloop	16, b3a, -16*128+80*128, rdi, -16*ZMM_SCD4 ;; Test loop counter, next source pointer, restore sin/cos pointer

	loops_reset			;; 7 more iters of 16
	zr5rb_five_complex_djbfft_preload
b3b:	zr5rb_five_complex_djbfft rsi, 128, 16*128, rdi, ZMM_SCD4, 16
	zloop	16, b3b, -16*128+80*128, rdi, -16*ZMM_SCD4 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	8, b3b, -8*80*128	;; Test loop counter, restore source pointer
	end_timer 6

	start_timer 7
	bump	rdi, -8*ZMM_SCD1
	zloop_init 320			;; 40 iterations of 8
	zr2_four_reals_two_complex_djbfft_preload
b4a:	zr2_four_reals_two_complex_djbfft rsi, 128, 8*128, rdi, ZMM_SCD1, 8
	zloop	8, b4a, -8*128+16*128, rdi, -8*ZMM_SCD1 ;; Test loop counter, next source pointer, restore sin/cos pointer

	loops_reset			;; 39 more iters of 8
	zr2rb_two_complex_djbfft_preload
b4b:	zr2rb_two_complex_djbfft rsi, 128, 8*128, rdi, ZMM_SCD1, 8
	zloop	8, b4b, -8*128+16*128, rdi, -8*ZMM_SCD1 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	40, b4b, -40*16*128	;; Test loop counter, restore source pointer
	end_timer 7
ELSE
	start_timer 6
	lea	r10, [rdi-80*ZMM_SCD7-16*ZMM_SCD4]
	lea	rdi, [r10-8*ZMM_SCD1]
	zloop_init 64			;; 8 iterations of 8
	zr25_twenty_reals_ten_complex_djbfft_preload
b3a:	zr25_twenty_reals_ten_complex_djbfft rsi, 128, 8*128, r10, 8*ZMM_SCD4, ZMM_SCD4, rdi, 0*ZMM_SCD1, ZMM_SCD1, 8
	zloop	8, b3a, -8*128+80*128, r10, -8*ZMM_SCD4, rdi, -8*ZMM_SCD1 ;; Test loop counter, next source pointer, restore sin/cos pointers
	loops_reset			;; 7 more iters of 8
	zr25rb_ten_complex_djbfft_preload
b3b:	zr25rb_ten_complex_djbfft rsi, 128, 8*128, r10, 8*ZMM_SCD4, ZMM_SCD4, rdi, 0*ZMM_SCD1, ZMM_SCD1, 8
	zloop	8, b3b, -8*128+80*128, r10, -8*ZMM_SCD4, rdi, -8*ZMM_SCD1 ;; Test loop counter, next source pointer, restore sin/cos pointers
	zloop	8, b3b, -8*80*128	;; Test loop counter, restore source pointer
	end_timer 6
ENDIF

;; Do FFT levels 12,13,14
;; On input and output the 128-byte cache line pairs hold these data values:
;;	0	+1280	...	8	+1280	...
;;	...
;;	7
;;	16	...
;;	32	...
;;	...

;; Do 1 sixteen_reals_fft operation
;; Do 639 eight_complex_fft operations
;;	distance between fft data elements is 1
;;	do 80 macros processing 128 data values

zmid:	start_timer 8
	mov	rcx, 80
	call	zreal_onepass_middle
	end_timer 8

;; Do inverse FFT levels 11
;; On input the 64-byte cache lines hold these data values:
;;	0	+1280	...	8	+1280	...
;;	...
;;	7
;;	16	...
;;	32	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+1280	...	16	+1280	...
;;	...
;;	15
;;	32	...
;;	...

;; Do 8 four_reals_unfft operations
;; Do 2552 two_complex_unfft operations
;;	distance between fft data elements is 8
;;	do 320 macros each processing 32 data values

;; Do inverse FFT levels 8,9,10
;; On input the 64-byte cache lines hold these data values:
;;	0	+1280	...	16	+1280	...
;;	...
;;	15
;;	32	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+1280	...	80	+1280	...
;;	...
;;	79
;;	160	...
;;	320	...
;;	...

;; Do 16 ten_reals_unfft operations
;; Do 1008 five_complex_unfft operations
;;	distance between fft data elements is 16
;;	do 128 macros each processing 80 data values

IFDEF OLDWAY
	start_timer 9
	mov	rdi, sincos1		;; Load sin/cos pointer
	zloop_init 320			;; 40 iterations of 8
	zr2_four_reals_two_complex_djbunfft_preload
c4a:	zr2_four_reals_two_complex_djbunfft rsi, 128, 8*128, rdi, ZMM_SCD1, 8
	zloop	8, c4a, -8*128+16*128, rdi, -8*ZMM_SCD1 ;; Test loop counter, restore source pointer, restore sin/cos pointer

	loops_reset			;; 39 more iters of 8
	zr2rb_two_complex_djbunfft_preload
c4b:	zr2rb_two_complex_djbunfft rsi, 128, 8*128, rdi, ZMM_SCD1, 8
	zloop	8, c4b, -8*128+16*128, rdi, -8*ZMM_SCD1 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	40, c4b, -40*16*128, rdi, 8*ZMM_SCD1 ;; Test loop counter, restore source pointer, next sin/cos ptr
	end_timer 9

	start_timer 10
	zloop_init 128			;; 8 iterations of 16
	zr5_ten_reals_five_complex_djbunfft_preload
c3a:	zr5_ten_reals_five_complex_djbunfft rsi, 128, 16*128, rdi, ZMM_SCD4, 16
	zloop	16, c3a, -16*128+80*128, rdi, -16*ZMM_SCD4 ;; Test loop counter, restore source pointer, restore sin/cos pointer

	loops_reset			;; 7 more iters of 16
	zr5rb_five_complex_djbunfft_preload
c3b:	zr5rb_five_complex_djbunfft rsi, 128, 16*128, rdi, ZMM_SCD4, 16
	zloop	16, c3b, -16*128+80*128, rdi, -16*ZMM_SCD4 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	8, c3b, -8*80*128, rdi, 16*ZMM_SCD4 ;; Test loop counter, restore source pointer, next sin/cos ptr
	end_timer 10
ELSE
	start_timer 9
	mov	rdi, sincos1		;; Load sin/cos pointers
	lea	r10, [rdi+8*ZMM_SCD1]
	zloop_init 64			;; 8 iterations of 8
	zr25_twenty_reals_ten_complex_djbunfft_preload
c3a:	zr25_twenty_reals_ten_complex_djbunfft rsi, 128, 8*128, rdi, 0*ZMM_SCD1, ZMM_SCD1, r10, 8*ZMM_SCD4, ZMM_SCD4, 8
	zloop	8, c3a, -8*128+80*128, rdi, -8*ZMM_SCD1, r10, -8*ZMM_SCD4 ;; Test loop counter, next source pointer, restore sin/cos pointers
	loops_reset			;; 7 more iters of 8
	zr25rb_ten_complex_djbunfft_preload
c3b:	zr25rb_ten_complex_djbunfft rsi, 128, 8*128, rdi, 0*ZMM_SCD1, ZMM_SCD1, r10, 8*ZMM_SCD4, ZMM_SCD4, 8
	zloop	8, c3b, -8*128+80*128, rdi, -8*ZMM_SCD1, r10, -8*ZMM_SCD4 ;; Test loop counter, next source pointer, restore sin/cos pointers
	zloop	8, c3b, -8*80*128, rdi, 8*ZMM_SCD1+16*ZMM_SCD4 ;; Test loop counter, restore source pointer, next sin/cos ptr
	end_timer 9
ENDIF

;; Do inverse FFT levels 5,6,7
;; On input the 64-byte cache lines hold these data values:
;;	0	+1280	...	80	+1280	...
;;	...
;;	79
;;	160	...
;;	320	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+1280	...	640	+1280	...
;;	...
;;	639	...

;; Do 80 sixteen_reals_unfft operations
;; Do 560 eight_complex_unfft operations
;;	distance between fft data elements is 80
;;	do 80 macros each processing 128 data values

;; Do inverse FFT levels 1,2,3,4
;; On input the 128-byte cache line pairs hold these data values:
;;	0	+1280	...	640	+1280	...
;;	...
;;	639	...
;; These are swizzled to:
;;	0	+80	...	640	+80	...
;;	...
;;	79	...
;;	1280	...
;;	2560	...
;;	...
;; On output the 128-byte cache line pairs hold these data values:
;;	0	+80	...	5K	+80	...
;;	...
;;	79	...
;;	640	...
;;	1280	...
;;	...

;; Do 640 sixteen_reals_unfft operations
;;	distance between fft data elements is 640
;;	do 80 macros processing 128 data values

IFDEF OLDWAY
	start_timer 11
	zloop_init 80			;; 80 iterations
	zr8_sixteen_reals_eight_complex_djbunfft_preload
c2b:	zr8_sixteen_reals_eight_complex_djbunfft rsi, 128, 80*128, 160*128, 320*128, rdi, ZMM_SCD7, 80
	zloop	80, c2b, -80*128	;; Test loop counter, restore source pointer, restore sin/cos pointer
	end_timer 11

	start_timer 12
	zloop_init 80			;; 80 iterations
	zr8s_sixteen_reals_last_unfft_preload
c1b:	zr8s_sixteen_reals_last_unfft rsi, 128, 80*128, 160*128, 320*128, rdi, ZMM_SCD7, 80, L1PREFETCH_ALL
	zloop	80, c1b, -80*128	;; Test loop counter, restore source pointer
	end_timer 12
ELSE
	start_timer 12
	lea	r10, [rdi+80*ZMM_SCD7]	;; Second sin/cos pointer
	zloop_init 80			;; 80 iterations
	zr8s_hundredtwentyeight_reals_last_unfft_preload
c1b:	zr8s_hundredtwentyeight_reals_last_unfft rsi, 128, 80*128, 160*128, 320*128, rdi, ZMM_SCD7, r10, ZMM_SCD7, 80, L1PREFETCH_ALL
	zloop	80, c1b, -80*128	;; Test loop counter, restore source pointer
	end_timer 12
ENDIF

	zfft_3_ret
	ENDM

;; Perform a 10K-element negacyclic FFT

zfft10Kp MACRO type
	LOCAL	b1b, b2b, b3b, b4b, zmid, c4b, c3b, c2b, c1b

;; Do a multiply with pre-FFTed inputs

	cmp	ffttype, 4
	je	zmid

;; Do FFT levels 1,2,3,4
;; On input the 128-byte cache line pairs hold these data values:
;;	0	+80	...	5K	+80	...
;;	...
;;	79	...
;;	640	...
;;	1280	...
;;	...
;; On output the 128-byte cache line pairs hold these data values:
;;	0	+80	...	640	+80	...
;;	...
;;	79	...
;;	1280	...
;;	2560	...
;;	...
;; These are swizzled to:
;;	0	+1280	...	640	+1280	...
;;	...
;;	639	...

;; Do 640 eight_complex_first_fft operations
;;	distance between fft data elements is 640
;;	do 80 macros processing 128 data values

;; Do FFT levels 5,6,7
;; On input the 64-byte cache lines hold these data values:
;;	0	+1280	...	640	+1280	...
;;	...
;;	639	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+1280	...	80	+1280	...
;;	...
;;	79
;;	160	...
;;	320	...
;;	...

;; Do 640 eight_complex_fft operations
;;	distance between fft data elements is 80
;;	do 80 macros each processing 128 data values

IFDEF OLDWAY
	start_timer 4
	mov	rdi, sincos1		;; Load sin/cos pointer
	bump	rdi, 80*ZMM_SCD4/8+16*ZMM_SCD2/8+8*ZMM_SCD1/8
	mov	r9, norm_col_mults	;; Load weights pointer
	zloop_init 80			;; 80 iterations
	zr8fs_eight_complex_first_fft_preload
b1b:	zr8fs_eight_complex_first_fft rsi, 128, 80*128, 160*128, 320*128, rdi, ZMM_CD8SCD4, r9, 8*128, 80, L1PREFETCH_ALL
	zloop	80, b1b, -80*128	;; Test loop counter, restore source pointer
	end_timer 4

	start_timer 5
	bump	rdi, -80*ZMM_CD8SCD4-80*ZMM_SCD4/8 ;; Load sin/cos pointer
	zloop_init 80			;; 80 iterations
	zr8b_eight_complex_djbfft_preload
b2b:	zr8b_eight_complex_djbfft rsi, 128, 80*128, 160*128, 320*128, rdi, ZMM_SCD4/8, 80, L1PREFETCH_ALL, 128
	zloop	80, b2b, -80*128	;; Test loop counter, restore source pointer
	end_timer 5
ELSE
	start_timer 4
	mov	rdi, sincos1		;; Load sin/cos pointer
	bump	rdi, 16*ZMM_SCD2/8+8*ZMM_SCD1/8
	lea	r10, [rdi+80*ZMM_SCD4/8] ;; Second sin/cos pointer
	mov	r9, norm_col_mults	;; Load weights pointer
	zloop_init 80			;; 80 iterations
	zr8fs_sixtyfour_complex_first_fft_preload
b1b:	zr8fs_sixtyfour_complex_first_fft rsi, 128, 80*128, 160*128, 320*128, r10, ZMM_CD8SCD4, rdi, ZMM_SCD4/8, r9, 8*128, 80, L1PREFETCH_ALL
	zloop	80, b1b, -80*128	;; Test loop counter, restore source pointer
	end_timer 4
ENDIF

;; Do FFT levels 8,9,10
;; On input the 64-byte cache lines hold these data values:
;;	0	+1280	...	80	+1280	...
;;	...
;;	79
;;	160	...
;;	320	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+1280	...	16	+1280	...
;;	...
;;	15
;;	32	...
;;	...

;; Do 1024 five_complex_fft operations
;;	distance between fft data elements is 16
;;	do 128 macros each processing 80 data values

;; Do FFT levels 11
;; On input the 64-byte cache lines hold these data values:
;;	0	+1280	...	16	+1280	...
;;	...
;;	15
;;	32	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+1280	...	8	+1280	...
;;	...
;;	7
;;	16	...
;;	32	...
;;	...

;; Do 2560 two_complex_fft operations
;;	distance between fft data elements is 8
;;	do 320 macros each processing 32 data values

IFDEF OLDWAY
	start_timer 6
	bump	rdi, -80*ZMM_SCD4/8-16*ZMM_SCD2/8 ;; Load sin/cos pointer
	zloop_init 128			;; 8 iterations of 16
	zr5b_five_complex_djbfft_preload
b3b:	zr5b_five_complex_djbfft rsi, 128, 16*128, rdi, ZMM_SCD2/8, 16
	zloop	16, b3b, -16*128+80*128, rdi, -16*ZMM_SCD2/8 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	8, b3b, -8*80*128	;; Test loop counter, restore source pointer
	end_timer 6

	start_timer 7
	bump	rdi, -8*ZMM_SCD1/8	;; Load sin/cos pointer
	zloop_init 320			;; 40 iterations of 8
	zr2b_two_complex_djbfft_preload
b4b:	zr2b_two_complex_djbfft rsi, 128, 8*128, rdi, ZMM_SCD1/8, 8
	zloop	8, b4b, -8*128+16*128, rdi, -8*ZMM_SCD1/8 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	40, b4b, -40*16*128	;; Test loop counter, restore source pointer
	end_timer 7
ELSE
	start_timer 6
	lea	r10, [rdi-80*ZMM_SCD4/8-16*ZMM_SCD2/8] ;; Load sin/cos pointers
	lea	rdi, [r10-8*ZMM_SCD1/8]
	zloop_init 64			;; 8 iterations of 8
	zr25b_ten_complex_djbfft_preload
b3b:	zr25b_ten_complex_djbfft rsi, 128, 8*128, r10, 8*ZMM_SCD2/8, ZMM_SCD2/8, rdi, 0*ZMM_SCD1/8, ZMM_SCD1/8, 8
	zloop	8, b3b, -8*128+80*128, r10, -8*ZMM_SCD2/8, rdi, -8*ZMM_SCD1/8 ;; Test loop counter, next source pointer, restore sin/cos pointers
	zloop	8, b3b, -8*80*128	;; Test loop counter, restore source pointer
	end_timer 6
ENDIF

;; Do FFT levels 12,13,14
;; On input and output the 128-byte cache line pairs hold these data values:
;;	0	+1280	...	8	+1280	...
;;	...
;;	7
;;	16	...
;;	32	...
;;	...

;; Do 640 eight_complex_fft operations
;;	distance between fft data elements is 1
;;	do 80 macros processing 128 data values

zmid:	start_timer 8
	mov	rcx, 80
	call	zcomplex_onepass_middle
	end_timer 8

;; Do inverse FFT levels 11
;; On input the 64-byte cache lines hold these data values:
;;	0	+1280	...	8	+1280	...
;;	...
;;	7
;;	16	...
;;	32	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+1280	...	16	+1280	...
;;	...
;;	15
;;	32	...
;;	...

;; Do 2560 two_complex_unfft operations
;;	distance between fft data elements is 8
;;	do 320 macros each processing 32 data values

;; Do inverse FFT levels 8,9,10
;; On input the 64-byte cache lines hold these data values:
;;	0	+1280	...	16	+1280	...
;;	...
;;	15
;;	32	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+1280	...	80	+1280	...
;;	...
;;	79
;;	160	...
;;	320	...
;;	...

;; Do 1024 five_complex_unfft operations
;;	distance between fft data elements is 32
;;	do 128 macros each processing 80 data values

IFDEF OLDWAY
	start_timer 9
	mov	rdi, sincos1		;; Load sin/cos pointer
	zloop_init 320			;; 40 iterations of 8
	zr2b_two_complex_djbunfft_preload
c4b:	zr2b_two_complex_djbunfft rsi, 128, 8*128, rdi, ZMM_SCD1/8, 8
	zloop	8, c4b, -8*128+16*128, rdi, -8*ZMM_SCD1/8 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	40, c4b, -40*16*128, rdi, 8*ZMM_SCD1/8	;; Test loop counter, restore source pointer, next sin/cos ptr
	end_timer 9

	start_timer 10
	zloop_init 128			;; 8 iterations of 16
	zr5b_five_complex_djbunfft_preload
c3b:	zr5b_five_complex_djbunfft rsi, 128, 16*128, rdi, ZMM_SCD2/8, 8
	zloop	16, c3b, -16*128+80*128, rdi, -16*ZMM_SCD2/8 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	8, c3b, -8*80*128, rdi, 16*ZMM_SCD2/8 ;; Test loop counter, restore source pointer, next sin/cos ptr
	end_timer 10
ELSE
	start_timer 9
	mov	rdi, sincos1		;; Load sin/cos pointers
	lea	r10, [rdi+8*ZMM_SCD1/8]
	zloop_init 64			;; 8 iterations of 8
	zr25b_ten_complex_djbunfft_preload
c4b:	zr25b_ten_complex_djbunfft rsi, 128, 8*128, rdi, 0*ZMM_SCD1/8, ZMM_SCD1/8, r10, 8*ZMM_SCD2/8, ZMM_SCD2/8, 8
	zloop	8, c4b, -8*128+80*128, rdi, -8*ZMM_SCD1/8, r10, -8*ZMM_SCD2/8 ;; Test loop counter, next source pointer, restore sin/cos pointers
	zloop	8, c4b, -8*80*128, rdi, 8*ZMM_SCD1/8+16*ZMM_SCD2/8 ;; Test loop counter, restore source pointer, next sin/cos ptr
	end_timer 9
ENDIF

;; Do inverse FFT levels 5,6,7
;; On input the 64-byte cache lines hold these data values:
;;	0	+1280	...	80	+1280	...
;;	...
;;	79
;;	160	...
;;	320	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+1280	...	640	+1280	...
;;	...
;;	639	...

;; Do 640 eight_complex_unfft operations
;;	distance between fft data elements is 80
;;	do 80 macros each processing 128 data values

;; Do inverse FFT levels 1,2,3,4
;; On input the 128-byte cache line pairs hold these data values:
;;	0	+1280	...	640	+1280	...
;;	...
;;	639	...
;; These are swizzled to:
;;	0	+80	...	640	+80	...
;;	...
;;	79	...
;;	1280	...
;;	2560	...
;;	...
;; On output the 128-byte cache line pairs hold these data values:
;;	0	+80	...	5K	+80	...
;;	...
;;	79	...
;;	640	...
;;	1280	...
;;	...

;; Do 640 eight_complex_last_unfft operations
;;	distance between fft data elements is 640
;;	do 80 macros processing 128 data values

IFDEF OLDWAY
	start_timer 11
	zloop_init 80			;; 80 iterations
	zr8b_eight_complex_djbunfft_preload
c2b:	zr8b_eight_complex_djbunfft rsi, 128, 80*128, 160*128, 320*128, rdi, ZMM_SCD4/8, 80, L1PREFETCH_ALL, 128
	zloop	80, c2b, -80*128	;; Test loop counter, restore source pointer
	end_timer 11

	start_timer 12
	zloop_init 80			;; 80 iterations
	zr8s_eight_complex_last_unfft_preload
c1b:	zr8s_eight_complex_last_unfft rsi, 128, 80*128, 160*128, 320*128, rdi, ZMM_CD8SCD4, 80, L1PREFETCH_ALL
	zloop	80, c1b, -80*128	;; Test loop counter, restore source pointer
	end_timer 12
ELSE
	start_timer 12
	lea	r10, [rdi+80*ZMM_SCD4/8] ;; Second sin/cos pointer
	zloop_init 80			;; 80 iterations
	zr8s_sixtyfour_complex_last_unfft_preload
c1b:	zr8s_sixtyfour_complex_last_unfft rsi, 128, 80*128, 160*128, 320*128, rdi, ZMM_SCD4/8, r10, ZMM_CD8SCD4, 80, L1PREFETCH_ALL
	zloop	80, c1b, -80*128	;; Test loop counter, restore source pointer
	end_timer 12
ENDIF

	zfft_3_ret
	ENDM


;; Perform a 12K-element FFT

zfft12K MACRO type
	LOCAL	b1b, b2b, b3a, b3b, b4a, b4b, zmid, c4a, c4b, c3a, c3b, c2b, c1b

;; Check for a multiply with pre-FFTed inputs

	cmp	ffttype, 4
	je	zmid

;; Do FFT levels 1,2,3,4
;; On input the 128-byte cache line pairs hold these data values:
;;	0	+96	...	6K	+96	...
;;	...
;;	95	...
;;	768	...
;;	1.5K	...
;;	...
;; On output the 128-byte cache line pairs hold these data values:
;;	0	+96	...	768	+96	...
;;	...
;;	95	...
;;	1.5K	...
;;	3K	...
;;	...
;; These are swizzled to:
;;	0	+1.5K	...	768	+1.5K	...
;;	...
;;	767	...

;; Do 768 sixteen_reals_fft operations
;;	distance between fft data elements is 768
;;	do 96 macros processing 128 data values

;; Do FFT levels 5,6,7
;; On input the 64-byte cache lines hold these data values:
;;	0	+1.5K	...	768	+1.5K	...
;;	...
;;	767	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+1.5K	...	96	+1.5K	...
;;	...
;;	95
;;	192	...
;;	384	...
;;	...

;; Do 96 sixteen_reals_fft operation
;; Do 672 eight_complex_fft operations
;;	distance between fft data elements is 96
;;	do 96 macros each processing 128 data values

IFDEF OLDWAY
	start_timer 4
	mov	rdi, sincos1		;; Load sin/cos data ptr
	bump	rdi, 96*ZMM_SCD7+32*ZMM_SCD2+8*ZMM_SCD3
	mov	r9, norm_col_mults	;; Load weights pointer
	zloop_init 96			;; 96 iterations
	zr8fs_sixteen_reals_first_fft_preload
b1b:	zr8fs_sixteen_reals_first_fft rsi, 128, 96*128, 192*128, 384*128, rdi, ZMM_SCD7, r9, 8*128, 96, L1PREFETCH_ALL
	zloop	96, b1b, -96*128	;; Test loop counter, restore source pointer
	end_timer 4

	start_timer 5
	bump	rdi, -96*ZMM_SCD7-96*ZMM_SCD7 ;; Load sin/cos pointer
	zloop_init 96			;; 96 iterations
	zr8_sixteen_reals_eight_complex_djbfft_preload
b2b:	zr8_sixteen_reals_eight_complex_djbfft rsi, 128, 96*128, 192*128, 384*128, rdi, ZMM_SCD7, 96, L1PREFETCH_ALL
	zloop	96, b2b, -96*128	;; Test loop counter, restore source pointer
	end_timer 5
ELSE
	start_timer 4
	mov	rdi, sincos1		;; Load sin/cos data ptr
	bump	rdi, 32*ZMM_SCD2+8*ZMM_SCD3
	lea	r10, [rdi+96*ZMM_SCD7]	;; Second sin/cos pointer
	mov	r9, norm_col_mults	;; Load weights pointer
	zloop_init 96			;; 96 iterations
	zr8fs_hundredtwentyeight_reals_first_fft_preload
b1b:	zr8fs_hundredtwentyeight_reals_first_fft rsi, 128, 96*128, 192*128, 384*128, r10, ZMM_SCD7, rdi, ZMM_SCD7, r9, 8*128, 96, L1PREFETCH_ALL
	zloop	96, b1b, -96*128	;; Test loop counter, restore source pointer
	end_timer 4
ENDIF

;; Do FFT levels 8,9
;; On input the 64-byte cache lines hold these data values:
;;	0	+1.5K	...	96	+1.5K	...
;;	...
;;	95
;;	192	...
;;	384	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+1.5K	...	32	+1.5K	...
;;	...
;;	31
;;	64	...
;;	...

;; Do 32 six_reals_fft operation
;; Do 2016 three_complex_fft operations
;;	distance between fft data elements is 32
;;	do 256 macros each processing 48 data values

;; Do FFT levels 10,11
;; On input the 64-byte cache lines hold these data values:
;;	0	+1.5K	...	32	+1.5K	...
;;	...
;;	31
;;	64	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+1.5K	...	8	+1.5K	...
;;	...
;;	7
;;	16	...
;;	32	...
;;	...

;; Do 8 eight_reals_fft operation
;; Do 1528 four_complex_fft operations
;;	distance between fft data elements is 8
;;	do 192 macros each processing 64 data values

IFDEF OLDWAY
	start_timer 6
	bump	rdi, -96*ZMM_SCD7-32*ZMM_SCD2
	zloop_init 256			;; 8 iterations of 32
	zr3_six_reals_three_complex_djbfft_preload
b3a:	zr3_six_reals_three_complex_djbfft rsi, 128, 32*128, rdi, ZMM_SCD2, 32
	zloop	32, b3a, -32*128+96*128, rdi, -32*ZMM_SCD2 ;; Test loop counter, next source pointer, restore sin/cos pointer

	loops_reset			;; 7 more iters of 32
	zr3rb_three_complex_djbfft_preload
b3b:	zr3rb_three_complex_djbfft rsi, 128, 32*128, rdi, ZMM_SCD2, 32
	zloop	32, b3b, -32*128+96*128, rdi, -32*ZMM_SCD2 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	8, b3b, -8*96*128	;; Test loop counter, restore source pointer
	end_timer 6

	start_timer 7
	bump	rdi, -8*ZMM_SCD3
	zloop_init 192			;; 24 iterations of 8
	zr4_eight_reals_four_complex_djbfft_preload
b4a:	zr4_eight_reals_four_complex_djbfft rsi, 128, 8*128, 16*128, rdi, ZMM_SCD3, 8
	zloop	8, b4a, -8*128+32*128, rdi, -8*ZMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer

	loops_reset			;; 23 more iters of 8
	zr4rb_four_complex_djbfft_preload
b4b:	zr4rb_four_complex_djbfft rsi, 128, 8*128, 16*128, rdi, ZMM_SCD3, 8
	zloop	8, b4b, -8*128+32*128, rdi, -8*ZMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	24, b4b, -24*32*128	;; Test loop counter, restore source pointer
	end_timer 7
ELSE
	start_timer 6
	lea	r10, [rdi-96*ZMM_SCD7-32*ZMM_SCD2]
	lea	rdi, [r10-8*ZMM_SCD3]
	zloop_init 64			;; 8 iterations of 8
	zr34_twentyfour_reals_twelve_complex_djbfft_preload
b3a:	zr34_twentyfour_reals_twelve_complex_djbfft rsi, 128, 8*128, r10, 8*ZMM_SCD2, ZMM_SCD2, rdi, 0*ZMM_SCD3, ZMM_SCD3, 8
	zloop	8, b3a, -8*128+96*128, r10, -8*ZMM_SCD2, rdi, -8*ZMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointers
	loops_reset			;; 7 more iters of 8
	zr34rb_twelve_complex_djbfft_preload
b3b:	zr34rb_twelve_complex_djbfft rsi, 128, 8*128, r10, 8*ZMM_SCD2, ZMM_SCD2, rdi, 0*ZMM_SCD3, ZMM_SCD3, 8
	zloop	8, b3b, -8*128+96*128, r10, -8*ZMM_SCD2, rdi, -8*ZMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointers
	zloop	8, b3b, -8*96*128	;; Test loop counter, restore source pointer
	end_timer 6
ENDIF

;; Do FFT levels 12,13,14
;; On input and output the 128-byte cache line pairs hold these data values:
;;	0	+1.5K	...	8	+1.5K	...
;;	...
;;	7
;;	16	...
;;	32	...
;;	...

;; Do 1 sixteen_reals_fft operation
;; Do 767 eight_complex_fft operations
;;	distance between fft data elements is 1
;;	do 96 macros processing 128 data values

zmid:	start_timer 8
	mov	rcx, 96
	call	zreal_onepass_middle
	end_timer 8

;; Do inverse FFT levels 10,11
;; On input the 64-byte cache lines hold these data values:
;;	0	+1.5K	...	8	+1.5K	...
;;	...
;;	7
;;	16	...
;;	32	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+1.5K	...	32	+1.5K	...
;;	...
;;	31
;;	64	...
;;	...

;; Do 8 eight_reals_unfft operations
;; Do 1528 four_complex_unfft operations
;;	distance between fft data elements is 8
;;	do 192 macros each processing 64 data values

;; Do inverse FFT levels 8,9
;; On input the 64-byte cache lines hold these data values:
;;	0	+1.5K	...	32	+1.5K	...
;;	...
;;	31
;;	64	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+1.5K	...	96	+1.5K	...
;;	...
;;	95
;;	192	...
;;	384	...
;;	...

;; Do 32 six_reals_unfft operations
;; Do 2016 three_complex_unfft operations
;;	distance between fft data elements is 32
;;	do 256 macros each processing 48 data values

IFDEF OLDWAY
	start_timer 9
	mov	rdi, sincos1		;; Load sin/cos pointer
	zloop_init 192			;; 24 iterations of 8
	zr4_eight_reals_four_complex_djbunfft_preload
c4a:	zr4_eight_reals_four_complex_djbunfft rsi, 128, 8*128, 16*128, rdi, ZMM_SCD3, 8
	zloop	8, c4a, -8*128+32*128, rdi, -8*ZMM_SCD3 ;; Test loop counter, restore source pointer, restore sin/cos pointer

	loops_reset			;; 23 more iters of 8
	zr4rb_four_complex_djbunfft_preload
c4b:	zr4rb_four_complex_djbunfft rsi, 128, 8*128, 16*128, rdi, ZMM_SCD3, 8
	zloop	8, c4b, -8*128+32*128, rdi, -8*ZMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	24, c4b, -24*32*128, rdi, 8*ZMM_SCD3 ;; Test loop counter, restore source pointer, next sin/cos ptr
	end_timer 9

	start_timer 10
	zloop_init 256			;; 8 iterations of 32
	zr3_six_reals_three_complex_djbunfft_preload
c3a:	zr3_six_reals_three_complex_djbunfft rsi, 128, 32*128, rdi, ZMM_SCD2, 32
	zloop	32, c3a, -32*128+96*128, rdi, -32*ZMM_SCD2 ;; Test loop counter, next source pointer, restore sin/cos pointer

	loops_reset			;; 7 more iters of 32
	zr3rb_three_complex_djbunfft_preload
c3b:	zr3rb_three_complex_djbunfft rsi, 128, 32*128, rdi, ZMM_SCD2, 32
	zloop	32, c3b, -32*128+96*128, rdi, -32*ZMM_SCD2 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	8, c3b, -8*96*128, rdi, 32*ZMM_SCD2 ;; Test loop counter, restore source pointer, next sin/cos ptr
	end_timer 10
ELSE
	start_timer 9
	mov	rdi, sincos1		;; Load sin/cos pointers
	lea	r10, [rdi+8*ZMM_SCD3]
	zloop_init 64			;; 8 iterations of 8
	zr34_twentyfour_reals_twelve_complex_djbunfft_preload
c3a:	zr34_twentyfour_reals_twelve_complex_djbunfft rsi, 128, 8*128, rdi, 0*ZMM_SCD3, ZMM_SCD3, r10, 8*ZMM_SCD2, ZMM_SCD2, 8
	zloop	8, c3a, -8*128+96*128, rdi, -8*ZMM_SCD3, r10, -8*ZMM_SCD2 ;; Test loop counter, next source pointer, restore sin/cos pointers
	loops_reset			;; 7 more iters of 8
	zr34rb_twelve_complex_djbunfft_preload
c3b:	zr34rb_twelve_complex_djbunfft rsi, 128, 8*128, rdi, 0*ZMM_SCD3, ZMM_SCD3, r10, 8*ZMM_SCD2, ZMM_SCD2, 8
	zloop	8, c3b, -8*128+96*128, rdi, -8*ZMM_SCD3, r10, -8*ZMM_SCD2 ;; Test loop counter, next source pointer, restore sin/cos pointers
	zloop	8, c3b, -8*96*128, rdi, 8*ZMM_SCD3+32*ZMM_SCD2 ;; Test loop counter, restore source pointer, next sin/cos ptr
	end_timer 9
ENDIF

;; Do inverse FFT levels 5,6,7
;; On input the 64-byte cache lines hold these data values:
;;	0	+1.5K	...	96	+1.5K	...
;;	...
;;	95
;;	192	...
;;	384	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+1.5K	...	768	+1.5K	...
;;	...
;;	767	...

;; Do 96 sixteen_reals_unfft operations
;; Do 672 eight_complex_unfft operations
;;	distance between fft data elements is 96
;;	do 96 macros each processing 128 data values

;; Do inverse FFT levels 1,2,3,4
;; On input the 128-byte cache line pairs hold these data values:
;;	0	+1.5K	...	768	+1.5K	...
;;	...
;;	767	...
;; These are swizzled to:
;;	0	+96	...	768	+96	...
;;	...
;;	95	...
;;	1.5K	...
;;	3K	...
;;	...
;; On output the 128-byte cache line pairs hold these data values:
;;	0	+96	...	6K	+96	...
;;	...
;;	95	...
;;	768	...
;;	1.5K	...
;;	...

;; Do 768 sixteen_reals_unfft operations
;;	distance between fft data elements is 768
;;	do 96 macros processing 128 data values

IFDEF OLDWAY
	start_timer 11
	zloop_init 96			;; 96 iterations
	zr8_sixteen_reals_eight_complex_djbunfft_preload
c2b:	zr8_sixteen_reals_eight_complex_djbunfft rsi, 128, 96*128, 192*128, 384*128, rdi, ZMM_SCD7, 96
	zloop	96, c2b, -96*128	;; Test loop counter, restore source pointer, restore sin/cos pointer
	end_timer 11

	start_timer 12
	zloop_init 96			;; 96 iterations
	zr8s_sixteen_reals_last_unfft_preload
c1b:	zr8s_sixteen_reals_last_unfft rsi, 128, 96*128, 192*128, 384*128, rdi, ZMM_SCD7, 96, L1PREFETCH_ALL
	zloop	96, c1b, -96*128	;; Test loop counter, restore source pointer
	end_timer 12
ELSE
	start_timer 12
	lea	r10, [rdi+96*ZMM_SCD7]	;; Second sin/cos pointer
	zloop_init 96			;; 96 iterations
	zr8s_hundredtwentyeight_reals_last_unfft_preload
c1b:	zr8s_hundredtwentyeight_reals_last_unfft rsi, 128, 96*128, 192*128, 384*128, rdi, ZMM_SCD7, r10, ZMM_SCD7, 96, L1PREFETCH_ALL
	zloop	96, c1b, -96*128	;; Test loop counter, restore source pointer
	end_timer 12
ENDIF

	zfft_3_ret
	ENDM

;; Perform a 12K-element negacyclic FFT

zfft12Kp MACRO type
	LOCAL	b1b, b2b, b3b, b4b, zmid, c4b, c3b, c2b, c1b

;; Do a multiply with pre-FFTed inputs

	cmp	ffttype, 4
	je	zmid

;; Do FFT levels 1,2,3,4
;; On input the 128-byte cache line pairs hold these data values:
;;	0	+96	...	6K	+96	...
;;	...
;;	95	...
;;	768	...
;;	1.5K	...
;;	...
;; On output the 128-byte cache line pairs hold these data values:
;;	0	+96	...	768	+96	...
;;	...
;;	95	...
;;	1.5K	...
;;	3K	...
;;	...
;; These are swizzled to:
;;	0	+1.5K	...	768	+1.5K	...
;;	...
;;	767	...

;; Do 768 eight_complex_first_fft operations
;;	distance between fft data elements is 768
;;	do 96 macros processing 128 data values

;; Do FFT levels 5,6,7
;; On input the 64-byte cache lines hold these data values:
;;	0	+1.5K	...	768	+1.5K	...
;;	...
;;	767	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+1.5K	...	96	+1.5K	...
;;	...
;;	95
;;	192	...
;;	384	...
;;	...

;; Do 768 eight_complex_fft operations
;;	distance between fft data elements is 96
;;	do 96 macros each processing 128 data values

IFDEF OLDWAY
	start_timer 4
	mov	rdi, sincos1		;; Load sin/cos pointer
	bump	rdi, 96*ZMM_SCD4/8+32*ZMM_SCD1/8+8*ZMM_SCD2/8
	mov	r9, norm_col_mults	;; Load weights pointer
	zloop_init 96			;; 96 iterations
	zr8fs_eight_complex_first_fft_preload
b1b:	zr8fs_eight_complex_first_fft rsi, 128, 96*128, 192*128, 384*128, rdi, ZMM_CD8SCD4, r9, 8*128, 96, L1PREFETCH_ALL
	zloop	96, b1b, -96*128	;; Test loop counter, restore source pointer
	end_timer 4

	start_timer 5
	bump	rdi, -96*ZMM_CD8SCD4-96*ZMM_SCD4/8 ;; Load sin/cos pointer
	zloop_init 96			;; 96 iterations
	zr8b_eight_complex_djbfft_preload
b2b:	zr8b_eight_complex_djbfft rsi, 128, 96*128, 192*128, 384*128, rdi, ZMM_SCD4/8, 96, L1PREFETCH_ALL, 128
	zloop	96, b2b, -96*128	;; Test loop counter, restore source pointer
	end_timer 5
ELSE
	start_timer 4
	mov	rdi, sincos1		;; Load sin/cos pointer
	bump	rdi, 32*ZMM_SCD1/8+8*ZMM_SCD2/8
	lea	r10, [rdi+96*ZMM_SCD4/8] ;; Second sin/cos pointer
	mov	r9, norm_col_mults	;; Load weights pointer
	zloop_init 96			;; 96 iterations
	zr8fs_sixtyfour_complex_first_fft_preload
b1b:	zr8fs_sixtyfour_complex_first_fft rsi, 128, 96*128, 192*128, 384*128, r10, ZMM_CD8SCD4, rdi, ZMM_SCD4/8, r9, 8*128, 96, L1PREFETCH_ALL
	zloop	96, b1b, -96*128	;; Test loop counter, restore source pointer
	end_timer 4
ENDIF

;; Do FFT levels 8,9
;; On input the 64-byte cache lines hold these data values:
;;	0	+1.5K	...	96	+1.5K	...
;;	...
;;	95
;;	192	...
;;	384	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+1.5K	...	32	+1.5K	...
;;	...
;;	31
;;	64	...
;;	...

;; Do 2048 three_complex_fft operations
;;	distance between fft data elements is 32
;;	do 256 macros each processing 48 data values

;; Do FFT levels 10,11
;; On input the 64-byte cache lines hold these data values:
;;	0	+1.5K	...	32	+1.5K	...
;;	...
;;	31
;;	64	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+1.5K	...	8	+1.5K	...
;;	...
;;	7
;;	16	...
;;	32	...
;;	...

;; Do 1536 four_complex_fft operations
;;	distance between fft data elements is 8
;;	do 192 macros each processing 64 data values

IFDEF OLDWAY
	start_timer 6
	bump	rdi, -96*ZMM_SCD4/8-32*ZMM_SCD1/8 ;; Load sin/cos pointer
	zloop_init 256			;; 8 iterations of 32
	zr3b_three_complex_djbfft_preload
b3b:	zr3b_three_complex_djbfft rsi, 128, 32*128, rdi, ZMM_SCD1/8, 32
	zloop	32, b3b, -32*128+96*128, rdi, -32*ZMM_SCD1/8 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	8, b3b, -8*96*128	;; Test loop counter, restore source pointer
	end_timer 6

	start_timer 7
	bump	rdi, -8*ZMM_SCD2/8	;; Load sin/cos pointer
	zloop_init 192			;; 24 iterations of 8
	zr4b_four_complex_djbfft_preload
b4b:	zr4b_four_complex_djbfft rsi, 128, 8*128, 16*128, rdi, ZMM_SCD2/8, 8
	zloop	8, b4b, -8*128+32*128, rdi, -8*ZMM_SCD2/8 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	24, b4b, -24*32*128	;; Test loop counter, restore source pointer
	end_timer 7
ELSE
	start_timer 6
	lea	r10, [rdi-96*ZMM_SCD4/8-32*ZMM_SCD1/8] ;; Load sin/cos pointers
	lea	rdi, [r10-8*ZMM_SCD2/8]
	zloop_init 64			;; 8 iterations of 8
	zr34b_twelve_complex_djbfft_preload
b3b:	zr34b_twelve_complex_djbfft rsi, 128, 8*128, r10, 8*ZMM_SCD1/8, ZMM_SCD1/8, rdi, 0*ZMM_SCD2/8, ZMM_SCD2/8, 8
	zloop	8, b3b, -8*128+96*128, r10, -8*ZMM_SCD1/8, rdi, -8*ZMM_SCD2/8 ;; Test loop counter, next source pointer, restore sin/cos pointers
	zloop	8, b3b, -8*96*128	;; Test loop counter, restore source pointer
	end_timer 6
ENDIF

;; Do FFT levels 12,13,14
;; On input and output the 128-byte cache line pairs hold these data values:
;;	0	+1.5K	...	8	+1.5K	...
;;	...
;;	7
;;	16	...
;;	32	...
;;	...

;; Do 768 eight_complex_fft operations
;;	distance between fft data elements is 1
;;	do 96 macros processing 128 data values

zmid:	start_timer 8
	mov	rcx, 96
	call	zcomplex_onepass_middle
	end_timer 8

;; Do inverse FFT levels 10,11
;; On input the 64-byte cache lines hold these data values:
;;	0	+1.5K	...	8	+1.5K	...
;;	...
;;	7
;;	16	...
;;	32	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+1.5K	...	32	+1.5K	...
;;	...
;;	31
;;	64	...
;;	...

;; Do 1536 four_complex_unfft operations
;;	distance between fft data elements is 8
;;	do 192 macros each processing 64 data values

;; Do inverse FFT levels 8,9
;; On input the 64-byte cache lines hold these data values:
;;	0	+1.5K	...	32	+1.5K	...
;;	...
;;	31
;;	64	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+1.5K	...	96	+1.5K	...
;;	...
;;	95
;;	192	...
;;	384	...
;;	...

;; Do 2048 three_complex_unfft operations
;;	distance between fft data elements is 32
;;	do 256 macros each processing 48 data values

IFDEF OLDWAY
	start_timer 9
	mov	rdi, sincos1		;; Load sin/cos pointer
	zloop_init 192			;; 24 iterations of 8
	zr4b_four_complex_djbunfft_preload
c4b:	zr4b_four_complex_djbunfft rsi, 128, 8*128, 16*128, rdi, ZMM_SCD2/8, 8
	zloop	8, c4b, -8*128+32*128, rdi, -8*ZMM_SCD2/8 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	24, c4b, -24*32*128, rdi, 8*ZMM_SCD2/8	;; Test loop counter, restore source pointer, next sin/cos ptr
	end_timer 9

	start_timer 10
	zloop_init 256			;; 8 iterations of 32
	zr3b_three_complex_djbunfft_preload
c3b:	zr3b_three_complex_djbunfft rsi, 128, 32*128, rdi, ZMM_SCD1/8, 32
	zloop	32, c3b, -32*128+96*128, rdi, -32*ZMM_SCD1/8 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	8, c3b, -8*96*128, rdi, 32*ZMM_SCD1/8 ;; Test loop counter, restore source pointer, next sin/cos ptr
	end_timer 10
ELSE
	start_timer 9
	mov	rdi, sincos1		;; Load sin/cos pointers
	lea	r10, [rdi+8*ZMM_SCD2/8]
	zloop_init 64			;; 8 iterations of 8
	zr34b_twelve_complex_djbunfft_preload
c4b:	zr34b_twelve_complex_djbunfft rsi, 128, 8*128, rdi, 0*ZMM_SCD2/8, ZMM_SCD2/8, r10, 8*ZMM_SCD1/8, ZMM_SCD1/8, 8
	zloop	8, c4b, -8*128+96*128, rdi, -8*ZMM_SCD2/8, r10, -8*ZMM_SCD1/8 ;; Test loop counter, next source pointer, restore sin/cos pointers
	zloop	8, c4b, -8*96*128, rdi, 8*ZMM_SCD2/8+32*ZMM_SCD1/8 ;; Test loop counter, restore source pointer, next sin/cos ptr
	end_timer 9
ENDIF

;; Do inverse FFT levels 5,6,7
;; On input the 64-byte cache lines hold these data values:
;;	0	+1.5K	...	96	+1.5K	...
;;	...
;;	95
;;	192	...
;;	384	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+1.5K	...	768	+1.5K	...
;;	...
;;	767	...

;; Do 768 eight_complex_unfft operations
;;	distance between fft data elements is 96
;;	do 96 macros each processing 128 data values

;; Do inverse FFT levels 1,2,3,4
;; On input the 128-byte cache line pairs hold these data values:
;;	0	+1.5K	...	768	+1.5K	...
;;	...
;;	767	...
;; These are swizzled to:
;;	0	+96	...	768	+96	...
;;	...
;;	95	...
;;	1.5K	...
;;	3K	...
;;	...
;; On output the 128-byte cache line pairs hold these data values:
;;	0	+96	...	6K	+96	...
;;	...
;;	95	...
;;	768	...
;;	1.5K	...
;;	...

;; Do 768 eight_complex_last_unfft operations
;;	distance between fft data elements is 768
;;	do 96 macros processing 128 data values

IFDEF OLDWAY
	start_timer 11
	zloop_init 96			;; 96 iterations
	zr8b_eight_complex_djbunfft_preload
c2b:	zr8b_eight_complex_djbunfft rsi, 128, 96*128, 192*128, 384*128, rdi, ZMM_SCD4/8, 96, L1PREFETCH_ALL, 128
	zloop	96, c2b, -96*128	;; Test loop counter, restore source pointer
	end_timer 11

	start_timer 12
	zloop_init 96			;; 96 iterations
	zr8s_eight_complex_last_unfft_preload
c1b:	zr8s_eight_complex_last_unfft rsi, 128, 96*128, 192*128, 384*128, rdi, ZMM_CD8SCD4, 96, L1PREFETCH_ALL
	zloop	96, c1b, -96*128	;; Test loop counter, restore source pointer
	end_timer 12
ELSE
	start_timer 12
	lea	r10, [rdi+96*ZMM_SCD4/8] ;; Second sin/cos pointer
	zloop_init 96			;; 96 iterations
	zr8s_sixtyfour_complex_last_unfft_preload
c1b:	zr8s_sixtyfour_complex_last_unfft rsi, 128, 96*128, 192*128, 384*128, rdi, ZMM_SCD4/8, r10, ZMM_CD8SCD4, 96, L1PREFETCH_ALL
	zloop	96, c1b, -96*128	;; Test loop counter, restore source pointer
	end_timer 12
ENDIF

	zfft_3_ret
	ENDM


;; Perform a 14K-element FFT

zfft14K MACRO type
	LOCAL	b1b, b2b, b3a, b3b, b4a, b4b, zmid, c4a, c4b, c3a, c3b, c2b, c1b

;; Check for a multiply with pre-FFTed inputs

	cmp	ffttype, 4
	je	zmid

;; Do FFT levels 1,2,3,4
;; On input the 128-byte cache line pairs hold these data values:
;;	0	+112	...	7K	+112	...
;;	...
;;	111	...
;;	896	...
;;	1792	...
;;	...
;; On output the 128-byte cache line pairs hold these data values:
;;	0	+112	...	896	+112	...
;;	...
;;	111	...
;;	1792	...
;;	3584	...
;;	...
;; These are swizzled to:
;;	0	+1792	...	896	+1792	...
;;	...
;;	895	...

;; Do 896 sixteen_reals_fft operations
;;	distance between fft data elements is 896
;;	do 112 macros processing 128 data values

;; Do FFT levels 5,6,7
;; On input the 64-byte cache lines hold these data values:
;;	0	+1792	...	896	+1792	...
;;	...
;;	895	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+1792	...	112	+1792	...
;;	...
;;	111
;;	448	...
;;	896	...
;;	...

;; Do 112 sixteen_reals_fft operation
;; Do 784 eight_complex_fft operations
;;	distance between fft data elements is 112
;;	do 112 macros each processing 128 data values

IFDEF OLDWAY
	start_timer 4
	mov	rdi, sincos1		;; Load sin/cos data ptr
	bump	rdi, 112*ZMM_SCD7+16*ZMM_SCD6+8*ZMM_SCD1
	mov	r9, norm_col_mults	;; Load weights pointer
	zloop_init 112			;; 112 iterations
	zr8fs_sixteen_reals_first_fft_preload
b1b:	zr8fs_sixteen_reals_first_fft rsi, 128, 112*128, 224*128, 448*128, rdi, ZMM_SCD7, r9, 8*128, 112, L1PREFETCH_ALL
	zloop	112, b1b, -112*128	;; Test loop counter, restore source pointer
	end_timer 4

	start_timer 5
	bump	rdi, -112*ZMM_SCD7-112*ZMM_SCD7 ;; Load sin/cos pointer
	zloop_init 112			;; 112 iterations
	zr8_sixteen_reals_eight_complex_djbfft_preload
b2b:	zr8_sixteen_reals_eight_complex_djbfft rsi, 128, 112*128, 224*128, 448*128, rdi, ZMM_SCD7, 112, L1PREFETCH_ALL
	zloop	112, b2b, -112*128	;; Test loop counter, restore source pointer
	end_timer 5
ELSE
	start_timer 4
	mov	rdi, sincos1		;; Load sin/cos data ptr
	bump	rdi, 16*ZMM_SCD6+8*ZMM_SCD1
	lea	r10, [rdi+112*ZMM_SCD7]	;; Second sin/cos pointer
	mov	r9, norm_col_mults	;; Load weights pointer
	zloop_init 112			;; 112 iterations
	zr8fs_hundredtwentyeight_reals_first_fft_preload
b1b:	zr8fs_hundredtwentyeight_reals_first_fft rsi, 128, 112*128, 224*128, 448*128, r10, ZMM_SCD7, rdi, ZMM_SCD7, r9, 8*128, 112, L1PREFETCH_ALL
	zloop	112, b1b, -112*128	;; Test loop counter, restore source pointer
	end_timer 4
ENDIF

;; Do FFT levels 8,9,10
;; On input the 64-byte cache lines hold these data values:
;;	0	+1792	...	112	+1792	...
;;	...
;;	111
;;	448	...
;;	896	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+1792	...	16	+1792	...
;;	...
;;	15
;;	32	...
;;	...

;; Do 16 fourteen_reals_fft operation
;; Do 1008 seven_complex_fft operations
;;	distance between fft data elements is 16
;;	do 128 macros each processing 112 data values

;; Do FFT levels 11
;; On input the 64-byte cache lines hold these data values:
;;	0	+1792	...	16	+1792	...
;;	...
;;	15
;;	32	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+1792	...	8	+1792	...
;;	...
;;	7
;;	16	...
;;	32	...
;;	...

;; Do 8 four_reals_fft operation
;; Do 3576 two_complex_fft operations
;;	distance between fft data elements is 8
;;	do 448 macros each processing 32 data values

IFDEF OLDWAY
	start_timer 6
	bump	rdi, -112*ZMM_SCD7-16*ZMM_SCD6
	zloop_init 128			;; 8 iterations of 16
	zr7_fourteen_reals_seven_complex_djbfft_preload
b3a:	zr7_fourteen_reals_seven_complex_djbfft rsi, 128, 16*128, rdi, ZMM_SCD6, 16
	zloop	16, b3a, -16*128+112*128, rdi, -16*ZMM_SCD6 ;; Test loop counter, next source pointer, restore sin/cos pointer

	loops_reset			;; 7 more iters of 16
	zr7rb_seven_complex_djbfft_preload
b3b:	zr7rb_seven_complex_djbfft rsi, 128, 16*128, rdi, ZMM_SCD6, 16
	zloop	16, b3b, -16*128+112*128, rdi, -16*ZMM_SCD6 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	8, b3b, -8*112*128	;; Test loop counter, restore source pointer
	end_timer 6

	start_timer 7
	bump	rdi, -8*ZMM_SCD1
	zloop_init 448			;; 56 iterations of 8
	zr2_four_reals_two_complex_djbfft_preload
b4a:	zr2_four_reals_two_complex_djbfft rsi, 128, 8*128, rdi, ZMM_SCD1, 8
	zloop	8, b4a, -8*128+16*128, rdi, -8*ZMM_SCD1 ;; Test loop counter, next source pointer, restore sin/cos pointer

	loops_reset			;; 55 more iters of 8
	zr2rb_two_complex_djbfft_preload
b4b:	zr2rb_two_complex_djbfft rsi, 128, 8*128, rdi, ZMM_SCD1, 8
	zloop	8, b4b, -8*128+16*128, rdi, -8*ZMM_SCD1 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	56, b4b, -56*16*128	;; Test loop counter, restore source pointer
	end_timer 7
ELSE
	start_timer 6
	lea	r10, [rdi-112*ZMM_SCD7-16*ZMM_SCD6]
	lea	rdi, [r10-8*ZMM_SCD1]
	zloop_init 16			;; 16 iterations
	zr7_fourteen_reals_seven_complex_djbfft_preload
b3a:	zr7_fourteen_reals_seven_complex_djbfft rsi, 128, 16*128, r10, ZMM_SCD6, 16
	zloop	16, b3a, -16*128, r10, -16*ZMM_SCD6 ;; Test loop counter, restore source pointer, restore sin/cos pointer

	zloop_init 56			;; 7 iterations of 8
	zr2_four_reals_two_complex_djbfft_preload
b4a:	zr2_four_reals_two_complex_djbfft rsi, 128, 8*128, rdi, ZMM_SCD1, 8
	zloop	8, b4a, -8*128+16*128, rdi, -8*ZMM_SCD1 ;; Test loop counter, next source pointer, restore sin/cos pointer

	loops_reset			;; 6 more iters of 8
	zr2rb_two_complex_djbfft_preload
b4b:	zr2rb_two_complex_djbfft rsi, 128, 8*128, rdi, ZMM_SCD1, 8
	zloop	8, b4b, -8*128+16*128, rdi, -8*ZMM_SCD1 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	7, b4b, -7*16*128+112*128 ;; Test loop counter, next source pointer
	end_timer 6

	start_timer 7
	zloop_init 56			;; 7 more iterations of 8
	zr27rb_fourteen_complex_djbfft_preload
b3b:	zr27rb_fourteen_complex_djbfft rsi, 128, 8*128, r10, 8*ZMM_SCD6, ZMM_SCD6, rdi, 0*ZMM_SCD1, ZMM_SCD1, 8
	zloop	8, b3b, -8*128+112*128, r10, -8*ZMM_SCD6, rdi, -8*ZMM_SCD1 ;; Test loop counter, next source pointer, restore sin/cos pointers
	zloop	7, b3b, -8*112*128	;; Test loop counter, restore source pointer
	end_timer 7
ENDIF

;; Do FFT levels 12,13,14
;; On input and output the 128-byte cache line pairs hold these data values:
;;	0	+1792	...	8	+1792	...
;;	...
;;	7
;;	16	...
;;	32	...
;;	...

;; Do 1 sixteen_reals_fft operation
;; Do 895 eight_complex_fft operations
;;	distance between fft data elements is 1
;;	do 112 macros processing 128 data values

zmid:	start_timer 8
	mov	rcx, 112
	call	zreal_onepass_middle
	end_timer 8

;; Do inverse FFT levels 11
;; On input the 64-byte cache lines hold these data values:
;;	0	+1792	...	8	+1792	...
;;	...
;;	7
;;	16	...
;;	32	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+1792	...	16	+1792	...
;;	...
;;	15
;;	32	...
;;	...

;; Do 8 four_reals_unfft operations
;; Do 3576 two_complex_unfft operations
;;	distance between fft data elements is 8
;;	do 448 macros each processing 32 data values

;; Do inverse FFT levels 8,9,10
;; On input the 64-byte cache lines hold these data values:
;;	0	+1792	...	16	+1792	...
;;	...
;;	15
;;	32	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+1792	...	112	+1792	...
;;	...
;;	111
;;	448	...
;;	896	...
;;	...

;; Do 16 fourteen_reals_unfft operations
;; Do 1008 seven_complex_unfft operations
;;	distance between fft data elements is 16
;;	do 128 macros each processing 112 data values

IFDEF OLDWAY
	start_timer 9
	mov	rdi, sincos1		;; Load sin/cos pointer
	zloop_init 448			;; 56 iterations of 8
	zr2_four_reals_two_complex_djbunfft_preload
c4a:	zr2_four_reals_two_complex_djbunfft rsi, 128, 8*128, rdi, ZMM_SCD1, 8
	zloop	8, c4a, -8*128+16*128, rdi, -8*ZMM_SCD1 ;; Test loop counter, restore source pointer, restore sin/cos pointer

	loops_reset			;; 55 more iters of 8
	zr2rb_two_complex_djbunfft_preload
c4b:	zr2rb_two_complex_djbunfft rsi, 128, 8*128, rdi, ZMM_SCD1, 8
	zloop	8, c4b, -8*128+16*128, rdi, -8*ZMM_SCD1 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	56, c4b, -56*16*128, rdi, 8*ZMM_SCD1 ;; Test loop counter, restore source pointer, next sin/cos ptr
	end_timer 9

	start_timer 10
	zloop_init 128			;; 8 iterations of 16
	zr7_fourteen_reals_seven_complex_djbunfft_preload
c3a:	zr7_fourteen_reals_seven_complex_djbunfft rsi, 128, 16*128, rdi, ZMM_SCD6, 16
	zloop	16, c3a, -16*128+112*128, rdi, -16*ZMM_SCD6 ;; Test loop counter, restore source pointer, restore sin/cos pointer

	loops_reset			;; 7 more iters of 16
	zr7rb_seven_complex_djbunfft_preload
c3b:	zr7rb_seven_complex_djbunfft rsi, 128, 16*128, rdi, ZMM_SCD6, 16
	zloop	16, c3b, -16*128+112*128, rdi, -16*ZMM_SCD6 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	8, c3b, -8*112*128, rdi, 16*ZMM_SCD6 ;; Test loop counter, restore source pointer, next sin/cos ptr
	end_timer 10
ELSE
	start_timer 9
	mov	rdi, sincos1		;; Load sin/cos pointers
	lea	r10, [rdi+8*ZMM_SCD1]
	zloop_init 56			;; 7 iterations of 8
	zr2_four_reals_two_complex_djbunfft_preload
c4a:	zr2_four_reals_two_complex_djbunfft rsi, 128, 8*128, rdi, ZMM_SCD1, 8
	zloop	8, c4a, -8*128+16*128, rdi, -8*ZMM_SCD1 ;; Test loop counter, restore source pointer, restore sin/cos pointer

	loops_reset			;; 6 more iters of 8
	zr2rb_two_complex_djbunfft_preload
c4b:	zr2rb_two_complex_djbunfft rsi, 128, 8*128, rdi, ZMM_SCD1, 8
	zloop	8, c4b, -8*128+16*128, rdi, -8*ZMM_SCD1 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	7, c4b, -7*16*128	;; Test loop counter, restore source pointer

	zloop_init 16			;; 16 iterations
	zr7_fourteen_reals_seven_complex_djbunfft_preload
c3a:	zr7_fourteen_reals_seven_complex_djbunfft rsi, 128, 16*128, r10, ZMM_SCD6, 16
	zloop	16, c3a, -16*128+112*128, r10, -16*ZMM_SCD6 ;; Test loop counter, next source pointer, restore sin/cos pointer
	end_timer 9

	start_timer 10
	zloop_init 56			;; 7 more iterations of 8
	zr27rb_fourteen_complex_djbunfft_preload
c3b:	zr27rb_fourteen_complex_djbunfft rsi, 128, 8*128, rdi, 0*ZMM_SCD1, ZMM_SCD1, r10, 8*ZMM_SCD6, ZMM_SCD6, 8
	zloop	8, c3b, -8*128+112*128, rdi, -8*ZMM_SCD1, r10, -8*ZMM_SCD6 ;; Test loop counter, next source pointer, restore sin/cos pointers
	zloop	7, c3b, -8*112*128, rdi, 8*ZMM_SCD1+16*ZMM_SCD6 ;; Test loop counter, restore source pointer, next sin/cos ptr
	end_timer 10
ENDIF

;; Do inverse FFT levels 5,6,7
;; On input the 64-byte cache lines hold these data values:
;;	0	+1792	...	112	+1792	...
;;	...
;;	111
;;	448	...
;;	896	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+1792	...	896	+1792	...
;;	...
;;	895	...

;; Do 112 sixteen_reals_unfft operations
;; Do 784 eight_complex_unfft operations
;;	distance between fft data elements is 112
;;	do 112 macros each processing 128 data values

;; Do inverse FFT levels 1,2,3,4
;; On input the 128-byte cache line pairs hold these data values:
;;	0	+1792	...	896	+1792	...
;;	...
;;	895	...
;; These are swizzled to:
;;	0	+112	...	896	+112	...
;;	...
;;	111	...
;;	1792	...
;;	3584	...
;;	...
;; On output the 128-byte cache line pairs hold these data values:
;;	0	+112	...	7K	+112	...
;;	...
;;	111	...
;;	896	...
;;	1792	...
;;	...

;; Do 896 sixteen_reals_unfft operations
;;	distance between fft data elements is 896
;;	do 112 macros processing 128 data values

IFDEF OLDWAY
	start_timer 11
	zloop_init 112			;; 112 iterations
	zr8_sixteen_reals_eight_complex_djbunfft_preload
c2b:	zr8_sixteen_reals_eight_complex_djbunfft rsi, 128, 112*128, 224*128, 448*128, rdi, ZMM_SCD7, 112
	zloop	112, c2b, -112*128	;; Test loop counter, restore source pointer, restore sin/cos pointer
	end_timer 11

	start_timer 12
	zloop_init 112			;; 112 iterations
	zr8s_sixteen_reals_last_unfft_preload
c1b:	zr8s_sixteen_reals_last_unfft rsi, 128, 112*128, 224*128, 448*128, rdi, ZMM_SCD7, 112, L1PREFETCH_ALL
	zloop	112, c1b, -112*128	;; Test loop counter, restore source pointer
	end_timer 12
ELSE
	start_timer 12
	lea	r10, [rdi+112*ZMM_SCD7]	;; Second sin/cos pointer
	zloop_init 112			;; 112 iterations
	zr8s_hundredtwentyeight_reals_last_unfft_preload
c1b:	zr8s_hundredtwentyeight_reals_last_unfft rsi, 128, 112*128, 224*128, 448*128, rdi, ZMM_SCD7, r10, ZMM_SCD7, 112, L1PREFETCH_ALL
	zloop	112, c1b, -112*128	;; Test loop counter, restore source pointer
	end_timer 12
ENDIF

	zfft_3_ret
	ENDM

;; Perform a 14K-element negacyclic FFT

zfft14Kp MACRO type
	LOCAL	b1b, b2b, b3b, b4b, zmid, c4b, c3b, c2b, c1b

;; Do a multiply with pre-FFTed inputs

	cmp	ffttype, 4
	je	zmid

;; Do FFT levels 1,2,3,4
;; On input the 128-byte cache line pairs hold these data values:
;;	0	+112	...	7K	+112	...
;;	...
;;	111	...
;;	896	...
;;	1792	...
;;	...
;; On output the 128-byte cache line pairs hold these data values:
;;	0	+112	...	896	+112	...
;;	...
;;	111	...
;;	1792	...
;;	3584	...
;;	...
;; These are swizzled to:
;;	0	+1792	...	896	+1792	...
;;	...
;;	895	...

;; Do 896 eight_complex_first_fft operations
;;	distance between fft data elements is 896
;;	do 112 macros processing 128 data values

;; Do FFT levels 5,6,7
;; On input the 64-byte cache lines hold these data values:
;;	0	+1792	...	896	+1792	...
;;	...
;;	895	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+1792	...	112	+1792	...
;;	...
;;	111
;;	448	...
;;	896	...
;;	...

;; Do 896 eight_complex_fft operations
;;	distance between fft data elements is 112
;;	do 112 macros each processing 128 data values

IFDEF OLDWAY
	start_timer 4
	mov	rdi, sincos1		;; Load sin/cos pointer
	bump	rdi, 112*ZMM_SCD4/8+16*ZMM_SCD3/8+8*ZMM_SCD1/8
	mov	r9, norm_col_mults	;; Load weights pointer
	zloop_init 112			;; 112 iterations
	zr8fs_eight_complex_first_fft_preload
b1b:	zr8fs_eight_complex_first_fft rsi, 128, 112*128, 224*128, 448*128, rdi, ZMM_CD8SCD4, r9, 8*128, 112, L1PREFETCH_ALL
	zloop	112, b1b, -112*128	;; Test loop counter, restore source pointer
	end_timer 4

	start_timer 5
	bump	rdi, -112*ZMM_CD8SCD4-112*ZMM_SCD4/8 ;; Load sin/cos pointer
	zloop_init 112			;; 112 iterations
	zr8b_eight_complex_djbfft_preload
b2b:	zr8b_eight_complex_djbfft rsi, 128, 112*128, 224*128, 448*128, rdi, ZMM_SCD4/8, 112, L1PREFETCH_ALL, 128
	zloop	112, b2b, -112*128	;; Test loop counter, restore source pointer
	end_timer 5
ELSE
	start_timer 4
	mov	rdi, sincos1		;; Load sin/cos pointer
	bump	rdi, 16*ZMM_SCD3/8+8*ZMM_SCD1/8
	lea	r10, [rdi+112*ZMM_SCD4/8] ;; Second sin/cos pointer
	mov	r9, norm_col_mults	;; Load weights pointer
	zloop_init 112			;; 112 iterations
	zr8fs_sixtyfour_complex_first_fft_preload
b1b:	zr8fs_sixtyfour_complex_first_fft rsi, 128, 112*128, 224*128, 448*128, r10, ZMM_CD8SCD4, rdi, ZMM_SCD4/8, r9, 8*128, 112, L1PREFETCH_ALL
	zloop	112, b1b, -112*128	;; Test loop counter, restore source pointer
	end_timer 4
ENDIF

;; Do FFT levels 8,9,10
;; On input the 64-byte cache lines hold these data values:
;;	0	+1792	...	112	+1792	...
;;	...
;;	111
;;	448	...
;;	896	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+1792	...	16	+1792	...
;;	...
;;	15
;;	32	...
;;	...

;; Do 1024 seven_complex_fft operations
;;	distance between fft data elements is 16
;;	do 128 macros each processing 112 data values

;; Do FFT levels 11
;; On input the 64-byte cache lines hold these data values:
;;	0	+1792	...	16	+1792	...
;;	...
;;	15
;;	32	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+1792	...	8	+1792	...
;;	...
;;	7
;;	16	...
;;	32	...
;;	...

;; Do 3584 two_complex_fft operations
;;	distance between fft data elements is 8
;;	do 448 macros each processing 32 data values

IFDEF OLDWAY
	start_timer 6
	bump	rdi, -112*ZMM_SCD4/8-16*ZMM_SCD3/8 ;; Load sin/cos pointer
	zloop_init 128			;; 8 iterations of 16
	zr7b_seven_complex_djbfft_preload
b3b:	zr7b_seven_complex_djbfft rsi, 128, 16*128, rdi, ZMM_SCD3/8, 16
	zloop	16, b3b, -16*128+112*128, rdi, -16*ZMM_SCD3/8 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	8, b3b, -8*112*128	;; Test loop counter, restore source pointer
	end_timer 6

	start_timer 7
	bump	rdi, -8*ZMM_SCD1/8	;; Load sin/cos pointer
	zloop_init 448			;; 56 iterations of 8
	zr2b_two_complex_djbfft_preload
b4b:	zr2b_two_complex_djbfft rsi, 128, 8*128, rdi, ZMM_SCD1/8, 8
	zloop	8, b4b, -8*128+16*128, rdi, -8*ZMM_SCD1/8 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	56, b4b, -56*16*128	;; Test loop counter, restore source pointer
	end_timer 7
ELSE
	start_timer 6
	lea	r10, [rdi-112*ZMM_SCD4/8-16*ZMM_SCD3/8] ;; Load sin/cos pointers
	lea	rdi, [r10-8*ZMM_SCD1/8]
	zloop_init 64			;; 8 iterations of 8
	zr27b_fourteen_complex_djbfft_preload
b3b:	zr27b_fourteen_complex_djbfft rsi, 128, 8*128, r10, 8*ZMM_SCD3/8, ZMM_SCD3/8, rdi, 0*ZMM_SCD1/8, ZMM_SCD1/8, 8
	zloop	8, b3b, -8*128+112*128, r10, -8*ZMM_SCD3/8, rdi, -8*ZMM_SCD1/8 ;; Test loop counter, next source pointer, restore sin/cos pointers
	zloop	8, b3b, -8*112*128	;; Test loop counter, restore source pointer
	end_timer 6
ENDIF

;; Do FFT levels 12,13,14
;; On input and output the 128-byte cache line pairs hold these data values:
;;	0	+1792	...	8	+1792	...
;;	...
;;	7
;;	16	...
;;	32	...
;;	...

;; Do 896 eight_complex_fft operations
;;	distance between fft data elements is 1
;;	do 112 macros processing 128 data values

zmid:	start_timer 8
	mov	rcx, 112
	call	zcomplex_onepass_middle
	end_timer 8

;; Do inverse FFT levels 11
;; On input the 64-byte cache lines hold these data values:
;;	0	+1792	...	8	+1792	...
;;	...
;;	7
;;	16	...
;;	32	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+1792	...	16	+1792	...
;;	...
;;	15
;;	32	...
;;	...

;; Do 3584 two_complex_unfft operations
;;	distance between fft data elements is 8
;;	do 448 macros each processing 32 data values

;; Do inverse FFT levels 8,9,10
;; On input the 64-byte cache lines hold these data values:
;;	0	+1792	...	16	+1792	...
;;	...
;;	15
;;	32	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+1792	...	112	+1792	...
;;	...
;;	111
;;	448	...
;;	896	...
;;	...

;; Do 1024 seven_complex_unfft operations
;;	distance between fft data elements is 32
;;	do 128 macros each processing 112 data values

IFDEF OLDWAY
	start_timer 9
	mov	rdi, sincos1		;; Load sin/cos pointer
	zloop_init 448			;; 56 iterations of 8
	zr2b_two_complex_djbunfft_preload
c4b:	zr2b_two_complex_djbunfft rsi, 128, 8*128, rdi, ZMM_SCD1/8, 8
	zloop	8, c4b, -8*128+16*128, rdi, -8*ZMM_SCD1/8 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	56, c4b, -56*16*128, rdi, 8*ZMM_SCD1/8	;; Test loop counter, restore source pointer, next sin/cos ptr
	end_timer 9

	start_timer 10
	zloop_init 128			;; 8 iterations of 16
	zr7b_seven_complex_djbunfft_preload
c3b:	zr7b_seven_complex_djbunfft rsi, 128, 16*128, rdi, ZMM_SCD3/8, 8
	zloop	16, c3b, -16*128+112*128, rdi, -16*ZMM_SCD3/8 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	8, c3b, -8*112*128, rdi, 16*ZMM_SCD3/8 ;; Test loop counter, restore source pointer, next sin/cos ptr
	end_timer 10
ELSE
	start_timer 9
	mov	rdi, sincos1		;; Load sin/cos pointers
	lea	r10, [rdi+8*ZMM_SCD1/8]
	zloop_init 64			;; 8 iterations of 8
	zr27b_fourteen_complex_djbunfft_preload
c4b:	zr27b_fourteen_complex_djbunfft rsi, 128, 8*128, rdi, 0*ZMM_SCD1/8, ZMM_SCD1/8, r10, 8*ZMM_SCD3/8, ZMM_SCD3/8, 8
	zloop	8, c4b, -8*128+112*128, rdi, -8*ZMM_SCD1/8, r10, -8*ZMM_SCD3/8 ;; Test loop counter, next source pointer, restore sin/cos pointers
	zloop	8, c4b, -8*112*128, rdi, 8*ZMM_SCD1/8+16*ZMM_SCD3/8 ;; Test loop counter, restore source pointer, next sin/cos ptr
	end_timer 9
ENDIF

;; Do inverse FFT levels 5,6,7
;; On input the 64-byte cache lines hold these data values:
;;	0	+1792	...	112	+1792	...
;;	...
;;	111
;;	448	...
;;	896	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+1792	...	896	+1792	...
;;	...
;;	895	...

;; Do 896 eight_complex_unfft operations
;;	distance between fft data elements is 112
;;	do 112 macros each processing 128 data values

;; Do inverse FFT levels 1,2,3,4
;; On input the 128-byte cache line pairs hold these data values:
;;	0	+1792	...	896	+1792	...
;;	...
;;	895	...
;; These are swizzled to:
;;	0	+112	...	896	+112	...
;;	...
;;	111	...
;;	1792	...
;;	3584	...
;;	...
;; On output the 128-byte cache line pairs hold these data values:
;;	0	+112	...	7K	+112	...
;;	...
;;	111	...
;;	896	...
;;	1792	...
;;	...

;; Do 896 eight_complex_last_unfft operations
;;	distance between fft data elements is 896
;;	do 112 macros processing 128 data values

IFDEF OLDWAY
	start_timer 11
	zloop_init 112			;; 112 iterations
	zr8b_eight_complex_djbunfft_preload
c2b:	zr8b_eight_complex_djbunfft rsi, 128, 112*128, 224*128, 448*128, rdi, ZMM_SCD4/8, 112, L1PREFETCH_ALL, 128
	zloop	112, c2b, -112*128	;; Test loop counter, restore source pointer
	end_timer 11

	start_timer 12
	zloop_init 112			;; 112 iterations
	zr8s_eight_complex_last_unfft_preload
c1b:	zr8s_eight_complex_last_unfft rsi, 128, 112*128, 224*128, 448*128, rdi, ZMM_CD8SCD4, 112, L1PREFETCH_ALL
	zloop	112, c1b, -112*128	;; Test loop counter, restore source pointer
	end_timer 12
ELSE
	start_timer 12
	lea	r10, [rdi+112*ZMM_SCD4/8] ;; Second sin/cos pointer
	zloop_init 112			;; 112 iterations
	zr8s_sixtyfour_complex_last_unfft_preload
c1b:	zr8s_sixtyfour_complex_last_unfft rsi, 128, 112*128, 224*128, 448*128, rdi, ZMM_SCD4/8, r10, ZMM_CD8SCD4, 112, L1PREFETCH_ALL
	zloop	112, c1b, -112*128	;; Test loop counter, restore source pointer
	end_timer 12
ENDIF

	zfft_3_ret
	ENDM


;; Perform a 15K-element FFT

zfft15K MACRO type
	LOCAL	b1b, b2b, b3a, b3b, b4a, b4b, zmid, c4a, c4b, c3a, c3b, c2b, c1b

;; Check for a multiply with pre-FFTed inputs

	cmp	ffttype, 4
	je	zmid

;; Do FFT levels 1,2,3,4
;; On input the 128-byte cache line pairs hold these data values:
;;	0	+120	...	7.5K	+120	...
;;	...
;;	119	...
;;	960	...
;;	...
;; On output the 128-byte cache line pairs hold these data values:
;;	0	+120	...	960	+120	...
;;	...
;;	119	...
;;	1920	...
;;	...
;; These are swizzled to:
;;	0	+1920	...	960	+1920	...
;;	...
;;	959	...

;; Do 960 sixteen_reals_fft operations
;;	distance between fft data elements is 960
;;	do 120 macros processing 128 data values

;; Do FFT levels 5,6,7
;; On input the 64-byte cache lines hold these data values:
;;	0	+1920	...	960	+1920	...
;;	...
;;	959	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+1920	...	120	+1920	...
;;	...
;;	119
;;	240	...
;;	...

;; Do 120 sixteen_reals_fft operation
;; Do 840 eight_complex_fft operations
;;	distance between fft data elements is 120
;;	do 120 macros each processing 128 data values

IFDEF OLDWAY
	start_timer 4
	mov	rdi, sincos1		;; Load sin/cos data ptr
	bump	rdi, 120*ZMM_SCD7+40*ZMM_SCD2+8*ZMM_SCD4
	mov	r9, norm_col_mults	;; Load weights pointer
	zloop_init 120			;; 120 iterations
	zr8fs_sixteen_reals_first_fft_preload
b1b:	zr8fs_sixteen_reals_first_fft rsi, 128, 120*128, 240*128, 480*128, rdi, ZMM_SCD7, r9, 8*128, 120, L1PREFETCH_ALL
	zloop	120, b1b, -120*128	;; Test loop counter, restore source pointer
	end_timer 4

	start_timer 5
	bump	rdi, -120*ZMM_SCD7-120*ZMM_SCD7 ;; Load sin/cos pointer
	zloop_init 120			;; 120 iterations
	zr8_sixteen_reals_eight_complex_djbfft_preload
b2b:	zr8_sixteen_reals_eight_complex_djbfft rsi, 128, 120*128, 240*128, 480*128, rdi, ZMM_SCD7, 120, L1PREFETCH_ALL
	zloop	120, b2b, -120*128	;; Test loop counter, restore source pointer
	end_timer 5
ELSE
	start_timer 4
	mov	rdi, sincos1		;; Load sin/cos data ptr
	bump	rdi, 40*ZMM_SCD2+8*ZMM_SCD4
	lea	r10, [rdi+120*ZMM_SCD7]	;; Second sin/cos pointer
	mov	r9, norm_col_mults	;; Load weights pointer
	zloop_init 120			;; 120 iterations
	zr8fs_hundredtwentyeight_reals_first_fft_preload
b1b:	zr8fs_hundredtwentyeight_reals_first_fft rsi, 128, 120*128, 240*128, 480*128, r10, ZMM_SCD7, rdi, ZMM_SCD7, r9, 8*128, 120, L1PREFETCH_ALL
	zloop	120, b1b, -120*128	;; Test loop counter, restore source pointer
	end_timer 4
ENDIF

;; Do FFT levels 8,9
;; On input the 64-byte cache lines hold these data values:
;;	0	+1920	...	120	+1920	...
;;	...
;;	119
;;	240	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+1920	...	40	+1920	...
;;	...
;;	39
;;	80	...
;;	...

;; Do 40 six_reals_fft operation
;; Do 2520 three_complex_fft operations
;;	distance between fft data elements is 40
;;	do 320 macros each processing 48 data values

;; Do FFT levels 10,11
;; On input the 64-byte cache lines hold these data values:
;;	0	+1920	...	40	+1920	...
;;	...
;;	39
;;	80	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+1920	...	8	+1920	...
;;	...
;;	7
;;	16	...
;;	32	...
;;	...

;; Do 8 ten_reals_fft operation
;; Do 1528 five_complex_fft operations
;;	distance between fft data elements is 8
;;	do 192 macros each processing 80 data values

IFDEF OLDWAY
	start_timer 6
	bump	rdi, -120*ZMM_SCD7-40*ZMM_SCD2
	zloop_init 320			;; 8 iterations of 40
	zr3_six_reals_three_complex_djbfft_preload
b3a:	zr3_six_reals_three_complex_djbfft rsi, 128, 40*128, rdi, ZMM_SCD2, 40
	zloop	40, b3a, -40*128+120*128, rdi, -40*ZMM_SCD2 ;; Test loop counter, next source pointer, restore sin/cos pointer

	loops_reset			;; 7 more iters of 40
	zr3rb_three_complex_djbfft_preload
b3b:	zr3rb_three_complex_djbfft rsi, 128, 40*128, rdi, ZMM_SCD2, 40
	zloop	40, b3b, -40*128+120*128, rdi, -40*ZMM_SCD2 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	8, b3b, -8*120*128	;; Test loop counter, restore source pointer
	end_timer 6

	start_timer 7
	bump	rdi, -8*ZMM_SCD4
	zloop_init 192			;; 24 iterations of 8
	zr5_ten_reals_five_complex_djbfft_preload
b4a:	zr5_ten_reals_five_complex_djbfft rsi, 128, 8*128, rdi, ZMM_SCD4, 8
	zloop	8, b4a, -8*128+40*128, rdi, -8*ZMM_SCD4 ;; Test loop counter, next source pointer, restore sin/cos pointer

	loops_reset			;; 23 more iters of 8
	zr5rb_five_complex_djbfft_preload
b4b:	zr5rb_five_complex_djbfft rsi, 128, 8*128, rdi, ZMM_SCD4, 8
	zloop	8, b4b, -8*128+40*128, rdi, -8*ZMM_SCD4 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	24, b4b, -24*40*128	;; Test loop counter, restore source pointer
	end_timer 7
ELSE
	start_timer 6
	lea	r10, [rdi-120*ZMM_SCD7-40*ZMM_SCD2] ;; Load sin/cos pointers
	lea	rdi, [r10-8*ZMM_SCD4]
	zloop_init 40			;; 40 iterations
	zr3_six_reals_three_complex_djbfft_preload
b3a:	zr3_six_reals_three_complex_djbfft rsi, 128, 40*128, r10, ZMM_SCD2, 40
	zloop	40, b3a, -40*128, r10, -40*ZMM_SCD2 ;; Test loop counter, restore source pointer, restore sin/cos pointer
	zloop_init 24			;; 3 iterations of 8
	zr5_ten_reals_five_complex_djbfft_preload
b4a:	zr5_ten_reals_five_complex_djbfft rsi, 128, 8*128, rdi, ZMM_SCD4, 8
	zloop	8, b4a, -8*128+40*128, rdi, -8*ZMM_SCD4 ;; Test loop counter, next source pointer, restore sin/cos pointer
	loops_reset			;; 2 more iters of 8
	zr5rb_five_complex_djbfft_preload
b4b:	zr5rb_five_complex_djbfft rsi, 128, 8*128, rdi, ZMM_SCD4, 8
	zloop	8, b4b, -8*128+40*128, rdi, -8*ZMM_SCD4 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	3, b4b, -3*40*128+120*128 ;; Test loop counter, next source pointer
	end_timer 6

	start_timer 7
	zloop_init 56			;; 7 more iterations of 8
	zr35rb_fifteen_complex_djbfft_preload
b3b:	zr35rb_fifteen_complex_djbfft rsi, 128, 8*128, r10, 8*ZMM_SCD2, ZMM_SCD2, rdi, 0*ZMM_SCD4, ZMM_SCD4, 8
	zloop	8, b3b, -8*128+120*128, r10, -8*ZMM_SCD2, rdi, -8*ZMM_SCD4 ;; Test loop counter, next source pointer, restore sin/cos pointers
	zloop	7, b3b, -8*120*128	;; Test loop counter, restore source pointer
	end_timer 7
ENDIF

;; Do FFT levels 12,13,14
;; On input and output the 128-byte cache line pairs hold these data values:
;;	0	+1920	...	8	+1920	...
;;	...
;;	7
;;	16	...
;;	32	...
;;	...

;; Do 1 sixteen_reals_fft operation
;; Do 959 eight_complex_fft operations
;;	distance between fft data elements is 1
;;	do 120 macros processing 128 data values

zmid:	start_timer 8
	mov	rcx, 120
	call	zreal_onepass_middle
	end_timer 8

;; Do inverse FFT levels 10,11
;; On input the 64-byte cache lines hold these data values:
;;	0	+1920	...	8	+1920	...
;;	...
;;	7
;;	16	...
;;	32	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+1920	...	40	+1920	...
;;	...
;;	39
;;	80	...
;;	...

;; Do 8 ten_reals_unfft operations
;; Do 1528 five_complex_unfft operations
;;	distance between fft data elements is 8
;;	do 192 macros each processing 80 data values

;; Do inverse FFT levels 8,9
;; On input the 64-byte cache lines hold these data values:
;;	0	+1920	...	40	+1920	...
;;	...
;;	39
;;	80	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+1920	...	120	+1920	...
;;	...
;;	119
;;	240	...
;;	...

;; Do 40 six_reals_unfft operations
;; Do 2520 three_complex_unfft operations
;;	distance between fft data elements is 40
;;	do 320 macros each processing 48 data values

IFDEF OLDWAY
	start_timer 9
	mov	rdi, sincos1		;; Load sin/cos pointer
	zloop_init 192			;; 24 iterations of 8
	zr5_ten_reals_five_complex_djbunfft_preload
c4a:	zr5_ten_reals_five_complex_djbunfft rsi, 128, 8*128, rdi, ZMM_SCD4, 8
	zloop	8, c4a, -8*128+40*128, rdi, -8*ZMM_SCD4 ;; Test loop counter, restore source pointer, restore sin/cos pointer

	loops_reset			;; 23 more iters of 8
	zr5rb_five_complex_djbunfft_preload
c4b:	zr5rb_five_complex_djbunfft rsi, 128, 8*128, rdi, ZMM_SCD4, 8
	zloop	8, c4b, -8*128+40*128, rdi, -8*ZMM_SCD4 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	24, c4b, -24*40*128, rdi, 8*ZMM_SCD4 ;; Test loop counter, restore source pointer, next sin/cos ptr
	end_timer 9

	start_timer 10
	zloop_init 320			;; 8 iterations of 40
	zr3_six_reals_three_complex_djbunfft_preload
c3a:	zr3_six_reals_three_complex_djbunfft rsi, 128, 40*128, rdi, ZMM_SCD2, 40
	zloop	40, c3a, -40*128+120*128, rdi, -40*ZMM_SCD2 ;; Test loop counter, restore source pointer, restore sin/cos pointer

	loops_reset			;; 7 more iters of 40
	zr3rb_three_complex_djbunfft_preload
c3b:	zr3rb_three_complex_djbunfft rsi, 128, 40*128, rdi, ZMM_SCD2, 40
	zloop	40, c3b, -40*128+120*128, rdi, -40*ZMM_SCD2 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	8, c3b, -8*120*128, rdi, 40*ZMM_SCD2 ;; Test loop counter, restore source pointer, next sin/cos ptr
	end_timer 10
ELSE
	start_timer 9
	mov	rdi, sincos1		;; Load sin/cos pointers
	lea	r10, [rdi+8*ZMM_SCD4]
	zloop_init 24			;; 3 iterations of 8
	zr5_ten_reals_five_complex_djbunfft_preload
c4a:	zr5_ten_reals_five_complex_djbunfft rsi, 128, 8*128, rdi, ZMM_SCD4, 8
	zloop	8, c4a, -8*128+40*128, rdi, -8*ZMM_SCD4 ;; Test loop counter, restore source pointer, restore sin/cos pointer
	loops_reset			;; 2 more iters of 8
	zr5rb_five_complex_djbunfft_preload
c4b:	zr5rb_five_complex_djbunfft rsi, 128, 8*128, rdi, ZMM_SCD4, 8
	zloop	8, c4b, -8*128+40*128, rdi, -8*ZMM_SCD4 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	3, c4b, -3*40*128	;; Test loop counter, restore source pointer
	zloop_init 40			;; 40 iterations
	zr3_six_reals_three_complex_djbunfft_preload
c3a:	zr3_six_reals_three_complex_djbunfft rsi, 128, 40*128, r10, ZMM_SCD2, 40
	zloop	40, c3a, -40*128+120*128, r10, -40*ZMM_SCD2 ;; Test loop counter, next source pointer, restore sin/cos pointer
	end_timer 9

	start_timer 10
	zloop_init 56			;; 7 more iterations of 8
	zr35rb_fifteen_complex_djbunfft_preload
c3b:	zr35rb_fifteen_complex_djbunfft rsi, 128, 8*128, rdi, 0*ZMM_SCD4, ZMM_SCD4, r10, 8*ZMM_SCD2, ZMM_SCD2, 8
	zloop	8, c3b, -8*128+120*128, rdi, -8*ZMM_SCD4, r10, -8*ZMM_SCD2 ;; Test loop counter, next source pointer, restore sin/cos pointers
	zloop	7, c3b, -8*120*128, rdi, 8*ZMM_SCD4+40*ZMM_SCD2 ;; Test loop counter, restore source pointer, next sin/cos ptr
	end_timer 10
ENDIF

;; Do inverse FFT levels 5,6,7
;; On input the 64-byte cache lines hold these data values:
;;	0	+1920	...	120	+1920	...
;;	...
;;	119
;;	240	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+1920	...	960	+1920	...
;;	...
;;	959	...

;; Do 120 sixteen_reals_unfft operations
;; Do 840 eight_complex_unfft operations
;;	distance between fft data elements is 120
;;	do 120 macros each processing 128 data values

;; Do inverse FFT levels 1,2,3,4
;; On input the 128-byte cache line pairs hold these data values:
;;	0	+1920	...	960	+1920	...
;;	...
;;	959	...
;; These are swizzled to:
;;	0	+120	...	960	+120	...
;;	...
;;	119	...
;;	1920	...
;;	...
;; On output the 128-byte cache line pairs hold these data values:
;;	0	+120	...	7.5K	+120	...
;;	...
;;	119	...
;;	960	...
;;	...

;; Do 960 sixteen_reals_unfft operations
;;	distance between fft data elements is 960
;;	do 120 macros processing 128 data values

IFDEF OLDWAY
	start_timer 11
	zloop_init 120			;; 120 iterations
	zr8_sixteen_reals_eight_complex_djbunfft_preload
c2b:	zr8_sixteen_reals_eight_complex_djbunfft rsi, 128, 120*128, 240*128, 480*128, rdi, ZMM_SCD7, 120
	zloop	120, c2b, -120*128	;; Test loop counter, restore source pointer, restore sin/cos pointer
	end_timer 11

	start_timer 12
	zloop_init 120			;; 120 iterations
	zr8s_sixteen_reals_last_unfft_preload
c1b:	zr8s_sixteen_reals_last_unfft rsi, 128, 120*128, 240*128, 480*128, rdi, ZMM_SCD7, 120, L1PREFETCH_ALL
	zloop	120, c1b, -120*128	;; Test loop counter, restore source pointer
	end_timer 12
ELSE
	start_timer 12
	lea	r10, [rdi+120*ZMM_SCD7]	;; Second sin/cos pointer
	zloop_init 120			;; 120 iterations
	zr8s_hundredtwentyeight_reals_last_unfft_preload
c1b:	zr8s_hundredtwentyeight_reals_last_unfft rsi, 128, 120*128, 240*128, 480*128, rdi, ZMM_SCD7, r10, ZMM_SCD7, 120, L1PREFETCH_ALL
	zloop	120, c1b, -120*128	;; Test loop counter, restore source pointer
	end_timer 12
ENDIF

	zfft_3_ret
	ENDM

;; Perform a 15K-element negacyclic FFT

zfft15Kp MACRO type
	LOCAL	b1b, b2b, b3b, b4b, zmid, c4b, c3b, c2b, c1b

;; Do a multiply with pre-FFTed inputs

	cmp	ffttype, 4
	je	zmid

;; Do FFT levels 1,2,3,4
;; On input the 128-byte cache line pairs hold these data values:
;;	0	+120	...	7.5K	+120	...
;;	...
;;	119	...
;;	960	...
;;	...
;; On output the 128-byte cache line pairs hold these data values:
;;	0	+120	...	960	+120	...
;;	...
;;	119	...
;;	1920	...
;;	...
;; These are swizzled to:
;;	0	+1920	...	960	+1920	...
;;	...
;;	959	...

;; Do 960 eight_complex_first_fft operations
;;	distance between fft data elements is 960
;;	do 120 macros processing 128 data values

;; Do FFT levels 5,6,7
;; On input the 64-byte cache lines hold these data values:
;;	0	+1920	...	960	+1920	...
;;	...
;;	959	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+1920	...	120	+1920	...
;;	...
;;	119
;;	240	...
;;	...

;; Do 960 eight_complex_fft operations
;;	distance between fft data elements is 120
;;	do 120 macros each processing 128 data values

IFDEF OLDWAY
	start_timer 4
	mov	rdi, sincos1		;; Load sin/cos pointer
	bump	rdi, 120*ZMM_SCD4/8+40*ZMM_SCD1/8+8*ZMM_SCD2/8
	mov	r9, norm_col_mults	;; Load weights pointer
	zloop_init 120			;; 120 iterations
	zr8fs_eight_complex_first_fft_preload
b1b:	zr8fs_eight_complex_first_fft rsi, 128, 120*128, 240*128, 480*128, rdi, ZMM_CD8SCD4, r9, 8*128, 120, L1PREFETCH_ALL
	zloop	120, b1b, -120*128	;; Test loop counter, restore source pointer
	end_timer 4

	start_timer 5
	bump	rdi, -120*ZMM_CD8SCD4-120*ZMM_SCD4/8 ;; Load sin/cos pointer
	zloop_init 120			;; 120 iterations
	zr8b_eight_complex_djbfft_preload
b2b:	zr8b_eight_complex_djbfft rsi, 128, 120*128, 240*128, 480*128, rdi, ZMM_SCD4/8, 120, L1PREFETCH_ALL, 128
	zloop	120, b2b, -120*128	;; Test loop counter, restore source pointer
	end_timer 5
ELSE
	start_timer 4
	mov	rdi, sincos1		;; Load sin/cos pointer
	bump	rdi, 40*ZMM_SCD1/8+8*ZMM_SCD2/8
	lea	r10, [rdi+120*ZMM_SCD4/8] ;; Second sin/cos pointer
	mov	r9, norm_col_mults	;; Load weights pointer
	zloop_init 120			;; 120 iterations
	zr8fs_sixtyfour_complex_first_fft_preload
b1b:	zr8fs_sixtyfour_complex_first_fft rsi, 128, 120*128, 240*128, 480*128, r10, ZMM_CD8SCD4, rdi, ZMM_SCD4/8, r9, 8*128, 120, L1PREFETCH_ALL
	zloop	120, b1b, -120*128	;; Test loop counter, restore source pointer
	end_timer 4
ENDIF

;; Do FFT levels 8,9
;; On input the 64-byte cache lines hold these data values:
;;	0	+1920	...	120	+1920	...
;;	...
;;	119
;;	240	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+1920	...	40	+1920	...
;;	...
;;	39
;;	80	...
;;	...

;; Do 2560 three_complex_fft operations
;;	distance between fft data elements is 40
;;	do 320 macros each processing 48 data values

;; Do FFT levels 10,11
;; On input the 64-byte cache lines hold these data values:
;;	0	+1920	...	40	+1920	...
;;	...
;;	39
;;	80	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+1920	...	8	+1920	...
;;	...
;;	7
;;	16	...
;;	32	...
;;	...

;; Do 1536 five_complex_fft operations
;;	distance between fft data elements is 8
;;	do 192 macros each processing 80 data values

IFDEF OLDWAY
	start_timer 6
	bump	rdi, -120*ZMM_SCD4/8-40*ZMM_SCD1/8 ;; Load sin/cos pointer
	zloop_init 320			;; 8 iterations of 40
	zr3b_three_complex_djbfft_preload
b3b:	zr3b_three_complex_djbfft rsi, 128, 40*128, rdi, ZMM_SCD1/8, 40
	zloop	40, b3b, -40*128+120*128, rdi, -40*ZMM_SCD1/8 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	8, b3b, -8*120*128	;; Test loop counter, restore source pointer
	end_timer 6

	start_timer 7
	bump	rdi, -8*ZMM_SCD2/8	;; Load sin/cos pointer
	zloop_init 192			;; 24 iterations of 8
	zr5b_five_complex_djbfft_preload
b4b:	zr5b_five_complex_djbfft rsi, 128, 8*128, rdi, ZMM_SCD2/8, 8
	zloop	8, b4b, -8*128+40*128, rdi, -8*ZMM_SCD2/8 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	24, b4b, -24*40*128	;; Test loop counter, restore source pointer
	end_timer 7
ELSE
	start_timer 6
	lea	r10, [rdi-120*ZMM_SCD4/8-40*ZMM_SCD1/8] ;; Load sin/cos pointers
	lea	rdi, [r10-8*ZMM_SCD2/8]
	zloop_init 64			;; 8 iterations of 8
	zr35b_fifteen_complex_djbfft_preload
b4b:	zr35b_fifteen_complex_djbfft rsi, 128, 8*128, r10, 8*ZMM_SCD1/8, ZMM_SCD1/8, rdi, 0*ZMM_SCD2/8, ZMM_SCD2/8, 8
	zloop	8, b4b, -8*128+120*128, r10, -8*ZMM_SCD1/8, rdi, -8*ZMM_SCD2/8 ;; Test loop counter, next source pointer, restore sin/cos pointers
	zloop	8, b4b, -8*120*128	;; Test loop counter, restore source pointer
	end_timer 6
ENDIF

;; Do FFT levels 12,13,14
;; On input and output the 128-byte cache line pairs hold these data values:
;;	0	+1920	...	8	+1920	...
;;	...
;;	7
;;	16	...
;;	32	...
;;	...

;; Do 960 eight_complex_fft operations
;;	distance between fft data elements is 1
;;	do 120 macros processing 128 data values

zmid:	start_timer 8
	mov	rcx, 120
	call	zcomplex_onepass_middle
	end_timer 8

;; Do inverse FFT levels 10,11
;; On input the 64-byte cache lines hold these data values:
;;	0	+1920	...	8	+1920	...
;;	...
;;	7
;;	16	...
;;	32	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+1920	...	40	+1920	...
;;	...
;;	39
;;	80	...
;;	...

;; Do 1536 five_complex_unfft operations
;;	distance between fft data elements is 8
;;	do 192 macros each processing 80 data values

;; Do inverse FFT levels 8,9
;; On input the 64-byte cache lines hold these data values:
;;	0	+1920	...	40	+1920	...
;;	...
;;	39
;;	80	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+1920	...	120	+1920	...
;;	...
;;	119
;;	240	...
;;	...

;; Do 2560 three_complex_unfft operations
;;	distance between fft data elements is 40
;;	do 320 macros each processing 48 data values

IFDEF OLDWAY
	start_timer 9
	mov	rdi, sincos1		;; Load sin/cos pointer
	zloop_init 192			;; 24 iterations of 8
	zr5b_five_complex_djbunfft_preload
c4b:	zr5b_five_complex_djbunfft rsi, 128, 8*128, rdi, ZMM_SCD2/8, 8
	zloop	8, c4b, -8*128+40*128, rdi, -8*ZMM_SCD2/8 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	24, c4b, -24*40*128, rdi, 8*ZMM_SCD2/8	;; Test loop counter, restore source pointer, next sin/cos ptr
	end_timer 9

	start_timer 10
	zloop_init 320			;; 8 iterations of 40
	zr3b_three_complex_djbunfft_preload
c3b:	zr3b_three_complex_djbunfft rsi, 128, 40*128, rdi, ZMM_SCD1/8, 40
	zloop	40, c3b, -40*128+120*128, rdi, -40*ZMM_SCD1/8 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	8, c3b, -8*120*128, rdi, 40*ZMM_SCD1/8 ;; Test loop counter, restore source pointer, next sin/cos ptr
	end_timer 10
ELSE
	start_timer 9
	mov	rdi, sincos1		;; Load sin/cos pointers
	lea	r10, [rdi+8*ZMM_SCD2/8]
	zloop_init 64			;; 8 iterations of 8
	zr35b_fifteen_complex_djbunfft_preload
c4b:	zr35b_fifteen_complex_djbunfft rsi, 128, 8*128, rdi, 0*ZMM_SCD2/8, ZMM_SCD2/8, r10, 8*ZMM_SCD1/8, ZMM_SCD1/8, 8
	zloop	8, c4b, -8*128+120*128, rdi, -8*ZMM_SCD2/8, r10, -8*ZMM_SCD1/8 ;; Test loop counter, next source pointer, restore sin/cos pointers
	zloop	8, c4b, -8*120*128, rdi, 8*ZMM_SCD2/8+40*ZMM_SCD1/8 ;; Test loop counter, restore source pointer, next sin/cos ptr
	end_timer 9
ENDIF

;; Do inverse FFT levels 5,6,7
;; On input the 64-byte cache lines hold these data values:
;;	0	+1920	...	120	+1920	...
;;	...
;;	119
;;	240	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+1920	...	960	+1920	...
;;	...
;;	959	...

;; Do 960 eight_complex_unfft operations
;;	distance between fft data elements is 120
;;	do 120 macros each processing 128 data values

;; Do inverse FFT levels 1,2,3,4
;; On input the 128-byte cache line pairs hold these data values:
;;	0	+1920	...	960	+1920	...
;;	...
;;	959	...
;; These are swizzled to:
;;	0	+120	...	960	+120	...
;;	...
;;	119	...
;;	1920	...
;;	...
;; On output the 128-byte cache line pairs hold these data values:
;;	0	+120	...	7.5K	+120	...
;;	...
;;	119	...
;;	960	...
;;	...

;; Do 960 eight_complex_last_unfft operations
;;	distance between fft data elements is 960
;;	do 120 macros processing 128 data values

IFDEF OLDWAY
	start_timer 11
	zloop_init 120			;; 120 iterations
	zr8b_eight_complex_djbunfft_preload
c2b:	zr8b_eight_complex_djbunfft rsi, 128, 120*128, 240*128, 480*128, rdi, ZMM_SCD4/8, 120, L1PREFETCH_ALL, 128
	zloop	120, c2b, -120*128	;; Test loop counter, restore source pointer
	end_timer 11

	start_timer 12
	zloop_init 120			;; 120 iterations
	zr8s_eight_complex_last_unfft_preload
c1b:	zr8s_eight_complex_last_unfft rsi, 128, 120*128, 240*128, 480*128, rdi, ZMM_CD8SCD4, 120, L1PREFETCH_ALL
	zloop	120, c1b, -120*128	;; Test loop counter, restore source pointer
	end_timer 12
ELSE
	start_timer 12
	lea	r10, [rdi+120*ZMM_SCD4/8] ;; Second sin/cos pointer
	zloop_init 120			;; 120 iterations
	zr8s_sixtyfour_complex_last_unfft_preload
c1b:	zr8s_sixtyfour_complex_last_unfft rsi, 128, 120*128, 240*128, 480*128, rdi, ZMM_SCD4/8, r10, ZMM_CD8SCD4, 120, L1PREFETCH_ALL
	zloop	120, c1b, -120*128	;; Test loop counter, restore source pointer
	end_timer 12
ENDIF

	zfft_3_ret
	ENDM


;; Perform a 16K-element FFT

zfft16K MACRO type
	LOCAL	b1b, b2b, b3a, b3b, b4a, b4b, zmid, c4a, c4b, c3a, c3b, c2b, c1b

;; Check for a multiply with pre-FFTed inputs

	cmp	ffttype, 4
	je	zmid

;; Do FFT levels 1,2,3,4
;; On input the 128-byte cache line pairs hold these data values:
;;	0	+128	...	8K	+128	...
;;	...
;;	127	...
;;	1024	...
;;	...
;; On output the 128-byte cache line pairs hold these data values:
;;	0	+128	...	1024	+128	...
;;	...
;;	127	...
;;	2K	...
;;	...
;; These are swizzled to:
;;	0	+2K	...	1024	+2K	...
;;	...
;;	1023	...

;; Do 1024 sixteen_reals_fft operations
;;	distance between fft data elements is 1024
;;	do 128 macros processing 128 data values

;; Do FFT levels 5,6,7
;; On input the 64-byte cache lines hold these data values:
;;	0	+2K	...	1024	+2K	...
;;	...
;;	1023	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+2K	...	128	+2K	...
;;	...
;;	127
;;	256	...
;;	...

;; Do 128 sixteen_reals_fft operation
;; Do 896 eight_complex_fft operations
;;	distance between fft data elements is 128
;;	do 128 macros each processing 128 data values

IFDEF OLDWAY
	start_timer 4
	mov	rdi, sincos1		;; Load sin/cos data ptr
	bump	rdi, 128*ZMM_SCD7+32*ZMM_SCD3
	mov	r9, norm_col_mults	;; Load weights pointer
	zloop_init 128			;; 128 iterations
	zr8fs_sixteen_reals_first_fft_preload
b1b:	zr8fs_sixteen_reals_first_fft rsi, 128, 128*128, 256*128, 512*128, rdi, ZMM_SCD7, r9, 8*128, 128, L1PREFETCH_ALL
	zloop	128, b1b, -128*128	;; Test loop counter, restore source pointer
	end_timer 4

	start_timer 5
	bump	rdi, -128*ZMM_SCD7-128*ZMM_SCD7 ;; Load sin/cos pointer
	zloop_init 128			;; 128 iterations
	zr8_sixteen_reals_eight_complex_djbfft_preload
b2b:	zr8_sixteen_reals_eight_complex_djbfft rsi, 128, 128*128, 256*128, 512*128, rdi, ZMM_SCD7, 128, L1PREFETCH_ALL
	zloop	128, b2b, -128*128	;; Test loop counter, restore source pointer
	end_timer 5
ELSE
	start_timer 4
	mov	rdi, sincos1		;; Load sin/cos data ptr
	bump	rdi, 32*ZMM_SCD3
	lea	r10, [rdi+128*ZMM_SCD7]	;; Second sin/cos pointer
	mov	r9, norm_col_mults	;; Load weights pointer
	zloop_init 128			;; 128 iterations
	zr8fs_hundredtwentyeight_reals_first_fft_preload
b1b:	zr8fs_hundredtwentyeight_reals_first_fft rsi, 128, 128*128, 256*128, 512*128, r10, ZMM_SCD7, rdi, ZMM_SCD7, r9, 8*128, 128, L1PREFETCH_ALL
	zloop	128, b1b, -128*128	;; Test loop counter, restore source pointer
	end_timer 4
ENDIF

;; Do FFT levels 8,9
;; On input the 64-byte cache lines hold these data values:
;;	0	+2K	...	128	+2K	...
;;	...
;;	127
;;	256	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+2K	...	32	+2K	...
;;	...
;;	31
;;	64	...
;;	...

;; Do 32 eight_reals_fft operation
;; Do 2016 four_complex_fft operations
;;	distance between fft data elements is 32
;;	do 256 macros each processing 64 data values

;; Do FFT levels 10,11
;; On input the 64-byte cache lines hold these data values:
;;	0	+2K	...	32	+2K	...
;;	...
;;	31
;;	64	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+2K	...	8	+2K	...
;;	...
;;	7
;;	16	...
;;	32	...
;;	...

;; Do 8 eight_reals_fft operation
;; Do 2040 four_complex_fft operations
;;	distance between fft data elements is 8
;;	do 256 macros each processing 64 data values

IFDEF OLDWAY
	start_timer 6
	bump	rdi, -128*ZMM_SCD7-32*ZMM_SCD3
	zloop_init 256			;; 8 iterations of 32
	zr4_eight_reals_four_complex_djbfft_preload
b3a:	zr4_eight_reals_four_complex_djbfft rsi, 128, 32*128, 64*128, rdi, ZMM_SCD3, 32
	zloop	32, b3a, -32*128+128*128, rdi, -32*ZMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer

	loops_reset			;; 7 more iters of 32
	zr4rb_four_complex_djbfft_preload
b3b:	zr4rb_four_complex_djbfft rsi, 128, 32*128, 64*128, rdi, ZMM_SCD3, 32
	zloop	32, b3b, -32*128+128*128, rdi, -32*ZMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	8, b3b, -8*128*128	;; Test loop counter, restore source pointer
	end_timer 6

	start_timer 7
	zloop_init 256			;; 32 iterations of 8
	zr4_eight_reals_four_complex_djbfft_preload
b4a:	zr4_eight_reals_four_complex_djbfft rsi, 128, 8*128, 16*128, rdi, 4*ZMM_SCD3, 8
	zloop	8, b4a, -8*128+32*128, rdi, -8*4*ZMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer

	loops_reset			;; 31 more iters of 8
	zr4rb_four_complex_djbfft_preload
b4b:	zr4rb_four_complex_djbfft rsi, 128, 8*128, 16*128, rdi, 4*ZMM_SCD3, 8
	zloop	8, b4b, -8*128+32*128, rdi, -8*4*ZMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	32, b4b, -32*32*128	;; Test loop counter, restore source pointer
	end_timer 7
ELSE
	start_timer 6
	bump	rdi, -128*ZMM_SCD7-32*ZMM_SCD3
	mov	r10, rdi
	zloop_init 32			;; 32 iterations
	zr4_eight_reals_four_complex_djbfft_preload
b3a:	zr4_eight_reals_four_complex_djbfft rsi, 128, 32*128, 64*128, r10, ZMM_SCD3, 32
	zloop	32, b3a, -32*128, r10, -32*ZMM_SCD3 ;; Test loop counter, restore source pointer, restore sin/cos pointer
	zloop_init 32			;; 32 iterations
	zr4_eight_reals_four_complex_djbfft_preload
b4a:	zr4_eight_reals_four_complex_djbfft rsi, 128, 8*128, 16*128, rdi, 4*ZMM_SCD3, 8
	zloop	8, b4a, -8*128+32*128, rdi, -8*4*ZMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer
	loops_reset			;; 3 more iters of 8
	zr4rb_four_complex_djbfft_preload
b4b:	zr4rb_four_complex_djbfft rsi, 128, 8*128, 16*128, rdi, 4*ZMM_SCD3, 8
	zloop	8, b4b, -8*128+32*128, rdi, -8*4*ZMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	4, b4b, -4*32*128+128*128 ;; Test loop counter, next source pointer
	end_timer 6

	start_timer 7
	zloop_init 56			;; 7 more iterations of 8
	zr44rb_sixteen_complex_djbfft_preload
b3b:	zr44rb_sixteen_complex_djbfft rsi, 128, 8*128, 16*128, 32*128, 64*128, r10, 8*ZMM_SCD3, ZMM_SCD3, rdi, 0*4*ZMM_SCD3, 4*ZMM_SCD3, 8
	zloop	8, b3b, -8*128+128*128, r10, -8*ZMM_SCD3, rdi, -8*4*ZMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointers
	zloop	7, b3b, -8*128*128	;; Test loop counter, restore source pointer
	end_timer 7
ENDIF

;; Do FFT levels 12,13,14
;; On input and output the 128-byte cache line pairs hold these data values:
;;	0	+2K	...	8	+2K	...
;;	...
;;	7
;;	16	...
;;	32	...
;;	...

;; Do 1 sixteen_reals_fft operation
;; Do 1023 eight_complex_fft operations
;;	distance between fft data elements is 1
;;	do 128 macros processing 128 data values

zmid:	start_timer 8
	mov	rcx, 128
	call	zreal_onepass_middle
	end_timer 8

;; Do inverse FFT levels 10,11
;; On input the 64-byte cache lines hold these data values:
;;	0	+2K	...	8	+2K	...
;;	...
;;	7
;;	16	...
;;	32	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+2K	...	32	+2K	...
;;	...
;;	31
;;	64	...
;;	...

;; Do 8 eight_reals_unfft operations
;; Do 2040 four_complex_unfft operations
;;	distance between fft data elements is 8
;;	do 256 macros each processing 64 data values

;; Do inverse FFT levels 8,9
;; On input the 64-byte cache lines hold these data values:
;;	0	+2K	...	32	+2K	...
;;	...
;;	31
;;	64	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+2K	...	128	+2K	...
;;	...
;;	127
;;	256	...
;;	...

;; Do 32 eight_reals_unfft operations
;; Do 2016 four_complex_unfft operations
;;	distance between fft data elements is 32
;;	do 256 macros each processing 64 data values

IFDEF OLDWAY
	start_timer 9
	mov	rdi, sincos1		;; Load sin/cos pointer
	zloop_init 256			;; 32 iterations of 8
	zr4_eight_reals_four_complex_djbunfft_preload
c4a:	zr4_eight_reals_four_complex_djbunfft rsi, 128, 8*128, 16*128, rdi, 4*ZMM_SCD3, 8
	zloop	8, c4a, -8*128+32*128, rdi, -8*4*ZMM_SCD3 ;; Test loop counter, restore source pointer, restore sin/cos pointer

	loops_reset			;; 31 more iters of 8
	zr4rb_four_complex_djbunfft_preload
c4b:	zr4rb_four_complex_djbunfft rsi, 128, 8*128, 16*128, rdi, 4*ZMM_SCD3, 8
	zloop	8, c4b, -8*128+32*128, rdi, -8*4*ZMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	32, c4b, -32*32*128	;; Test loop counter, restore source pointer
	end_timer 9

	start_timer 10
	zloop_init 256			;; 8 iterations of 32
	zr4_eight_reals_four_complex_djbunfft_preload
c3a:	zr4_eight_reals_four_complex_djbunfft rsi, 128, 32*128, 64*128, rdi, ZMM_SCD3, 32
	zloop	32, c3a, -32*128+128*128, rdi, -32*ZMM_SCD3 ;; Test loop counter, restore source pointer, restore sin/cos pointer

	loops_reset			;; 7 more iters of 32
	zr4rb_four_complex_djbunfft_preload
c3b:	zr4rb_four_complex_djbunfft rsi, 128, 32*128, 64*128, rdi, ZMM_SCD3, 32
	zloop	32, c3b, -32*128+128*128, rdi, -32*ZMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	8, c3b, -8*128*128, rdi, 32*ZMM_SCD3 ;; Test loop counter, restore source pointer, next sin/cos ptr
	end_timer 10
ELSE
	start_timer 9
	mov	rdi, sincos1		;; Load sin/cos pointers
	mov	r10, rdi
	zloop_init 32			;; 4 iterations of 8
	zr4_eight_reals_four_complex_djbunfft_preload
c4a:	zr4_eight_reals_four_complex_djbunfft rsi, 128, 8*128, 16*128, rdi, 4*ZMM_SCD3, 8
	zloop	8, c4a, -8*128+32*128, rdi, -8*4*ZMM_SCD3 ;; Test loop counter, restore source pointer, restore sin/cos pointer
	loops_reset			;; 3 more iters of 8
	zr4rb_four_complex_djbunfft_preload
c4b:	zr4rb_four_complex_djbunfft rsi, 128, 8*128, 16*128, rdi, 4*ZMM_SCD3, 8
	zloop	8, c4b, -8*128+32*128, rdi, -8*4*ZMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	4, c4b, -4*32*128	;; Test loop counter, restore source pointer
	zloop_init 32			;; 32 iterations
	zr4_eight_reals_four_complex_djbunfft_preload
c3a:	zr4_eight_reals_four_complex_djbunfft rsi, 128, 32*128, 64*128, r10, ZMM_SCD3, 32
	zloop	32, c3a, -32*128+128*128, r10, -32*ZMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointers
	end_timer 9

	start_timer 10
	zloop_init 56			;; 7 more iterations of 8
	zr44rb_sixteen_complex_djbunfft_preload
c3b:	zr44rb_sixteen_complex_djbunfft rsi, 128, 8*128, 16*128, 32*128, 64*128, rdi, 0*4*ZMM_SCD3, 4*ZMM_SCD3, r10, 8*ZMM_SCD3, ZMM_SCD3, 8
	zloop	8, c3b, -8*128+128*128, rdi, -8*4*ZMM_SCD3, r10, -8*ZMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointers
	zloop	7, c3b, -8*128*128, rdi, 32*ZMM_SCD3 ;; Test loop counter, restore source pointer, next sin/cos ptr
	end_timer 10
ENDIF

;; Do inverse FFT levels 5,6,7
;; On input the 64-byte cache lines hold these data values:
;;	0	+2K	...	128	+2K	...
;;	...
;;	127
;;	256	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+2K	...	1024	+2K	...
;;	...
;;	1023	...

;; Do 128 sixteen_reals_unfft operations
;; Do 896 eight_complex_unfft operations
;;	distance between fft data elements is 128
;;	do 128 macros each processing 128 data values

;; Do inverse FFT levels 1,2,3,4
;; On input the 128-byte cache line pairs hold these data values:
;;	0	+2K	...	1024	+2K	...
;;	...
;;	1023	...
;; These are swizzled to:
;;	0	+128	...	1024	+128	...
;;	...
;;	127	...
;;	2K	...
;;	...
;; On output the 128-byte cache line pairs hold these data values:
;;	0	+128	...	8K	+128	...
;;	...
;;	127	...
;;	1024	...
;;	...

;; Do 1024 sixteen_reals_unfft operations
;;	distance between fft data elements is 1024
;;	do 128 macros processing 128 data values

IFDEF OLDWAY
	start_timer 11
	zloop_init 128			;; 128 iterations
	zr8_sixteen_reals_eight_complex_djbunfft_preload
c2b:	zr8_sixteen_reals_eight_complex_djbunfft rsi, 128, 128*128, 256*128, 512*128, rdi, ZMM_SCD7, 128
	zloop	128, c2b, -128*128	;; Test loop counter, restore source pointer, restore sin/cos pointer
	end_timer 11

	start_timer 12
	zloop_init 128			;; 128 iterations
	zr8s_sixteen_reals_last_unfft_preload
c1b:	zr8s_sixteen_reals_last_unfft rsi, 128, 128*128, 256*128, 512*128, rdi, ZMM_SCD7, 128, L1PREFETCH_ALL
	zloop	128, c1b, -128*128	;; Test loop counter, restore source pointer
	end_timer 12
ELSE
	start_timer 12
	lea	r10, [rdi+128*ZMM_SCD7]	;; Second sin/cos pointer
	zloop_init 128			;; 128 iterations
	zr8s_hundredtwentyeight_reals_last_unfft_preload
c1b:	zr8s_hundredtwentyeight_reals_last_unfft rsi, 128, 128*128, 256*128, 512*128, rdi, ZMM_SCD7, r10, ZMM_SCD7, 128, L1PREFETCH_ALL
	zloop	128, c1b, -128*128	;; Test loop counter, restore source pointer
	end_timer 12
ENDIF

	zfft_3_ret
	ENDM

;; Perform a 16K-element negacyclic FFT

zfft16Kp MACRO type
	LOCAL	b1b, b2b, b3b, b4b, zmid, c4b, c3b, c2b, c1b

;; Do a multiply with pre-FFTed inputs

	cmp	ffttype, 4
	je	zmid

;; Do FFT levels 1,2,3,4
;; On input the 128-byte cache line pairs hold these data values:
;;	0	+128	...	8K	+128	...
;;	...
;;	127	...
;;	1024	...
;;	...
;; On output the 128-byte cache line pairs hold these data values:
;;	0	+128	...	1024	+128	...
;;	...
;;	127	...
;;	2K	...
;;	...
;; These are swizzled to:
;;	0	+2K	...	1024	+2K	...
;;	...
;;	1023	...

;; Do 1024 eight_complex_first_fft operations
;;	distance between fft data elements is 1024
;;	do 128 macros processing 128 data values

;; Do FFT levels 5,6,7
;; On input the 64-byte cache lines hold these data values:
;;	0	+2K	...	1024	+2K	...
;;	...
;;	1023	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+2K	...	128	+2K	...
;;	...
;;	127
;;	256	...
;;	...

;; Do 1024 eight_complex_fft operations
;;	distance between fft data elements is 128
;;	do 128 macros each processing 128 data values

IFDEF OLDWAY
	start_timer 4
	mov	rdi, sincos1		;; Load sin/cos pointer
	bump	rdi, 128*ZMM_SCD4/8+32*ZMM_SCD2/8
	mov	r9, norm_col_mults	;; Load weights pointer
	zloop_init 128			;; 128 iterations
	zr8fs_eight_complex_first_fft_preload
b1b:	zr8fs_eight_complex_first_fft rsi, 128, 128*128, 256*128, 512*128, rdi, ZMM_CD8SCD4, r9, 8*128, 128, L1PREFETCH_ALL
	zloop	128, b1b, -128*128	;; Test loop counter, restore source pointer
	end_timer 4

	start_timer 5
	bump	rdi, -128*ZMM_CD8SCD4-128*ZMM_SCD4/8 ;; Load sin/cos pointer
	zloop_init 128			;; 128 iterations
	zr8b_eight_complex_djbfft_preload
b2b:	zr8b_eight_complex_djbfft rsi, 128, 128*128, 256*128, 512*128, rdi, ZMM_SCD4/8, 128, L1PREFETCH_ALL, 128
	zloop	128, b2b, -128*128	;; Test loop counter, restore source pointer
	end_timer 5
ELSE
	start_timer 4
	mov	rdi, sincos1		;; Load sin/cos pointer
	bump	rdi, 32*ZMM_SCD2/8
	lea	r10, [rdi+128*ZMM_SCD4/8] ;; Second sin/cos pointer
	mov	r9, norm_col_mults	;; Load weights pointer
	zloop_init 128			;; 128 iterations
	zr8fs_sixtyfour_complex_first_fft_preload
b1b:	zr8fs_sixtyfour_complex_first_fft rsi, 128, 128*128, 256*128, 512*128, r10, ZMM_CD8SCD4, rdi, ZMM_SCD4/8, r9, 8*128, 128, L1PREFETCH_ALL
	zloop	128, b1b, -128*128	;; Test loop counter, restore source pointer
	end_timer 4
ENDIF

;; Do FFT levels 8,9
;; On input the 64-byte cache lines hold these data values:
;;	0	+2K	...	128	+2K	...
;;	...
;;	127
;;	256	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+2K	...	32	+2K	...
;;	...
;;	31
;;	64	...
;;	...

;; Do 2048 four_complex_fft operations
;;	distance between fft data elements is 32
;;	do 256 macros each processing 64 data values

;; Do FFT levels 10,11
;; On input the 64-byte cache lines hold these data values:
;;	0	+2K	...	32	+2K	...
;;	...
;;	31
;;	64	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+2K	...	8	+2K	...
;;	...
;;	7
;;	16	...
;;	32	...
;;	...

;; Do 2048 four_complex_fft operations
;;	distance between fft data elements is 8
;;	do 256 macros each processing 64 data values

IFDEF OLDWAY
	start_timer 6
	bump	rdi, -128*ZMM_SCD4/8-32*ZMM_SCD2/8 ;; Load sin/cos pointer
	zloop_init 256			;; 8 iterations of 32
	zr4b_four_complex_djbfft_preload
b3b:	zr4b_four_complex_djbfft rsi, 128, 32*128, 64*128, rdi, ZMM_SCD2/8, 32
	zloop	32, b3b, -32*128+128*128, rdi, -32*ZMM_SCD2/8 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	8, b3b, -8*128*128	;; Test loop counter, restore source pointer
	end_timer 6
	start_timer 7
	zloop_init 256			;; 32 iterations of 8
	zr4b_four_complex_djbfft_preload
b4b:	zr4b_four_complex_djbfft rsi, 128, 8*128, 16*128, rdi, 4*ZMM_SCD2/8, 8
	zloop	8, b4b, -8*128+32*128, rdi, -8*4*ZMM_SCD2/8 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	32, b4b, -32*32*128	;; Test loop counter, restore source pointer
	end_timer 7
ELSE
	start_timer 6
	bump	rdi, -128*ZMM_SCD4/8-32*ZMM_SCD2/8 ;; Load sin/cos pointers
	mov	r10, rdi
	zloop_init 64			;; 8 iterations of 8
	zr44b_sixteen_complex_djbfft_preload
b4b:	zr44b_sixteen_complex_djbfft rsi, 128, 8*128, 16*128, 32*128, 64*128, r10, 8*ZMM_SCD2/8, ZMM_SCD2/8, rdi, 0*4*ZMM_SCD2/8, 4*ZMM_SCD2/8, 8
	zloop	8, b4b, -8*128+128*128, r10, -8*ZMM_SCD2/8, rdi, -8*4*ZMM_SCD2/8 ;; Test loop counter, next source pointer, restore sin/cos pointers
	zloop	8, b4b, -8*128*128	;; Test loop counter, restore source pointer
	end_timer 6
ENDIF

;; Do FFT levels 12,13,14
;; On input and output the 128-byte cache line pairs hold these data values:
;;	0	+2K	...	8	+2K	...
;;	...
;;	7
;;	16	...
;;	32	...
;;	...

;; Do 1024 eight_complex_fft operations
;;	distance between fft data elements is 1
;;	do 128 macros processing 128 data values

zmid:	start_timer 8
	mov	rcx, 128
	call	zcomplex_onepass_middle
	end_timer 8

;; Do inverse FFT levels 10,11
;; On input the 64-byte cache lines hold these data values:
;;	0	+2K	...	8	+2K	...
;;	...
;;	7
;;	16	...
;;	32	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+2K	...	32	+2K	...
;;	...
;;	31
;;	64	...
;;	...

;; Do 2048 four_complex_unfft operations
;;	distance between fft data elements is 8
;;	do 256 macros each processing 64 data values

;; Do inverse FFT levels 8,9
;; On input the 64-byte cache lines hold these data values:
;;	0	+2K	...	32	+2K	...
;;	...
;;	31
;;	64	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+2K	...	128	+2K	...
;;	...
;;	127
;;	256	...
;;	...

;; Do 2048 four_complex_unfft operations
;;	distance between fft data elements is 32
;;	do 256 macros each processing 64 data values

IFDEF OLDWAY
	start_timer 9
	mov	rdi, sincos1		;; Load sin/cos pointer
	zloop_init 256			;; 32 iterations of 8
	zr4b_four_complex_djbunfft_preload
c4b:	zr4b_four_complex_djbunfft rsi, 128, 8*128, 16*128, rdi, 4*ZMM_SCD2/8, 8
	zloop	8, c4b, -8*128+32*128, rdi, -8*4*ZMM_SCD2/8 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	32, c4b, -32*32*128	;; Test loop counter, restore source pointer
	end_timer 9

	start_timer 10
	zloop_init 256			;; 8 iterations of 32
	zr4b_four_complex_djbunfft_preload
c3b:	zr4b_four_complex_djbunfft rsi, 128, 32*128, 64*128, rdi, ZMM_SCD2/8, 32
	zloop	32, c3b, -32*128+128*128, rdi, -32*ZMM_SCD2/8 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	8, c3b, -8*128*128, rdi, 32*ZMM_SCD2/8 ;; Test loop counter, restore source pointer, next sin/cos ptr
	end_timer 10
ELSE
	start_timer 9
	mov	rdi, sincos1		;; Load sin/cos pointers
	mov	r10, rdi
	zloop_init 64			;; 8 iterations of 8
	zr44b_sixteen_complex_djbunfft_preload
c3b:	zr44b_sixteen_complex_djbunfft rsi, 128, 8*128, 16*128, 32*128, 64*128, rdi, 0*4*ZMM_SCD2/8, 4*ZMM_SCD2/8, r10, 8*ZMM_SCD2/8, ZMM_SCD2/8, 8
	zloop	8, c3b, -8*128+128*128, rdi, -8*4*ZMM_SCD2/8, r10, -8*ZMM_SCD2/8 ;; Test loop counter, next source pointer, restore sin/cos pointers
	zloop	8, c3b, -8*128*128, rdi, 32*ZMM_SCD2/8 ;; Test loop counter, restore source pointer, next sin/cos ptr
	end_timer 9
ENDIF

;; Do inverse FFT levels 5,6,7
;; On input the 64-byte cache lines hold these data values:
;;	0	+2K	...	128	+2K	...
;;	...
;;	127
;;	256	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+2K	...	1024	+2K	...
;;	...
;;	1023	...

;; Do 1024 eight_complex_unfft operations
;;	distance between fft data elements is 128
;;	do 128 macros each processing 128 data values

;; Do inverse FFT levels 1,2,3,4
;; On input the 128-byte cache line pairs hold these data values:
;;	0	+2K	...	1024	+2K	...
;;	...
;;	1023	...
;; These are swizzled to:
;;	0	+128	...	1024	+128	...
;;	...
;;	127	...
;;	2K	...
;;	...
;; On output the 128-byte cache line pairs hold these data values:
;;	0	+128	...	8K	+128	...
;;	...
;;	127	...
;;	1024	...
;;	...

;; Do 1024 eight_complex_last_unfft operations
;;	distance between fft data elements is 1024
;;	do 128 macros processing 128 data values

IFDEF OLDWAY
	start_timer 11
	zloop_init 128			;; 128 iterations
	zr8b_eight_complex_djbunfft_preload
c2b:	zr8b_eight_complex_djbunfft rsi, 128, 128*128, 256*128, 512*128, rdi, ZMM_SCD4/8, 128, L1PREFETCH_ALL, 128
	zloop	128, c2b, -128*128	;; Test loop counter, restore source pointer
	end_timer 11

	start_timer 12
	zloop_init 128			;; 128 iterations
	zr8s_eight_complex_last_unfft_preload
c1b:	zr8s_eight_complex_last_unfft rsi, 128, 128*128, 256*128, 512*128, rdi, ZMM_CD8SCD4, 128, L1PREFETCH_ALL
	zloop	128, c1b, -128*128	;; Test loop counter, restore source pointer
	end_timer 12
ELSE
	start_timer 12
	lea	r10, [rdi+128*ZMM_SCD4/8] ;; Second sin/cos pointer
	zloop_init 128			;; 128 iterations
	zr8s_sixtyfour_complex_last_unfft_preload
c1b:	zr8s_sixtyfour_complex_last_unfft rsi, 128, 128*128, 256*128, 512*128, rdi, ZMM_SCD4/8, r10, ZMM_CD8SCD4, 128, L1PREFETCH_ALL
	zloop	128, c1b, -128*128	;; Test loop counter, restore source pointer
	end_timer 12
ENDIF

	zfft_3_ret
	ENDM


;; Perform a 18K-element FFT

zfft18K MACRO type
	LOCAL	b1b, b2b, b3a, b3b, b4a, b4b, zmid, c4a, c4b, c3a, c3b, c2b, c1b

;; Check for a multiply with pre-FFTed inputs

	cmp	ffttype, 4
	je	zmid

;; Do FFT levels 1,2,3,4
;; On input the 128-byte cache line pairs hold these data values:
;;	0	+144	...	9K	+144	...
;;	...
;;	143	...
;;	1152	...
;;	...
;; On output the 128-byte cache line pairs hold these data values:
;;	0	+144	...	1152	+144	...
;;	...
;;	143	...
;;	2304	...
;;	...
;; These are swizzled to:
;;	0	+2304	...	1152	+2304	...
;;	...
;;	1151	...

;; Do 1152 sixteen_reals_fft operations
;;	distance between fft data elements is 1152
;;	do 144 macros processing 128 data values

;; Do FFT levels 5,6,7
;; On input the 64-byte cache lines hold these data values:
;;	0	+2304	...	1152	+2304	...
;;	...
;;	1151	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+2304	...	144	+2304	...
;;	...
;;	143
;;	288	...
;;	...

;; Do 144 sixteen_reals_fft operation
;; Do 1008 eight_complex_fft operations
;;	distance between fft data elements is 144
;;	do 144 macros each processing 128 data values

IFDEF OLDWAY
	start_timer 4
	mov	rdi, sincos1		;; Load sin/cos data ptr
	bump	rdi, 144*ZMM_SCD7+48*ZMM_SCD2+8*ZMM_SCD5
	mov	r9, norm_col_mults	;; Load weights pointer
	zloop_init 144			;; 144 iterations
	zr8fs_sixteen_reals_first_fft_preload
b1b:	zr8fs_sixteen_reals_first_fft rsi, 128, 144*128, 288*128, 576*128, rdi, ZMM_SCD7, r9, 8*128, 144, L1PREFETCH_ALL
	zloop	144, b1b, -144*128	;; Test loop counter, restore source pointer
	end_timer 4

	start_timer 5
	bump	rdi, -144*ZMM_SCD7-144*ZMM_SCD7 ;; Load sin/cos pointer
	zloop_init 144			;; 144 iterations
	zr8_sixteen_reals_eight_complex_djbfft_preload
b2b:	zr8_sixteen_reals_eight_complex_djbfft rsi, 128, 144*128, 288*128, 576*128, rdi, ZMM_SCD7, 144, L1PREFETCH_ALL
	zloop	144, b2b, -144*128	;; Test loop counter, restore source pointer
	end_timer 5
ELSE
	start_timer 4
	mov	rdi, sincos1		;; Load sin/cos data ptr
	bump	rdi, 48*ZMM_SCD2+8*ZMM_SCD5
	lea	r10, [rdi+144*ZMM_SCD7]	;; Second sin/cos pointer
	mov	r9, norm_col_mults	;; Load weights pointer
	zloop_init 144			;; 144 iterations
	zr8fs_hundredtwentyeight_reals_first_fft_preload
b1b:	zr8fs_hundredtwentyeight_reals_first_fft rsi, 128, 144*128, 288*128, 576*128, r10, ZMM_SCD7, rdi, ZMM_SCD7, r9, 8*128, 144, L1PREFETCH_ALL
	zloop	144, b1b, -144*128	;; Test loop counter, restore source pointer
	end_timer 4
ENDIF

;; Do FFT levels 8,9
;; On input the 64-byte cache lines hold these data values:
;;	0	+2304	...	144	+2304	...
;;	...
;;	143
;;	288	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+2304	...	48	+2304	...
;;	...
;;	47
;;	96	...
;;	...

;; Do 48 six_reals_fft operation
;; Do 3024 three_complex_fft operations
;;	distance between fft data elements is 48
;;	do 384 macros each processing 48 data values

	start_timer 6
	bump	rdi, -144*ZMM_SCD7-48*ZMM_SCD2
	zloop_init 384			;; 8 iterations of 48
	zr3_six_reals_three_complex_djbfft_preload
b3a:	zr3_six_reals_three_complex_djbfft rsi, 128, 48*128, rdi, ZMM_SCD2, 48
	zloop	48, b3a, -48*128+144*128, rdi, -48*ZMM_SCD2 ;; Test loop counter, next source pointer, restore sin/cos pointer

	loops_reset			;; 7 more iters of 48
	zr3rb_three_complex_djbfft_preload
b3b:	zr3rb_three_complex_djbfft rsi, 128, 48*128, rdi, ZMM_SCD2, 48
	zloop	48, b3b, -48*128+144*128, rdi, -48*ZMM_SCD2 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	8, b3b, -8*144*128	;; Test loop counter, restore source pointer
	end_timer 6

;; Do FFT levels 10,11,12
;; On input the 64-byte cache lines hold these data values:
;;	0	+2304	...	48	+2304	...
;;	...
;;	47
;;	96	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+2304	...	8	+2304	...
;;	...
;;	7
;;	16	...
;;	32	...
;;	...

;; Do 8 twelve_reals_fft operation
;; Do 1528 six_complex_fft operations
;;	distance between fft data elements is 8
;;	do 192 macros each processing 96 data values

	start_timer 7
	bump	rdi, -8*ZMM_SCD5
	zloop_init 192			;; 24 iterations of 8
	zr6_twelve_reals_six_complex_djbfft_preload
b4a:	zr6_twelve_reals_six_complex_djbfft rsi, 128, 8*128, rdi, ZMM_SCD5, 8
	zloop	8, b4a, -8*128+48*128, rdi, -8*ZMM_SCD5 ;; Test loop counter, next source pointer, restore sin/cos pointer

	loops_reset			;; 23 more iters of 8
	zr6rb_six_complex_djbfft_preload
b4b:	zr6rb_six_complex_djbfft rsi, 128, 8*128, rdi, ZMM_SCD5, 8
	zloop	8, b4b, -8*128+48*128, rdi, -8*ZMM_SCD5 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	24, b4b, -24*48*128	;; Test loop counter, restore source pointer
	end_timer 7

;; Do FFT levels 13,14,15
;; On input and output the 128-byte cache line pairs hold these data values:
;;	0	+2304	...	8	+2304	...
;;	...
;;	7
;;	16	...
;;	32	...
;;	...

;; Do 1 sixteen_reals_fft operation
;; Do 1151 eight_complex_fft operations
;;	distance between fft data elements is 1
;;	do 144 macros processing 128 data values

zmid:	start_timer 8
	mov	rcx, 144
	call	zreal_onepass_middle
	end_timer 8

;; Do inverse FFT levels 10,11,12
;; On input the 64-byte cache lines hold these data values:
;;	0	+2304	...	8	+2304	...
;;	...
;;	7
;;	16	...
;;	32	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+2304	...	48	+2304	...
;;	...
;;	47
;;	96	...
;;	...

;; Do 8 twelve_reals_unfft operations
;; Do 1528 six_complex_unfft operations
;;	distance between fft data elements is 8
;;	do 192 macros each processing 96 data values

	start_timer 9
	mov	rdi, sincos1		;; Load sin/cos pointer
	zloop_init 192			;; 24 iterations of 8
	zr6_twelve_reals_six_complex_djbunfft_preload
c4a:	zr6_twelve_reals_six_complex_djbunfft rsi, 128, 8*128, rdi, ZMM_SCD5, 8
	zloop	8, c4a, -8*128+48*128, rdi, -8*ZMM_SCD5 ;; Test loop counter, restore source pointer, restore sin/cos pointer

	loops_reset			;; 23 more iters of 8
	zr6rb_six_complex_djbunfft_preload
c4b:	zr6rb_six_complex_djbunfft rsi, 128, 8*128, rdi, ZMM_SCD5, 8
	zloop	8, c4b, -8*128+48*128, rdi, -8*ZMM_SCD5 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	24, c4b, -24*48*128, rdi, 8*ZMM_SCD5 ;; Test loop counter, restore source pointer, next sin/cos ptr
	end_timer 9

;; Do inverse FFT levels 8,9
;; On input the 64-byte cache lines hold these data values:
;;	0	+2304	...	48	+2304	...
;;	...
;;	47
;;	96	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+2304	...	144	+2304	...
;;	...
;;	143
;;	288	...
;;	...

;; Do 48 six_reals_unfft operations
;; Do 3024 three_complex_unfft operations
;;	distance between fft data elements is 48
;;	do 384 macros each processing 48 data values

	start_timer 10
	zloop_init 384			;; 8 iterations of 48
	zr3_six_reals_three_complex_djbunfft_preload
c3a:	zr3_six_reals_three_complex_djbunfft rsi, 128, 48*128, rdi, ZMM_SCD2, 48
	zloop	48, c3a, -48*128+144*128, rdi, -48*ZMM_SCD2 ;; Test loop counter, restore source pointer, restore sin/cos pointer

	loops_reset			;; 7 more iters of 48
	zr3rb_three_complex_djbunfft_preload
c3b:	zr3rb_three_complex_djbunfft rsi, 128, 48*128, rdi, ZMM_SCD2, 48
	zloop	48, c3b, -48*128+144*128, rdi, -48*ZMM_SCD2 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	8, c3b, -8*144*128, rdi, 48*ZMM_SCD2 ;; Test loop counter, restore source pointer, next sin/cos ptr
	end_timer 10

;; Do inverse FFT levels 5,6,7
;; On input the 64-byte cache lines hold these data values:
;;	0	+2304	...	144	+2304	...
;;	...
;;	143
;;	288	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+2304	...	1152	+2304	...
;;	...
;;	1151	...

;; Do 144 sixteen_reals_unfft operations
;; Do 1008 eight_complex_unfft operations
;;	distance between fft data elements is 144
;;	do 144 macros each processing 128 data values

;; Do inverse FFT levels 1,2,3,4
;; On input the 128-byte cache line pairs hold these data values:
;;	0	+2304	...	1152	+2304	...
;;	...
;;	1151	...
;; These are swizzled to:
;;	0	+144	...	1152	+144	...
;;	...
;;	143	...
;;	2304	...
;;	...
;; On output the 128-byte cache line pairs hold these data values:
;;	0	+144	...	9K	+144	...
;;	...
;;	143	...
;;	1152	...
;;	...

;; Do 1152 sixteen_reals_unfft operations
;;	distance between fft data elements is 1152
;;	do 144 macros processing 128 data values

IFDEF OLDWAY
	start_timer 11
	zloop_init 144			;; 144 iterations
	zr8_sixteen_reals_eight_complex_djbunfft_preload
c2b:	zr8_sixteen_reals_eight_complex_djbunfft rsi, 128, 144*128, 288*128, 576*128, rdi, ZMM_SCD7, 144
	zloop	144, c2b, -144*128	;; Test loop counter, restore source pointer, restore sin/cos pointer
	end_timer 11

	start_timer 12
	zloop_init 144			;; 144 iterations
	zr8s_sixteen_reals_last_unfft_preload
c1b:	zr8s_sixteen_reals_last_unfft rsi, 128, 144*128, 288*128, 576*128, rdi, ZMM_SCD7, 144, L1PREFETCH_ALL
	zloop	144, c1b, -144*128	;; Test loop counter, restore source pointer
	end_timer 12
ELSE
	start_timer 12
	lea	r10, [rdi+144*ZMM_SCD7]	;; Second sin/cos pointer
	zloop_init 144			;; 144 iterations
	zr8s_hundredtwentyeight_reals_last_unfft_preload
c1b:	zr8s_hundredtwentyeight_reals_last_unfft rsi, 128, 144*128, 288*128, 576*128, rdi, ZMM_SCD7, r10, ZMM_SCD7, 144, L1PREFETCH_ALL
	zloop	144, c1b, -144*128	;; Test loop counter, restore source pointer
	end_timer 12
ENDIF

	zfft_3_ret
	ENDM

;; Perform a 18K-element negacyclic FFT

zfft18Kp MACRO type
	LOCAL	b1b, b2b, b3b, b4b, zmid, c4b, c3b, c2b, c1b

;; Do a multiply with pre-FFTed inputs

	cmp	ffttype, 4
	je	zmid

;; Do FFT levels 1,2,3,4
;; On input the 128-byte cache line pairs hold these data values:
;;	0	+144	...	9K	+144	...
;;	...
;;	143	...
;;	1152	...
;;	...
;; On output the 128-byte cache line pairs hold these data values:
;;	0	+144	...	1152	+144	...
;;	...
;;	143	...
;;	2304	...
;;	...
;; These are swizzled to:
;;	0	+2304	...	1152	+2304	...
;;	...
;;	1151	...

;; Do 1152 eight_complex_first_fft operations
;;	distance between fft data elements is 1152
;;	do 144 macros processing 128 data values

;; Do FFT levels 5,6,7
;; On input the 64-byte cache lines hold these data values:
;;	0	+2304	...	1152	+2304	...
;;	...
;;	1151	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+2304	...	144	+2304	...
;;	...
;;	143
;;	288	...
;;	...

;; Do 1152 eight_complex_fft operations
;;	distance between fft data elements is 144
;;	do 144 macros each processing 128 data values

IFDEF OLDWAY
	start_timer 4
	mov	rdi, sincos1		;; Load sin/cos pointer
	bump	rdi, 144*ZMM_SCD4/8+48*ZMM_SCD1/8+8*ZMM_SCD3/8
	mov	r9, norm_col_mults	;; Load weights pointer
	zloop_init 144			;; 144 iterations
	zr8fs_eight_complex_first_fft_preload
b1b:	zr8fs_eight_complex_first_fft rsi, 128, 144*128, 288*128, 576*128, rdi, ZMM_CD8SCD4, r9, 8*128, 144, L1PREFETCH_ALL
	zloop	144, b1b, -144*128	;; Test loop counter, restore source pointer
	end_timer 4

	start_timer 5
	bump	rdi, -144*ZMM_CD8SCD4-144*ZMM_SCD4/8 ;; Load sin/cos pointer
	zloop_init 144			;; 144 iterations
	zr8b_eight_complex_djbfft_preload
b2b:	zr8b_eight_complex_djbfft rsi, 128, 144*128, 288*128, 576*128, rdi, ZMM_SCD4/8, 144, L1PREFETCH_ALL, 128
	zloop	144, b2b, -144*128	;; Test loop counter, restore source pointer
	end_timer 5
ELSE
	start_timer 4
	mov	rdi, sincos1		;; Load sin/cos pointer
	bump	rdi, 48*ZMM_SCD1/8+8*ZMM_SCD3/8
	lea	r10, [rdi+144*ZMM_SCD4/8] ;; Second sin/cos pointer
	mov	r9, norm_col_mults	;; Load weights pointer
	zloop_init 144			;; 144 iterations
	zr8fs_sixtyfour_complex_first_fft_preload
b1b:	zr8fs_sixtyfour_complex_first_fft rsi, 128, 144*128, 288*128, 576*128, r10, ZMM_CD8SCD4, rdi, ZMM_SCD4/8, r9, 8*128, 144, L1PREFETCH_ALL
	zloop	144, b1b, -144*128	;; Test loop counter, restore source pointer
	end_timer 4
ENDIF

;; Do FFT levels 8,9
;; On input the 64-byte cache lines hold these data values:
;;	0	+2304	...	144	+2304	...
;;	...
;;	143
;;	288	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+2304	...	48	+2304	...
;;	...
;;	47
;;	96	...
;;	...

;; Do 3072 three_complex_fft operations
;;	distance between fft data elements is 48
;;	do 384 macros each processing 48 data values

	start_timer 6
	bump	rdi, -144*ZMM_SCD4/8-48*ZMM_SCD1/8 ;; Load sin/cos pointer
	zloop_init 384			;; 8 iterations of 48
	zr3b_three_complex_djbfft_preload
b3b:	zr3b_three_complex_djbfft rsi, 128, 48*128, rdi, ZMM_SCD1/8, 48
	zloop	48, b3b, -48*128+144*128, rdi, -48*ZMM_SCD1/8 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	8, b3b, -8*144*128	;; Test loop counter, restore source pointer
	end_timer 6

;; Do FFT levels 10,11,12
;; On input the 64-byte cache lines hold these data values:
;;	0	+2304	...	48	+2304	...
;;	...
;;	47
;;	96	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+2304	...	8	+2304	...
;;	...
;;	7
;;	16	...
;;	32	...
;;	...

;; Do 1536 six_complex_fft operations
;;	distance between fft data elements is 8
;;	do 192 macros each processing 96 data values

	start_timer 7
	bump	rdi, -8*ZMM_SCD3/8	;; Load sin/cos pointer
	zloop_init 192			;; 24 iterations of 8
	zr6b_six_complex_djbfft_preload
b4b:	zr6b_six_complex_djbfft rsi, 128, 8*128, rdi, ZMM_SCD3/8, 8
	zloop	8, b4b, -8*128+48*128, rdi, -8*ZMM_SCD3/8 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	24, b4b, -24*48*128	;; Test loop counter, restore source pointer
	end_timer 7

;; Do FFT levels 13,14,15
;; On input and output the 128-byte cache line pairs hold these data values:
;;	0	+2304	...	8	+2304	...
;;	...
;;	7
;;	16	...
;;	32	...
;;	...

;; Do 1152 eight_complex_fft operations
;;	distance between fft data elements is 1
;;	do 144 macros processing 128 data values

zmid:	start_timer 8
	mov	rcx, 144
	call	zcomplex_onepass_middle
	end_timer 8

;; Do inverse FFT levels 10,11,12
;; On input the 64-byte cache lines hold these data values:
;;	0	+2304	...	8	+2304	...
;;	...
;;	7
;;	16	...
;;	32	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+2304	...	48	+2304	...
;;	...
;;	47
;;	96	...
;;	...

;; Do 1536 six_complex_unfft operations
;;	distance between fft data elements is 8
;;	do 192 macros each processing 96 data values

	start_timer 9
	mov	rdi, sincos1		;; Load sin/cos pointer
	zloop_init 192			;; 24 iterations of 8
	zr6b_six_complex_djbunfft_preload
c4b:	zr6b_six_complex_djbunfft rsi, 128, 8*128, rdi, ZMM_SCD3/8, 8
	zloop	8, c4b, -8*128+48*128, rdi, -8*ZMM_SCD3/8 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	24, c4b, -24*48*128, rdi, 8*ZMM_SCD3/8	;; Test loop counter, restore source pointer, next sin/cos ptr
	end_timer 9

;; Do inverse FFT levels 8,9
;; On input the 64-byte cache lines hold these data values:
;;	0	+2304	...	48	+2304	...
;;	...
;;	47
;;	96	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+2304	...	144	+2304	...
;;	...
;;	143
;;	288	...
;;	...

;; Do 3072 three_complex_unfft operations
;;	distance between fft data elements is 48
;;	do 384 macros each processing 48 data values

	start_timer 10
	zloop_init 384			;; 8 iterations of 48
	zr3b_three_complex_djbunfft_preload
c3b:	zr3b_three_complex_djbunfft rsi, 128, 48*128, rdi, ZMM_SCD1/8, 48
	zloop	48, c3b, -48*128+144*128, rdi, -48*ZMM_SCD1/8 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	8, c3b, -8*144*128, rdi, 48*ZMM_SCD1/8 ;; Test loop counter, restore source pointer, next sin/cos ptr
	end_timer 10

;; Do inverse FFT levels 5,6,7
;; On input the 64-byte cache lines hold these data values:
;;	0	+2304	...	144	+2304	...
;;	...
;;	143
;;	288	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+2304	...	1152	+2304	...
;;	...
;;	1151	...

;; Do 1152 eight_complex_unfft operations
;;	distance between fft data elements is 144
;;	do 144 macros each processing 128 data values

;; Do inverse FFT levels 1,2,3,4
;; On input the 128-byte cache line pairs hold these data values:
;;	0	+2304	...	1152	+2304	...
;;	...
;;	1151	...
;; These are swizzled to:
;;	0	+144	...	1152	+144	...
;;	...
;;	143	...
;;	2304	...
;;	...
;; On output the 128-byte cache line pairs hold these data values:
;;	0	+144	...	9K	+144	...
;;	...
;;	143	...
;;	1152	...
;;	...

;; Do 1152 eight_complex_last_unfft operations
;;	distance between fft data elements is 1152
;;	do 144 macros processing 128 data values

IFDEF OLDWAY
	start_timer 11
	zloop_init 144			;; 144 iterations
	zr8b_eight_complex_djbunfft_preload
c2b:	zr8b_eight_complex_djbunfft rsi, 128, 144*128, 288*128, 576*128, rdi, ZMM_SCD4/8, 144, L1PREFETCH_ALL, 128
	zloop	144, c2b, -144*128	;; Test loop counter, restore source pointer
	end_timer 11

	start_timer 12
	zloop_init 144			;; 144 iterations
	zr8s_eight_complex_last_unfft_preload
c1b:	zr8s_eight_complex_last_unfft rsi, 128, 144*128, 288*128, 576*128, rdi, ZMM_CD8SCD4, 144, L1PREFETCH_ALL
	zloop	144, c1b, -144*128	;; Test loop counter, restore source pointer
	end_timer 12
ELSE
	start_timer 12
	lea	r10, [rdi+144*ZMM_SCD4/8] ;; Second sin/cos pointer
	zloop_init 144			;; 144 iterations
	zr8s_sixtyfour_complex_last_unfft_preload
c1b:	zr8s_sixtyfour_complex_last_unfft rsi, 128, 144*128, 288*128, 576*128, rdi, ZMM_SCD4/8, r10, ZMM_CD8SCD4, 144, L1PREFETCH_ALL
	zloop	144, c1b, -144*128	;; Test loop counter, restore source pointer
	end_timer 12
ENDIF

	zfft_3_ret
	ENDM


;; Perform a 20K-element FFT

zfft20K MACRO type
	LOCAL	b1b, b2b, b3a, b3b, b4a, b4b, zmid, c4a, c4b, c3a, c3b, c2b, c1b

;; Check for a multiply with pre-FFTed inputs

	cmp	ffttype, 4
	je	zmid

;; Do FFT levels 1,2,3,4
;; On input the 128-byte cache line pairs hold these data values:
;;	0	+160	...	10K	+160	...
;;	...
;;	159	...
;;	1280	...
;;	...
;; On output the 128-byte cache line pairs hold these data values:
;;	0	+160	...	1280	+160	...
;;	...
;;	159	...
;;	2560	...
;;	...
;; These are swizzled to:
;;	0	+2560	...	1280	+2560	...
;;	...
;;	1279	...

;; Do 1280 sixteen_reals_fft operations
;;	distance between fft data elements is 1280
;;	do 160 macros processing 128 data values

;; Do FFT levels 5,6,7
;; On input the 64-byte cache lines hold these data values:
;;	0	+2560	...	1280	+2560	...
;;	...
;;	1279	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+2560	...	160	+2560	...
;;	...
;;	159
;;	320	...
;;	...

;; Do 160 sixteen_reals_fft operation
;; Do 1120 eight_complex_fft operations
;;	distance between fft data elements is 160
;;	do 160 macros each processing 128 data values

IFDEF OLDWAY
	start_timer 4
	mov	rdi, sincos1		;; Load sin/cos data ptr
	bump	rdi, 160*ZMM_SCD7+32*ZMM_SCD4+8*ZMM_SCD3
	mov	r9, norm_col_mults	;; Load weights pointer
	zloop_init 160			;; 160 iterations
	zr8fs_sixteen_reals_first_fft_preload
b1b:	zr8fs_sixteen_reals_first_fft rsi, 128, 160*128, 320*128, 640*128, rdi, ZMM_SCD7, r9, 8*128, 160, L1PREFETCH_ALL
	zloop	160, b1b, -160*128	;; Test loop counter, restore source pointer
	end_timer 4

	start_timer 5
	bump	rdi, -160*ZMM_SCD7-160*ZMM_SCD7 ;; Load sin/cos pointer
	zloop_init 160			;; 160 iterations
	zr8_sixteen_reals_eight_complex_djbfft_preload
b2b:	zr8_sixteen_reals_eight_complex_djbfft rsi, 128, 160*128, 320*128, 640*128, rdi, ZMM_SCD7, 160, L1PREFETCH_ALL
	zloop	160, b2b, -160*128	;; Test loop counter, restore source pointer
	end_timer 5
ELSE
	start_timer 4
	mov	rdi, sincos1		;; Load sin/cos data ptr
	bump	rdi, 32*ZMM_SCD4+8*ZMM_SCD3
	lea	r10, [rdi+160*ZMM_SCD7]	;; Second sin/cos pointer
	mov	r9, norm_col_mults	;; Load weights pointer
	zloop_init 160			;; 160 iterations
	zr8fs_hundredtwentyeight_reals_first_fft_preload
b1b:	zr8fs_hundredtwentyeight_reals_first_fft rsi, 128, 160*128, 320*128, 640*128, r10, ZMM_SCD7, rdi, ZMM_SCD7, r9, 8*128, 160, L1PREFETCH_ALL
	zloop	160, b1b, -160*128	;; Test loop counter, restore source pointer
	end_timer 4
ENDIF

;; Do FFT levels 8,9,10
;; On input the 64-byte cache lines hold these data values:
;;	0	+2560	...	160	+2560	...
;;	...
;;	159
;;	320	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+2560	...	32	+2560	...
;;	...
;;	31
;;	64	...
;;	...

;; Do 32 ten_reals_fft operation
;; Do 2016 five_complex_fft operations
;;	distance between fft data elements is 32
;;	do 256 macros each processing 80 data values

	start_timer 6
	bump	rdi, -160*ZMM_SCD7-32*ZMM_SCD4
	zloop_init 256			;; 8 iterations of 32
	zr5_ten_reals_five_complex_djbfft_preload
b3a:	zr5_ten_reals_five_complex_djbfft rsi, 128, 32*128, rdi, ZMM_SCD4, 32
	zloop	32, b3a, -32*128+160*128, rdi, -32*ZMM_SCD4 ;; Test loop counter, next source pointer, restore sin/cos pointer

	loops_reset			;; 7 more iters of 32
	zr5rb_five_complex_djbfft_preload
b3b:	zr5rb_five_complex_djbfft rsi, 128, 32*128, rdi, ZMM_SCD4, 32
	zloop	32, b3b, -32*128+160*128, rdi, -32*ZMM_SCD4 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	8, b3b, -8*160*128	;; Test loop counter, restore source pointer
	end_timer 6

;; Do FFT levels 11,12
;; On input the 64-byte cache lines hold these data values:
;;	0	+2560	...	32	+2560	...
;;	...
;;	31
;;	64	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+2560	...	8	+2560	...
;;	...
;;	7
;;	16	...
;;	32	...
;;	...

;; Do 8 eight_reals_fft operation
;; Do 2552 four_complex_fft operations
;;	distance between fft data elements is 8
;;	do 320 macros each processing 64 data values

	start_timer 7
	bump	rdi, -8*ZMM_SCD3
	zloop_init 320			;; 40 iterations of 8
	zr4_eight_reals_four_complex_djbfft_preload
b4a:	zr4_eight_reals_four_complex_djbfft rsi, 128, 8*128, 16*128, rdi, ZMM_SCD3, 8
	zloop	8, b4a, -8*128+32*128, rdi, -8*ZMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer

	loops_reset			;; 39 more iters of 8
	zr4rb_four_complex_djbfft_preload
b4b:	zr4rb_four_complex_djbfft rsi, 128, 8*128, 16*128, rdi, ZMM_SCD3, 8
	zloop	8, b4b, -8*128+32*128, rdi, -8*ZMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	40, b4b, -40*32*128	;; Test loop counter, restore source pointer
	end_timer 7

;; Do FFT levels 13,14,15
;; On input and output the 128-byte cache line pairs hold these data values:
;;	0	+2560	...	8	+2560	...
;;	...
;;	7
;;	16	...
;;	32	...
;;	...

;; Do 1 sixteen_reals_fft operation
;; Do 1279 eight_complex_fft operations
;;	distance between fft data elements is 1
;;	do 160 macros processing 128 data values

zmid:	start_timer 8
	mov	rcx, 160
	call	zreal_onepass_middle
	end_timer 8

;; Do inverse FFT levels 11,12
;; On input the 64-byte cache lines hold these data values:
;;	0	+2560	...	8	+2560	...
;;	...
;;	7
;;	16	...
;;	32	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+2560	...	32	+2560	...
;;	...
;;	31
;;	64	...
;;	...

;; Do 8 eight_reals_unfft operations
;; Do 2552 four_complex_unfft operations
;;	distance between fft data elements is 8
;;	do 320 macros each processing 64 data values

	start_timer 9
	mov	rdi, sincos1		;; Load sin/cos pointer
	zloop_init 320			;; 40 iterations of 8
	zr4_eight_reals_four_complex_djbunfft_preload
c4a:	zr4_eight_reals_four_complex_djbunfft rsi, 128, 8*128, 16*128, rdi, ZMM_SCD3, 8
	zloop	8, c4a, -8*128+32*128, rdi, -8*ZMM_SCD3 ;; Test loop counter, restore source pointer, restore sin/cos pointer

	loops_reset			;; 39 more iters of 8
	zr4rb_four_complex_djbunfft_preload
c4b:	zr4rb_four_complex_djbunfft rsi, 128, 8*128, 16*128, rdi, ZMM_SCD3, 8
	zloop	8, c4b, -8*128+32*128, rdi, -8*ZMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	40, c4b, -40*32*128, rdi, 8*ZMM_SCD3 ;; Test loop counter, restore source pointer, next sin/cos ptr
	end_timer 9

;; Do inverse FFT levels 8,9,10
;; On input the 64-byte cache lines hold these data values:
;;	0	+2560	...	32	+2560	...
;;	...
;;	31
;;	64	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+2560	...	160	+2560	...
;;	...
;;	159
;;	320	...
;;	...

;; Do 32 ten_reals_unfft operations
;; Do 2016 five_complex_unfft operations
;;	distance between fft data elements is 32
;;	do 256 macros each processing 80 data values

	start_timer 10
	zloop_init 256			;; 8 iterations of 32
	zr5_ten_reals_five_complex_djbunfft_preload
c3a:	zr5_ten_reals_five_complex_djbunfft rsi, 128, 32*128, rdi, ZMM_SCD4, 32
	zloop	32, c3a, -32*128+160*128, rdi, -32*ZMM_SCD4 ;; Test loop counter, restore source pointer, restore sin/cos pointer

	loops_reset			;; 7 more iters of 32
	zr5rb_five_complex_djbunfft_preload
c3b:	zr5rb_five_complex_djbunfft rsi, 128, 32*128, rdi, ZMM_SCD4, 32
	zloop	32, c3b, -32*128+160*128, rdi, -32*ZMM_SCD4 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	8, c3b, -8*160*128, rdi, 32*ZMM_SCD4 ;; Test loop counter, restore source pointer, next sin/cos ptr
	end_timer 10

;; Do inverse FFT levels 5,6,7
;; On input the 64-byte cache lines hold these data values:
;;	0	+2560	...	160	+2560	...
;;	...
;;	159
;;	320	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+2560	...	1280	+2560	...
;;	...
;;	1279	...

;; Do 160 sixteen_reals_unfft operations
;; Do 1120 eight_complex_unfft operations
;;	distance between fft data elements is 160
;;	do 160 macros each processing 128 data values

;; Do inverse FFT levels 1,2,3,4
;; On input the 128-byte cache line pairs hold these data values:
;;	0	+2560	...	1280	+2560	...
;;	...
;;	1279	...
;; These are swizzled to:
;;	0	+160	...	1280	+160	...
;;	...
;;	159	...
;;	2560	...
;;	...
;; On output the 128-byte cache line pairs hold these data values:
;;	0	+160	...	10K	+160	...
;;	...
;;	159	...
;;	1280	...
;;	...

;; Do 1280 sixteen_reals_unfft operations
;;	distance between fft data elements is 1280
;;	do 160 macros processing 128 data values

IFDEF OLDWAY
	start_timer 11
	zloop_init 160			;; 160 iterations
	zr8_sixteen_reals_eight_complex_djbunfft_preload
c2b:	zr8_sixteen_reals_eight_complex_djbunfft rsi, 128, 160*128, 320*128, 640*128, rdi, ZMM_SCD7, 160
	zloop	160, c2b, -160*128	;; Test loop counter, restore source pointer, restore sin/cos pointer
	end_timer 11

	start_timer 12
	zloop_init 160			;; 160 iterations
	zr8s_sixteen_reals_last_unfft_preload
c1b:	zr8s_sixteen_reals_last_unfft rsi, 128, 160*128, 320*128, 640*128, rdi, ZMM_SCD7, 160, L1PREFETCH_ALL
	zloop	160, c1b, -160*128	;; Test loop counter, restore source pointer
	end_timer 12
ELSE
	start_timer 12
	lea	r10, [rdi+160*ZMM_SCD7]	;; Second sin/cos pointer
	zloop_init 160			;; 160 iterations
	zr8s_hundredtwentyeight_reals_last_unfft_preload
c1b:	zr8s_hundredtwentyeight_reals_last_unfft rsi, 128, 160*128, 320*128, 640*128, rdi, ZMM_SCD7, r10, ZMM_SCD7, 160, L1PREFETCH_ALL
	zloop	160, c1b, -160*128	;; Test loop counter, restore source pointer
	end_timer 12
ENDIF

	zfft_3_ret
	ENDM

;; Perform a 20K-element negacyclic FFT

zfft20Kp MACRO type
	LOCAL	b1b, b2b, b3b, b4b, zmid, c4b, c3b, c2b, c1b

;; Do a multiply with pre-FFTed inputs

	cmp	ffttype, 4
	je	zmid

;; Do FFT levels 1,2,3,4
;; On input the 128-byte cache line pairs hold these data values:
;;	0	+160	...	10K	+160	...
;;	...
;;	159	...
;;	1280	...
;;	...
;; On output the 128-byte cache line pairs hold these data values:
;;	0	+160	...	1280	+160	...
;;	...
;;	159	...
;;	2560	...
;;	...
;; These are swizzled to:
;;	0	+2560	...	1280	+2560	...
;;	...
;;	1279	...

;; Do 1280 eight_complex_first_fft operations
;;	distance between fft data elements is 1280
;;	do 160 macros processing 128 data values

;; Do FFT levels 5,6,7
;; On input the 64-byte cache lines hold these data values:
;;	0	+2560	...	1280	+2560	...
;;	...
;;	1279	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+2560	...	160	+2560	...
;;	...
;;	159
;;	320	...
;;	...

;; Do 1280 eight_complex_fft operations
;;	distance between fft data elements is 160
;;	do 160 macros each processing 128 data values

IFDEF OLDWAY
	start_timer 4
	mov	rdi, sincos1		;; Load sin/cos pointer
	bump	rdi, 160*ZMM_SCD4/8+32*ZMM_SCD2/8+8*ZMM_SCD2/8
	mov	r9, norm_col_mults	;; Load weights pointer
	zloop_init 160			;; 160 iterations
	zr8fs_eight_complex_first_fft_preload
b1b:	zr8fs_eight_complex_first_fft rsi, 128, 160*128, 320*128, 640*128, rdi, ZMM_CD8SCD4, r9, 8*128, 160, L1PREFETCH_ALL
	zloop	160, b1b, -160*128	;; Test loop counter, restore source pointer
	end_timer 4

	start_timer 5
	bump	rdi, -160*ZMM_CD8SCD4-160*ZMM_SCD4/8 ;; Load sin/cos pointer
	zloop_init 160			;; 160 iterations
	zr8b_eight_complex_djbfft_preload
b2b:	zr8b_eight_complex_djbfft rsi, 128, 160*128, 320*128, 640*128, rdi, ZMM_SCD4/8, 160, L1PREFETCH_ALL, 128
	zloop	160, b2b, -160*128	;; Test loop counter, restore source pointer
	end_timer 5
ELSE
	start_timer 4
	mov	rdi, sincos1		;; Load sin/cos pointer
	bump	rdi, 32*ZMM_SCD2/8+8*ZMM_SCD2/8
	lea	r10, [rdi+160*ZMM_SCD4/8] ;; Second sin/cos pointer
	mov	r9, norm_col_mults	;; Load weights pointer
	zloop_init 160			;; 160 iterations
	zr8fs_sixtyfour_complex_first_fft_preload
b1b:	zr8fs_sixtyfour_complex_first_fft rsi, 128, 160*128, 320*128, 640*128, r10, ZMM_CD8SCD4, rdi, ZMM_SCD4/8, r9, 8*128, 160, L1PREFETCH_ALL
	zloop	160, b1b, -160*128	;; Test loop counter, restore source pointer
	end_timer 4
ENDIF

;; Do FFT levels 8,9,10
;; On input the 64-byte cache lines hold these data values:
;;	0	+2560	...	160	+2560	...
;;	...
;;	159
;;	320	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+2560	...	32	+2560	...
;;	...
;;	31
;;	64	...
;;	...

;; Do 2048 five_complex_fft operations
;;	distance between fft data elements is 32
;;	do 256 macros each processing 80 data values

	start_timer 6
	bump	rdi, -160*ZMM_SCD4/8-32*ZMM_SCD2/8 ;; Load sin/cos pointer
	zloop_init 256			;; 8 iterations of 32
	zr5b_five_complex_djbfft_preload
b3b:	zr5b_five_complex_djbfft rsi, 128, 32*128, rdi, ZMM_SCD2/8, 32
	zloop	32, b3b, -32*128+160*128, rdi, -32*ZMM_SCD2/8 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	8, b3b, -8*160*128	;; Test loop counter, restore source pointer
	end_timer 6

;; Do FFT levels 11,12
;; On input the 64-byte cache lines hold these data values:
;;	0	+2560	...	32	+2560	...
;;	...
;;	31
;;	64	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+2560	...	8	+2560	...
;;	...
;;	7
;;	16	...
;;	32	...
;;	...

;; Do 2560 four_complex_fft operations
;;	distance between fft data elements is 8
;;	do 320 macros each processing 64 data values

	start_timer 7
	bump	rdi, -8*ZMM_SCD2/8	;; Load sin/cos pointer
	zloop_init 320			;; 40 iterations of 8
	zr4b_four_complex_djbfft_preload
b4b:	zr4b_four_complex_djbfft rsi, 128, 8*128, 16*128, rdi, ZMM_SCD2/8, 8
	zloop	8, b4b, -8*128+32*128, rdi, -8*ZMM_SCD2/8 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	40, b4b, -40*32*128	;; Test loop counter, restore source pointer
	end_timer 7

;; Do FFT levels 13,14,15
;; On input and output the 128-byte cache line pairs hold these data values:
;;	0	+2560	...	8	+2560	...
;;	...
;;	7
;;	16	...
;;	32	...
;;	...

;; Do 1280 eight_complex_fft operations
;;	distance between fft data elements is 1
;;	do 160 macros processing 128 data values

zmid:	start_timer 8
	mov	rcx, 160
	call	zcomplex_onepass_middle
	end_timer 8

;; Do inverse FFT levels 11,12
;; On input the 64-byte cache lines hold these data values:
;;	0	+2560	...	8	+2560	...
;;	...
;;	7
;;	16	...
;;	32	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+2560	...	32	+2560	...
;;	...
;;	31
;;	64	...
;;	...

;; Do 2560 four_complex_unfft operations
;;	distance between fft data elements is 8
;;	do 320 macros each processing 64 data values

	start_timer 9
	mov	rdi, sincos1		;; Load sin/cos pointer
	zloop_init 320			;; 40 iterations of 8
	zr4b_four_complex_djbunfft_preload
c4b:	zr4b_four_complex_djbunfft rsi, 128, 8*128, 16*128, rdi, ZMM_SCD2/8, 8
	zloop	8, c4b, -8*128+32*128, rdi, -8*ZMM_SCD2/8 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	40, c4b, -40*32*128, rdi, 8*ZMM_SCD2/8	;; Test loop counter, restore source pointer, next sin/cos ptr
	end_timer 9

;; Do inverse FFT levels 8,9,10
;; On input the 64-byte cache lines hold these data values:
;;	0	+2560	...	32	+2560	...
;;	...
;;	31
;;	64	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+2560	...	160	+2560	...
;;	...
;;	159
;;	320	...
;;	...

;; Do 2048 five_complex_unfft operations
;;	distance between fft data elements is 32
;;	do 256 macros each processing 80 data values

	start_timer 10
	zloop_init 256			;; 8 iterations of 32
	zr5b_five_complex_djbunfft_preload
c3b:	zr5b_five_complex_djbunfft rsi, 128, 32*128, rdi, ZMM_SCD2/8, 32
	zloop	32, c3b, -32*128+160*128, rdi, -32*ZMM_SCD2/8 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	8, c3b, -8*160*128, rdi, 32*ZMM_SCD2/8 ;; Test loop counter, restore source pointer, next sin/cos ptr
	end_timer 10

;; Do inverse FFT levels 5,6,7
;; On input the 64-byte cache lines hold these data values:
;;	0	+2560	...	160	+2560	...
;;	...
;;	159
;;	320	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+2560	...	1280	+2560	...
;;	...
;;	1279	...

;; Do 1280 eight_complex_unfft operations
;;	distance between fft data elements is 160
;;	do 160 macros each processing 128 data values

;; Do inverse FFT levels 1,2,3,4
;; On input the 128-byte cache line pairs hold these data values:
;;	0	+2560	...	1280	+2560	...
;;	...
;;	1279	...
;; These are swizzled to:
;;	0	+160	...	1280	+160	...
;;	...
;;	159	...
;;	2560	...
;;	...
;; On output the 128-byte cache line pairs hold these data values:
;;	0	+160	...	10K	+160	...
;;	...
;;	159	...
;;	1280	...
;;	...

;; Do 1280 eight_complex_last_unfft operations
;;	distance between fft data elements is 1280
;;	do 160 macros processing 128 data values

IFDEF OLDWAY
	start_timer 11
	zloop_init 160			;; 160 iterations
	zr8b_eight_complex_djbunfft_preload
c2b:	zr8b_eight_complex_djbunfft rsi, 128, 160*128, 320*128, 640*128, rdi, ZMM_SCD4/8, 160, L1PREFETCH_ALL, 128
	zloop	160, c2b, -160*128	;; Test loop counter, restore source pointer
	end_timer 11

	start_timer 12
	zloop_init 160			;; 160 iterations
	zr8s_eight_complex_last_unfft_preload
c1b:	zr8s_eight_complex_last_unfft rsi, 128, 160*128, 320*128, 640*128, rdi, ZMM_CD8SCD4, 160, L1PREFETCH_ALL
	zloop	160, c1b, -160*128	;; Test loop counter, restore source pointer
	end_timer 12
ELSE
	start_timer 12
	lea	r10, [rdi+160*ZMM_SCD4/8] ;; Second sin/cos pointer
	zloop_init 160			;; 160 iterations
	zr8s_sixtyfour_complex_last_unfft_preload
c1b:	zr8s_sixtyfour_complex_last_unfft rsi, 128, 160*128, 320*128, 640*128, rdi, ZMM_SCD4/8, r10, ZMM_CD8SCD4, 160, L1PREFETCH_ALL
	zloop	160, c1b, -160*128	;; Test loop counter, restore source pointer
	end_timer 12
ENDIF

	zfft_3_ret
	ENDM


;; Perform a 21K-element FFT

zfft21K MACRO type
	LOCAL	b1b, b2b, b3a, b3b, b4a, b4b, zmid, c4a, c4b, c3a, c3b, c2b, c1b

;; Check for a multiply with pre-FFTed inputs

	cmp	ffttype, 4
	je	zmid

;; Do FFT levels 1,2,3,4
;; On input the 128-byte cache line pairs hold these data values:
;;	0	+168	...	10.5K	+168	...
;;	...
;;	167	...
;;	1344	...
;;	...
;; On output the 128-byte cache line pairs hold these data values:
;;	0	+168	...	1344	+168	...
;;	...
;;	167	...
;;	2688	...
;;	...
;; These are swizzled to:
;;	0	+2688	...	1344	+2688	...
;;	...
;;	1343	...

;; Do 1344 sixteen_reals_fft operations
;;	distance between fft data elements is 1344
;;	do 168 macros processing 128 data values

;; Do FFT levels 5,6,7
;; On input the 64-byte cache lines hold these data values:
;;	0	+2688	...	1344	+2688	...
;;	...
;;	1343	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+2688	...	168	+2688	...
;;	...
;;	167
;;	336	...
;;	...

;; Do 168 sixteen_reals_fft operation
;; Do 1176 eight_complex_fft operations
;;	distance between fft data elements is 168
;;	do 168 macros each processing 128 data values

	start_timer 4
	mov	rdi, sincos1		;; Load sin/cos data ptr
	bump	rdi, 56*ZMM_SCD2+8*ZMM_SCD6
	lea	r10, [rdi+168*ZMM_SCD7]	;; Second sin/cos pointer
	mov	r9, norm_col_mults	;; Load weights pointer
	zloop_init 168			;; 168 iterations
	zr8fs_hundredtwentyeight_reals_first_fft_preload
b1b:	zr8fs_hundredtwentyeight_reals_first_fft rsi, 128, 168*128, 336*128, 672*128, r10, ZMM_SCD7, rdi, ZMM_SCD7, r9, 8*128, 168, L1PREFETCH_ALL
	zloop	168, b1b, -168*128	;; Test loop counter, restore source pointer
	end_timer 4

;; Do FFT levels 8,9
;; On input the 64-byte cache lines hold these data values:
;;	0	+2688	...	168	+2688	...
;;	...
;;	167
;;	336	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+2688	...	56	+2688	...
;;	...
;;	55
;;	112	...
;;	...

;; Do 56 six_reals_fft operation
;; Do 3528 three_complex_fft operations
;;	distance between fft data elements is 56
;;	do 448 macros each processing 48 data values

	start_timer 6
	bump	rdi, -168*ZMM_SCD7-56*ZMM_SCD2
	zloop_init 448			;; 8 iterations of 56
	zr3_six_reals_three_complex_djbfft_preload
b3a:	zr3_six_reals_three_complex_djbfft rsi, 128, 56*128, rdi, ZMM_SCD2, 56
	zloop	56, b3a, -56*128+168*128, rdi, -56*ZMM_SCD2 ;; Test loop counter, next source pointer, restore sin/cos pointer

	loops_reset			;; 7 more iters of 56
	zr3rb_three_complex_djbfft_preload
b3b:	zr3rb_three_complex_djbfft rsi, 128, 56*128, rdi, ZMM_SCD2, 56
	zloop	56, b3b, -56*128+168*128, rdi, -56*ZMM_SCD2 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	8, b3b, -8*168*128	;; Test loop counter, restore source pointer
	end_timer 6

;; Do FFT levels 10,11,12
;; On input the 64-byte cache lines hold these data values:
;;	0	+2688	...	56	+2688	...
;;	...
;;	55
;;	112	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+2688	...	8	+2688	...
;;	...
;;	7
;;	16	...
;;	32	...
;;	...

;; Do 8 fourteen_reals_fft operation
;; Do 1528 seven_complex_fft operations
;;	distance between fft data elements is 8
;;	do 192 macros each processing 112 data values

	start_timer 7
	bump	rdi, -8*ZMM_SCD6
	zloop_init 192			;; 24 iterations of 8
	zr7_fourteen_reals_seven_complex_djbfft_preload
b4a:	zr7_fourteen_reals_seven_complex_djbfft rsi, 128, 8*128, rdi, ZMM_SCD6, 8
	zloop	8, b4a, -8*128+56*128, rdi, -8*ZMM_SCD6 ;; Test loop counter, next source pointer, restore sin/cos pointer

	loops_reset			;; 23 more iters of 8
	zr7rb_seven_complex_djbfft_preload
b4b:	zr7rb_seven_complex_djbfft rsi, 128, 8*128, rdi, ZMM_SCD6, 8
	zloop	8, b4b, -8*128+56*128, rdi, -8*ZMM_SCD6 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	24, b4b, -24*56*128	;; Test loop counter, restore source pointer
	end_timer 7

;; Do FFT levels 13,14,15
;; On input and output the 128-byte cache line pairs hold these data values:
;;	0	+2688	...	8	+2688	...
;;	...
;;	7
;;	16	...
;;	32	...
;;	...

;; Do 1 sixteen_reals_fft operation
;; Do 1343 eight_complex_fft operations
;;	distance between fft data elements is 1
;;	do 168 macros processing 128 data values

zmid:	start_timer 8
	mov	rcx, 168
	call	zreal_onepass_middle
	end_timer 8

;; Do inverse FFT levels 10,11,12
;; On input the 64-byte cache lines hold these data values:
;;	0	+2688	...	8	+2688	...
;;	...
;;	7
;;	16	...
;;	32	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+2688	...	56	+2688	...
;;	...
;;	55
;;	112	...
;;	...

;; Do 8 fourteen_reals_unfft operations
;; Do 1528 seven_complex_unfft operations
;;	distance between fft data elements is 8
;;	do 192 macros each processing 112 data values

	start_timer 9
	mov	rdi, sincos1		;; Load sin/cos pointer
	zloop_init 192			;; 24 iterations of 8
	zr7_fourteen_reals_seven_complex_djbunfft_preload
c4a:	zr7_fourteen_reals_seven_complex_djbunfft rsi, 128, 8*128, rdi, ZMM_SCD6, 8
	zloop	8, c4a, -8*128+56*128, rdi, -8*ZMM_SCD6 ;; Test loop counter, restore source pointer, restore sin/cos pointer

	loops_reset			;; 23 more iters of 8
	zr7rb_seven_complex_djbunfft_preload
c4b:	zr7rb_seven_complex_djbunfft rsi, 128, 8*128, rdi, ZMM_SCD6, 8
	zloop	8, c4b, -8*128+56*128, rdi, -8*ZMM_SCD6 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	24, c4b, -24*56*128, rdi, 8*ZMM_SCD6 ;; Test loop counter, restore source pointer, next sin/cos ptr
	end_timer 9

;; Do inverse FFT levels 8,9
;; On input the 64-byte cache lines hold these data values:
;;	0	+2688	...	56	+2688	...
;;	...
;;	55
;;	112	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+2688	...	168	+2688	...
;;	...
;;	167
;;	336	...
;;	...

;; Do 56 six_reals_unfft operations
;; Do 3528 three_complex_unfft operations
;;	distance between fft data elements is 56
;;	do 448 macros each processing 48 data values

	start_timer 10
	zloop_init 448			;; 8 iterations of 56
	zr3_six_reals_three_complex_djbunfft_preload
c3a:	zr3_six_reals_three_complex_djbunfft rsi, 128, 56*128, rdi, ZMM_SCD2, 56
	zloop	56, c3a, -56*128+168*128, rdi, -56*ZMM_SCD2 ;; Test loop counter, restore source pointer, restore sin/cos pointer

	loops_reset			;; 7 more iters of 56
	zr3rb_three_complex_djbunfft_preload
c3b:	zr3rb_three_complex_djbunfft rsi, 128, 56*128, rdi, ZMM_SCD2, 56
	zloop	56, c3b, -56*128+168*128, rdi, -56*ZMM_SCD2 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	8, c3b, -8*168*128, rdi, 56*ZMM_SCD2 ;; Test loop counter, restore source pointer, next sin/cos ptr
	end_timer 10

;; Do inverse FFT levels 5,6,7
;; On input the 64-byte cache lines hold these data values:
;;	0	+2688	...	168	+2688	...
;;	...
;;	167
;;	336	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+2688	...	1344	+2688	...
;;	...
;;	1343	...

;; Do 168 sixteen_reals_unfft operations
;; Do 1176 eight_complex_unfft operations
;;	distance between fft data elements is 168
;;	do 168 macros each processing 128 data values

;; Do inverse FFT levels 1,2,3,4
;; On input the 128-byte cache line pairs hold these data values:
;;	0	+2688	...	1344	+2688	...
;;	...
;;	1343	...
;; These are swizzled to:
;;	0	+168	...	1344	+168	...
;;	...
;;	167	...
;;	2688	...
;;	...
;; On output the 128-byte cache line pairs hold these data values:
;;	0	+168	...	10.5K	+168	...
;;	...
;;	167	...
;;	1344	...
;;	...

;; Do 1344 sixteen_reals_unfft operations
;;	distance between fft data elements is 1344
;;	do 168 macros processing 128 data values

	start_timer 12
	lea	r10, [rdi+168*ZMM_SCD7]	;; Second sin/cos pointer
	zloop_init 168			;; 168 iterations
	zr8s_hundredtwentyeight_reals_last_unfft_preload
c1b:	zr8s_hundredtwentyeight_reals_last_unfft rsi, 128, 168*128, 336*128, 672*128, rdi, ZMM_SCD7, r10, ZMM_SCD7, 168, L1PREFETCH_ALL
	zloop	168, c1b, -168*128	;; Test loop counter, restore source pointer
	end_timer 12

	zfft_3_ret
	ENDM

;; Perform a 21K-element negacyclic FFT

zfft21Kp MACRO type
	LOCAL	b1b, b2b, b3b, b4b, zmid, c4b, c3b, c2b, c1b

;; Do a multiply with pre-FFTed inputs

	cmp	ffttype, 4
	je	zmid

;; Do FFT levels 1,2,3,4
;; On input the 128-byte cache line pairs hold these data values:
;;	0	+168	...	10.5K	+168	...
;;	...
;;	167	...
;;	1344	...
;;	...
;; On output the 128-byte cache line pairs hold these data values:
;;	0	+168	...	1344	+168	...
;;	...
;;	167	...
;;	2688	...
;;	...
;; These are swizzled to:
;;	0	+2688	...	1344	+2688	...
;;	...
;;	1343	...

;; Do 1344 eight_complex_first_fft operations
;;	distance between fft data elements is 1344
;;	do 168 macros processing 128 data values

;; Do FFT levels 5,6,7
;; On input the 64-byte cache lines hold these data values:
;;	0	+2688	...	1344	+2688	...
;;	...
;;	1343	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+2688	...	168	+2688	...
;;	...
;;	167
;;	336	...
;;	...

;; Do 1344 eight_complex_fft operations
;;	distance between fft data elements is 168
;;	do 168 macros each processing 128 data values

	start_timer 4
	mov	rdi, sincos1		;; Load sin/cos pointer
	bump	rdi, 56*ZMM_SCD1/8+8*ZMM_SCD3/8
	lea	r10, [rdi+168*ZMM_SCD4/8] ;; Second sin/cos pointer
	mov	r9, norm_col_mults	;; Load weights pointer
	zloop_init 168			;; 168 iterations
	zr8fs_sixtyfour_complex_first_fft_preload
b1b:	zr8fs_sixtyfour_complex_first_fft rsi, 128, 168*128, 336*128, 672*128, r10, ZMM_CD8SCD4, rdi, ZMM_SCD4/8, r9, 8*128, 168, L1PREFETCH_ALL
	zloop	168, b1b, -168*128	;; Test loop counter, restore source pointer
	end_timer 4

;; Do FFT levels 8,9
;; On input the 64-byte cache lines hold these data values:
;;	0	+2688	...	168	+2688	...
;;	...
;;	167
;;	336	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+2688	...	56	+2688	...
;;	...
;;	55
;;	112	...
;;	...

;; Do 3584 three_complex_fft operations
;;	distance between fft data elements is 56
;;	do 448 macros each processing 48 data values

	start_timer 6
	bump	rdi, -168*ZMM_SCD4/8-56*ZMM_SCD1/8 ;; Load sin/cos pointer
	zloop_init 448			;; 8 iterations of 56
	zr3b_three_complex_djbfft_preload
b3b:	zr3b_three_complex_djbfft rsi, 128, 56*128, rdi, ZMM_SCD1/8, 56
	zloop	56, b3b, -56*128+168*128, rdi, -56*ZMM_SCD1/8 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	8, b3b, -8*168*128	;; Test loop counter, restore source pointer
	end_timer 6

;; Do FFT levels 10,11,12
;; On input the 64-byte cache lines hold these data values:
;;	0	+2688	...	56	+2688	...
;;	...
;;	55
;;	112	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+2688	...	8	+2688	...
;;	...
;;	7
;;	16	...
;;	32	...
;;	...

;; Do 1536 seven_complex_fft operations
;;	distance between fft data elements is 8
;;	do 192 macros each processing 112 data values

	start_timer 7
	bump	rdi, -8*ZMM_SCD3/8	;; Load sin/cos pointer
	zloop_init 192			;; 24 iterations of 8
	zr7b_seven_complex_djbfft_preload
b4b:	zr7b_seven_complex_djbfft rsi, 128, 8*128, rdi, ZMM_SCD3/8, 8
	zloop	8, b4b, -8*128+56*128, rdi, -8*ZMM_SCD3/8 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	24, b4b, -24*56*128	;; Test loop counter, restore source pointer
	end_timer 7

;; Do FFT levels 13,14,15
;; On input and output the 128-byte cache line pairs hold these data values:
;;	0	+2688	...	8	+2688	...
;;	...
;;	7
;;	16	...
;;	32	...
;;	...

;; Do 1344 eight_complex_fft operations
;;	distance between fft data elements is 1
;;	do 168 macros processing 128 data values

zmid:	start_timer 8
	mov	rcx, 168
	call	zcomplex_onepass_middle
	end_timer 8

;; Do inverse FFT levels 10,11,12
;; On input the 64-byte cache lines hold these data values:
;;	0	+2688	...	8	+2688	...
;;	...
;;	7
;;	16	...
;;	32	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+2688	...	56	+2688	...
;;	...
;;	55
;;	112	...
;;	...

;; Do 1536 seven_complex_unfft operations
;;	distance between fft data elements is 8
;;	do 192 macros each processing 112 data values

	start_timer 9
	mov	rdi, sincos1		;; Load sin/cos pointer
	zloop_init 192			;; 24 iterations of 8
	zr7b_seven_complex_djbunfft_preload
c4b:	zr7b_seven_complex_djbunfft rsi, 128, 8*128, rdi, ZMM_SCD3/8, 8
	zloop	8, c4b, -8*128+56*128, rdi, -8*ZMM_SCD3/8 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	24, c4b, -24*56*128, rdi, 8*ZMM_SCD3/8	;; Test loop counter, restore source pointer, next sin/cos ptr
	end_timer 9

;; Do inverse FFT levels 8,9
;; On input the 64-byte cache lines hold these data values:
;;	0	+2688	...	56	+2688	...
;;	...
;;	55
;;	112	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+2688	...	168	+2688	...
;;	...
;;	167
;;	336	...
;;	...

;; Do 3584 three_complex_unfft operations
;;	distance between fft data elements is 56
;;	do 448 macros each processing 48 data values

	start_timer 10
	zloop_init 448			;; 8 iterations of 56
	zr3b_three_complex_djbunfft_preload
c3b:	zr3b_three_complex_djbunfft rsi, 128, 56*128, rdi, ZMM_SCD1/8, 56
	zloop	56, c3b, -56*128+168*128, rdi, -56*ZMM_SCD1/8 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	8, c3b, -8*168*128, rdi, 56*ZMM_SCD1/8 ;; Test loop counter, restore source pointer, next sin/cos ptr
	end_timer 10

;; Do inverse FFT levels 5,6,7
;; On input the 64-byte cache lines hold these data values:
;;	0	+2688	...	168	+2688	...
;;	...
;;	167
;;	336	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+2688	...	1344	+2688	...
;;	...
;;	1343	...

;; Do 1344 eight_complex_unfft operations
;;	distance between fft data elements is 168
;;	do 168 macros each processing 128 data values

;; Do inverse FFT levels 1,2,3,4
;; On input the 128-byte cache line pairs hold these data values:
;;	0	+2688	...	1344	+2688	...
;;	...
;;	1343	...
;; These are swizzled to:
;;	0	+168	...	1344	+168	...
;;	...
;;	167	...
;;	2688	...
;;	...
;; On output the 128-byte cache line pairs hold these data values:
;;	0	+168	...	10.5K	+168	...
;;	...
;;	167	...
;;	1344	...
;;	...

;; Do 1344 eight_complex_last_unfft operations
;;	distance between fft data elements is 1344
;;	do 168 macros processing 128 data values

	start_timer 12
	lea	r10, [rdi+168*ZMM_SCD4/8] ;; Second sin/cos pointer
	zloop_init 168			;; 168 iterations
	zr8s_sixtyfour_complex_last_unfft_preload
c1b:	zr8s_sixtyfour_complex_last_unfft rsi, 128, 168*128, 336*128, 672*128, rdi, ZMM_SCD4/8, r10, ZMM_CD8SCD4, 168, L1PREFETCH_ALL
	zloop	168, c1b, -168*128	;; Test loop counter, restore source pointer
	end_timer 12

	zfft_3_ret
	ENDM


;; Perform a 24K-element FFT

zfft24K MACRO type
	LOCAL	b1b, b2b, b3a, b3b, b4a, b4b, zmid, c4a, c4b, c3a, c3b, c2b, c1b

;; Check for a multiply with pre-FFTed inputs

	cmp	ffttype, 4
	je	zmid

;; Do FFT levels 1,2,3,4
;; On input the 128-byte cache line pairs hold these data values:
;;	0	+192	...	12K	+192	...
;;	...
;;	191	...
;;	1536	...
;;	...
;; On output the 128-byte cache line pairs hold these data values:
;;	0	+192	...	1536	+192	...
;;	...
;;	191	...
;;	3072	...
;;	...
;; These are swizzled to:
;;	0	+3072	...	1536	+3072	...
;;	...
;;	1535	...

;; Do 1536 sixteen_reals_fft operations
;;	distance between fft data elements is 1536
;;	do 192 macros processing 128 data values

;; Do FFT levels 5,6,7
;; On input the 64-byte cache lines hold these data values:
;;	0	+3072	...	1536	+3072	...
;;	...
;;	1535	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+3072	...	192	+3072	...
;;	...
;;	191
;;	384	...
;;	...

;; Do 192 sixteen_reals_fft operation
;; Do 1344 eight_complex_fft operations
;;	distance between fft data elements is 192
;;	do 192 macros each processing 128 data values

	start_timer 4
	mov	rdi, sincos1		;; Load sin/cos data ptr
	bump	rdi, 32*ZMM_SCD5+8*ZMM_SCD3
	lea	r10, [rdi+192*ZMM_SCD7]	;; Second sin/cos pointer
	mov	r9, norm_col_mults	;; Load weights pointer
	zloop_init 192			;; 192 iterations
	zr8fs_hundredtwentyeight_reals_first_fft_preload
b1b:	zr8fs_hundredtwentyeight_reals_first_fft rsi, 128, 192*128, 384*128, 768*128, r10, ZMM_SCD7, rdi, ZMM_SCD7, r9, 8*128, 192, L1PREFETCH_ALL
	zloop	192, b1b, -192*128	;; Test loop counter, restore source pointer
	end_timer 4

;; Do FFT levels 8,9,10
;; On input the 64-byte cache lines hold these data values:
;;	0	+3072	...	192	+3072	...
;;	...
;;	191
;;	384	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+3072	...	32	+3072	...
;;	...
;;	31
;;	64	...
;;	...

;; Do 32 twelve_reals_fft operation
;; Do 2016 six_complex_fft operations
;;	distance between fft data elements is 32
;;	do 256 macros each processing 96 data values

	start_timer 6
	bump	rdi, -192*ZMM_SCD7-32*ZMM_SCD5
	zloop_init 256			;; 8 iterations of 32
	zr6_twelve_reals_six_complex_djbfft_preload
b3a:	zr6_twelve_reals_six_complex_djbfft rsi, 128, 32*128, rdi, ZMM_SCD5, 32
	zloop	32, b3a, -32*128+192*128, rdi, -32*ZMM_SCD5 ;; Test loop counter, next source pointer, restore sin/cos pointer

	loops_reset			;; 7 more iters of 32
	zr6rb_six_complex_djbfft_preload
b3b:	zr6rb_six_complex_djbfft rsi, 128, 32*128, rdi, ZMM_SCD5, 32
	zloop	32, b3b, -32*128+192*128, rdi, -32*ZMM_SCD5 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	8, b3b, -8*192*128	;; Test loop counter, restore source pointer
	end_timer 6

;; Do FFT levels 10,11,12
;; On input the 64-byte cache lines hold these data values:
;;	0	+3072	...	32	+3072	...
;;	...
;;	31
;;	64	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+3072	...	8	+3072	...
;;	...
;;	7
;;	16	...
;;	32	...
;;	...

;; Do 8 eight_reals_fft operation
;; Do 3064 four_complex_fft operations
;;	distance between fft data elements is 8
;;	do 384 macros each processing 64 data values

	start_timer 7
	bump	rdi, -8*ZMM_SCD3
	zloop_init 384			;; 48 iterations of 8
	zr4_eight_reals_four_complex_djbfft_preload
b4a:	zr4_eight_reals_four_complex_djbfft rsi, 128, 8*128, 16*128, rdi, ZMM_SCD3, 8
	zloop	8, b4a, -8*128+32*128, rdi, -8*ZMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer

	loops_reset			;; 47 more iters of 8
	zr4rb_four_complex_djbfft_preload
b4b:	zr4rb_four_complex_djbfft rsi, 128, 8*128, 16*128, rdi, ZMM_SCD3, 8
	zloop	8, b4b, -8*128+32*128, rdi, -8*ZMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	48, b4b, -48*32*128	;; Test loop counter, restore source pointer
	end_timer 7

;; Do FFT levels 13,14,15
;; On input and output the 128-byte cache line pairs hold these data values:
;;	0	+3072	...	8	+3072	...
;;	...
;;	7
;;	16	...
;;	32	...
;;	...

;; Do 1 sixteen_reals_fft operation
;; Do 1535 eight_complex_fft operations
;;	distance between fft data elements is 1
;;	do 192 macros processing 128 data values

zmid:	start_timer 8
	mov	rcx, 192
	call	zreal_onepass_middle
	end_timer 8

;; Do inverse FFT levels 10,11,12
;; On input the 64-byte cache lines hold these data values:
;;	0	+3072	...	8	+3072	...
;;	...
;;	7
;;	16	...
;;	32	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+3072	...	32	+3072	...
;;	...
;;	31
;;	64	...
;;	...

;; Do 8 eight_reals_unfft operations
;; Do 3064 four_complex_unfft operations
;;	distance between fft data elements is 8
;;	do 384 macros each processing 64 data values

	start_timer 9
	mov	rdi, sincos1		;; Load sin/cos pointer
	zloop_init 384			;; 48 iterations of 8
	zr4_eight_reals_four_complex_djbunfft_preload
c4a:	zr4_eight_reals_four_complex_djbunfft rsi, 128, 8*128, 16*128, rdi, ZMM_SCD3, 8
	zloop	8, c4a, -8*128+32*128, rdi, -8*ZMM_SCD3 ;; Test loop counter, restore source pointer, restore sin/cos pointer

	loops_reset			;; 47 more iters of 8
	zr4rb_four_complex_djbunfft_preload
c4b:	zr4rb_four_complex_djbunfft rsi, 128, 8*128, 16*128, rdi, ZMM_SCD3, 8
	zloop	8, c4b, -8*128+32*128, rdi, -8*ZMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	48, c4b, -48*32*128, rdi, 8*ZMM_SCD3 ;; Test loop counter, restore source pointer, next sin/cos ptr
	end_timer 9

;; Do inverse FFT levels 8,9
;; On input the 64-byte cache lines hold these data values:
;;	0	+3072	...	32	+3072	...
;;	...
;;	31
;;	64	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+3072	...	192	+3072	...
;;	...
;;	191
;;	384	...
;;	...

;; Do 32 twelve_reals_unfft operations
;; Do 2016 six_complex_unfft operations
;;	distance between fft data elements is 32
;;	do 256 macros each processing 96 data values

	start_timer 10
	zloop_init 256			;; 8 iterations of 32
	zr6_twelve_reals_six_complex_djbunfft_preload
c3a:	zr6_twelve_reals_six_complex_djbunfft rsi, 128, 32*128, rdi, ZMM_SCD5, 32
	zloop	32, c3a, -32*128+192*128, rdi, -32*ZMM_SCD5 ;; Test loop counter, restore source pointer, restore sin/cos pointer

	loops_reset			;; 7 more iters of 32
	zr6rb_six_complex_djbunfft_preload
c3b:	zr6rb_six_complex_djbunfft rsi, 128, 32*128, rdi, ZMM_SCD5, 32
	zloop	32, c3b, -32*128+192*128, rdi, -32*ZMM_SCD5 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	8, c3b, -8*192*128, rdi, 32*ZMM_SCD5 ;; Test loop counter, restore source pointer, next sin/cos ptr
	end_timer 10

;; Do inverse FFT levels 5,6,7
;; On input the 64-byte cache lines hold these data values:
;;	0	+3072	...	192	+3072	...
;;	...
;;	191
;;	384	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+3072	...	1536	+3072	...
;;	...
;;	1535	...

;; Do 192 sixteen_reals_unfft operations
;; Do 1344 eight_complex_unfft operations
;;	distance between fft data elements is 192
;;	do 192 macros each processing 128 data values

;; Do inverse FFT levels 1,2,3,4
;; On input the 128-byte cache line pairs hold these data values:
;;	0	+3072	...	1536	+3072	...
;;	...
;;	1535	...
;; These are swizzled to:
;;	0	+192	...	1536	+192	...
;;	...
;;	191	...
;;	3072	...
;;	...
;; On output the 128-byte cache line pairs hold these data values:
;;	0	+192	...	12K	+192	...
;;	...
;;	191	...
;;	1536	...
;;	...

;; Do 1536 sixteen_reals_unfft operations
;;	distance between fft data elements is 1536
;;	do 192 macros processing 128 data values

	start_timer 12
	lea	r10, [rdi+192*ZMM_SCD7]	;; Second sin/cos pointer
	zloop_init 192			;; 192 iterations
	zr8s_hundredtwentyeight_reals_last_unfft_preload
c1b:	zr8s_hundredtwentyeight_reals_last_unfft rsi, 128, 192*128, 384*128, 768*128, rdi, ZMM_SCD7, r10, ZMM_SCD7, 192, L1PREFETCH_ALL
	zloop	192, c1b, -192*128	;; Test loop counter, restore source pointer
	end_timer 12

	zfft_3_ret
	ENDM

;; Perform a 24K-element negacyclic FFT

zfft24Kp MACRO type
	LOCAL	b1b, b2b, b3b, b4b, zmid, c4b, c3b, c2b, c1b

;; Do a multiply with pre-FFTed inputs

	cmp	ffttype, 4
	je	zmid

;; Do FFT levels 1,2,3,4
;; On input the 128-byte cache line pairs hold these data values:
;;	0	+192	...	12K	+192	...
;;	...
;;	191	...
;;	1536	...
;;	...
;; On output the 128-byte cache line pairs hold these data values:
;;	0	+192	...	1536	+192	...
;;	...
;;	191	...
;;	3072	...
;;	...
;; These are swizzled to:
;;	0	+3072	...	1536	+3072	...
;;	...
;;	1535	...

;; Do 1536 eight_complex_first_fft operations
;;	distance between fft data elements is 1536
;;	do 192 macros processing 128 data values

;; Do FFT levels 5,6,7
;; On input the 64-byte cache lines hold these data values:
;;	0	+3072	...	1536	+3072	...
;;	...
;;	1535	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+3072	...	192	+3072	...
;;	...
;;	191
;;	384	...
;;	...

;; Do 1536 eight_complex_fft operations
;;	distance between fft data elements is 192
;;	do 192 macros each processing 128 data values

	start_timer 4
	mov	rdi, sincos1		;; Load sin/cos pointer
	bump	rdi, 32*ZMM_SCD3/8+8*ZMM_SCD2/8
	lea	r10, [rdi+192*ZMM_SCD4/8] ;; Second sin/cos pointer
	mov	r9, norm_col_mults	;; Load weights pointer
	zloop_init 192			;; 192 iterations
	zr8fs_sixtyfour_complex_first_fft_preload
b1b:	zr8fs_sixtyfour_complex_first_fft rsi, 128, 192*128, 384*128, 768*128, r10, ZMM_CD8SCD4, rdi, ZMM_SCD4/8, r9, 8*128, 192, L1PREFETCH_ALL
	zloop	192, b1b, -192*128	;; Test loop counter, restore source pointer
	end_timer 4

;; Do FFT levels 8,9,10
;; On input the 64-byte cache lines hold these data values:
;;	0	+3072	...	192	+3072	...
;;	...
;;	191
;;	384	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+3072	...	32	+3072	...
;;	...
;;	31
;;	64	...
;;	...

;; Do 2048 six_complex_fft operations
;;	distance between fft data elements is 32
;;	do 256 macros each processing 96 data values

	start_timer 6
	bump	rdi, -192*ZMM_SCD4/8-32*ZMM_SCD3/8 ;; Load sin/cos pointer
	zloop_init 256			;; 8 iterations of 32
	zr6b_six_complex_djbfft_preload
b3b:	zr6b_six_complex_djbfft rsi, 128, 32*128, rdi, ZMM_SCD3/8, 32
	zloop	32, b3b, -32*128+192*128, rdi, -32*ZMM_SCD3/8 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	8, b3b, -8*192*128	;; Test loop counter, restore source pointer
	end_timer 6

;; Do FFT levels 10,11,12
;; On input the 64-byte cache lines hold these data values:
;;	0	+3072	...	32	+3072	...
;;	...
;;	31
;;	64	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+3072	...	8	+3072	...
;;	...
;;	7
;;	16	...
;;	32	...
;;	...

;; Do 3072 four_complex_fft operations
;;	distance between fft data elements is 8
;;	do 384 macros each processing 64 data values

	start_timer 7
	bump	rdi, -8*ZMM_SCD2/8	;; Load sin/cos pointer
	zloop_init 384			;; 48 iterations of 8
	zr4b_four_complex_djbfft_preload
b4b:	zr4b_four_complex_djbfft rsi, 128, 8*128, 16*128, rdi, ZMM_SCD2/8, 8
	zloop	8, b4b, -8*128+32*128, rdi, -8*ZMM_SCD2/8 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	48, b4b, -48*32*128	;; Test loop counter, restore source pointer
	end_timer 7

;; Do FFT levels 13,14,15
;; On input and output the 128-byte cache line pairs hold these data values:
;;	0	+3072	...	8	+3072	...
;;	...
;;	7
;;	16	...
;;	32	...
;;	...

;; Do 1536 eight_complex_fft operations
;;	distance between fft data elements is 1
;;	do 192 macros processing 128 data values

zmid:	start_timer 8
	mov	rcx, 192
	call	zcomplex_onepass_middle
	end_timer 8

;; Do inverse FFT levels 10,11,12
;; On input the 64-byte cache lines hold these data values:
;;	0	+3072	...	8	+3072	...
;;	...
;;	7
;;	16	...
;;	32	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+3072	...	32	+3072	...
;;	...
;;	31
;;	64	...
;;	...

;; Do 3072 four_complex_unfft operations
;;	distance between fft data elements is 8
;;	do 384 macros each processing 64 data values

	start_timer 9
	mov	rdi, sincos1		;; Load sin/cos pointer
	zloop_init 384			;; 48 iterations of 8
	zr4b_four_complex_djbunfft_preload
c4b:	zr4b_four_complex_djbunfft rsi, 128, 8*128, 16*128, rdi, ZMM_SCD2/8, 8
	zloop	8, c4b, -8*128+32*128, rdi, -8*ZMM_SCD2/8 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	48, c4b, -48*32*128, rdi, 8*ZMM_SCD2/8	;; Test loop counter, restore source pointer, next sin/cos ptr
	end_timer 9

;; Do inverse FFT levels 8,9
;; On input the 64-byte cache lines hold these data values:
;;	0	+3072	...	32	+3072	...
;;	...
;;	31
;;	64	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+3072	...	192	+3072	...
;;	...
;;	191
;;	384	...
;;	...

;; Do 2048 six_complex_unfft operations
;;	distance between fft data elements is 32
;;	do 256 macros each processing 96 data values

	start_timer 10
	zloop_init 256			;; 8 iterations of 32
	zr6b_six_complex_djbunfft_preload
c3b:	zr6b_six_complex_djbunfft rsi, 128, 32*128, rdi, ZMM_SCD3/8, 32
	zloop	32, c3b, -32*128+192*128, rdi, -32*ZMM_SCD3/8 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	8, c3b, -8*192*128, rdi, 32*ZMM_SCD3/8 ;; Test loop counter, restore source pointer, next sin/cos ptr
	end_timer 10

;; Do inverse FFT levels 5,6,7
;; On input the 64-byte cache lines hold these data values:
;;	0	+3072	...	192	+3072	...
;;	...
;;	191
;;	384	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+3072	...	1536	+3072	...
;;	...
;;	1535	...

;; Do 1536 eight_complex_unfft operations
;;	distance between fft data elements is 192
;;	do 192 macros each processing 128 data values

;; Do inverse FFT levels 1,2,3,4
;; On input the 128-byte cache line pairs hold these data values:
;;	0	+3072	...	1536	+3072	...
;;	...
;;	1535	...
;; These are swizzled to:
;;	0	+192	...	1536	+192	...
;;	...
;;	191	...
;;	3072	...
;;	...
;; On output the 128-byte cache line pairs hold these data values:
;;	0	+192	...	12K	+192	...
;;	...
;;	191	...
;;	1536	...
;;	...

;; Do 1536 eight_complex_last_unfft operations
;;	distance between fft data elements is 1536
;;	do 192 macros processing 128 data values

	start_timer 12
	lea	r10, [rdi+192*ZMM_SCD4/8] ;; Second sin/cos pointer
	zloop_init 192			;; 192 iterations
	zr8s_sixtyfour_complex_last_unfft_preload
c1b:	zr8s_sixtyfour_complex_last_unfft rsi, 128, 192*128, 384*128, 768*128, rdi, ZMM_SCD4/8, r10, ZMM_CD8SCD4, 192, L1PREFETCH_ALL
	zloop	192, c1b, -192*128	;; Test loop counter, restore source pointer
	end_timer 12

	zfft_3_ret
	ENDM


;; Perform a 25K-element FFT

zfft25K MACRO type
	LOCAL	b1b, b2b, b3a, b3b, b4a, b4b, zmid, c4a, c4b, c3a, c3b, c2b, c1b

;; Check for a multiply with pre-FFTed inputs

	cmp	ffttype, 4
	je	zmid

;; Do FFT levels 1,2,3,4
;; On input the 128-byte cache line pairs hold these data values:
;;	0	+200	...	12.5K	+200	...
;;	...
;;	199	...
;;	1600	...
;;	...
;; On output the 128-byte cache line pairs hold these data values:
;;	0	+200	...	1600	+200	...
;;	...
;;	199	...
;;	3200	...
;;	...
;; These are swizzled to:
;;	0	+3200	...	1600	+3200	...
;;	...
;;	1599	...

;; Do 1600 sixteen_reals_fft operations
;;	distance between fft data elements is 1600
;;	do 200 macros processing 128 data values

;; Do FFT levels 5,6,7
;; On input the 64-byte cache lines hold these data values:
;;	0	+3200	...	1600	+3200	...
;;	...
;;	1599	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+3200	...	200	+3200	...
;;	...
;;	199
;;	400	...
;;	...

;; Do 200 sixteen_reals_fft operation
;; Do 1400 eight_complex_fft operations
;;	distance between fft data elements is 200
;;	do 200 macros each processing 128 data values

	start_timer 4
	mov	rdi, sincos1		;; Load sin/cos data ptr
	bump	rdi, 40*ZMM_SCD4
	lea	r10, [rdi+200*ZMM_SCD7]	;; Second sin/cos pointer
	mov	r9, norm_col_mults	;; Load weights pointer
	zloop_init 200			;; 200 iterations
	zr8fs_hundredtwentyeight_reals_first_fft_preload
b1b:	zr8fs_hundredtwentyeight_reals_first_fft rsi, 128, 200*128, 400*128, 800*128, r10, ZMM_SCD7, rdi, ZMM_SCD7, r9, 8*128, 200, L1PREFETCH_ALL
	zloop	200, b1b, -200*128	;; Test loop counter, restore source pointer
	end_timer 4

;; Do FFT levels 8,9
;; On input the 64-byte cache lines hold these data values:
;;	0	+3200	...	200	+3200	...
;;	...
;;	199
;;	400	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+3200	...	40	+3200	...
;;	...
;;	39
;;	80	...
;;	...

;; Do 40 ten_reals_fft operation
;; Do 2520 five_complex_fft operations
;;	distance between fft data elements is 40
;;	do 320 macros each processing 80 data values

	start_timer 6
	bump	rdi, -200*ZMM_SCD7-40*ZMM_SCD4
	zloop_init 320			;; 8 iterations of 40
	zr5_ten_reals_five_complex_djbfft_preload
b3a:	zr5_ten_reals_five_complex_djbfft rsi, 128, 40*128, rdi, ZMM_SCD4, 40
	zloop	40, b3a, -40*128+200*128, rdi, -40*ZMM_SCD4 ;; Test loop counter, next source pointer, restore sin/cos pointer

	loops_reset			;; 7 more iters of 40
	zr5rb_five_complex_djbfft_preload
b3b:	zr5rb_five_complex_djbfft rsi, 128, 40*128, rdi, ZMM_SCD4, 40
	zloop	40, b3b, -40*128+200*128, rdi, -40*ZMM_SCD4 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	8, b3b, -8*200*128	;; Test loop counter, restore source pointer
	end_timer 6

;; Do FFT levels 10,11,12
;; On input the 64-byte cache lines hold these data values:
;;	0	+3200	...	40	+3200	...
;;	...
;;	39
;;	80	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+3200	...	8	+3200	...
;;	...
;;	7
;;	16	...
;;	32	...
;;	...

;; Do 8 ten_reals_fft operation
;; Do 2552 five_complex_fft operations
;;	distance between fft data elements is 8
;;	do 320 macros each processing 80 data values

	start_timer 7
	zloop_init 320			;; 40 iterations of 8
	zr5_ten_reals_five_complex_djbfft_preload
b4a:	zr5_ten_reals_five_complex_djbfft rsi, 128, 8*128, rdi, 5*ZMM_SCD4, 8
	zloop	8, b4a, -8*128+40*128, rdi, -8*5*ZMM_SCD4 ;; Test loop counter, next source pointer, restore sin/cos pointer

	loops_reset			;; 39 more iters of 8
	zr5rb_five_complex_djbfft_preload
b4b:	zr5rb_five_complex_djbfft rsi, 128, 8*128, rdi, 5*ZMM_SCD4, 8
	zloop	8, b4b, -8*128+40*128, rdi, -8*5*ZMM_SCD4 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	40, b4b, -40*40*128	;; Test loop counter, restore source pointer
	end_timer 7

;; Do FFT levels 13,14,15
;; On input and output the 128-byte cache line pairs hold these data values:
;;	0	+3200	...	8	+3200	...
;;	...
;;	7
;;	16	...
;;	32	...
;;	...

;; Do 1 sixteen_reals_fft operation
;; Do 1599 eight_complex_fft operations
;;	distance between fft data elements is 1
;;	do 200 macros processing 128 data values

zmid:	start_timer 8
	mov	rcx, 200
	call	zreal_onepass_middle
	end_timer 8

;; Do inverse FFT levels 10,11,12
;; On input the 64-byte cache lines hold these data values:
;;	0	+3200	...	8	+3200	...
;;	...
;;	7
;;	16	...
;;	32	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+3200	...	40	+3200	...
;;	...
;;	39
;;	80	...
;;	...

;; Do 8 ten_reals_unfft operations
;; Do 2552 five_complex_unfft operations
;;	distance between fft data elements is 8
;;	do 320 macros each processing 80 data values

	start_timer 9
	mov	rdi, sincos1		;; Load sin/cos pointer
	zloop_init 320			;; 40 iterations of 8
	zr5_ten_reals_five_complex_djbunfft_preload
c4a:	zr5_ten_reals_five_complex_djbunfft rsi, 128, 8*128, rdi, 5*ZMM_SCD4, 8
	zloop	8, c4a, -8*128+40*128, rdi, -8*5*ZMM_SCD4 ;; Test loop counter, restore source pointer, restore sin/cos pointer

	loops_reset			;; 39 more iters of 8
	zr5rb_five_complex_djbunfft_preload
c4b:	zr5rb_five_complex_djbunfft rsi, 128, 8*128, rdi, 5*ZMM_SCD4, 8
	zloop	8, c4b, -8*128+40*128, rdi, -8*5*ZMM_SCD4 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	40, c4b, -40*40*128	;; Test loop counter, restore source pointer
	end_timer 9

;; Do inverse FFT levels 8,9
;; On input the 64-byte cache lines hold these data values:
;;	0	+3200	...	40	+3200	...
;;	...
;;	39
;;	80	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+3200	...	200	+3200	...
;;	...
;;	199
;;	400	...
;;	...

;; Do 40 ten_reals_unfft operations
;; Do 2520 five_complex_unfft operations
;;	distance between fft data elements is 40
;;	do 320 macros each processing 80 data values

	start_timer 10
	zloop_init 320			;; 8 iterations of 40
	zr5_ten_reals_five_complex_djbunfft_preload
c3a:	zr5_ten_reals_five_complex_djbunfft rsi, 128, 40*128, rdi, ZMM_SCD4, 40
	zloop	40, c3a, -40*128+200*128, rdi, -40*ZMM_SCD4 ;; Test loop counter, restore source pointer, restore sin/cos pointer

	loops_reset			;; 7 more iters of 40
	zr5rb_five_complex_djbunfft_preload
c3b:	zr5rb_five_complex_djbunfft rsi, 128, 40*128, rdi, ZMM_SCD4, 40
	zloop	40, c3b, -40*128+200*128, rdi, -40*ZMM_SCD4 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	8, c3b, -8*200*128, rdi, 40*ZMM_SCD4 ;; Test loop counter, restore source pointer, next sin/cos ptr
	end_timer 10

;; Do inverse FFT levels 5,6,7
;; On input the 64-byte cache lines hold these data values:
;;	0	+3200	...	200	+3200	...
;;	...
;;	199
;;	400	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+3200	...	1600	+3200	...
;;	...
;;	1599	...

;; Do 200 sixteen_reals_unfft operations
;; Do 1400 eight_complex_unfft operations
;;	distance between fft data elements is 200
;;	do 200 macros each processing 128 data values

;; Do inverse FFT levels 1,2,3,4
;; On input the 128-byte cache line pairs hold these data values:
;;	0	+3200	...	1600	+3200	...
;;	...
;;	1599	...
;; These are swizzled to:
;;	0	+200	...	1600	+200	...
;;	...
;;	199	...
;;	3200	...
;;	...
;; On output the 128-byte cache line pairs hold these data values:
;;	0	+200	...	12.5K	+200	...
;;	...
;;	199	...
;;	1600	...
;;	...

;; Do 1600 sixteen_reals_unfft operations
;;	distance between fft data elements is 1600
;;	do 200 macros processing 128 data values

	start_timer 12
	lea	r10, [rdi+200*ZMM_SCD7]	;; Second sin/cos pointer
	zloop_init 200			;; 200 iterations
	zr8s_hundredtwentyeight_reals_last_unfft_preload
c1b:	zr8s_hundredtwentyeight_reals_last_unfft rsi, 128, 200*128, 400*128, 800*128, rdi, ZMM_SCD7, r10, ZMM_SCD7, 200, L1PREFETCH_ALL
	zloop	200, c1b, -200*128	;; Test loop counter, restore source pointer
	end_timer 12

	zfft_3_ret
	ENDM

;; Perform a 25K-element negacyclic FFT

zfft25Kp MACRO type
	LOCAL	b1b, b2b, b3b, b4b, zmid, c4b, c3b, c2b, c1b

;; Do a multiply with pre-FFTed inputs

	cmp	ffttype, 4
	je	zmid

;; Do FFT levels 1,2,3,4
;; On input the 128-byte cache line pairs hold these data values:
;;	0	+200	...	12.5K	+200	...
;;	...
;;	199	...
;;	1600	...
;;	...
;; On output the 128-byte cache line pairs hold these data values:
;;	0	+200	...	1600	+200	...
;;	...
;;	199	...
;;	3200	...
;;	...
;; These are swizzled to:
;;	0	+3200	...	1600	+3200	...
;;	...
;;	1599	...

;; Do 1600 eight_complex_first_fft operations
;;	distance between fft data elements is 1600
;;	do 200 macros processing 128 data values

;; Do FFT levels 5,6,7
;; On input the 64-byte cache lines hold these data values:
;;	0	+3200	...	1600	+3200	...
;;	...
;;	1599	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+3200	...	200	+3200	...
;;	...
;;	199
;;	400	...
;;	...

;; Do 1600 eight_complex_fft operations
;;	distance between fft data elements is 200
;;	do 200 macros each processing 128 data values

	start_timer 4
	mov	rdi, sincos1		;; Load sin/cos pointer
	bump	rdi, 40*ZMM_SCD2/8
	lea	r10, [rdi+200*ZMM_SCD4/8] ;; Second sin/cos pointer
	mov	r9, norm_col_mults	;; Load weights pointer
	zloop_init 200			;; 200 iterations
	zr8fs_sixtyfour_complex_first_fft_preload
b1b:	zr8fs_sixtyfour_complex_first_fft rsi, 128, 200*128, 400*128, 800*128, r10, ZMM_CD8SCD4, rdi, ZMM_SCD4/8, r9, 8*128, 200, L1PREFETCH_ALL
	zloop	200, b1b, -200*128	;; Test loop counter, restore source pointer
	end_timer 4

;; Do FFT levels 8,9
;; On input the 64-byte cache lines hold these data values:
;;	0	+3200	...	200	+3200	...
;;	...
;;	199
;;	400	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+3200	...	40	+3200	...
;;	...
;;	39
;;	80	...
;;	...

;; Do 2560 five_complex_fft operations
;;	distance between fft data elements is 40
;;	do 320 macros each processing 80 data values

	start_timer 6
	bump	rdi, -200*ZMM_SCD4/8-40*ZMM_SCD2/8 ;; Load sin/cos pointer
	zloop_init 320			;; 8 iterations of 40
	zr5b_five_complex_djbfft_preload
b3b:	zr5b_five_complex_djbfft rsi, 128, 40*128, rdi, ZMM_SCD2/8, 40
	zloop	40, b3b, -40*128+200*128, rdi, -40*ZMM_SCD2/8 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	8, b3b, -8*200*128	;; Test loop counter, restore source pointer
	end_timer 6

;; Do FFT levels 10,11,12
;; On input the 64-byte cache lines hold these data values:
;;	0	+3200	...	40	+3200	...
;;	...
;;	39
;;	80	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+3200	...	8	+3200	...
;;	...
;;	7
;;	16	...
;;	32	...
;;	...

;; Do 2560 five_complex_fft operations
;;	distance between fft data elements is 8
;;	do 320 macros each processing 80 data values

	start_timer 7
	zloop_init 320			;; 40 iterations of 8
	zr5b_five_complex_djbfft_preload
b4b:	zr5b_five_complex_djbfft rsi, 128, 8*128, rdi, 5*ZMM_SCD2/8, 8
	zloop	8, b4b, -8*128+40*128, rdi, -8*5*ZMM_SCD2/8 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	40, b4b, -40*40*128	;; Test loop counter, restore source pointer
	end_timer 7

;; Do FFT levels 13,14,15
;; On input and output the 128-byte cache line pairs hold these data values:
;;	0	+3200	...	8	+3200	...
;;	...
;;	7
;;	16	...
;;	32	...
;;	...

;; Do 1600 eight_complex_fft operations
;;	distance between fft data elements is 1
;;	do 200 macros processing 128 data values

zmid:	start_timer 8
	mov	rcx, 200
	call	zcomplex_onepass_middle
	end_timer 8

;; Do inverse FFT levels 10,11,12
;; On input the 64-byte cache lines hold these data values:
;;	0	+3200	...	8	+3200	...
;;	...
;;	7
;;	16	...
;;	32	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+3200	...	40	+3200	...
;;	...
;;	39
;;	80	...
;;	...

;; Do 2560 five_complex_unfft operations
;;	distance between fft data elements is 8
;;	do 320 macros each processing 80 data values

	start_timer 9
	mov	rdi, sincos1		;; Load sin/cos pointer
	zloop_init 320			;; 40 iterations of 8
	zr5b_five_complex_djbunfft_preload
c4b:	zr5b_five_complex_djbunfft rsi, 128, 8*128, rdi, 5*ZMM_SCD2/8, 8
	zloop	8, c4b, -8*128+40*128, rdi, -8*5*ZMM_SCD2/8 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	40, c4b, -40*40*128	;; Test loop counter, restore source pointer
	end_timer 9

;; Do inverse FFT levels 8,9
;; On input the 64-byte cache lines hold these data values:
;;	0	+3200	...	40	+3200	...
;;	...
;;	39
;;	80	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+3200	...	200	+3200	...
;;	...
;;	199
;;	400	...
;;	...

;; Do 2560 five_complex_unfft operations
;;	distance between fft data elements is 40
;;	do 320 macros each processing 48 data values

	start_timer 10
	zloop_init 320			;; 8 iterations of 40
	zr5b_five_complex_djbunfft_preload
c3b:	zr5b_five_complex_djbunfft rsi, 128, 40*128, rdi, ZMM_SCD2/8, 40
	zloop	40, c3b, -40*128+200*128, rdi, -40*ZMM_SCD2/8 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	8, c3b, -8*200*128, rdi, 40*ZMM_SCD2/8 ;; Test loop counter, restore source pointer, next sin/cos ptr
	end_timer 10

;; Do inverse FFT levels 5,6,7
;; On input the 64-byte cache lines hold these data values:
;;	0	+3200	...	200	+3200	...
;;	...
;;	199
;;	400	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+3200	...	1600	+3200	...
;;	...
;;	1599	...

;; Do 1600 eight_complex_unfft operations
;;	distance between fft data elements is 200
;;	do 200 macros each processing 128 data values

;; Do inverse FFT levels 1,2,3,4
;; On input the 128-byte cache line pairs hold these data values:
;;	0	+3200	...	1600	+3200	...
;;	...
;;	1599	...
;; These are swizzled to:
;;	0	+200	...	1600	+200	...
;;	...
;;	199	...
;;	3200	...
;;	...
;; On output the 128-byte cache line pairs hold these data values:
;;	0	+200	...	12.5K	+200	...
;;	...
;;	199	...
;;	1600	...
;;	...

;; Do 1600 eight_complex_last_unfft operations
;;	distance between fft data elements is 1600
;;	do 200 macros processing 128 data values

	start_timer 12
	lea	r10, [rdi+200*ZMM_SCD4/8] ;; Second sin/cos pointer
	zloop_init 200			;; 200 iterations
	zr8s_sixtyfour_complex_last_unfft_preload
c1b:	zr8s_sixtyfour_complex_last_unfft rsi, 128, 200*128, 400*128, 800*128, rdi, ZMM_SCD4/8, r10, ZMM_CD8SCD4, 200, L1PREFETCH_ALL
	zloop	200, c1b, -200*128	;; Test loop counter, restore source pointer
	end_timer 12

	zfft_3_ret
	ENDM


;; Perform a 28K-element FFT

zfft28K MACRO type
	LOCAL	b1b, b2b, b3a, b3b, b4a, b4b, zmid, c4a, c4b, c3a, c3b, c2b, c1b

;; Check for a multiply with pre-FFTed inputs

	cmp	ffttype, 4
	je	zmid

;; Do FFT levels 1,2,3,4
;; On input the 128-byte cache line pairs hold these data values:
;;	0	+224	...	14K	+224	...
;;	...
;;	223	...
;;	1792	...
;;	...
;; On output the 128-byte cache line pairs hold these data values:
;;	0	+224	...	1792	+224	...
;;	...
;;	223	...
;;	3584	...
;;	...
;; These are swizzled to:
;;	0	+3584	...	1792	+3584	...
;;	...
;;	1791	...

;; Do 1792 sixteen_reals_fft operations
;;	distance between fft data elements is 1792
;;	do 224 macros processing 128 data values

;; Do FFT levels 5,6,7
;; On input the 64-byte cache lines hold these data values:
;;	0	+3584	...	1792	+3584	...
;;	...
;;	1791	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+3584	...	224	+3584	...
;;	...
;;	223
;;	448	...
;;	...

;; Do 224 sixteen_reals_fft operation
;; Do 1568 eight_complex_fft operations
;;	distance between fft data elements is 224
;;	do 224 macros each processing 128 data values

	start_timer 4
	mov	rdi, sincos1		;; Load sin/cos data ptr
	bump	rdi, 32*ZMM_SCD6+8*ZMM_SCD3
	lea	r10, [rdi+224*ZMM_SCD7]	;; Second sin/cos pointer
	mov	r9, norm_col_mults	;; Load weights pointer
	zloop_init 224			;; 224 iterations
	zr8fs_hundredtwentyeight_reals_first_fft_preload
b1b:	zr8fs_hundredtwentyeight_reals_first_fft rsi, 128, 224*128, 448*128, 896*128, r10, ZMM_SCD7, rdi, ZMM_SCD7, r9, 8*128, 224, L1PREFETCH_ALL
	zloop	224, b1b, -224*128	;; Test loop counter, restore source pointer
	end_timer 4

;; Do FFT levels 8,9,10
;; On input the 64-byte cache lines hold these data values:
;;	0	+3584	...	224	+3584	...
;;	...
;;	223
;;	448	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+3584	...	32	+3584	...
;;	...
;;	31
;;	64	...
;;	...

;; Do 32 fourteen_reals_fft operation
;; Do 2016 seven_complex_fft operations
;;	distance between fft data elements is 32
;;	do 256 macros each processing 112 data values

	start_timer 6
	bump	rdi, -224*ZMM_SCD7-32*ZMM_SCD6
	zloop_init 256			;; 8 iterations of 32
	zr7_fourteen_reals_seven_complex_djbfft_preload
b3a:	zr7_fourteen_reals_seven_complex_djbfft rsi, 128, 32*128, rdi, ZMM_SCD6, 32
	zloop	32, b3a, -32*128+224*128, rdi, -32*ZMM_SCD6 ;; Test loop counter, next source pointer, restore sin/cos pointer

	loops_reset			;; 7 more iters of 32
	zr7rb_seven_complex_djbfft_preload
b3b:	zr7rb_seven_complex_djbfft rsi, 128, 32*128, rdi, ZMM_SCD6, 32
	zloop	32, b3b, -32*128+224*128, rdi, -32*ZMM_SCD6 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	8, b3b, -8*224*128	;; Test loop counter, restore source pointer
	end_timer 6

;; Do FFT levels 10,11,12
;; On input the 64-byte cache lines hold these data values:
;;	0	+3584	...	32	+3584	...
;;	...
;;	31
;;	64	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+3584	...	8	+3584	...
;;	...
;;	7
;;	16	...
;;	32	...
;;	...

;; Do 8 eight_reals_fft operation
;; Do 3576 four_complex_fft operations
;;	distance between fft data elements is 8
;;	do 448 macros each processing 64 data values

	start_timer 7
	bump	rdi, -8*ZMM_SCD3
	zloop_init 448			;; 56 iterations of 8
	zr4_eight_reals_four_complex_djbfft_preload
b4a:	zr4_eight_reals_four_complex_djbfft rsi, 128, 8*128, 16*128, rdi, ZMM_SCD3, 8
	zloop	8, b4a, -8*128+32*128, rdi, -8*ZMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer

	loops_reset			;; 55 more iters of 8
	zr4rb_four_complex_djbfft_preload
b4b:	zr4rb_four_complex_djbfft rsi, 128, 8*128, 16*128, rdi, ZMM_SCD3, 8
	zloop	8, b4b, -8*128+32*128, rdi, -8*ZMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	56, b4b, -56*32*128	;; Test loop counter, restore source pointer
	end_timer 7

;; Do FFT levels 13,14,15
;; On input and output the 128-byte cache line pairs hold these data values:
;;	0	+3584	...	8	+3584	...
;;	...
;;	7
;;	16	...
;;	32	...
;;	...

;; Do 1 sixteen_reals_fft operation
;; Do 1791 eight_complex_fft operations
;;	distance between fft data elements is 1
;;	do 224 macros processing 128 data values

zmid:	start_timer 8
	mov	rcx, 224
	call	zreal_onepass_middle
	end_timer 8

;; Do inverse FFT levels 10,11,12
;; On input the 64-byte cache lines hold these data values:
;;	0	+3584	...	8	+3584	...
;;	...
;;	7
;;	16	...
;;	32	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+3584	...	32	+3584	...
;;	...
;;	31
;;	64	...
;;	...

;; Do 8 eight_reals_unfft operations
;; Do 3576 four_complex_unfft operations
;;	distance between fft data elements is 8
;;	do 448 macros each processing 64 data values

	start_timer 9
	mov	rdi, sincos1		;; Load sin/cos pointer
	zloop_init 448			;; 56 iterations of 8
	zr4_eight_reals_four_complex_djbunfft_preload
c4a:	zr4_eight_reals_four_complex_djbunfft rsi, 128, 8*128, 16*128, rdi, ZMM_SCD3, 8
	zloop	8, c4a, -8*128+32*128, rdi, -8*ZMM_SCD3 ;; Test loop counter, restore source pointer, restore sin/cos pointer

	loops_reset			;; 55 more iters of 8
	zr4rb_four_complex_djbunfft_preload
c4b:	zr4rb_four_complex_djbunfft rsi, 128, 8*128, 16*128, rdi, ZMM_SCD3, 8
	zloop	8, c4b, -8*128+32*128, rdi, -8*ZMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	56, c4b, -56*32*128, rdi, 8*ZMM_SCD3 ;; Test loop counter, restore source pointer, next sin/cos ptr
	end_timer 9

;; Do inverse FFT levels 8,9
;; On input the 64-byte cache lines hold these data values:
;;	0	+3584	...	32	+3584	...
;;	...
;;	31
;;	64	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+3584	...	224	+3584	...
;;	...
;;	223
;;	448	...
;;	...

;; Do 32 fourteen_reals_unfft operations
;; Do 2016 seven_complex_unfft operations
;;	distance between fft data elements is 32
;;	do 256 macros each processing 112 data values

	start_timer 10
	zloop_init 256			;; 8 iterations of 32
	zr7_fourteen_reals_seven_complex_djbunfft_preload
c3a:	zr7_fourteen_reals_seven_complex_djbunfft rsi, 128, 32*128, rdi, ZMM_SCD6, 32
	zloop	32, c3a, -32*128+224*128, rdi, -32*ZMM_SCD6 ;; Test loop counter, restore source pointer, restore sin/cos pointer

	loops_reset			;; 7 more iters of 32
	zr7rb_seven_complex_djbunfft_preload
c3b:	zr7rb_seven_complex_djbunfft rsi, 128, 32*128, rdi, ZMM_SCD6, 32
	zloop	32, c3b, -32*128+224*128, rdi, -32*ZMM_SCD6 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	8, c3b, -8*224*128, rdi, 32*ZMM_SCD6 ;; Test loop counter, restore source pointer, next sin/cos ptr
	end_timer 10

;; Do inverse FFT levels 5,6,7
;; On input the 64-byte cache lines hold these data values:
;;	0	+3584	...	224	+3584	...
;;	...
;;	223
;;	448	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+3584	...	1792	+3584	...
;;	...
;;	1791	...

;; Do 224 sixteen_reals_unfft operations
;; Do 1568 eight_complex_unfft operations
;;	distance between fft data elements is 224
;;	do 224 macros each processing 128 data values

;; Do inverse FFT levels 1,2,3,4
;; On input the 128-byte cache line pairs hold these data values:
;;	0	+3584	...	1792	+3584	...
;;	...
;;	1791	...
;; These are swizzled to:
;;	0	+224	...	1792	+224	...
;;	...
;;	223	...
;;	3584	...
;;	...
;; On output the 128-byte cache line pairs hold these data values:
;;	0	+224	...	14K	+224	...
;;	...
;;	223	...
;;	1792	...
;;	...

;; Do 1792 sixteen_reals_unfft operations
;;	distance between fft data elements is 1792
;;	do 224 macros processing 128 data values

	start_timer 12
	lea	r10, [rdi+224*ZMM_SCD7]	;; Second sin/cos pointer
	zloop_init 224			;; 224 iterations
	zr8s_hundredtwentyeight_reals_last_unfft_preload
c1b:	zr8s_hundredtwentyeight_reals_last_unfft rsi, 128, 224*128, 448*128, 896*128, rdi, ZMM_SCD7, r10, ZMM_SCD7, 224, L1PREFETCH_ALL
	zloop	224, c1b, -224*128	;; Test loop counter, restore source pointer
	end_timer 12

	zfft_3_ret
	ENDM

;; Perform a 28K-element negacyclic FFT

zfft28Kp MACRO type
	LOCAL	b1b, b2b, b3b, b4b, zmid, c4b, c3b, c2b, c1b

;; Do a multiply with pre-FFTed inputs

	cmp	ffttype, 4
	je	zmid

;; Do FFT levels 1,2,3,4
;; On input the 128-byte cache line pairs hold these data values:
;;	0	+224	...	14K	+224	...
;;	...
;;	223	...
;;	1792	...
;;	...
;; On output the 128-byte cache line pairs hold these data values:
;;	0	+224	...	1792	+224	...
;;	...
;;	223	...
;;	3584	...
;;	...
;; These are swizzled to:
;;	0	+3584	...	1792	+3584	...
;;	...
;;	1791	...

;; Do 1792 eight_complex_first_fft operations
;;	distance between fft data elements is 1792
;;	do 224 macros processing 128 data values

;; Do FFT levels 5,6,7
;; On input the 64-byte cache lines hold these data values:
;;	0	+3584	...	1792	+3584	...
;;	...
;;	1791	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+3584	...	224	+3584	...
;;	...
;;	223
;;	448	...
;;	...

;; Do 1792 eight_complex_fft operations
;;	distance between fft data elements is 224
;;	do 224 macros each processing 128 data values

	start_timer 4
	mov	rdi, sincos1		;; Load sin/cos pointer
	bump	rdi, 32*ZMM_SCD3/8+8*ZMM_SCD2/8
	lea	r10, [rdi+224*ZMM_SCD4/8] ;; Second sin/cos pointer
	mov	r9, norm_col_mults	;; Load weights pointer
	zloop_init 224			;; 224 iterations
	zr8fs_sixtyfour_complex_first_fft_preload
b1b:	zr8fs_sixtyfour_complex_first_fft rsi, 128, 224*128, 448*128, 896*128, r10, ZMM_CD8SCD4, rdi, ZMM_SCD4/8, r9, 8*128, 224, L1PREFETCH_ALL
	zloop	224, b1b, -224*128	;; Test loop counter, restore source pointer
	end_timer 4

;; Do FFT levels 8,9,10
;; On input the 64-byte cache lines hold these data values:
;;	0	+3584	...	224	+3584	...
;;	...
;;	223
;;	448	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+3584	...	32	+3584	...
;;	...
;;	31
;;	64	...
;;	...

;; Do 2048 seven_complex_fft operations
;;	distance between fft data elements is 32
;;	do 256 macros each processing 112 data values

	start_timer 6
	bump	rdi, -224*ZMM_SCD4/8-32*ZMM_SCD3/8 ;; Load sin/cos pointer
	zloop_init 256			;; 8 iterations of 32
	zr7b_seven_complex_djbfft_preload
b3b:	zr7b_seven_complex_djbfft rsi, 128, 32*128, rdi, ZMM_SCD3/8, 32
	zloop	32, b3b, -32*128+224*128, rdi, -32*ZMM_SCD3/8 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	8, b3b, -8*224*128	;; Test loop counter, restore source pointer
	end_timer 6

;; Do FFT levels 10,11,12
;; On input the 64-byte cache lines hold these data values:
;;	0	+3584	...	32	+3584	...
;;	...
;;	31
;;	64	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+3584	...	8	+3584	...
;;	...
;;	7
;;	16	...
;;	32	...
;;	...

;; Do 3584 four_complex_fft operations
;;	distance between fft data elements is 8
;;	do 448 macros each processing 64 data values

	start_timer 7
	bump	rdi, -8*ZMM_SCD2/8	;; Load sin/cos pointer
	zloop_init 448			;; 56 iterations of 8
	zr4b_four_complex_djbfft_preload
b4b:	zr4b_four_complex_djbfft rsi, 128, 8*128, 16*128, rdi, ZMM_SCD2/8, 8
	zloop	8, b4b, -8*128+32*128, rdi, -8*ZMM_SCD2/8 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	56, b4b, -56*32*128	;; Test loop counter, restore source pointer
	end_timer 7

;; Do FFT levels 13,14,15
;; On input and output the 128-byte cache line pairs hold these data values:
;;	0	+3584	...	8	+3584	...
;;	...
;;	7
;;	16	...
;;	32	...
;;	...

;; Do 1792 eight_complex_fft operations
;;	distance between fft data elements is 1
;;	do 224 macros processing 128 data values

zmid:	start_timer 8
	mov	rcx, 224
	call	zcomplex_onepass_middle
	end_timer 8

;; Do inverse FFT levels 10,11,12
;; On input the 64-byte cache lines hold these data values:
;;	0	+3584	...	8	+3584	...
;;	...
;;	7
;;	16	...
;;	32	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+3584	...	32	+3584	...
;;	...
;;	31
;;	64	...
;;	...

;; Do 3584 four_complex_unfft operations
;;	distance between fft data elements is 8
;;	do 448 macros each processing 64 data values

	start_timer 9
	mov	rdi, sincos1		;; Load sin/cos pointer
	zloop_init 448			;; 56 iterations of 8
	zr4b_four_complex_djbunfft_preload
c4b:	zr4b_four_complex_djbunfft rsi, 128, 8*128, 16*128, rdi, ZMM_SCD2/8, 8
	zloop	8, c4b, -8*128+32*128, rdi, -8*ZMM_SCD2/8 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	56, c4b, -56*32*128, rdi, 8*ZMM_SCD2/8	;; Test loop counter, restore source pointer, next sin/cos ptr
	end_timer 9

;; Do inverse FFT levels 8,9
;; On input the 64-byte cache lines hold these data values:
;;	0	+3584	...	32	+3584	...
;;	...
;;	31
;;	64	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+3584	...	224	+3584	...
;;	...
;;	223
;;	448	...
;;	...

;; Do 2048 seven_complex_unfft operations
;;	distance between fft data elements is 32
;;	do 256 macros each processing 112 data values

	start_timer 10
	zloop_init 256			;; 8 iterations of 32
	zr7b_seven_complex_djbunfft_preload
c3b:	zr7b_seven_complex_djbunfft rsi, 128, 32*128, rdi, ZMM_SCD3/8, 32
	zloop	32, c3b, -32*128+224*128, rdi, -32*ZMM_SCD3/8 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	8, c3b, -8*224*128, rdi, 32*ZMM_SCD3/8 ;; Test loop counter, restore source pointer, next sin/cos ptr
	end_timer 10

;; Do inverse FFT levels 5,6,7
;; On input the 64-byte cache lines hold these data values:
;;	0	+3584	...	224	+3584	...
;;	...
;;	223
;;	448	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+3584	...	1792	+3584	...
;;	...
;;	1791	...

;; Do 1792 eight_complex_unfft operations
;;	distance between fft data elements is 224
;;	do 224 macros each processing 128 data values

;; Do inverse FFT levels 1,2,3,4
;; On input the 128-byte cache line pairs hold these data values:
;;	0	+3584	...	1792	+3584	...
;;	...
;;	1791	...
;; These are swizzled to:
;;	0	+224	...	1792	+224	...
;;	...
;;	223	...
;;	3584	...
;;	...
;; On output the 128-byte cache line pairs hold these data values:
;;	0	+224	...	14K	+224	...
;;	...
;;	223	...
;;	1792	...
;;	...

;; Do 1792 eight_complex_last_unfft operations
;;	distance between fft data elements is 1792
;;	do 224 macros processing 128 data values

	start_timer 12
	lea	r10, [rdi+224*ZMM_SCD4/8] ;; Second sin/cos pointer
	zloop_init 224			;; 224 iterations
	zr8s_sixtyfour_complex_last_unfft_preload
c1b:	zr8s_sixtyfour_complex_last_unfft rsi, 128, 224*128, 448*128, 896*128, rdi, ZMM_SCD4/8, r10, ZMM_CD8SCD4, 224, L1PREFETCH_ALL
	zloop	224, c1b, -224*128	;; Test loop counter, restore source pointer
	end_timer 12

	zfft_3_ret
	ENDM


;; Perform a 30K-element FFT

zfft30K MACRO type
	LOCAL	b1b, b2b, b3a, b3b, b4a, b4b, zmid, c4a, c4b, c3a, c3b, c2b, c1b

;; Check for a multiply with pre-FFTed inputs

	cmp	ffttype, 4
	je	zmid

;; Do FFT levels 1,2,3,4
;; On input the 128-byte cache line pairs hold these data values:
;;	0	+240	...	15K	+240	...
;;	...
;;	239	...
;;	1920	...
;;	...
;; On output the 128-byte cache line pairs hold these data values:
;;	0	+240	...	1920	+240	...
;;	...
;;	239	...
;;	3840	...
;;	...
;; These are swizzled to:
;;	0	+3840	...	1920	+3840	...
;;	...
;;	1919	...

;; Do 1920 sixteen_reals_fft operations
;;	distance between fft data elements is 1920
;;	do 240 macros processing 128 data values

;; Do FFT levels 5,6,7
;; On input the 64-byte cache lines hold these data values:
;;	0	+3840	...	1920	+3840	...
;;	...
;;	1919	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+3840	...	240	+3840	...
;;	...
;;	239
;;	480	...
;;	...

;; Do 240 sixteen_reals_fft operation
;; Do 1680 eight_complex_fft operations
;;	distance between fft data elements is 240
;;	do 240 macros each processing 128 data values

	start_timer 4
	mov	rdi, sincos1		;; Load sin/cos data ptr
	bump	rdi, 48*ZMM_SCD4+8*ZMM_SCD5
	lea	r10, [rdi+240*ZMM_SCD7]	;; Second sin/cos pointer
	mov	r9, norm_col_mults	;; Load weights pointer
	zloop_init 240			;; 240 iterations
	zr8fs_hundredtwentyeight_reals_first_fft_preload
b1b:	zr8fs_hundredtwentyeight_reals_first_fft rsi, 128, 240*128, 480*128, 960*128, r10, ZMM_SCD7, rdi, ZMM_SCD7, r9, 8*128, 240, L1PREFETCH_ALL
	zloop	240, b1b, -240*128	;; Test loop counter, restore source pointer
	end_timer 4

;; Do FFT levels 8,9,10
;; On input the 64-byte cache lines hold these data values:
;;	0	+3840	...	240	+3840	...
;;	...
;;	239
;;	480	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+3840	...	48	+3840	...
;;	...
;;	47
;;	96	...
;;	...

;; Do 48 ten_reals_fft operation
;; Do 3024 five_complex_fft operations
;;	distance between fft data elements is 48
;;	do 384 macros each processing 80 data values

	start_timer 6
	bump	rdi, -240*ZMM_SCD7-48*ZMM_SCD4
	zloop_init 384			;; 8 iterations of 48
	zr5_ten_reals_five_complex_djbfft_preload
b3a:	zr5_ten_reals_five_complex_djbfft rsi, 128, 48*128, rdi, ZMM_SCD4, 48
	zloop	48, b3a, -48*128+240*128, rdi, -48*ZMM_SCD4 ;; Test loop counter, next source pointer, restore sin/cos pointer

	loops_reset			;; 7 more iters of 48
	zr5rb_five_complex_djbfft_preload
b3b:	zr5rb_five_complex_djbfft rsi, 128, 48*128, rdi, ZMM_SCD4, 48
	zloop	48, b3b, -48*128+240*128, rdi, -48*ZMM_SCD4 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	8, b3b, -8*240*128	;; Test loop counter, restore source pointer
	end_timer 6

;; Do FFT levels 11,12
;; On input the 64-byte cache lines hold these data values:
;;	0	+3840	...	48	+3840	...
;;	...
;;	47
;;	96	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+3840	...	8	+3840	...
;;	...
;;	7
;;	16	...
;;	32	...
;;	...

;; Do 8 twelve_reals_fft operation
;; Do 2552 six_complex_fft operations
;;	distance between fft data elements is 8
;;	do 320 macros each processing 96 data values

	start_timer 7
	bump	rdi, -8*ZMM_SCD5
	zloop_init 320			;; 40 iterations of 8
	zr6_twelve_reals_six_complex_djbfft_preload
b4a:	zr6_twelve_reals_six_complex_djbfft rsi, 128, 8*128, rdi, ZMM_SCD5, 8
	zloop	8, b4a, -8*128+48*128, rdi, -8*ZMM_SCD5 ;; Test loop counter, next source pointer, restore sin/cos pointer

	loops_reset			;; 39 more iters of 8
	zr6rb_six_complex_djbfft_preload
b4b:	zr6rb_six_complex_djbfft rsi, 128, 8*128, rdi, ZMM_SCD5, 8
	zloop	8, b4b, -8*128+48*128, rdi, -8*ZMM_SCD5 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	40, b4b, -40*48*128	;; Test loop counter, restore source pointer
	end_timer 7

;; Do FFT levels 13,14,15
;; On input and output the 128-byte cache line pairs hold these data values:
;;	0	+3840	...	8	+3840	...
;;	...
;;	7
;;	16	...
;;	32	...
;;	...

;; Do 1 sixteen_reals_fft operation
;; Do 1919 eight_complex_fft operations
;;	distance between fft data elements is 1
;;	do 240 macros processing 128 data values

zmid:	start_timer 8
	mov	rcx, 240
	call	zreal_onepass_middle
	end_timer 8

;; Do inverse FFT levels 11,12
;; On input the 64-byte cache lines hold these data values:
;;	0	+3840	...	8	+3840	...
;;	...
;;	7
;;	16	...
;;	32	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+3840	...	48	+3840	...
;;	...
;;	47
;;	96	...
;;	...

;; Do 8 twelve_reals_unfft operations
;; Do 2552 six_complex_unfft operations
;;	distance between fft data elements is 8
;;	do 320 macros each processing 96 data values

	start_timer 9
	mov	rdi, sincos1		;; Load sin/cos pointer
	zloop_init 320			;; 40 iterations of 8
	zr6_twelve_reals_six_complex_djbunfft_preload
c4a:	zr6_twelve_reals_six_complex_djbunfft rsi, 128, 8*128, rdi, ZMM_SCD5, 8
	zloop	8, c4a, -8*128+48*128, rdi, -8*ZMM_SCD5 ;; Test loop counter, restore source pointer, restore sin/cos pointer

	loops_reset			;; 39 more iters of 8
	zr6rb_six_complex_djbunfft_preload
c4b:	zr6rb_six_complex_djbunfft rsi, 128, 8*128, rdi, ZMM_SCD5, 8
	zloop	8, c4b, -8*128+48*128, rdi, -8*ZMM_SCD5 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	40, c4b, -40*48*128, rdi, 8*ZMM_SCD5 ;; Test loop counter, restore source pointer, next sin/cos ptr
	end_timer 9

;; Do inverse FFT levels 8,9,10
;; On input the 64-byte cache lines hold these data values:
;;	0	+3840	...	48	+3840	...
;;	...
;;	47
;;	96	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+3840	...	240	+3840	...
;;	...
;;	239
;;	480	...
;;	...

;; Do 48 ten_reals_unfft operations
;; Do 3024 five_complex_unfft operations
;;	distance between fft data elements is 48
;;	do 384 macros each processing 80 data values

	start_timer 10
	zloop_init 384			;; 8 iterations of 48
	zr5_ten_reals_five_complex_djbunfft_preload
c3a:	zr5_ten_reals_five_complex_djbunfft rsi, 128, 48*128, rdi, ZMM_SCD4, 48
	zloop	48, c3a, -48*128+240*128, rdi, -48*ZMM_SCD4 ;; Test loop counter, restore source pointer, restore sin/cos pointer

	loops_reset			;; 7 more iters of 48
	zr5rb_five_complex_djbunfft_preload
c3b:	zr5rb_five_complex_djbunfft rsi, 128, 48*128, rdi, ZMM_SCD4, 48
	zloop	48, c3b, -48*128+240*128, rdi, -48*ZMM_SCD4 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	8, c3b, -8*240*128, rdi, 48*ZMM_SCD4 ;; Test loop counter, restore source pointer, next sin/cos ptr
	end_timer 10

;; Do inverse FFT levels 5,6,7
;; On input the 64-byte cache lines hold these data values:
;;	0	+3840	...	240	+3840	...
;;	...
;;	239
;;	480	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+3840	...	1920	+3840	...
;;	...
;;	1919	...

;; Do 240 sixteen_reals_unfft operations
;; Do 1680 eight_complex_unfft operations
;;	distance between fft data elements is 240
;;	do 240 macros each processing 128 data values

;; Do inverse FFT levels 1,2,3,4
;; On input the 128-byte cache line pairs hold these data values:
;;	0	+3840	...	1920	+3840	...
;;	...
;;	1919	...
;; These are swizzled to:
;;	0	+120	...	960	+120	...
;;	...
;;	119	...
;;	1920	...
;;	...
;; On output the 128-byte cache line pairs hold these data values:
;;	0	+240	...	15K	+240	...
;;	...
;;	239	...
;;	1920	...
;;	...

;; Do 1920 sixteen_reals_unfft operations
;;	distance between fft data elements is 1920
;;	do 240 macros processing 128 data values

	start_timer 12
	lea	r10, [rdi+240*ZMM_SCD7]	;; Second sin/cos pointer
	zloop_init 240			;; 240 iterations
	zr8s_hundredtwentyeight_reals_last_unfft_preload
c1b:	zr8s_hundredtwentyeight_reals_last_unfft rsi, 128, 240*128, 480*128, 960*128, rdi, ZMM_SCD7, r10, ZMM_SCD7, 240, L1PREFETCH_ALL
	zloop	240, c1b, -240*128	;; Test loop counter, restore source pointer
	end_timer 12

	zfft_3_ret
	ENDM

;; Perform a 30K-element negacyclic FFT

zfft30Kp MACRO type
	LOCAL	b1b, b2b, b3b, b4b, zmid, c4b, c3b, c2b, c1b

;; Do a multiply with pre-FFTed inputs

	cmp	ffttype, 4
	je	zmid

;; Do FFT levels 1,2,3,4
;; On input the 128-byte cache line pairs hold these data values:
;;	0	+240	...	15K	+240	...
;;	...
;;	239	...
;;	1920	...
;;	...
;; On output the 128-byte cache line pairs hold these data values:
;;	0	+240	...	1920	+240	...
;;	...
;;	239	...
;;	3840	...
;;	...
;; These are swizzled to:
;;	0	+3840	...	1920	+3840	...
;;	...
;;	1919	...

;; Do 1920 eight_complex_first_fft operations
;;	distance between fft data elements is 1920
;;	do 240 macros processing 128 data values

;; Do FFT levels 5,6,7
;; On input the 64-byte cache lines hold these data values:
;;	0	+3840	...	1920	+3840	...
;;	...
;;	1919	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+3840	...	240	+3840	...
;;	...
;;	239
;;	480	...
;;	...

;; Do 1920 eight_complex_fft operations
;;	distance between fft data elements is 1920
;;	do 240 macros each processing 128 data values

	start_timer 4
	mov	rdi, sincos1		;; Load sin/cos pointer
	bump	rdi, 48*ZMM_SCD2/8+8*ZMM_SCD3/8
	lea	r10, [rdi+240*ZMM_SCD4/8] ;; Second sin/cos pointer
	mov	r9, norm_col_mults	;; Load weights pointer
	zloop_init 240			;; 240 iterations
	zr8fs_sixtyfour_complex_first_fft_preload
b1b:	zr8fs_sixtyfour_complex_first_fft rsi, 128, 240*128, 480*128, 960*128, r10, ZMM_CD8SCD4, rdi, ZMM_SCD4/8, r9, 8*128, 240, L1PREFETCH_ALL
	zloop	240, b1b, -240*128	;; Test loop counter, restore source pointer
	end_timer 4

;; Do FFT levels 8,9,10
;; On input the 64-byte cache lines hold these data values:
;;	0	+3840	...	240	+3840	...
;;	...
;;	239
;;	480	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+3840	...	48	+3840	...
;;	...
;;	47
;;	96	...
;;	...

;; Do 3072 five_complex_fft operations
;;	distance between fft data elements is 48
;;	do 384 macros each processing 80 data values

	start_timer 6
	bump	rdi, -240*ZMM_SCD4/8-48*ZMM_SCD2/8 ;; Load sin/cos pointer
	zloop_init 384			;; 8 iterations of 48
	zr5b_five_complex_djbfft_preload
b3b:	zr5b_five_complex_djbfft rsi, 128, 48*128, rdi, ZMM_SCD2/8, 48
	zloop	48, b3b, -48*128+240*128, rdi, -48*ZMM_SCD2/8 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	8, b3b, -8*240*128	;; Test loop counter, restore source pointer
	end_timer 6

;; Do FFT levels 11,12
;; On input the 64-byte cache lines hold these data values:
;;	0	+3840	...	48	+3840	...
;;	...
;;	47
;;	96	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+3840	...	8	+3840	...
;;	...
;;	7
;;	16	...
;;	32	...
;;	...

;; Do 2560 six_complex_fft operations
;;	distance between fft data elements is 8
;;	do 320 macros each processing 96 data values

	start_timer 7
	bump	rdi, -8*ZMM_SCD3/8	;; Load sin/cos pointer
	zloop_init 320			;; 40 iterations of 8
	zr6b_six_complex_djbfft_preload
b4b:	zr6b_six_complex_djbfft rsi, 128, 8*128, rdi, ZMM_SCD3/8, 8
	zloop	8, b4b, -8*128+48*128, rdi, -8*ZMM_SCD3/8 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	40, b4b, -40*48*128	;; Test loop counter, restore source pointer
	end_timer 7

;; Do FFT levels 13,14,15
;; On input and output the 128-byte cache line pairs hold these data values:
;;	0	+3840	...	8	+3840	...
;;	...
;;	7
;;	16	...
;;	32	...
;;	...

;; Do 1920 eight_complex_fft operations
;;	distance between fft data elements is 1
;;	do 240 macros processing 128 data values

zmid:	start_timer 8
	mov	rcx, 240
	call	zcomplex_onepass_middle
	end_timer 8

;; Do inverse FFT levels 11,12
;; On input the 64-byte cache lines hold these data values:
;;	0	+3840	...	8	+3840	...
;;	...
;;	7
;;	16	...
;;	32	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+3840	...	48	+3840	...
;;	...
;;	47
;;	96	...
;;	...

;; Do 2560 six_complex_unfft operations
;;	distance between fft data elements is 8
;;	do 320 macros each processing 96 data values

	start_timer 9
	mov	rdi, sincos1		;; Load sin/cos pointer
	zloop_init 320			;; 40 iterations of 8
	zr6b_six_complex_djbunfft_preload
c4b:	zr6b_six_complex_djbunfft rsi, 128, 8*128, rdi, ZMM_SCD3/8, 8
	zloop	8, c4b, -8*128+48*128, rdi, -8*ZMM_SCD3/8 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	40, c4b, -40*48*128, rdi, 8*ZMM_SCD3/8	;; Test loop counter, restore source pointer, next sin/cos ptr
	end_timer 9

;; Do inverse FFT levels 8,9,10
;; On input the 64-byte cache lines hold these data values:
;;	0	+3840	...	48	+3840	...
;;	...
;;	47
;;	96	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+3840	...	240	+3840	...
;;	...
;;	239
;;	480	...
;;	...

;; Do 3072 five_complex_unfft operations
;;	distance between fft data elements is 48
;;	do 384 macros each processing 80 data values

	start_timer 10
	zloop_init 384			;; 8 iterations of 48
	zr5b_five_complex_djbunfft_preload
c3b:	zr5b_five_complex_djbunfft rsi, 128, 48*128, rdi, ZMM_SCD2/8, 48
	zloop	48, c3b, -48*128+240*128, rdi, -48*ZMM_SCD2/8 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	8, c3b, -8*240*128, rdi, 48*ZMM_SCD2/8 ;; Test loop counter, restore source pointer, next sin/cos ptr
	end_timer 10

;; Do inverse FFT levels 5,6,7
;; On input the 64-byte cache lines hold these data values:
;;	0	+3840	...	240	+3840	...
;;	...
;;	239
;;	480	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+3840	...	1920	+3840	...
;;	...
;;	1919	...

;; Do 1920 eight_complex_unfft operations
;;	distance between fft data elements is 240
;;	do 240 macros each processing 128 data values

;; Do inverse FFT levels 1,2,3,4
;; On input the 128-byte cache line pairs hold these data values:
;;	0	+3840	...	1920	+3840	...
;;	...
;;	1919	...
;; These are swizzled to:
;;	0	+120	...	960	+120	...
;;	...
;;	119	...
;;	1920	...
;;	...
;; On output the 128-byte cache line pairs hold these data values:
;;	0	+240	...	15K	+240	...
;;	...
;;	239	...
;;	1920	...
;;	...

;; Do 1920 eight_complex_last_unfft operations
;;	distance between fft data elements is 1920
;;	do 240 macros processing 128 data values

	start_timer 12
	lea	r10, [rdi+240*ZMM_SCD4/8] ;; Second sin/cos pointer
	zloop_init 240			;; 240 iterations
	zr8s_sixtyfour_complex_last_unfft_preload
c1b:	zr8s_sixtyfour_complex_last_unfft rsi, 128, 240*128, 480*128, 960*128, rdi, ZMM_SCD4/8, r10, ZMM_CD8SCD4, 240, L1PREFETCH_ALL
	zloop	240, c1b, -240*128	;; Test loop counter, restore source pointer
	end_timer 12

	zfft_3_ret
	ENDM


;; Perform a 32K-element FFT

zfft32K MACRO type
	LOCAL	b1b, b2b, b3a, b3b, b4a, b4b, zmid, c4a, c4b, c3a, c3b, c2b, c1b

;; Check for a multiply with pre-FFTed inputs

	cmp	ffttype, 4
	je	zmid

;; Do FFT levels 1,2,3,4
;; On input the 128-byte cache line pairs hold these data values:
;;	0	+256	...	16K	+256	...
;;	...
;;	255	...
;;	2048	...
;;	...
;; On output the 128-byte cache line pairs hold these data values:
;;	0	+256	...	2048	+256	...
;;	...
;;	255	...
;;	4K	...
;;	...
;; These are swizzled to:
;;	0	+4K	...	2048	+4K	...
;;	...
;;	2047	...

;; Do 2048 sixteen_reals_fft operations
;;	distance between fft data elements is 2048
;;	do 256 macros processing 128 data values

;; Do FFT levels 5,6,7
;; On input the 64-byte cache lines hold these data values:
;;	0	+4K	...	2048	+4K	...
;;	...
;;	2047	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+4K	...	256	+4K	...
;;	...
;;	255
;;	512	...
;;	...

;; Do 256 sixteen_reals_fft operation
;; Do 1792 eight_complex_fft operations
;;	distance between fft data elements is 256
;;	do 256 macros each processing 128 data values

	start_timer 4
	mov	rdi, sincos1		;; Load sin/cos data ptr
	bump	rdi, 8*ZMM_SCD3
	lea	r10, [rdi+256*ZMM_SCD7]	;; Second sin/cos pointer
	mov	r9, norm_col_mults	;; Load weights pointer
	zloop_init 256			;; 256 iterations
	zr8fs_hundredtwentyeight_reals_first_fft_preload
b1b:	zr8fs_hundredtwentyeight_reals_first_fft rsi, 128, 256*128, 512*128, 1024*128, r10, ZMM_SCD7, rdi, ZMM_SCD7, r9, 8*128, 256, L1PREFETCH_ALL
	zloop	256, b1b, -256*128	;; Test loop counter, restore source pointer
	end_timer 4

;; Do FFT levels 8,9,10
;; On input the 64-byte cache lines hold these data values:
;;	0	+4K	...	256	+4K	...
;;	...
;;	255
;;	512	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+4K	...	32	+4K	...
;;	...
;;	31
;;	64	...
;;	...

;; Do 32 sixteen_reals_fft operation
;; Do 2016 eight_complex_fft operations
;;	distance between fft data elements is 32
;;	do 256 macros each processing 128 data values

	start_timer 6
	bump	rdi, -256*ZMM_SCD7
	zloop_init 256			;; 8 iterations of 32
	zr8_sixteen_reals_eight_complex_djbfft_preload
b3a:	zr8_sixteen_reals_eight_complex_djbfft rsi, 128, 32*128, 64*128, 128*128, rdi, 8*ZMM_SCD7, 32
	zloop	32, b3a, -32*128+256*128, rdi, -32*8*ZMM_SCD7 ;; Test loop counter, next source pointer, restore sin/cos pointer

	loops_reset			;; 7 more iters of 32
	zr8rb_eight_complex_djbfft_preload
b3b:	zr8rb_eight_complex_djbfft rsi, 128, 32*128, 64*128, 128*128, rdi, 8*ZMM_SCD7, 32
	zloop	32, b3b, -32*128+256*128, rdi, -32*8*ZMM_SCD7 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	8, b3b, -8*256*128	;; Test loop counter, restore source pointer
	end_timer 6

;; Do FFT levels 11,12
;; On input the 64-byte cache lines hold these data values:
;;	0	+4K	...	32	+4K	...
;;	...
;;	31
;;	64	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+4K	...	8	+4K	...
;;	...
;;	7
;;	16	...
;;	32	...
;;	...

;; Do 8 eight_reals_fft operation
;; Do 4088 four_complex_fft operations
;;	distance between fft data elements is 8
;;	do 512 macros each processing 64 data values

	start_timer 7
	bump	rdi, -8*ZMM_SCD3
	zloop_init 512			;; 64 iterations of 8
	zr4_eight_reals_four_complex_djbfft_preload
b4a:	zr4_eight_reals_four_complex_djbfft rsi, 128, 8*128, 16*128, rdi, ZMM_SCD3, 8
	zloop	8, b4a, -8*128+32*128, rdi, -8*ZMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer

	loops_reset			;; 63 more iters of 8
	zr4rb_four_complex_djbfft_preload
b4b:	zr4rb_four_complex_djbfft rsi, 128, 8*128, 16*128, rdi, ZMM_SCD3, 8
	zloop	8, b4b, -8*128+32*128, rdi, -8*ZMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	64, b4b, -64*32*128	;; Test loop counter, restore source pointer
	end_timer 7

;; Do FFT levels 13,14,15
;; On input and output the 128-byte cache line pairs hold these data values:
;;	0	+4K	...	8	+4K	...
;;	...
;;	7
;;	16	...
;;	32	...
;;	...

;; Do 1 sixteen_reals_fft operation
;; Do 2047 eight_complex_fft operations
;;	distance between fft data elements is 1
;;	do 256 macros processing 128 data values

zmid:	start_timer 8
	mov	rcx, 256
	call	zreal_onepass_middle
	end_timer 8

;; Do inverse FFT levels 11,12
;; On input the 64-byte cache lines hold these data values:
;;	0	+4K	...	8	+4K	...
;;	...
;;	7
;;	16	...
;;	32	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+4K	...	32	+4K	...
;;	...
;;	31
;;	64	...
;;	...

;; Do 8 eight_reals_unfft operations
;; Do 4088 four_complex_unfft operations
;;	distance between fft data elements is 8
;;	do 512 macros each processing 64 data values

	start_timer 9
	mov	rdi, sincos1		;; Load sin/cos pointer
	zloop_init 512			;; 64 iterations of 8
	zr4_eight_reals_four_complex_djbunfft_preload
c4a:	zr4_eight_reals_four_complex_djbunfft rsi, 128, 8*128, 16*128, rdi, ZMM_SCD3, 8
	zloop	8, c4a, -8*128+32*128, rdi, -8*ZMM_SCD3 ;; Test loop counter, restore source pointer, restore sin/cos pointer

	loops_reset			;; 63 more iters of 8
	zr4rb_four_complex_djbunfft_preload
c4b:	zr4rb_four_complex_djbunfft rsi, 128, 8*128, 16*128, rdi, ZMM_SCD3, 8
	zloop	8, c4b, -8*128+32*128, rdi, -8*ZMM_SCD3 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	64, c4b, -64*32*128, rdi, 8*ZMM_SCD3 ;; Test loop counter, restore source pointer, next sin/cos ptr
	end_timer 9

;; Do inverse FFT levels 8,9,10
;; On input the 64-byte cache lines hold these data values:
;;	0	+4K	...	32	+4K	...
;;	...
;;	31
;;	64	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+4K	...	256	+4K	...
;;	...
;;	255
;;	512	...
;;	...

;; Do 32 sixteen_reals_unfft operations
;; Do 2016 eight_complex_unfft operations
;;	distance between fft data elements is 32
;;	do 256 macros each processing 128 data values

	start_timer 10
	zloop_init 256			;; 8 iterations of 32
	zr8_sixteen_reals_eight_complex_djbunfft_preload
c3a:	zr8_sixteen_reals_eight_complex_djbunfft rsi, 128, 32*128, 64*128, 128*128, rdi, 8*ZMM_SCD7, 32
	zloop	32, c3a, -32*128+256*128, rdi, -32*8*ZMM_SCD7 ;; Test loop counter, restore source pointer, restore sin/cos pointer

	loops_reset			;; 7 more iters of 32
	zr8rb_eight_complex_djbunfft_preload
c3b:	zr8rb_eight_complex_djbunfft rsi, 128, 32*128, 64*128, 128*128, rdi, 8*ZMM_SCD7, 32
	zloop	32, c3b, -32*128+256*128, rdi, -32*8*ZMM_SCD7 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	8, c3b, -8*256*128	;; Test loop counter, restore source pointer
	end_timer 10

;; Do inverse FFT levels 5,6,7
;; On input the 64-byte cache lines hold these data values:
;;	0	+4K	...	256	+4K	...
;;	...
;;	255
;;	512	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+4K	...	2048	+4K	...
;;	...
;;	2047	...

;; Do 256 sixteen_reals_unfft operations
;; Do 1792 eight_complex_unfft operations
;;	distance between fft data elements is 256
;;	do 256 macros each processing 128 data values

;; Do inverse FFT levels 1,2,3,4
;; On input the 128-byte cache line pairs hold these data values:
;;	0	+4K	...	2048	+4K	...
;;	...
;;	2047	...
;; These are swizzled to:
;;	0	+256	...	2048	+256	...
;;	...
;;	255	...
;;	4K	...
;;	...
;; On output the 128-byte cache line pairs hold these data values:
;;	0	+256	...	16K	+256	...
;;	...
;;	255	...
;;	2048	...
;;	...

;; Do 2048 sixteen_reals_unfft operations
;;	distance between fft data elements is 2048
;;	do 256 macros processing 128 data values

	start_timer 12
	lea	r10, [rdi+256*ZMM_SCD7]	;; Second sin/cos pointer
	zloop_init 256			;; 256 iterations
	zr8s_hundredtwentyeight_reals_last_unfft_preload
c1b:	zr8s_hundredtwentyeight_reals_last_unfft rsi, 128, 256*128, 512*128, 1024*128, rdi, ZMM_SCD7, r10, ZMM_SCD7, 256, L1PREFETCH_ALL
	zloop	256, c1b, -256*128	;; Test loop counter, restore source pointer
	end_timer 12

	zfft_3_ret
	ENDM

;; Perform a 32K-element negacyclic FFT

zfft32Kp MACRO type
	LOCAL	b1b, b2b, b3b, b4b, zmid, c4b, c3b, c2b, c1b

;; Do a multiply with pre-FFTed inputs

	cmp	ffttype, 4
	je	zmid

;; Do FFT levels 1,2,3,4
;; On input the 128-byte cache line pairs hold these data values:
;;	0	+256	...	16K	+256	...
;;	...
;;	255	...
;;	2048	...
;;	...
;; On output the 128-byte cache line pairs hold these data values:
;;	0	+256	...	2048	+256	...
;;	...
;;	255	...
;;	4K	...
;;	...
;; These are swizzled to:
;;	0	+4K	...	2048	+4K	...
;;	...
;;	2047	...

;; Do 2048 eight_complex_first_fft operations
;;	distance between fft data elements is 2048
;;	do 256 macros processing 128 data values

;; Do FFT levels 5,6,7
;; On input the 64-byte cache lines hold these data values:
;;	0	+4K	...	2048	+4K	...
;;	...
;;	2047	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+4K	...	256	+4K	...
;;	...
;;	255
;;	512	...
;;	...

;; Do 2048 eight_complex_fft operations
;;	distance between fft data elements is 256
;;	do 256 macros each processing 128 data values

	start_timer 4
	mov	rdi, sincos1		;; Load sin/cos pointer
	bump	rdi, 8*ZMM_SCD2/8
	lea	r10, [rdi+256*ZMM_SCD4/8] ;; Second sin/cos pointer
	mov	r9, norm_col_mults	;; Load weights pointer
	zloop_init 256			;; 256 iterations
	zr8fs_sixtyfour_complex_first_fft_preload
b1b:	zr8fs_sixtyfour_complex_first_fft rsi, 128, 256*128, 512*128, 1024*128, r10, ZMM_CD8SCD4, rdi, ZMM_SCD4/8, r9, 8*128, 256, L1PREFETCH_ALL
	zloop	256, b1b, -256*128	;; Test loop counter, restore source pointer
	end_timer 4

;; Do FFT levels 8,9,10
;; On input the 64-byte cache lines hold these data values:
;;	0	+4K	...	256	+4K	...
;;	...
;;	255
;;	512	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+4K	...	32	+4K	...
;;	...
;;	31
;;	64	...
;;	...

;; Do 2048 eight_complex_fft operations
;;	distance between fft data elements is 32
;;	do 256 macros each processing 128 data values

	start_timer 6
	bump	rdi, -256*ZMM_SCD4/8	;; Load sin/cos pointer
	zloop_init 256			;; 8 iterations of 32
	zr8b_eight_complex_djbfft_preload
b3b:	zr8b_eight_complex_djbfft rsi, 128, 32*128, 64*128, 128*128, rdi, 8*ZMM_SCD4/8, 32
	zloop	32, b3b, -32*128+256*128, rdi, -32*8*ZMM_SCD4/8 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	8, b3b, -8*256*128	;; Test loop counter, restore source pointer
	end_timer 6

;; Do FFT levels 11,12
;; On input the 64-byte cache lines hold these data values:
;;	0	+4K	...	32	+4K	...
;;	...
;;	31
;;	64	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+4K	...	8	+4K	...
;;	...
;;	7
;;	16	...
;;	32	...
;;	...

;; Do 4096 four_complex_fft operations
;;	distance between fft data elements is 8
;;	do 512 macros each processing 64 data values

	start_timer 7
	bump	rdi, -8*ZMM_SCD2/8	;; Load sin/cos pointer
	zloop_init 512			;; 64 iterations of 8
	zr4b_four_complex_djbfft_preload
b4b:	zr4b_four_complex_djbfft rsi, 128, 8*128, 16*128, rdi, ZMM_SCD2/8, 8
	zloop	8, b4b, -8*128+32*128, rdi, -8*ZMM_SCD2/8 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	64, b4b, -64*32*128	;; Test loop counter, restore source pointer
	end_timer 7

;; Do FFT levels 13,14,15
;; On input and output the 128-byte cache line pairs hold these data values:
;;	0	+4K	...	8	+4K	...
;;	...
;;	7
;;	16	...
;;	32	...
;;	...

;; Do 2048 eight_complex_fft operations
;;	distance between fft data elements is 1
;;	do 256 macros processing 128 data values

zmid:	start_timer 8
	mov	rcx, 256
	call	zcomplex_onepass_middle
	end_timer 8

;; Do inverse FFT levels 11,12
;; On input the 64-byte cache lines hold these data values:
;;	0	+4K	...	8	+4K	...
;;	...
;;	7
;;	16	...
;;	32	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+4K	...	32	+4K	...
;;	...
;;	31
;;	64	...
;;	...

;; Do 4096 four_complex_unfft operations
;;	distance between fft data elements is 8
;;	do 512 macros each processing 64 data values

	start_timer 9
	mov	rdi, sincos1		;; Load sin/cos pointer
	zloop_init 512			;; 64 iterations of 8
	zr4b_four_complex_djbunfft_preload
c4b:	zr4b_four_complex_djbunfft rsi, 128, 8*128, 16*128, rdi, ZMM_SCD2/8, 8
	zloop	8, c4b, -8*128+32*128, rdi, -8*ZMM_SCD2/8 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	64, c4b, -64*32*128, rdi, 8*ZMM_SCD2/8 ;; Test loop counter, restore source pointer, next sin/cos ptr
	end_timer 9

;; Do inverse FFT levels 8,9,10
;; On input the 64-byte cache lines hold these data values:
;;	0	+4K	...	32	+4K	...
;;	...
;;	31
;;	64	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+4K	...	256	+4K	...
;;	...
;;	255
;;	512	...
;;	...

;; Do 2048 eight_complex_unfft operations
;;	distance between fft data elements is 32
;;	do 256 macros each processing 128 data values

	start_timer 10
	zloop_init 256			;; 8 iterations of 32
	zr8b_eight_complex_djbunfft_preload
c3b:	zr8b_eight_complex_djbunfft rsi, 128, 32*128, 64*128, 128*128, rdi, 8*ZMM_SCD4/8, 32
	zloop	32, c3b, -32*128+256*128, rdi, -32*8*ZMM_SCD4/8 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	8, c3b, -8*256*128	;; Test loop counter, restore source pointer
	end_timer 10

;; Do inverse FFT levels 5,6,7
;; On input the 64-byte cache lines hold these data values:
;;	0	+4K	...	256	+4K	...
;;	...
;;	255
;;	512	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+4K	...	2048	+4K	...
;;	...
;;	2047	...

;; Do 2048 eight_complex_unfft operations
;;	distance between fft data elements is 256
;;	do 256 macros each processing 128 data values

;; Do inverse FFT levels 1,2,3,4
;; On input the 128-byte cache line pairs hold these data values:
;;	0	+4K	...	2048	+4K	...
;;	...
;;	2047	...
;; These are swizzled to:
;;	0	+256	...	2048	+256	...
;;	...
;;	255	...
;;	4K	...
;;	...
;; On output the 128-byte cache line pairs hold these data values:
;;	0	+256	...	16K	+256	...
;;	...
;;	255	...
;;	2048	...
;;	...

;; Do 2048 eight_complex_last_unfft operations
;;	distance between fft data elements is 2048
;;	do 256 macros processing 128 data values

	start_timer 12
	lea	r10, [rdi+256*ZMM_SCD4/8] ;; Second sin/cos pointer
	zloop_init 256			;; 256 iterations
	zr8s_sixtyfour_complex_last_unfft_preload
c1b:	zr8s_sixtyfour_complex_last_unfft rsi, 128, 256*128, 512*128, 1024*128, rdi, ZMM_SCD4/8, r10, ZMM_CD8SCD4, 256, L1PREFETCH_ALL
	zloop	256, c1b, -256*128	;; Test loop counter, restore source pointer
	end_timer 12

	zfft_3_ret
	ENDM


;; Perform a 35K-element FFT

zfft35K MACRO type
	LOCAL	b1b, b2b, b3a, b3b, b4a, b4b, zmid, c4a, c4b, c3a, c3b, c2b, c1b

;; Check for a multiply with pre-FFTed inputs

	cmp	ffttype, 4
	je	zmid

;; Do FFT levels 1,2,3,4
;; On input the 128-byte cache line pairs hold these data values:
;;	0	+280	...	17.5K	+280	...
;;	...
;;	279	...
;;	2240	...
;;	...
;; On output the 128-byte cache line pairs hold these data values:
;;	0	+280	...	2240	+280	...
;;	...
;;	279	...
;;	4480	...
;;	...
;; These are swizzled to:
;;	0	+4480	...	2240	+4480	...
;;	...
;;	2239	...

;; Do 2240 sixteen_reals_fft operations
;;	distance between fft data elements is 2240
;;	do 280 macros processing 128 data values

;; Do FFT levels 5,6,7
;; On input the 64-byte cache lines hold these data values:
;;	0	+4480	...	2240	+4480	...
;;	...
;;	2239	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+4480	...	280	+4480	...
;;	...
;;	279
;;	560	...
;;	...

;; Do 280 sixteen_reals_fft operation
;; Do 1960 eight_complex_fft operations
;;	distance between fft data elements is 280
;;	do 280 macros each processing 128 data values

	start_timer 4
	mov	rdi, sincos1		;; Load sin/cos data ptr
	bump	rdi, 56*ZMM_SCD4+8*ZMM_SCD6
	lea	r10, [rdi+280*ZMM_SCD7]	;; Second sin/cos pointer
	mov	r9, norm_col_mults	;; Load weights pointer
	zloop_init 280			;; 280 iterations
	zr8fs_hundredtwentyeight_reals_first_fft_preload
b1b:	zr8fs_hundredtwentyeight_reals_first_fft rsi, 128, 280*128, 560*128, 1120*128, r10, ZMM_SCD7, rdi, ZMM_SCD7, r9, 8*128, 280, L1PREFETCH_ALL
	zloop	280, b1b, -280*128	;; Test loop counter, restore source pointer
	end_timer 4

;; Do FFT levels 8,9,10
;; On input the 64-byte cache lines hold these data values:
;;	0	+4480	...	280	+4480	...
;;	...
;;	279
;;	560	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+4480	...	56	+4480	...
;;	...
;;	55
;;	112	...
;;	...

;; Do 56 ten_reals_fft operation
;; Do 3528 five_complex_fft operations
;;	distance between fft data elements is 32
;;	do 448 macros each processing 80 data values

	start_timer 6
	bump	rdi, -280*ZMM_SCD7-56*ZMM_SCD4
	zloop_init 448			;; 8 iterations of 56
	zr5_ten_reals_five_complex_djbfft_preload
b3a:	zr5_ten_reals_five_complex_djbfft rsi, 128, 56*128, rdi, ZMM_SCD4, 56
	zloop	56, b3a, -56*128+280*128, rdi, -56*ZMM_SCD4 ;; Test loop counter, next source pointer, restore sin/cos pointer

	loops_reset			;; 7 more iters of 56
	zr5rb_five_complex_djbfft_preload
b3b:	zr5rb_five_complex_djbfft rsi, 128, 56*128, rdi, ZMM_SCD4, 56
	zloop	56, b3b, -56*128+280*128, rdi, -56*ZMM_SCD4 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	8, b3b, -8*280*128	;; Test loop counter, restore source pointer
	end_timer 6

;; Do FFT levels 10,11,12
;; On input the 64-byte cache lines hold these data values:
;;	0	+4480	...	56	+4480	...
;;	...
;;	55
;;	112	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+4480	...	8	+4480	...
;;	...
;;	7
;;	16	...
;;	32	...
;;	...

;; Do 8 fourteen_reals_fft operation
;; Do 2552 seven_complex_fft operations
;;	distance between fft data elements is 8
;;	do 320 macros each processing 112 data values

	start_timer 7
	bump	rdi, -8*ZMM_SCD6
	zloop_init 320			;; 40 iterations of 8
	zr7_fourteen_reals_seven_complex_djbfft_preload
b4a:	zr7_fourteen_reals_seven_complex_djbfft rsi, 128, 8*128, rdi, ZMM_SCD6, 8
	zloop	8, b4a, -8*128+56*128, rdi, -8*ZMM_SCD6 ;; Test loop counter, next source pointer, restore sin/cos pointer

	loops_reset			;; 39 more iters of 8
	zr7rb_seven_complex_djbfft_preload
b4b:	zr7rb_seven_complex_djbfft rsi, 128, 8*128, rdi, ZMM_SCD6, 8
	zloop	8, b4b, -8*128+56*128, rdi, -8*ZMM_SCD6 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	40, b4b, -40*56*128	;; Test loop counter, restore source pointer
	end_timer 7

;; Do FFT levels 13,14,15
;; On input and output the 128-byte cache line pairs hold these data values:
;;	0	+4480	...	8	+4480	...
;;	...
;;	7
;;	16	...
;;	32	...
;;	...

;; Do 1 sixteen_reals_fft operation
;; Do 2239 eight_complex_fft operations
;;	distance between fft data elements is 1
;;	do 280 macros processing 128 data values

zmid:	start_timer 8
	mov	rcx, 280
	call	zreal_onepass_middle
	end_timer 8

;; Do inverse FFT levels 10,11,12
;; On input the 64-byte cache lines hold these data values:
;;	0	+4480	...	8	+4480	...
;;	...
;;	7
;;	16	...
;;	32	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+4480	...	56	+4480	...
;;	...
;;	55
;;	112	...
;;	...

;; Do 8 fourteen_reals_unfft operations
;; Do 2552 seven_complex_unfft operations
;;	distance between fft data elements is 8
;;	do 320 macros each processing 112 data values

	start_timer 9
	mov	rdi, sincos1		;; Load sin/cos pointer
	zloop_init 320			;; 40 iterations of 8
	zr7_fourteen_reals_seven_complex_djbunfft_preload
c4a:	zr7_fourteen_reals_seven_complex_djbunfft rsi, 128, 8*128, rdi, ZMM_SCD6, 8
	zloop	8, c4a, -8*128+56*128, rdi, -8*ZMM_SCD6 ;; Test loop counter, restore source pointer, restore sin/cos pointer

	loops_reset			;; 39 more iters of 8
	zr7rb_seven_complex_djbunfft_preload
c4b:	zr7rb_seven_complex_djbunfft rsi, 128, 8*128, rdi, ZMM_SCD6, 8
	zloop	8, c4b, -8*128+56*128, rdi, -8*ZMM_SCD6 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	40, c4b, -40*56*128, rdi, 8*ZMM_SCD6 ;; Test loop counter, restore source pointer, next sin/cos ptr
	end_timer 9

;; Do inverse FFT levels 8,9
;; On input the 64-byte cache lines hold these data values:
;;	0	+4480	...	56	+4480	...
;;	...
;;	55
;;	112	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+4480	...	280	+4480	...
;;	...
;;	279
;;	560	...
;;	...

;; Do 56 ten_reals_unfft operations
;; Do 3528 five_complex_unfft operations
;;	distance between fft data elements is 56
;;	do 448 macros each processing 80 data values

	start_timer 10
	zloop_init 448			;; 8 iterations of 56
	zr5_ten_reals_five_complex_djbunfft_preload
c3a:	zr5_ten_reals_five_complex_djbunfft rsi, 128, 56*128, rdi, ZMM_SCD4, 56
	zloop	56, c3a, -56*128+280*128, rdi, -56*ZMM_SCD4 ;; Test loop counter, restore source pointer, restore sin/cos pointer

	loops_reset			;; 7 more iters of 56
	zr5rb_five_complex_djbunfft_preload
c3b:	zr5rb_five_complex_djbunfft rsi, 128, 56*128, rdi, ZMM_SCD4, 56
	zloop	56, c3b, -56*128+280*128, rdi, -56*ZMM_SCD4 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	8, c3b, -8*280*128, rdi, 56*ZMM_SCD4 ;; Test loop counter, restore source pointer, next sin/cos ptr
	end_timer 10

;; Do inverse FFT levels 5,6,7
;; On input the 64-byte cache lines hold these data values:
;;	0	+4480	...	280	+4480	...
;;	...
;;	279
;;	560	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+4480	...	2240	+4480	...
;;	...
;;	2239	...

;; Do 280 sixteen_reals_unfft operations
;; Do 1960 eight_complex_unfft operations
;;	distance between fft data elements is 280
;;	do 280 macros each processing 128 data values

;; Do inverse FFT levels 1,2,3,4
;; On input the 128-byte cache line pairs hold these data values:
;;	0	+4480	...	2240	+4480	...
;;	...
;;	2239	...
;; These are swizzled to:
;;	0	+280	...	2240	+280	...
;;	...
;;	279	...
;;	4480	...
;;	...
;; On output the 128-byte cache line pairs hold these data values:
;;	0	+280	...	17.5K	+280	...
;;	...
;;	279	...
;;	2240	...
;;	...

;; Do 2240 sixteen_reals_unfft operations
;;	distance between fft data elements is 2240
;;	do 280 macros processing 128 data values

	start_timer 12
	lea	r10, [rdi+280*ZMM_SCD7]	;; Second sin/cos pointer
	zloop_init 280			;; 280 iterations
	zr8s_hundredtwentyeight_reals_last_unfft_preload
c1b:	zr8s_hundredtwentyeight_reals_last_unfft rsi, 128, 280*128, 560*128, 1120*128, rdi, ZMM_SCD7, r10, ZMM_SCD7, 280, L1PREFETCH_ALL
	zloop	280, c1b, -280*128	;; Test loop counter, restore source pointer
	end_timer 12

	zfft_3_ret
	ENDM

;; Perform a 35K-element negacyclic FFT

zfft35Kp MACRO type
	LOCAL	b1b, b2b, b3b, b4b, zmid, c4b, c3b, c2b, c1b

;; Do a multiply with pre-FFTed inputs

	cmp	ffttype, 4
	je	zmid

;; Do FFT levels 1,2,3,4
;; On input the 128-byte cache line pairs hold these data values:
;;	0	+280	...	17.5K	+280	...
;;	...
;;	279	...
;;	2240	...
;;	...
;; On output the 128-byte cache line pairs hold these data values:
;;	0	+280	...	2240	+280	...
;;	...
;;	279	...
;;	4480	...
;;	...
;; These are swizzled to:
;;	0	+4480	...	2240	+4480	...
;;	...
;;	2239	...

;; Do 2240 eight_complex_first_fft operations
;;	distance between fft data elements is 2240
;;	do 280 macros processing 128 data values

;; Do FFT levels 5,6,7
;; On input the 64-byte cache lines hold these data values:
;;	0	+4480	...	2240	+4480	...
;;	...
;;	2239	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+4480	...	280	+4480	...
;;	...
;;	279
;;	560	...
;;	...

;; Do 2240 eight_complex_fft operations
;;	distance between fft data elements is 280
;;	do 280 macros each processing 128 data values

	start_timer 4
	mov	rdi, sincos1		;; Load sin/cos pointer
	bump	rdi, 56*ZMM_SCD2/8+8*ZMM_SCD3/8
	lea	r10, [rdi+280*ZMM_SCD4/8] ;; Second sin/cos pointer
	mov	r9, norm_col_mults	;; Load weights pointer
	zloop_init 280			;; 280 iterations
	zr8fs_sixtyfour_complex_first_fft_preload
b1b:	zr8fs_sixtyfour_complex_first_fft rsi, 128, 280*128, 560*128, 1120*128, r10, ZMM_CD8SCD4, rdi, ZMM_SCD4/8, r9, 8*128, 280, L1PREFETCH_ALL
	zloop	280, b1b, -280*128	;; Test loop counter, restore source pointer
	end_timer 4

;; Do FFT levels 8,9,10
;; On input the 64-byte cache lines hold these data values:
;;	0	+4480	...	280	+4480	...
;;	...
;;	279
;;	560	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+4480	...	56	+4480	...
;;	...
;;	55
;;	112	...
;;	...

;; Do 3584 five_complex_fft operations
;;	distance between fft data elements is 56
;;	do 448 macros each processing 80 data values

	start_timer 6
	bump	rdi, -280*ZMM_SCD4/8-56*ZMM_SCD2/8 ;; Load sin/cos pointer
	zloop_init 448			;; 8 iterations of 56
	zr5b_five_complex_djbfft_preload
b3b:	zr5b_five_complex_djbfft rsi, 128, 56*128, rdi, ZMM_SCD2/8, 56
	zloop	56, b3b, -56*128+280*128, rdi, -56*ZMM_SCD2/8 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	8, b3b, -8*280*128	;; Test loop counter, restore source pointer
	end_timer 6

;; Do FFT levels 10,11,12
;; On input the 64-byte cache lines hold these data values:
;;	0	+4480	...	56	+4480	...
;;	...
;;	55
;;	112	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+4480	...	8	+4480	...
;;	...
;;	7
;;	16	...
;;	32	...
;;	...

;; Do 2560 seven_complex_fft operations
;;	distance between fft data elements is 8
;;	do 320 macros each processing 112 data values

	start_timer 7
	bump	rdi, -8*ZMM_SCD3/8	;; Load sin/cos pointer
	zloop_init 320			;; 40 iterations of 8
	zr7b_seven_complex_djbfft_preload
b4b:	zr7b_seven_complex_djbfft rsi, 128, 8*128, rdi, ZMM_SCD3/8, 8
	zloop	8, b4b, -8*128+56*128, rdi, -8*ZMM_SCD3/8 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	40, b4b, -40*56*128	;; Test loop counter, restore source pointer
	end_timer 7

;; Do FFT levels 13,14,15
;; On input and output the 128-byte cache line pairs hold these data values:
;;	0	+4480	...	8	+4480	...
;;	...
;;	7
;;	16	...
;;	32	...
;;	...

;; Do 2240 eight_complex_fft operations
;;	distance between fft data elements is 1
;;	do 280 macros processing 128 data values

zmid:	start_timer 8
	mov	rcx, 280
	call	zcomplex_onepass_middle
	end_timer 8

;; Do inverse FFT levels 10,11,12
;; On input the 64-byte cache lines hold these data values:
;;	0	+4480	...	8	+4480	...
;;	...
;;	7
;;	16	...
;;	32	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+4480	...	56	+4480	...
;;	...
;;	55
;;	112	...
;;	...

;; Do 2560 seven_complex_unfft operations
;;	distance between fft data elements is 8
;;	do 320 macros each processing 112 data values

	start_timer 9
	mov	rdi, sincos1		;; Load sin/cos pointer
	zloop_init 320			;; 40 iterations of 8
	zr7b_seven_complex_djbunfft_preload
c4b:	zr7b_seven_complex_djbunfft rsi, 128, 8*128, rdi, ZMM_SCD3/8, 8
	zloop	8, c4b, -8*128+56*128, rdi, -8*ZMM_SCD3/8 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	40, c4b, -40*56*128, rdi, 8*ZMM_SCD3/8	;; Test loop counter, restore source pointer, next sin/cos ptr
	end_timer 9

;; Do inverse FFT levels 8,9
;; On input the 64-byte cache lines hold these data values:
;;	0	+4480	...	56	+4480	...
;;	...
;;	55
;;	112	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+4480	...	280	+4480	...
;;	...
;;	279
;;	560	...
;;	...

;; Do 3584 five_complex_unfft operations
;;	distance between fft data elements is 56
;;	do 448 macros each processing 80 data values

	start_timer 10
	zloop_init 448			;; 8 iterations of 56
	zr5b_five_complex_djbunfft_preload
c3b:	zr5b_five_complex_djbunfft rsi, 128, 56*128, rdi, ZMM_SCD2/8, 56
	zloop	56, c3b, -56*128+280*128, rdi, -56*ZMM_SCD2/8 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	8, c3b, -8*280*128, rdi, 56*ZMM_SCD2/8 ;; Test loop counter, restore source pointer, next sin/cos ptr
	end_timer 10

;; Do inverse FFT levels 5,6,7
;; On input the 64-byte cache lines hold these data values:
;;	0	+4480	...	280	+4480	...
;;	...
;;	279
;;	560	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+4480	...	2240	+4480	...
;;	...
;;	2239	...

;; Do 2240 eight_complex_unfft operations
;;	distance between fft data elements is 280
;;	do 280 macros each processing 128 data values

;; Do inverse FFT levels 1,2,3,4
;; On input the 128-byte cache line pairs hold these data values:
;;	0	+4480	...	2240	+4480	...
;;	...
;;	2239	...
;; These are swizzled to:
;;	0	+280	...	2240	+280	...
;;	...
;;	279	...
;;	4480	...
;;	...
;; On output the 128-byte cache line pairs hold these data values:
;;	0	+280	...	17.5K	+280	...
;;	...
;;	279	...
;;	2240	...
;;	...

;; Do 2240 eight_complex_last_unfft operations
;;	distance between fft data elements is 2240
;;	do 280 macros processing 128 data values

	start_timer 12
	lea	r10, [rdi+280*ZMM_SCD4/8] ;; Second sin/cos pointer
	zloop_init 280			;; 280 iterations
	zr8s_sixtyfour_complex_last_unfft_preload
c1b:	zr8s_sixtyfour_complex_last_unfft rsi, 128, 280*128, 560*128, 1120*128, rdi, ZMM_SCD4/8, r10, ZMM_CD8SCD4, 280, L1PREFETCH_ALL
	zloop	280, c1b, -280*128	;; Test loop counter, restore source pointer
	end_timer 12

	zfft_3_ret
	ENDM


;; Perform a 36K-element FFT

zfft36K MACRO type
	LOCAL	b1b, b2b, b3a, b3b, b4a, b4b, zmid, c4a, c4b, c3a, c3b, c2b, c1b

;; Check for a multiply with pre-FFTed inputs

	cmp	ffttype, 4
	je	zmid

;; Do FFT levels 1,2,3,4
;; On input the 128-byte cache line pairs hold these data values:
;;	0	+288	...	18K	+288	...
;;	...
;;	287	...
;;	2304	...
;;	...
;; On output the 128-byte cache line pairs hold these data values:
;;	0	+288	...	2304	+288	...
;;	...
;;	287	...
;;	4608	...
;;	...
;; These are swizzled to:
;;	0	+4608	...	2304	+4608	...
;;	...
;;	2303	...

;; Do 2304 sixteen_reals_fft operations
;;	distance between fft data elements is 2304
;;	do 288 macros processing 128 data values

;; Do FFT levels 5,6,7
;; On input the 64-byte cache lines hold these data values:
;;	0	+4608	...	2304	+4608	...
;;	...
;;	2303	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+4608	...	288	+4608	...
;;	...
;;	287
;;	576	...
;;	...

;; Do 288 sixteen_reals_fft operation
;; Do 2016 eight_complex_fft operations
;;	distance between fft data elements is 288
;;	do 288 macros each processing 128 data values

	start_timer 4
	mov	rdi, sincos1		;; Load sin/cos data ptr
	bump	rdi, 48*ZMM_SCD5
	lea	r10, [rdi+288*ZMM_SCD7]	;; Second sin/cos pointer
	mov	r9, norm_col_mults	;; Load weights pointer
	zloop_init 288			;; 288 iterations
	zr8fs_hundredtwentyeight_reals_first_fft_preload
b1b:	zr8fs_hundredtwentyeight_reals_first_fft rsi, 128, 288*128, 576*128, 1152*128, r10, ZMM_SCD7, rdi, ZMM_SCD7, r9, 8*128, 288, L1PREFETCH_ALL
	zloop	288, b1b, -288*128	;; Test loop counter, restore source pointer
	end_timer 4

;; Do FFT levels 8,9
;; On input the 64-byte cache lines hold these data values:
;;	0	+4608	...	288	+4608	...
;;	...
;;	287
;;	576	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+4608	...	48	+4608	...
;;	...
;;	47
;;	96	...
;;	...

;; Do 48 twelve_reals_fft operation
;; Do 3024 six_complex_fft operations
;;	distance between fft data elements is 48
;;	do 384 macros each processing 96 data values

	start_timer 6
	bump	rdi, -288*ZMM_SCD7-48*ZMM_SCD5
	zloop_init 384			;; 8 iterations of 48
	zr6_twelve_reals_six_complex_djbfft_preload
b3a:	zr6_twelve_reals_six_complex_djbfft rsi, 128, 48*128, rdi, ZMM_SCD5, 48
	zloop	48, b3a, -48*128+288*128, rdi, -48*ZMM_SCD5 ;; Test loop counter, next source pointer, restore sin/cos pointer

	loops_reset			;; 7 more iters of 48
	zr6rb_six_complex_djbfft_preload
b3b:	zr6rb_six_complex_djbfft rsi, 128, 48*128, rdi, ZMM_SCD5, 48
	zloop	48, b3b, -48*128+288*128, rdi, -48*ZMM_SCD5 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	8, b3b, -8*288*128	;; Test loop counter, restore source pointer
	end_timer 6

;; Do FFT levels 10,11,12
;; On input the 64-byte cache lines hold these data values:
;;	0	+4608	...	48	+4608	...
;;	...
;;	47
;;	96	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+4608	...	8	+4608	...
;;	...
;;	7
;;	16	...
;;	32	...
;;	...

;; Do 8 twelve_reals_fft operation
;; Do 3064 six_complex_fft operations
;;	distance between fft data elements is 8
;;	do 384 macros each processing 96 data values

	start_timer 7
	zloop_init 384			;; 48 iterations of 8
	zr6_twelve_reals_six_complex_djbfft_preload
b4a:	zr6_twelve_reals_six_complex_djbfft rsi, 128, 8*128, rdi, 6*ZMM_SCD5, 8
	zloop	8, b4a, -8*128+48*128, rdi, -8*6*ZMM_SCD5 ;; Test loop counter, next source pointer, restore sin/cos pointer

	loops_reset			;; 47 more iters of 8
	zr6rb_six_complex_djbfft_preload
b4b:	zr6rb_six_complex_djbfft rsi, 128, 8*128, rdi, 6*ZMM_SCD5, 8
	zloop	8, b4b, -8*128+48*128, rdi, -8*6*ZMM_SCD5 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	48, b4b, -48*48*128	;; Test loop counter, restore source pointer
	end_timer 7

;; Do FFT levels 13,14,15
;; On input and output the 128-byte cache line pairs hold these data values:
;;	0	+4608	...	8	+4608	...
;;	...
;;	7
;;	16	...
;;	32	...
;;	...

;; Do 1 sixteen_reals_fft operation
;; Do 2303 eight_complex_fft operations
;;	distance between fft data elements is 1
;;	do 288 macros processing 128 data values

zmid:	start_timer 8
	mov	rcx, 288
	call	zreal_onepass_middle
	end_timer 8

;; Do inverse FFT levels 10,11,12
;; On input the 64-byte cache lines hold these data values:
;;	0	+4608	...	8	+4608	...
;;	...
;;	7
;;	16	...
;;	32	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+4608	...	48	+4608	...
;;	...
;;	47
;;	96	...
;;	...

;; Do 8 twelve_reals_unfft operations
;; Do 3064 six_complex_unfft operations
;;	distance between fft data elements is 8
;;	do 384 macros each processing 96 data values

	start_timer 9
	mov	rdi, sincos1		;; Load sin/cos pointer
	zloop_init 384			;; 48 iterations of 8
	zr6_twelve_reals_six_complex_djbunfft_preload
c4a:	zr6_twelve_reals_six_complex_djbunfft rsi, 128, 8*128, rdi, 6*ZMM_SCD5, 8
	zloop	8, c4a, -8*128+48*128, rdi, -8*6*ZMM_SCD5 ;; Test loop counter, restore source pointer, restore sin/cos pointer

	loops_reset			;; 47 more iters of 8
	zr6rb_six_complex_djbunfft_preload
c4b:	zr6rb_six_complex_djbunfft rsi, 128, 8*128, rdi, 6*ZMM_SCD5, 8
	zloop	8, c4b, -8*128+48*128, rdi, -8*6*ZMM_SCD5 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	48, c4b, -48*48*128	;; Test loop counter, restore source pointer
	end_timer 9

;; Do inverse FFT levels 8,9
;; On input the 64-byte cache lines hold these data values:
;;	0	+4608	...	48	+4608	...
;;	...
;;	47
;;	96	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+4608	...	288	+4608	...
;;	...
;;	287
;;	576	...
;;	...

;; Do 48 twelve_reals_unfft operations
;; Do 3024 six_complex_unfft operations
;;	distance between fft data elements is 48
;;	do 384 macros each processing 96 data values

	start_timer 10
	zloop_init 384			;; 8 iterations of 48
	zr6_twelve_reals_six_complex_djbunfft_preload
c3a:	zr6_twelve_reals_six_complex_djbunfft rsi, 128, 48*128, rdi, ZMM_SCD5, 48
	zloop	48, c3a, -48*128+288*128, rdi, -48*ZMM_SCD5 ;; Test loop counter, restore source pointer, restore sin/cos pointer

	loops_reset			;; 7 more iters of 48
	zr6rb_six_complex_djbunfft_preload
c3b:	zr6rb_six_complex_djbunfft rsi, 128, 48*128, rdi, ZMM_SCD5, 48
	zloop	48, c3b, -48*128+288*128, rdi, -48*ZMM_SCD5 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	8, c3b, -8*288*128, rdi, 48*ZMM_SCD5 ;; Test loop counter, restore source pointer, next sin/cos ptr
	end_timer 10

;; Do inverse FFT levels 5,6,7
;; On input the 64-byte cache lines hold these data values:
;;	0	+4608	...	288	+4608	...
;;	...
;;	287
;;	576	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+4608	...	2304	+4608	...
;;	...
;;	2303	...

;; Do 288 sixteen_reals_unfft operations
;; Do 2016 eight_complex_unfft operations
;;	distance between fft data elements is 288
;;	do 288 macros each processing 128 data values

;; Do inverse FFT levels 1,2,3,4
;; On input the 128-byte cache line pairs hold these data values:
;;	0	+4608	...	2304	+4608	...
;;	...
;;	2303	...
;; These are swizzled to:
;;	0	+288	...	2304	+288	...
;;	...
;;	287	...
;;	4608	...
;;	...
;; On output the 128-byte cache line pairs hold these data values:
;;	0	+288	...	18K	+288	...
;;	...
;;	287	...
;;	2304	...
;;	...

;; Do 2304 sixteen_reals_unfft operations
;;	distance between fft data elements is 2304
;;	do 288 macros processing 128 data values

	start_timer 12
	lea	r10, [rdi+288*ZMM_SCD7]	;; Second sin/cos pointer
	zloop_init 288			;; 288 iterations
	zr8s_hundredtwentyeight_reals_last_unfft_preload
c1b:	zr8s_hundredtwentyeight_reals_last_unfft rsi, 128, 288*128, 576*128, 1152*128, rdi, ZMM_SCD7, r10, ZMM_SCD7, 288, L1PREFETCH_ALL
	zloop	288, c1b, -288*128	;; Test loop counter, restore source pointer
	end_timer 12

	zfft_3_ret
	ENDM

;; Perform a 36K-element negacyclic FFT

zfft36Kp MACRO type
	LOCAL	b1b, b2b, b3b, b4b, zmid, c4b, c3b, c2b, c1b

;; Do a multiply with pre-FFTed inputs

	cmp	ffttype, 4
	je	zmid

;; Do FFT levels 1,2,3,4
;; On input the 128-byte cache line pairs hold these data values:
;;	0	+288	...	18K	+288	...
;;	...
;;	287	...
;;	2304	...
;;	...
;; On output the 128-byte cache line pairs hold these data values:
;;	0	+288	...	2304	+288	...
;;	...
;;	287	...
;;	4608	...
;;	...
;; These are swizzled to:
;;	0	+4608	...	2304	+4608	...
;;	...
;;	2303	...

;; Do 2304 eight_complex_first_fft operations
;;	distance between fft data elements is 2304
;;	do 288 macros processing 128 data values

;; Do FFT levels 5,6,7
;; On input the 64-byte cache lines hold these data values:
;;	0	+4608	...	2304	+4608	...
;;	...
;;	2303	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+4608	...	288	+4608	...
;;	...
;;	287
;;	576	...
;;	...

;; Do 2304 eight_complex_fft operations
;;	distance between fft data elements is 288
;;	do 288 macros each processing 128 data values

	start_timer 4
	mov	rdi, sincos1		;; Load sin/cos pointer
	bump	rdi, 48*ZMM_SCD3/8
	lea	r10, [rdi+288*ZMM_SCD4/8] ;; Second sin/cos pointer
	mov	r9, norm_col_mults	;; Load weights pointer
	zloop_init 288			;; 288 iterations
	zr8fs_sixtyfour_complex_first_fft_preload
b1b:	zr8fs_sixtyfour_complex_first_fft rsi, 128, 288*128, 576*128, 1152*128, r10, ZMM_CD8SCD4, rdi, ZMM_SCD4/8, r9, 8*128, 288, L1PREFETCH_ALL
	zloop	288, b1b, -288*128	;; Test loop counter, restore source pointer
	end_timer 4

;; Do FFT levels 8,9
;; On input the 64-byte cache lines hold these data values:
;;	0	+4608	...	288	+4608	...
;;	...
;;	287
;;	576	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+4608	...	48	+4608	...
;;	...
;;	47
;;	96	...
;;	...

;; Do 3072 six_complex_fft operations
;;	distance between fft data elements is 48
;;	do 384 macros each processing 96 data values

	start_timer 6
	bump	rdi, -288*ZMM_SCD4/8-48*ZMM_SCD3/8 ;; Load sin/cos pointer
	zloop_init 384			;; 8 iterations of 48
	zr6b_six_complex_djbfft_preload
b3b:	zr6b_six_complex_djbfft rsi, 128, 48*128, rdi, ZMM_SCD3/8, 48
	zloop	48, b3b, -48*128+288*128, rdi, -48*ZMM_SCD3/8 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	8, b3b, -8*288*128	;; Test loop counter, restore source pointer
	end_timer 6

;; Do FFT levels 10,11,12
;; On input the 64-byte cache lines hold these data values:
;;	0	+4608	...	48	+4608	...
;;	...
;;	47
;;	96	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+4608	...	8	+4608	...
;;	...
;;	7
;;	16	...
;;	32	...
;;	...

;; Do 3072 six_complex_fft operations
;;	distance between fft data elements is 8
;;	do 384 macros each processing 96 data values

	start_timer 7
	zloop_init 384			;; 48 iterations of 8
	zr6b_six_complex_djbfft_preload
b4b:	zr6b_six_complex_djbfft rsi, 128, 8*128, rdi, 6*ZMM_SCD3/8, 8
	zloop	8, b4b, -8*128+48*128, rdi, -8*6*ZMM_SCD3/8 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	48, b4b, -48*48*128	;; Test loop counter, restore source pointer
	end_timer 7

;; Do FFT levels 13,14,15
;; On input and output the 128-byte cache line pairs hold these data values:
;;	0	+4608	...	8	+4608	...
;;	...
;;	7
;;	16	...
;;	32	...
;;	...

;; Do 2304 eight_complex_fft operations
;;	distance between fft data elements is 1
;;	do 288 macros processing 128 data values

zmid:	start_timer 8
	mov	rcx, 288
	call	zcomplex_onepass_middle
	end_timer 8

;; Do inverse FFT levels 10,11,12
;; On input the 64-byte cache lines hold these data values:
;;	0	+4608	...	8	+4608	...
;;	...
;;	7
;;	16	...
;;	32	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+4608	...	48	+4608	...
;;	...
;;	47
;;	96	...
;;	...

;; Do 3072 six_complex_unfft operations
;;	distance between fft data elements is 8
;;	do 384 macros each processing 96 data values

	start_timer 9
	mov	rdi, sincos1		;; Load sin/cos pointer
	zloop_init 384			;; 48 iterations of 8
	zr6b_six_complex_djbunfft_preload
c4b:	zr6b_six_complex_djbunfft rsi, 128, 8*128, rdi, 6*ZMM_SCD3/8, 8
	zloop	8, c4b, -8*128+48*128, rdi, -8*6*ZMM_SCD3/8 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	48, c4b, -48*48*128	;; Test loop counter, restore source pointer
	end_timer 9

;; Do inverse FFT levels 8,9
;; On input the 64-byte cache lines hold these data values:
;;	0	+4608	...	48	+4608	...
;;	...
;;	47
;;	96	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+4608	...	288	+4608	...
;;	...
;;	287
;;	576	...
;;	...

;; Do 3072 six_complex_unfft operations
;;	distance between fft data elements is 48
;;	do 384 macros each processing 96 data values

	start_timer 10
	zloop_init 384			;; 8 iterations of 48
	zr6b_six_complex_djbunfft_preload
c3b:	zr6b_six_complex_djbunfft rsi, 128, 48*128, rdi, ZMM_SCD3/8, 48
	zloop	48, c3b, -48*128+288*128, rdi, -48*ZMM_SCD3/8 ;; Test loop counter, next source pointer, restore sin/cos pointer
	zloop	8, c3b, -8*288*128, rdi, 48*ZMM_SCD3/8 ;; Test loop counter, restore source pointer, next sin/cos ptr
	end_timer 10

;; Do inverse FFT levels 5,6,7
;; On input the 64-byte cache lines hold these data values:
;;	0	+4608	...	288	+4608	...
;;	...
;;	287
;;	576	...
;;	...
;; On output the 64-byte cache lines hold these data values:
;;	0	+4608	...	2304	+4608	...
;;	...
;;	2303	...

;; Do 2304 eight_complex_unfft operations
;;	distance between fft data elements is 288
;;	do 288 macros each processing 128 data values

;; Do inverse FFT levels 1,2,3,4
;; On input the 128-byte cache line pairs hold these data values:
;;	0	+4608	...	2304	+4608	...
;;	...
;;	2303	...
;; These are swizzled to:
;;	0	+288	...	2304	+288	...
;;	...
;;	287	...
;;	4608	...
;;	...
;; On output the 128-byte cache line pairs hold these data values:
;;	0	+288	...	18K	+288	...
;;	...
;;	287	...
;;	2304	...
;;	...

;; Do 2304 eight_complex_last_unfft operations
;;	distance between fft data elements is 2304
;;	do 288 macros processing 128 data values

	start_timer 12
	lea	r10, [rdi+288*ZMM_SCD4/8] ;; Second sin/cos pointer
	zloop_init 288			;; 288 iterations
	zr8s_sixtyfour_complex_last_unfft_preload
c1b:	zr8s_sixtyfour_complex_last_unfft rsi, 128, 288*128, 576*128, 1152*128, rdi, ZMM_SCD4/8, r10, ZMM_CD8SCD4, 288, L1PREFETCH_ALL
	zloop	288, c1b, -288*128	;; Test loop counter, restore source pointer
	end_timer 12

	zfft_3_ret
	ENDM
