; Copyright 2011-2016 - Mersenne Research, Inc.  All rights reserved
; Author:  George Woltman
; Email: woltman@alum.mit.edu
;

;;
;; Macros for a radix-7 first step in an AVX real FFT.
;;

;;
;; ************************************* 28-reals-first-fft variants ******************************************
;;

;; These macros operate on 28 reals doing 4.807 levels of the FFT, applying
;; the sin/cos multipliers afterwards.  The output is 2 reals and 13 complex numbers.

;; To calculate a 28-reals FFT, we calculate 28 complex values in a brute force way (using a shorthand notation):
;; r1 + r2 + ... + r28	*  w^0000000000...
;; r1 + r2 + ... + r28	*  w^0123456789A...
;; r1 + r2 + ... + r28	*  w^02468ACE....
;;    ...
;; r1 + r2 + ... + r28	*  w^...A987654321
;; Note that Hermetian symmetry means we won't need to calculate the last 14 complex values.
;;
;; The sin/cos values (w = 28th root of unity) are:
;; w^1 = .975 + .223i
;; w^2 = .901 + .434i
;; w^3 = .782 + .623i
;; w^4 = .623 + .782i
;; w^5 = .434 + .901i
;; w^6 = .223 + .975i
;; w^7 = 0 + 1i
;; w^8 = -.223 + .975i
;; w^9 = -.434 + .901i
;; w^10 = -.623 + .782i
;; w^11 = -.782 + .623i
;; w^12 = -.901 + .434i
;; w^13 = -.975 + .223i
;; w^14 = -1
;;
;; Applying the sin/cos values above (and noting that combining r2 and r28, r3 and r27, etc. will simplify calculations):
;; reals:
;; r1     +(r2+r28)     +(r3+r27)     +(r4+r26)     +(r5+r25)     +(r6+r24)     +(r7+r23) + (r8+r22)     +(r9+r21)     +(r10+r20)     +(r11+r19)     +(r12+r18)     +(r13+r17)     +(r14+r16) + r15
;; r1 +.975(r2+r28) +.901(r3+r27) +.782(r4+r26) +.623(r5+r25) +.434(r6+r24) +.223(r7+r23)            -.223(r9+r21) -.434(r10+r20) -.623(r11+r19) -.782(r12+r18) -.901(r13+r17) -.975(r14+r16) - r15
;; r1 +.901(r2+r28) +.623(r3+r27) +.223(r4+r26) -.223(r5+r25) -.623(r6+r24) -.901(r7+r23) - (r8+r22) -.901(r9+r21) -.623(r10+r20) -.223(r11+r19) +.223(r12+r18) +.623(r13+r17) +.901(r14+r16) + r15
;; r1 +.782(r2+r28) +.223(r3+r27) -.434(r4+r26) -.901(r5+r25) -.975(r6+r24) -.623(r7+r23)            +.623(r9+r21) +.975(r10+r20) +.901(r11+r19) +.434(r12+r18) -.223(r13+r17) -.782(r14+r16) - r15
;; r1 +.623(r2+r28) -.223(r3+r27) -.901(r4+r26) -.901(r5+r25) -.223(r6+r24) +.623(r7+r23) + (r8+r22) +.623(r9+r21) -.223(r10+r20) -.901(r11+r19) -.901(r12+r18) -.223(r13+r17) +.623(r14+r16) + r15
;; r1 +.434(r2+r28) -.623(r3+r27) -.975(r4+r26) -.223(r5+r25) +.782(r6+r24) +.901(r7+r23)            -.901(r9+r21) -.782(r10+r20) +.223(r11+r19) +.975(r12+r18) +.623(r13+r17) -.434(r14+r16) - r15
;; r1 +.223(r2+r28) -.901(r3+r27) -.623(r4+r26) +.623(r5+r25) +.901(r6+r24) -.223(r7+r23) - (r8+r22) -.223(r9+r21) +.901(r10+r20) +.623(r11+r19) -.623(r12+r18) -.901(r13+r17) +.223(r14+r16) + r15
;; r1                   -(r3+r27)                   +(r5+r25)                   -(r7+r23)                +(r9+r21)                    -(r11+r19)                    +(r13+r17)                - r15
;; r1 -.223(r2+r28) -.901(r3+r27) +.623(r4+r26) +.623(r5+r25) -.901(r6+r24) -.223(r7+r23) + (r8+r22) -.223(r9+r21) -.901(r10+r20) +.623(r11+r19) +.623(r12+r18) -.901(r13+r17) -.223(r14+r16) + r15
;; r1 -.434(r2+r28) -.623(r3+r27) +.975(r4+r26) -.223(r5+r25) -.782(r6+r24) +.901(r7+r23)            -.901(r9+r21) +.782(r10+r20) +.223(r11+r19) -.975(r12+r18) +.623(r13+r17) +.434(r14+r16) - r15
;; r1 -.623(r2+r28) -.223(r3+r27) +.901(r4+r26) -.901(r5+r25) +.223(r6+r24) +.623(r7+r23) - (r8+r22) +.623(r9+r21) +.223(r10+r20) -.901(r11+r19) +.901(r12+r18) -.223(r13+r17) -.623(r14+r16) + r15
;; r1 -.782(r2+r28) +.223(r3+r27) +.434(r4+r26) -.901(r5+r25) +.975(r6+r24) -.623(r7+r23)            +.623(r9+r21) -.975(r10+r20) +.901(r11+r19) -.434(r12+r18) -.223(r13+r17) +.782(r14+r16) - r15
;; r1 -.901(r2+r28) +.623(r3+r27) -.223(r4+r26) -.223(r5+r25) +.623(r6+r24) -.901(r7+r23) + (r8+r22) -.901(r9+r21) +.623(r10+r20) -.223(r11+r19) -.223(r12+r18) +.623(r13+r17) -.901(r14+r16) + r15
;; r1 -.975(r2+r28) +.901(r3+r27) -.782(r4+r26) +.623(r5+r25) -.434(r6+r24) +.223(r7+r23)            -.223(r9+r21) +.434(r10+r20) -.623(r11+r19) +.782(r12+r18) -.901(r13+r17) +.975(r14+r16) - r15
;; r1     -(r2+r28)     +(r3+r27)     -(r4+r26)     +(r5+r25)     -(r6+r24)     +(r7+r23) - (r8+r22)     +(r9+r21)     -(r10+r20)     +(r11+r19)     -(r12+r18)     +(r13+r17)     -(r14+r16) + r15
;;
;; imaginarys:
;; 0
;; +.223(r2-r28) +.434(r3-r27) +.623(r4-r26) +.782(r5-r25) +.901(r6-r24) +.975(r7-r23) + (r8-r22) +.975(r9-r21) +.901(r10-r20) +.782(r11-r19) +.623(r12-r18) +.434(r13-r17) +.223(r14-r16)
;; +.434(r2-r28) +.782(r3-r27) +.975(r4-r26) +.975(r5-r25) +.782(r6-r24) +.434(r7-r23)            -.434(r9-r21) -.782(r10-r20) -.975(r11-r19) -.975(r12-r18) -.782(r13-r17) -.434(r14-r16)
;; +.623(r2-r28) +.975(r3-r27) +.901(r4-r26) +.434(r5-r25) -.223(r6-r24) -.782(r7-r23) - (r8-r22) -.782(r9-r21) -.223(r10-r20) +.434(r11-r19) +.901(r12-r18) +.975(r13-r17) +.623(r14-r16)
;; +.782(r2-r28) +.975(r3-r27) +.434(r4-r26) -.434(r5-r25) -.975(r6-r24) -.782(r7-r23)            +.782(r9-r21) +.975(r10-r20) +.434(r11-r19) -.434(r12-r18) -.975(r13-r17) -.782(r14-r16)
;; +.901(r2-r28) +.782(r3-r27) -.223(r4-r26) -.975(r5-r25) -.623(r6-r24) +.434(r7-r23) + (r8-r22) +.434(r9-r21) -.623(r10-r20) -.975(r11-r19) -.223(r12-r18) +.782(r13-r17) +.901(r14-r16)
;; +.975(r2-r28) +.434(r3-r27) -.782(r4-r26) -.782(r5-r25) +.434(r6-r24) +.975(r7-r23)            -.975(r9-r21) -.434(r10-r20) +.782(r11-r19) +.782(r12-r18) -.434(r13-r17) -.975(r14-r16)
;;      (r2-r28)                   -(r4-r26)                   +(r6-r24)               - (r8-r22)                   +(r10-r20)                    -(r12-r18)                    +(r14-r16)
;; +.975(r2-r28) -.434(r3-r27) -.782(r4-r26) +.782(r5-r25) +.434(r6-r24) -.975(r7-r23)            +.975(r9-r21) -.434(r10-r20) -.782(r11-r19) +.782(r12-r18) +.434(r13-r17) -.975(r14-r16)
;; +.901(r2-r28) -.782(r3-r27) -.223(r4-r26) +.975(r5-r25) -.623(r6-r24) -.434(r7-r23) + (r8-r22) -.434(r9-r21) -.623(r10-r20) +.975(r11-r19) -.223(r12-r18) -.782(r13-r17) +.901(r14-r16)
;; +.782(r2-r28) -.975(r3-r27) +.434(r4-r26) +.434(r5-r25) -.975(r6-r24) +.782(r7-r23)            -.782(r9-r21) +.975(r10-r20) -.434(r11-r19) -.434(r12-r18) +.975(r13-r17) -.782(r14-r16)
;; +.623(r2-r28) -.975(r3-r27) +.901(r4-r26) -.434(r5-r25) -.223(r6-r24) +.782(r7-r23) - (r8-r22) +.782(r9-r21) -.223(r10-r20) -.434(r11-r19) +.901(r12-r18) -.975(r13-r17) +.623(r14-r16)
;; +.434(r2-r28) -.782(r3-r27) +.975(r4-r26) -.975(r5-r25) +.782(r6-r24) -.434(r7-r23)            +.434(r9-r21) -.782(r10-r20) +.975(r11-r19) -.975(r12-r18) +.782(r13-r17) -.434(r14-r16)
;; +.223(r2-r28) -.434(r3-r27) +.623(r4-r26) -.782(r5-r25) +.901(r6-r24) -.975(r7-r23) + (r8-r22) -.975(r9-r21) +.901(r10-r20) -.782(r11-r19) +.623(r12-r18) -.434(r13-r17) +.223(r14-r16)
;; 0
;;
;; There are many more symmetries we can take advantage of.   For example, the (r2+/-r28) column
;; always has the same multiplier as the (r14+/-r16) column.  This is true for all the "even" columns.
;; Also the computations for the 2nd row are very similar to the computations for the 14th row,
;; the 3rd row are similar to the 13th, etc.  Finally, note that for the odd columns, there are
;; only three multipliers to apply and can be combined with every fourth column.
;;
;; Lastly, output would normally be 13 complex and 2 reals.  but the users of this routine
;; expect us to "back up" the 2 reals by one level.  That is:
;;	real #1A:  r1 + r3+r27 + r5+r25 + ...
;;	real #1B:  r2+r28 + r4+r26 + ...

;; Store intermediate results in YMM_TMPS

yr7_14cl_28_reals_fft_preload MACRO
	ENDM

yr7_14cl_28_reals_fft MACRO srcreg,srcinc,d1,d2,screg,scinc,maxrpt,L1pt,L1pd

	;; Do the odd columns for the real results

	vmovapd	ymm0, [srcreg+2*d2]		;; r5
	vaddpd	ymm0, ymm0, [srcreg+5*d2+32]	;; r5+r25
	vmovapd	ymm6, YMM_P623
	vmulpd	ymm1, ymm6, ymm0		;; .623(r5+r25)
	vmovapd	ymm7, YMM_P223
	vmulpd	ymm4, ymm7, ymm0		;; .223(r5+r25)
	vmulpd	ymm5, ymm0, YMM_P901		;; .901(r5+r25)
	vmovapd	ymm3, [srcreg]			;; r1
	vaddpd	ymm0, ymm3, ymm0		;; r1+(r5+r25)
	vaddpd	ymm1, ymm3, ymm1		;; r1+.623(r5+r25)
	L1prefetchw srcreg+L1pd, L1pt
	vsubpd	ymm2, ymm3, ymm4		;; r1-.223(r5+r25)
	vsubpd	ymm3, ymm3, ymm5		;; r1-.901(r5+r25)

	vmovapd	ymm4, [srcreg+4*d2]		;; r9
	vaddpd	ymm4, ymm4, [srcreg+3*d2+32]	;; r9+r21
	vaddpd	ymm0, ymm0, ymm4		;; r1+(r5+r25)+(r9+r21)
	vmulpd	ymm5, ymm7, ymm4		;; .223(r9+r21)
	vsubpd	ymm1, ymm1, ymm5		;; r1+.623(r5+r25)-.223(r9+r21)
	vmulpd	ymm5, ymm4, YMM_P901		;; .901(r9+r21)
	vsubpd	ymm2, ymm2, ymm5		;; r1-.223(r5+r25)-.901(r9+r21)
	L1prefetchw srcreg+d1+L1pd, L1pt
	vmulpd	ymm5, ymm6, ymm4		;; .623(r9+r21)
	vaddpd	ymm3, ymm3, ymm5		;; r1-.901(r5+r25)+.623(r9+r21)

	vmovapd	ymm4, [srcreg+6*d2]		;; r13
	vaddpd	ymm4, ymm4, [srcreg+d2+32]	;; r13+r17
	vaddpd	ymm0, ymm0, ymm4		;; r1+(r5+r25)+(r9+r21)+(r13+r17)
	vmulpd	ymm5, ymm4, YMM_P901		;; .901(r13+r17)
	vsubpd	ymm1, ymm1, ymm5		;; r1+.623(r5+r25)-.223(r9+r21)-.901(r13+r17)
	vmulpd	ymm5, ymm6, ymm4		;; .623(r13+r17)
	L1prefetchw srcreg+d2+L1pd, L1pt
	vaddpd	ymm2, ymm2, ymm5		;; r1-.223(r5+r25)-.901(r9+r21)+.623(r13+r17)
	vmulpd	ymm5, ymm7, ymm4		;; .223(r13+r17)
	vsubpd	ymm3, ymm3, ymm5		;; r1-.901(r5+r25)+.623(r9+r21)-.223(r13+r17)

	ystore	YMM_TMPS[0*32], ymm0
	ystore	YMM_TMPS[1*32], ymm1
	ystore	YMM_TMPS[2*32], ymm2
	ystore	YMM_TMPS[3*32], ymm3

	vmovapd	ymm0, [srcreg+d2]		;; r3
	vaddpd	ymm0, ymm0, [srcreg+6*d2+32]	;; r3+r27
	vmulpd	ymm1, ymm0, YMM_P901		;; .901(r3+r27)
	vmulpd	ymm2, ymm6, ymm0		;; .623(r3+r27)
	vmulpd	ymm3, ymm7, ymm0		;; .223(r3+r27)
	vmovapd	ymm4, [srcreg+32]		;; r15
	vaddpd	ymm0, ymm0, ymm4		;; (r3+r27)+r15
	vsubpd	ymm1, ymm1, ymm4		;; .901(r3+r27)-r15
	L1prefetchw srcreg+d2+d1+L1pd, L1pt
	vaddpd	ymm2, ymm2, ymm4		;; .623(r3+r27)+r15
	vsubpd	ymm3, ymm3, ymm4		;; .223(r3+r27)-r15

	vmovapd	ymm4, [srcreg+3*d2]		;; r7
	vaddpd	ymm4, ymm4, [srcreg+4*d2+32]	;; r7+r23
	vaddpd	ymm0, ymm0, ymm4		;; (r3+r27)+(r7+r23)+r15
	vmulpd	ymm5, ymm7, ymm4		;; .223(r7+r23)
	vaddpd	ymm1, ymm1, ymm5		;; .901(r3+r27)+.223(r7+r23)-r15
	vmulpd	ymm5, ymm4, YMM_P901		;; .901(r7+r23)
	vsubpd	ymm2, ymm2, ymm5		;; .623(r3+r27)-.901(r7+r23)+r15
	L1prefetchw srcreg+2*d2+L1pd, L1pt
	vmulpd	ymm5, ymm6, ymm4		;; .623(r7+r23)
	vsubpd	ymm3, ymm3, ymm5		;; .223(r3+r27)-.623(r7+r23)-r15

	yloop_optional_early_prefetch

	vmovapd	ymm4, [srcreg+5*d2]		;; r11
	vaddpd	ymm4, ymm4, [srcreg+2*d2+32]	;; r11+r19
	vaddpd	ymm0, ymm0, ymm4		;; (r3+r27)+(r7+r23)+(r11+r19)+r15
	vmulpd	ymm5, ymm6, ymm4		;; .623(r11+r19)
	vsubpd	ymm1, ymm1, ymm5		;; .901(r3+r27)+.223(r7+r23)-.623(r11+r19)-r15
	L1prefetchw srcreg+2*d2+d1+L1pd, L1pt
	vmulpd	ymm5, ymm7, ymm4		;; .223(r11+r19)
	vsubpd	ymm2, ymm2, ymm5		;; .623(r3+r27)-.901(r7+r23)-.223(r11+r19)+r15
	vmulpd	ymm5, ymm4, YMM_P901		;; .901(r11+r19)
	vaddpd	ymm3, ymm3, ymm5		;; .223(r3+r27)-.623(r7+r23)+.901(r11+r19)-r15

	vmovapd	ymm7, YMM_TMPS[0*32]
	vsubpd	ymm4, ymm7, ymm0		;; Real odd-cols row #8 (final real #8)
	vaddpd	ymm0, ymm7, ymm0		;; Real odd-cols row #1 (final real #1A)

	vmovapd	ymm7, YMM_TMPS[1*32]
	vsubpd	ymm5, ymm7, ymm1		;; Real odd-cols row #7
	vaddpd	ymm1, ymm7, ymm1		;; Real odd-cols row #2

	vmovapd	ymm7, YMM_TMPS[2*32]
	vsubpd	ymm6, ymm7, ymm2		;; Real odd-cols row #6
	vaddpd	ymm2, ymm7, ymm2		;; Real odd-cols row #3

	ystore	YMM_TMPS[6*32], ymm4		;; Real #8
	ystore	[srcreg], ymm0			;; Final real #1A

	vmovapd	ymm0, YMM_TMPS[3*32]
	vsubpd	ymm7, ymm0, ymm3		;; Real odd-cols row #5
	vaddpd	ymm3, ymm0, ymm3		;; Real odd-cols row #4

	ystore	YMM_TMPS[5*32], ymm5		;; Real odd-cols row #7
	ystore	YMM_TMPS[0*32], ymm1		;; Real odd-cols row #2
	ystore	YMM_TMPS[4*32], ymm6		;; Real odd-cols row #6
	ystore	YMM_TMPS[1*32], ymm2		;; Real odd-cols row #3
	ystore	YMM_TMPS[3*32], ymm7		;; Real odd-cols row #5
	ystore	YMM_TMPS[2*32], ymm3		;; Real odd-cols row #4

	;; Do the even columns for the real results

	vmovapd	ymm7, [srcreg+d1]		;; r2
	vaddpd	ymm7, ymm7, [srcreg+6*d2+d1+32]	;; r2+r28
	vmovapd	ymm0, [srcreg+6*d2+d1]		;; r14
	vaddpd	ymm0, ymm0, [srcreg+d1+32]	;; r14+r16
	vsubpd	ymm1, ymm7, ymm0		;; (r2+r28)-(r14+r16)
	vaddpd	ymm0, ymm7, ymm0		;; (r2+r28)+(r14+r16)

	vmulpd	ymm5, ymm1, YMM_P975		;; .975((r2+r28)-(r14+r16))
	vmovapd	ymm2, YMM_P782
	vmulpd	ymm6, ymm2, ymm1		;; .782((r2+r28)-(r14+r16))
	vmulpd	ymm7, ymm1, YMM_P434		;; .434((r2+r28)-(r14+r16))

	vmovapd	ymm4, [srcreg+d2+d1]		;; r4
	vaddpd	ymm4, ymm4, [srcreg+5*d2+d1+32]	;; r4+r26
	vmovapd	ymm1, [srcreg+5*d2+d1]		;; r12
	vaddpd	ymm1, ymm1, [srcreg+d2+d1+32]	;; r12+r18
	vsubpd	ymm3, ymm4, ymm1		;; (r4+r26)-(r12+r18)
	vaddpd	ymm1, ymm4, ymm1		;; (r4+r26)+(r12+r18)
	L1prefetchw srcreg+3*d2+L1pd, L1pt

	vmulpd	ymm4, ymm2, ymm3		;; .782((r4+r26)-(r12+r18))
	vaddpd	ymm5, ymm5, ymm4		;; .975((r2+r28)-(r14+r16))+.782((r4+r26)-(r12+r18))
	vmulpd	ymm4, ymm3, YMM_P434		;; .434((r4+r26)-(r12+r18))
	vsubpd	ymm6, ymm6, ymm4		;; .782((r2+r28)-(r14+r16))-.434((r4+r26)-(r12+r18))
	vmulpd	ymm4, ymm3, YMM_P975		;; .975((r4+r26)-(r12+r18))
	vsubpd	ymm7, ymm7, ymm4		;; .434((r2+r28)-(r14+r16))-.975((r4+r26)-(r12+r18))

	vmovapd	ymm4, [srcreg+2*d2+d1]		;; r6
	vaddpd	ymm4, ymm4, [srcreg+4*d2+d1+32]	;; r6+r24
	vmovapd	ymm2, [srcreg+4*d2+d1]		;; r10
	vaddpd	ymm2, ymm2, [srcreg+2*d2+d1+32]	;; r10+r20
	vsubpd	ymm3, ymm4, ymm2		;; (r6+r24)-(r10+r20)
	vaddpd	ymm2, ymm4, ymm2		;; (r6+r24)+(r10+r20)

	yloop_optional_early_prefetch

	vmulpd	ymm4, ymm3, YMM_P434		;; .434((r6+r24)-(r10+r20))
	vaddpd	ymm5, ymm5, ymm4		;; .975((r2+r28)-(r14+r16))+.782((r4+r26)-(r12+r18))+.434((r6+r24)-(r10+r20))
	vmulpd	ymm4, ymm3, YMM_P975		;; .975((r6+r24)-(r10+r20))
	vsubpd	ymm6, ymm6, ymm4		;; .782((r2+r28)-(r14+r16))-.434((r4+r26)-(r12+r18))-.975((r6+r24)-(r10+r20))
	L1prefetchw srcreg+3*d2+d1+L1pd, L1pt
	vmulpd	ymm4, ymm3, YMM_P782		;; .782((r6+r24)-(r10+r20))
	vaddpd	ymm7, ymm7, ymm4		;; .434((r2+r28)-(r14+r16))-.975((r4+r26)-(r12+r18))+.782((r6+r24)-(r10+r20))

	ystore	YMM_TMPS[7*32], ymm5		;; Save real even-cols row #2
	ystore	YMM_TMPS[8*32], ymm6		;; Save real even-cols row #4
	ystore	YMM_TMPS[9*32], ymm7		;; Save real even-cols row #6

	vmulpd	ymm5, ymm0, YMM_P901		;; .901((r2+r28)+(r14+r16))
	vmulpd	ymm6, ymm0, YMM_P623		;; .623((r2+r28)+(r14+r16))
	vmovapd	ymm4, YMM_P223
	vmulpd	ymm7, ymm4, ymm0		;; .223((r2+r28)+(r14+r16))

	yloop_optional_early_prefetch

	vaddpd	ymm0, ymm0, ymm1		;; ((r2+r28)+(r14+r16))+((r4+r26)+(r12+r18))
	vmulpd	ymm3, ymm4, ymm1		;; .223((r4+r26)+(r12+r18))
	vaddpd	ymm5, ymm5, ymm3		;; .901((r2+r28)+(r14+r16))+.223((r4+r26)+(r12+r18))
	vmulpd	ymm3, ymm1, YMM_P901		;; .901((r4+r26)+(r12+r18))
	vsubpd	ymm6, ymm6, ymm3		;; .623((r2+r28)+(r14+r16))-.901((r4+r26)+(r12+r18))
	vmovapd	ymm3, YMM_P623			;; .623((r4+r26)+(r12+r18))
	vmulpd	ymm1, ymm3, ymm1		;; .623((r4+r26)+(r12+r18))
	vsubpd	ymm7, ymm7, ymm1		;; .223((r2+r28)+(r14+r16))-.623((r4+r26)+(r12+r18))
	L1prefetchw srcreg+4*d2+L1pd, L1pt

	vaddpd	ymm0, ymm0, ymm2		;; ((r2+r28)+(r14+r16))+((r4+r26)+(r12+r18))+((r6+r24)+(r10+r20))
	vmulpd	ymm1, ymm3, ymm2		;; .623((r6+r24)+(r10+r20))
	vsubpd	ymm5, ymm5, ymm1		;; .901((r2+r28)+(r14+r16))+.223((r4+r26)+(r12+r18))-.623((r6+r24)+(r10+r20))
	vmulpd	ymm1, ymm4, ymm2		;; .223((r6+r24)+(r10+r20))
	vsubpd	ymm6, ymm6, ymm1		;; .623((r2+r28)+(r14+r16))-.901((r4+r26)+(r12+r18))-.223((r6+r24)+(r10+r20))
	vmulpd	ymm1, ymm2, YMM_P901		;; .901((r6+r24)+(r10+r20))
	vaddpd	ymm7, ymm7, ymm1		;; .223((r2+r28)+(r14+r16))-.623((r4+r26)+(r12+r18))+.901((r6+r24)+(r10+r20))

	vmovapd	ymm1, [srcreg+3*d2+d1]		;; r8
	vmovapd	ymm2, [srcreg+3*d2+d1+32]	;; r22
	vaddpd	ymm3, ymm1, ymm2 		;; r8+r22
	vsubpd	ymm1, ymm1, ymm2		;; r8-r22
	vaddpd	ymm0, ymm0, ymm3		;; ((r2+r28)+(r14+r16))+((r4+r26)+(r12+r18))+((r6+r24)+(r10+r20))+(r8+r22)
	L1prefetchw srcreg+4*d2+d1+L1pd, L1pt
	vsubpd	ymm5, ymm5, ymm3		;; .901((r2+r28)+(r14+r16))+.223((r4+r26)+(r12+r18))-.623((r6+r24)+(r10+r20))-(r8+r22)
	vaddpd	ymm6, ymm6, ymm3		;; .623((r2+r28)+(r14+r16))-.901((r4+r26)+(r12+r18))-.223((r6+r24)+(r10+r20))+(r8+r22)
	vsubpd	ymm7, ymm7, ymm3		;; .223((r2+r28)+(r14+r16))-.623((r4+r26)+(r12+r18))+.901((r6+r24)+(r10+r20))-(r8+r22)

	ystore	[srcreg+32], ymm0		;; Save final real #1B (real even-cols row #1)
	ystore	YMM_TMPS[10*32], ymm5		;; Save real even-cols row #3
	ystore	YMM_TMPS[11*32], ymm6		;; Save real even-cols row #5
	ystore	YMM_TMPS[12*32], ymm7		;; Save real even-cols row #7

	;; Do the even columns for the imaginary results

	vmovapd	ymm0, [srcreg+d1]		;; r2
	vsubpd	ymm0, ymm0, [srcreg+6*d2+d1+32]	;; r2-r28
	vmovapd	ymm2, [srcreg+6*d2+d1]		;; r14
	vsubpd	ymm2, ymm2, [srcreg+d1+32]	;; r14-r16
	vaddpd	ymm3, ymm0, ymm2		;; (r2-r28)+(r14-r16)
	vsubpd	ymm0, ymm0, ymm2		;; (r2-r28)-(r14-r16)

	vmulpd	ymm5, ymm4, ymm3		;; .223((r2-r28)+(r14-r16))
	vmulpd	ymm6, ymm3, YMM_P623		;; .623((r2-r28)+(r14-r16))
	vmulpd	ymm7, ymm3, YMM_P901		;; .901((r2-r28)+(r14-r16))

	vsubpd	ymm4, ymm3, ymm1		;; ((r2-r28)+(r14-r16))-(r8-r22)
	vaddpd	ymm5, ymm5, ymm1		;; .223((r2-r28)+(r14-r16))+(r8-r22)
	vsubpd	ymm6, ymm6, ymm1		;; .623((r2-r28)+(r14-r16))-(r8-r22)
	vaddpd	ymm7, ymm7, ymm1		;; .901((r2-r28)+(r14-r16))+(r8-r22)
	L1prefetchw srcreg+5*d2+L1pd, L1pt

	vmovapd	ymm1, [srcreg+d2+d1]		;; r4
	vsubpd	ymm1, ymm1, [srcreg+5*d2+d1+32]	;; r4-r26
	vmovapd	ymm3, [srcreg+5*d2+d1]		;; r12
	vsubpd	ymm3, ymm3, [srcreg+d2+d1+32]	;; r12-r18
	vaddpd	ymm2, ymm1, ymm3		;; (r4-r26)+(r12-r18)
	vsubpd	ymm1, ymm1, ymm3		;; (r4-r26)-(r12-r18)

	vsubpd	ymm4, ymm4, ymm2		;; ((r2-r28)+(r14-r16))-((r4-r26)+(r12-r18))-(r8-r22)
	vmulpd	ymm3, ymm2, YMM_P623		;; .623((r4-r26)+(r12-r18))
	vaddpd	ymm5, ymm5, ymm3		;; .223((r2-r28)+(r14-r16))+.623((r4-r26)+(r12-r18))+(r8-r22)
	vmulpd	ymm3, ymm2, YMM_P901		;; .901((r4-r26)+(r12-r18))
	L1prefetchw srcreg+5*d2+d1+L1pd, L1pt
	vaddpd	ymm6, ymm6, ymm3		;; .623((r2-r28)+(r14-r16))+.901((r4-r26)+(r12-r18))-(r8-r22)
	vmulpd	ymm3, ymm2, YMM_P223		;; .223((r4-r26)+(r12-r18))
	vsubpd	ymm7, ymm7, ymm3		;; .901((r2-r28)+(r14-r16))-.223((r4-r26)+(r12-r18))+(r8-r22)

	vmovapd	ymm2, [srcreg+2*d2+d1]		;; r6
	vsubpd	ymm2, ymm2, [srcreg+4*d2+d1+32]	;; r6-r24
	vmovapd	ymm3, [srcreg+4*d2+d1]		;; r10
	vsubpd	ymm3, ymm3, [srcreg+2*d2+d1+32]	;; r10-r20
	vsubpd	ymm2, ymm2, ymm3		;; (r6-r24)-(r10-r20)
	vaddpd	ymm3, ymm3, ymm3		;; Mul by 2
	vaddpd	ymm3, ymm3, ymm2		;; (r6-r24)+(r10-r20)
	L1prefetchw srcreg+6*d2+L1pd, L1pt

	vaddpd	ymm4, ymm4, ymm3		;; ((r2-r28)+(r14-r16))-((r4-r26)+(r12-r18))+((r6-r24)+(r10-r20))-(r8-r22)
	ystore	YMM_TMPS[13*32], ymm4		;; Save imag row #8
	vmulpd	ymm4, ymm3, YMM_P901		;; .901((r6-r24)+(r10-r20))
	vaddpd	ymm5, ymm5, ymm4		;; .223((r2-r28)+(r14-r16))+.623((r4-r26)+(r12-r18))+.901((r6-r24)+(r10-r20))+(r8-r22)
	vmulpd	ymm4, ymm3, YMM_P223		;; .223((r6-r24)+(r10-r20))
	vsubpd	ymm6, ymm6, ymm4		;; .623((r2-r28)+(r14-r16))+.901((r4-r26)+(r12-r18))-.223((r6-r24)+(r10-r20))-(r8-r22)
	vmulpd	ymm4, ymm3, YMM_P623		;; .623((r6-r24)+(r10-r20))
	vsubpd	ymm7, ymm7, ymm4		;; .901((r2-r28)+(r14-r16))-.223((r4-r26)+(r12-r18))-.623((r6-r24)+(r10-r20))+(r8-r22)
	L1prefetchw srcreg+6*d2+d1+L1pd, L1pt

	ystore	YMM_TMPS[14*32], ymm5		;; Save imag even-cols row #2
	ystore	YMM_TMPS[15*32], ymm6		;; Save imag even-cols row #4
	ystore	YMM_TMPS[16*32], ymm7		;; Save imag even-cols row #6

	vmovapd	ymm4, YMM_P434
	vmulpd	ymm5, ymm4, ymm0		;; .434((r2-r28)-(r14-r16))
	vmulpd	ymm6, ymm0, YMM_P782		;; .782((r2-r28)-(r14-r16))
	vmovapd	ymm7, YMM_P975
	vmulpd	ymm0, ymm7, ymm0		;; .975((r2-r28)-(r14-r16))

	yloop_optional_early_prefetch

	vmulpd	ymm3, ymm7, ymm1		;; .975((r4-r26)-(r12-r18))
	vaddpd	ymm5, ymm5, ymm3		;; .434((r2-r28)-(r14-r16))+.975((r4-r26)-(r12-r18))
	vmulpd	ymm3, ymm4, ymm1		;; .434((r4-r26)-(r12-r18))
	vaddpd	ymm6, ymm6, ymm3		;; .782((r2-r28)-(r14-r16))+.434((r4-r26)-(r12-r18))
	vmovapd	ymm3, YMM_P782
	vmulpd	ymm1, ymm3, ymm1 		;; .782((r4-r26)-(r12-r18))
	vsubpd	ymm0, ymm0, ymm1		;; .975((r2-r28)-(r14-r16))-.782((r4-r26)-(r12-r18))

	vmulpd	ymm1, ymm3, ymm2		;; .782((r6-r24)-(r10-r20))
	vaddpd	ymm5, ymm5, ymm1		;; .434((r2-r28)-(r14-r16))+.975((r4-r26)-(r12-r18))+.782((r6-r24)-(r10-r20))
	vmulpd	ymm1, ymm7, ymm2		;; .975((r6-r24)-(r10-r20))
	vsubpd	ymm6, ymm6, ymm1		;; .782((r2-r28)-(r14-r16))+.434((r4-r26)-(r12-r18))-.975((r6-r24)-(r10-r20))
	vmulpd	ymm1, ymm4, ymm2		;; .434((r6-r24)-(r10-r20))
	vaddpd	ymm0, ymm0, ymm1		;; .975((r2-r28)-(r14-r16))-.782((r4-r26)-(r12-r18))+.434((r6-r24)-(r10-r20))

	ystore	YMM_TMPS[17*32], ymm5		;; Save imag even-cols row #3
	ystore	YMM_TMPS[18*32], ymm6		;; Save imag even-cols row #5
	ystore	YMM_TMPS[19*32], ymm0		;; Save imag even-cols row #7

	;; Do the odd columns for the imag results

	vmovapd	ymm2, [srcreg+2*d2]		;; r5
	vsubpd	ymm2, ymm2, [srcreg+5*d2+32]	;; r5-r25
	vmovapd	ymm3, YMM_P782
	vmulpd	ymm0, ymm3, ymm2		;; .782(r5-r25)
	vmovapd	ymm4, YMM_P975
	vmulpd	ymm1, ymm4, ymm2		;; .975(r5-r25)
	vmovapd	ymm5, YMM_P434
	vmulpd	ymm2, ymm5, ymm2		;; .434(r5-r25)

	vmovapd	ymm7, [srcreg+4*d2]		;; r9
	vsubpd	ymm7, ymm7, [srcreg+3*d2+32]	;; r9-r21
	vmulpd	ymm6, ymm4, ymm7		;; .975(r9-r21)
	vaddpd	ymm0, ymm0, ymm6		;; .782(r5-r25)+.975(r9-r21)
	vmulpd	ymm6, ymm5, ymm7		;; .434(r9-r21)
	vsubpd	ymm1, ymm1, ymm6		;; .975(r5-r25)-.434(r9-r21)
	vmulpd	ymm6, ymm3, ymm7		;; .782(r9-r21)
	vsubpd	ymm2, ymm2, ymm6		;; .434(r5-r25)-.782(r9-r21)

	vmovapd	ymm7, [srcreg+6*d2]		;; r13
	vsubpd	ymm7, ymm7, [srcreg+d2+32]	;; r13-r17
	vmulpd	ymm6, ymm5, ymm7		;; .434(r13-r17)
	vaddpd	ymm0, ymm0, ymm6		;; .782(r5-r25)+.975(r9-r21)+.434(r13-r17)
	vmulpd	ymm6, ymm3, ymm7		;; .782(r13-r17)
	vsubpd	ymm1, ymm1, ymm6		;; .975(r5-r25)-.434(r9-r21)-.782(r13-r17)
	vmulpd	ymm6, ymm4, ymm7		;; .975(r13-r17)
	vaddpd	ymm2, ymm2, ymm6		;; .434(r5-r25)-.782(r9-r21)+.975(r13-r17)

	ystore	YMM_TMPS[20*32], ymm0
	ystore	YMM_TMPS[21*32], ymm1
	ystore	YMM_TMPS[22*32], ymm2

	vmovapd	ymm2, [srcreg+d2]		;; r3
	vsubpd	ymm2, ymm2, [srcreg+6*d2+32]	;; r3-r27
	vmulpd	ymm0, ymm5, ymm2		;; .434(r3-r27)
	vmulpd	ymm1, ymm3, ymm2		;; .782(r3-r27)
	vmulpd	ymm2, ymm4, ymm2		;; .975(r3-r27)

	vmovapd	ymm7, [srcreg+3*d2]		;; r7
	vsubpd	ymm7, ymm7, [srcreg+4*d2+32]	;; r7-r23
	vmulpd	ymm6, ymm4, ymm7		;; .975(r7-r23)
	vaddpd	ymm0, ymm0, ymm6		;; .434(r3-r27)+.975(r7-r23)
	vmulpd	ymm6, ymm5, ymm7		;; .434(r7-r23)
	vaddpd	ymm1, ymm1, ymm6		;; .782(r3-r27)+.434(r7-r23)
	vmulpd	ymm6, ymm3, ymm7		;; .782(r7-r23)
	vsubpd	ymm2, ymm2, ymm6		;; .975(r3-r27)-.782(r7-r23)

	vmovapd	ymm7, [srcreg+5*d2]		;; r11
	vsubpd	ymm7, ymm7, [srcreg+2*d2+32]	;; r11-r19
	vmulpd	ymm6, ymm3, ymm7		;; .782(r11-r19)
	vaddpd	ymm0, ymm0, ymm6		;; .434(r3-r27)+.975(r7-r23)+.782(r11-r19)
	vmulpd	ymm6, ymm4, ymm7		;; .975(r11-r19)
	vsubpd	ymm1, ymm1, ymm6		;; .782(r3-r27)+.434(r7-r23)-.975(r11-r19)
	vmulpd	ymm6, ymm5, ymm7		;; .434(r11-r19)
	vaddpd	ymm2, ymm2, ymm6		;; .975(r3-r27)-.782(r7-r23)+.434(r11-r19)

	vmovapd	ymm7, YMM_TMPS[20*32]
	vaddpd	ymm5, ymm0, ymm7		;; Imag odd-cols row #2
	vsubpd	ymm0, ymm0, ymm7		;; Imag odd-cols row #7

	vmovapd	ymm7, YMM_TMPS[21*32]
	vaddpd	ymm6, ymm1, ymm7		;; Imag odd-cols row #3
	vsubpd	ymm1, ymm1, ymm7		;; Imag odd-cols row #6

	vmovapd	ymm7, YMM_TMPS[22*32]
	vaddpd	ymm3, ymm2, ymm7		;; Imag odd-cols row #4
	vsubpd	ymm2, ymm2, ymm7		;; Imag odd-cols row #5

;;	ystore	YMM_TMPS[20*32], ymm5		;; Imag odd-cols row #2
	ystore	YMM_TMPS[25*32], ymm0		;; Imag odd-cols row #7
	ystore	YMM_TMPS[21*32], ymm6		;; Imag odd-cols row #3
	ystore	YMM_TMPS[24*32], ymm1		;; Imag odd-cols row #6
	ystore	YMM_TMPS[22*32], ymm3		;; Imag odd-cols row #4
	ystore	YMM_TMPS[23*32], ymm2		;; Imag odd-cols row #5

	;; Now combine the even and odd columns then do the post-multiply by twiddle factors.

	vmovapd	ymm6, YMM_TMPS[0*32]		;; Real odd-cols row #2
	vmovapd	ymm7, YMM_TMPS[7*32]		;; Real even-cols row #2
	vsubpd	ymm0, ymm6, ymm7		;; Real #14
	vaddpd	ymm1, ymm6, ymm7		;; Real #2
	vmovapd	ymm4, YMM_TMPS[14*32]		;; Imag even-cols row #2
;;	vmovapd	ymm5, YMM_TMPS[20*32]		;; Imag odd-cols row #2
	vsubpd	ymm2, ymm4, ymm5		;; Imag #14
	vaddpd	ymm5, ymm4, ymm5		;; Imag #2

	vmovapd	ymm3, [screg+12*64+32]		;; cosine/sine
	vmulpd	ymm6, ymm0, ymm3		;; A14 = R14 * cosine/sine
	vmovapd	ymm4, [screg+32]		;; cosine/sine
	vmulpd	ymm7, ymm1, ymm4		;; A2 = R2 * cosine/sine
	vsubpd	ymm6, ymm6, ymm2		;; A14 = A14 - I14
	vmulpd	ymm2, ymm2, ymm3		;; B14 = I14 * cosine/sine
	vsubpd	ymm7, ymm7, ymm5		;; A2 = A2 - I2
	vmulpd	ymm5, ymm5, ymm4		;; B2 = I2 * cosine/sine
	vaddpd	ymm2, ymm2, ymm0		;; B14 = B14 + R14
	vmovapd	ymm3, [screg+12*64]		;; sine
	vmulpd	ymm6, ymm6, ymm3		;; A14 = A14 * sine (final R14)
	vaddpd	ymm5, ymm5, ymm1		;; B2 = B2 + R2
	vmovapd	ymm4, [screg]			;; sine
	vmulpd	ymm7, ymm7, ymm4		;; A2 = A2 * sine (final R2)
	vmulpd	ymm2, ymm2, ymm3		;; B14 = B14 * sine (final I14)
	vmulpd	ymm5, ymm5, ymm4		;; B2 = B2 * sine (final I2)
	ystore	[srcreg+6*d2+d1], ymm6		;; Save final R14
	ystore	[srcreg+6*d2+d1+32], ymm2	;; Save final I14
	ystore	[srcreg+d1], ymm7		;; Save final R2
	ystore	[srcreg+d1+32], ymm5		;; Save final I2

	vmovapd	ymm6, YMM_TMPS[1*32]		;; Real odd-cols row #3
	vmovapd	ymm7, YMM_TMPS[10*32]		;; Real even-cols row #3
	vsubpd	ymm0, ymm6, ymm7		;; Real #13
	vaddpd	ymm1, ymm6, ymm7		;; Real #3
	vmovapd	ymm6, YMM_TMPS[17*32]		;; Imag even-cols row #3
	vmovapd	ymm7, YMM_TMPS[21*32]		;; Imag odd-cols row #3
	vsubpd	ymm2, ymm6, ymm7		;; Imag #13
	vaddpd	ymm3, ymm6, ymm7		;; Imag #3

	vmovapd	ymm5, [screg+11*64+32]		;; cosine/sine
	vmulpd	ymm6, ymm0, ymm5		;; A13 = R13 * cosine/sine
	vmovapd	ymm4, [screg+64+32]		;; cosine/sine
	vmulpd	ymm7, ymm1, ymm4		;; A3 = R3 * cosine/sine
	vsubpd	ymm6, ymm6, ymm2		;; A13 = A13 - I13
	vmulpd	ymm2, ymm2, ymm5		;; B13 = I13 * cosine/sine
	vsubpd	ymm7, ymm7, ymm3		;; A3 = A3 - I3
	vmulpd	ymm3, ymm3, ymm4		;; B3 = I3 * cosine/sine
	vaddpd	ymm2, ymm2, ymm0		;; B13 = B13 + R13
	vmovapd	ymm5, [screg+11*64]		;; sine
	vmulpd	ymm6, ymm6, ymm5		;; A13 = A13 * sine (final R13)
	vaddpd	ymm3, ymm3, ymm1		;; B3 = B3 + R3
	vmovapd	ymm4, [screg+64]		;; sine
	vmulpd	ymm7, ymm7, ymm4		;; A3 = A3 * sine (final R3)
	vmulpd	ymm2, ymm2, ymm5		;; B13 = B13 * sine (final I13)
	vmulpd	ymm3, ymm3, ymm4		;; B3 = B3 * sine (final I3)
	ystore	[srcreg+6*d2], ymm6		;; Save final R13
	ystore	[srcreg+6*d2+32], ymm2		;; Save final I13
	ystore	[srcreg+d2], ymm7		;; Save final R3
	ystore	[srcreg+d2+32], ymm3		;; Save final I3

	vmovapd	ymm6, YMM_TMPS[2*32]		;; Real odd-cols row #4
	vmovapd	ymm7, YMM_TMPS[8*32]		;; Real even-cols row #4
	vsubpd	ymm0, ymm6, ymm7		;; Real #12
	vaddpd	ymm1, ymm6, ymm7		;; Real #4
	vmovapd	ymm6, YMM_TMPS[15*32]		;; Imag even-cols row #4
	vmovapd	ymm7, YMM_TMPS[22*32]		;; Imag odd-cols row #4
	vsubpd	ymm2, ymm6, ymm7		;; Imag #12
	vaddpd	ymm3, ymm6, ymm7		;; Imag #4

	vmovapd	ymm5, [screg+10*64+32]		;; cosine/sine
	vmulpd	ymm6, ymm0, ymm5		;; A12 = R12 * cosine/sine
	vmovapd	ymm4, [screg+2*64+32]		;; cosine/sine
	vmulpd	ymm7, ymm1, ymm4		;; A4 = R4 * cosine/sine
	vsubpd	ymm6, ymm6, ymm2		;; A12 = A12 - I12
	vmulpd	ymm2, ymm2, ymm5		;; B12 = I12 * cosine/sine
	vsubpd	ymm7, ymm7, ymm3		;; A4 = A4 - I4
	vmulpd	ymm3, ymm3, ymm4		;; B4 = I4 * cosine/sine
	vaddpd	ymm2, ymm2, ymm0		;; B12 = B12 + R12
	vmovapd ymm5, [screg+10*64]		;; sine
	vmulpd	ymm6, ymm6, ymm5		;; A12 = A12 * sine (final R12)
	vaddpd	ymm3, ymm3, ymm1		;; B4 = B4 + R4
	vmovapd	ymm4, [screg+2*64]		;; sine
	vmulpd	ymm7, ymm7, ymm4		;; A4 = A4 * sine (final R4)
	vmulpd	ymm2, ymm2, ymm5		;; B12 = B12 * sine (final I12)
	vmulpd	ymm3, ymm3, ymm4		;; B4 = B4 * sine (final I4)
	ystore	[srcreg+5*d2+d1], ymm6		;; Save final R12
	ystore	[srcreg+5*d2+d1+32], ymm2	;; Save final I12
	ystore	[srcreg+d2+d1], ymm7		;; Save final R4
	ystore	[srcreg+d2+d1+32], ymm3		;; Save final I4

	vmovapd	ymm6, YMM_TMPS[3*32]		;; Real odd-cols row #5
	vmovapd	ymm7, YMM_TMPS[11*32]		;; Real even-cols row #5
	vsubpd	ymm0, ymm6, ymm7		;; Real #11
	vaddpd	ymm1, ymm6, ymm7		;; Real #5
	vmovapd	ymm6, YMM_TMPS[18*32]		;; Imag even-cols row #5
	vmovapd	ymm7, YMM_TMPS[23*32]		;; Imag odd-cols row #5
	vsubpd	ymm2, ymm6, ymm7		;; Imag #11
	vaddpd	ymm3, ymm6, ymm7		;; Imag #5

	vmovapd	ymm5, [screg+9*64+32]		;; cosine/sine
	vmulpd	ymm6, ymm0, ymm5		;; A11 = R11 * cosine/sine
	vmovapd	ymm4, [screg+3*64+32]		;; cosine/sine
	vmulpd	ymm7, ymm1, ymm4		;; A5 = R5 * cosine/sine
	vsubpd	ymm6, ymm6, ymm2		;; A11 = A11 - I11
	vmulpd	ymm2, ymm2, ymm5		;; B11 = I11 * cosine/sine
	vsubpd	ymm7, ymm7, ymm3		;; A5 = A5 - I5
	vmulpd	ymm3, ymm3, ymm4		;; B5 = I5 * cosine/sine
	vaddpd	ymm2, ymm2, ymm0		;; B11 = B11 + R11
	vmovapd	ymm5, [screg+9*64]		;; sine
	vmulpd	ymm6, ymm6, ymm5		;; A11 = A11 * sine (final R11)
	vaddpd	ymm3, ymm3, ymm1		;; B5 = B5 + R5
	vmovapd	ymm4, [screg+3*64]		;; sine
	vmulpd	ymm7, ymm7, ymm4		;; A5 = A5 * sine (final R5)
	vmulpd	ymm2, ymm2, ymm5		;; B11 = B11 * sine (final I11)
	vmulpd	ymm3, ymm3, ymm4		;; B5 = B5 * sine (final I5)
	ystore	[srcreg+5*d2], ymm6		;; Save final R11
	ystore	[srcreg+5*d2+32], ymm2		;; Save final I11
	ystore	[srcreg+2*d2], ymm7		;; Save final R5
	ystore	[srcreg+2*d2+32], ymm3		;; Save final I5

	vmovapd	ymm6, YMM_TMPS[4*32]		;; Real odd-cols row #6
	vmovapd	ymm7, YMM_TMPS[9*32]		;; Real even-cols row #6
	vsubpd	ymm0, ymm6, ymm7		;; Real #10
	vaddpd	ymm1, ymm6, ymm7		;; Real #6
	vmovapd	ymm6, YMM_TMPS[16*32]		;; Imag even-cols row #6
	vmovapd	ymm7, YMM_TMPS[24*32]		;; Imag odd-cols row #6
	vsubpd	ymm2, ymm6, ymm7		;; Imag #10
	vaddpd	ymm3, ymm6, ymm7		;; Imag #6

	vmovapd	ymm5, [screg+8*64+32]		;; cosine/sine
	vmulpd	ymm6, ymm0, ymm5		;; A10 = R10 * cosine/sine
	vmovapd	ymm4, [screg+4*64+32]		;; cosine/sine
	vmulpd	ymm7, ymm1, ymm4		;; A6 = R6 * cosine/sine
	vsubpd	ymm6, ymm6, ymm2		;; A10 = A10 - I10
	vmulpd	ymm2, ymm2, ymm5		;; B10 = I10 * cosine/sine
	vsubpd	ymm7, ymm7, ymm3		;; A6 = A6 - I6
	vmulpd	ymm3, ymm3, ymm4		;; B6 = I6 * cosine/sine
	vaddpd	ymm2, ymm2, ymm0		;; B10 = B10 + R10
	vmovapd	ymm5, [screg+8*64]		;; sine
	vmulpd	ymm6, ymm6, ymm5		;; A10 = A10 * sine (final R10)
	vaddpd	ymm3, ymm3, ymm1		;; B6 = B6 + R6
	vmovapd	ymm4, [screg+4*64]		;; sine
	vmulpd	ymm7, ymm7, ymm4		;; A6 = A6 * sine (final R6)
	vmulpd	ymm2, ymm2, ymm5		;; B10 = B10 * sine (final I10)
	vmulpd	ymm3, ymm3, ymm4		;; B6 = B6 * sine (final I6)
	ystore	[srcreg+4*d2+d1], ymm6		;; Save final R10
	ystore	[srcreg+4*d2+d1+32], ymm2	;; Save final I10
	ystore	[srcreg+2*d2+d1], ymm7		;; Save final R6
	ystore	[srcreg+2*d2+d1+32], ymm3	;; Save final I6

	vmovapd	ymm6, YMM_TMPS[5*32]		;; Real odd-cols row #7
	vmovapd	ymm7, YMM_TMPS[12*32]		;; Real even-cols row #7
	vsubpd	ymm0, ymm6, ymm7		;; Real #9
	vaddpd	ymm1, ymm6, ymm7		;; Real #7
	vmovapd	ymm6, YMM_TMPS[19*32]		;; Imag even-cols row #7
	vmovapd	ymm7, YMM_TMPS[25*32]		;; Imag odd-cols row #7
	vsubpd	ymm2, ymm6, ymm7		;; Imag #9
	vaddpd	ymm3, ymm6, ymm7		;; Imag #7

	vmovapd	ymm5, [screg+7*64+32]		;; cosine/sine
	vmulpd	ymm6, ymm0, ymm5		;; A9 = R9 * cosine/sine
	vmovapd	ymm4, [screg+5*64+32]		;; cosine/sine
	vmulpd	ymm7, ymm1, ymm4		;; A7 = R7 * cosine/sine
	vsubpd	ymm6, ymm6, ymm2		;; A9 = A9 - I9
	vmulpd	ymm2, ymm2, ymm5		;; B9 = I9 * cosine/sine
	vsubpd	ymm7, ymm7, ymm3		;; A7 = A7 - I7
	vmulpd	ymm3, ymm3, ymm4		;; B7 = I7 * cosine/sine
	vaddpd	ymm2, ymm2, ymm0		;; B9 = B9 + R9
	vmovapd	ymm5, [screg+7*64]		;; sine
	vmulpd	ymm6, ymm6, ymm5		;; A9 = A9 * sine (final R9)
	vaddpd	ymm3, ymm3, ymm1		;; B7 = B7 + R7
	vmovapd	ymm4, [screg+5*64]		;; sine
	vmulpd	ymm7, ymm7, ymm4		;; A7 = A7 * sine (final R7)
	vmulpd	ymm2, ymm2, ymm5		;; B9 = B9 * sine (final I9)
	vmulpd	ymm3, ymm3, ymm4		;; B7 = B7 * sine (final I7)

	vmovapd	ymm0, YMM_TMPS[6*32]		;; Real #8
	vmovapd	ymm1, YMM_TMPS[13*32]		;; Imag #8
	vmovapd	ymm5, [screg+6*64+32]		;; cosine/sine
	vmulpd	ymm4, ymm0, ymm5		;; A8 = R8 * cosine/sine
	vsubpd	ymm4, ymm4, ymm1		;; A8 = A8 - I8
	vmulpd	ymm1, ymm1, ymm5		;; B8 = I8 * cosine/sine
	vaddpd	ymm1, ymm1, ymm0		;; B8 = B8 + R8
	vmovapd	ymm5, [screg+6*64]		;; sine
	vmulpd	ymm4, ymm4, ymm5		;; A8 = A8 * sine (final R8)
	vmulpd	ymm1, ymm1, ymm5		;; B8 = B8 * sine (final I8)

	ystore	[srcreg+4*d2], ymm6		;; Save final R9
	ystore	[srcreg+4*d2+32], ymm2		;; Save final I9
	ystore	[srcreg+3*d2], ymm7		;; Save final R7
	ystore	[srcreg+3*d2+32], ymm3		;; Save final I7
	ystore	[srcreg+3*d2+d1], ymm4		;; Save final R8
	ystore	[srcreg+3*d2+d1+32], ymm1	;; Save final I8

	bump	srcreg, srcinc
	bump	screg, scinc
	ENDM

;; 64-bit version

IFDEF X86_64

yr7_14cl_28_reals_fft_preload MACRO
	vbroadcastsd ymm13, Q YMM_P223
	vbroadcastsd ymm14, Q YMM_P623
	vbroadcastsd ymm15, Q YMM_P901
	ENDM

yr7_14cl_28_reals_fft MACRO srcreg,srcinc,d1,d2,screg,scinc,maxrpt,L1pt,L1pd

	;; Do the odd columns for the real results

	vmovapd	ymm0, [srcreg+2*d2]		;; r5
	vaddpd	ymm0, ymm0, [srcreg+5*d2+32]	;; r5+r25						; 1-3

	vmovapd	ymm1, [srcreg+4*d2]		;; r9
	vaddpd	ymm1, ymm1, [srcreg+3*d2+32]	;; r9+r21						; 2-4

	vmovapd	ymm2, [srcreg+6*d2]		;; r13
	vaddpd	ymm2, ymm2, [srcreg+d2+32]	;; r13+r17						; 3-5

	vmovapd	ymm3, [srcreg]			;; r1
	vaddpd	ymm4, ymm3, ymm0		;; r1+(r5+r25)						; 4-6
	vmulpd	ymm5, ymm14, ymm0		;; .623(r5+r25)						;	4-8

	vmovapd	ymm6, [srcreg+d2]		;; r3
	vaddpd	ymm6, ymm6, [srcreg+6*d2+32]	;; r3+r27						; 5-7
	vmulpd	ymm7, ymm13, ymm0		;; .223(r5+r25)						;	5-9

	vmovapd	ymm8, [srcreg+3*d2]		;; r7
	vaddpd	ymm8, ymm8, [srcreg+4*d2+32]	;; r7+r23						; 6-8
	vmulpd	ymm0, ymm15, ymm0		;; .901(r5+r25)						;	6-10

	vaddpd	ymm4, ymm4, ymm1		;; r1+(r5+r25)+(r9+r21)					; 7-9

	vmovapd	ymm9, [srcreg+5*d2]		;; r11
	vaddpd	ymm9, ymm9, [srcreg+2*d2+32]	;; r11+r19						; 8-10
	vmulpd	ymm10, ymm13, ymm1		;; .223(r9+r21)						;	8-12

	vaddpd	ymm5, ymm3, ymm5		;; r1+.623(r5+r25)					; 9-11
	vmulpd	ymm11, ymm15, ymm1		;; .901(r9+r21)						;	9-13
	vmovapd	ymm12, [srcreg+32]		;; r15

	vsubpd	ymm7, ymm3, ymm7		;; r1-.223(r5+r25)					; 10-12
	vmulpd	ymm1, ymm14, ymm1		;; .623(r9+r21)						;	10-14

	vsubpd	ymm3, ymm3, ymm0		;; r1-.901(r5+r25)					; 11-13
	vmulpd	ymm0, ymm15, ymm2		;; .901(r13+r17)					;	11-15

	vaddpd	ymm4, ymm4, ymm2		;; r1+(r5+r25)+(r9+r21)+(r13+r17)			; 12-14
	L1prefetchw srcreg+L1pd, L1pt

	vsubpd	ymm5, ymm5, ymm10		;; r1+.623(r5+r25)-.223(r9+r21)				; 13-15
	vmulpd	ymm10, ymm14, ymm2		;; .623(r13+r17)					;	13-17

	vsubpd	ymm7, ymm7, ymm11		;; r1-.223(r5+r25)-.901(r9+r21)				; 14-16
	vmulpd	ymm2, ymm13, ymm2		;; .223(r13+r17)					;	14-18

	yloop_optional_early_prefetch

	vaddpd	ymm3, ymm3, ymm1		;; r1-.901(r5+r25)+.623(r9+r21)				; 15-17

	vsubpd	ymm5, ymm5, ymm0		;; r1+.623(r5+r25)-.223(r9+r21)-.901(r13+r17)		; 16-18
	vmulpd	ymm0, ymm15, ymm6		;; .901(r3+r27)						;	16-20

	vaddpd	ymm1, ymm6, ymm12		;; (r3+r27)+r15						; 17-19
	vmulpd	ymm11, ymm14, ymm6		;; .623(r3+r27)						;	17-21
	L1prefetchw srcreg+d1+L1pd, L1pt

	vaddpd	ymm7, ymm7, ymm10		;; r1-.223(r5+r25)-.901(r9+r21)+.623(r13+r17)		; 18-20
	vmulpd	ymm6, ymm13, ymm6		;; .223(r3+r27)						;	18-22

	vsubpd	ymm3, ymm3, ymm2		;; r1-.901(r5+r25)+.623(r9+r21)-.223(r13+r17)		; 19-21

	vaddpd	ymm1, ymm1, ymm8		;; (r3+r27)+(r7+r23)+r15				; 20-22
	vmulpd	ymm2, ymm13, ymm8		;; .223(r7+r23)						;	20-24

	vsubpd	ymm0, ymm0, ymm12		;; .901(r3+r27)-r15					; 21-23
	vmulpd	ymm10, ymm15, ymm8		;; .901(r7+r23)						;	21-25

	vaddpd	ymm11, ymm11, ymm12		;; .623(r3+r27)+r15					; 22-24
	vmulpd	ymm8, ymm14, ymm8		;; .623(r7+r23)						;	22-26
	L1prefetchw srcreg+d2+L1pd, L1pt

	vsubpd	ymm6, ymm6, ymm12		;; .223(r3+r27)-r15					; 23-25
	vmulpd	ymm12, ymm14, ymm9		;; .623(r11+r19)					;	23-27

	vaddpd	ymm1, ymm1, ymm9		;; (r3+r27)+(r7+r23)+(r11+r19)+r15			; 24-26
	vaddpd	ymm0, ymm0, ymm2		;; .901(r3+r27)+.223(r7+r23)-r15			; 25-27
	vmulpd	ymm2, ymm13, ymm9		;; .223(r11+r19)					;	24-28

	vmulpd	ymm9, ymm15, ymm9		;; .901(r11+r19)					;	25-29

	vsubpd	ymm11, ymm11, ymm10		;; .623(r3+r27)-.901(r7+r23)+r15			; 26-28
	vmovapd	ymm10, [srcreg+d1]		;; r2

	vsubpd	ymm6, ymm6, ymm8		;; .223(r3+r27)-.623(r7+r23)-r15			; 27-29
	vmovapd	ymm8, [srcreg+6*d2+d1]		;; r14

	vsubpd	ymm0, ymm0, ymm12		;; .901(r3+r27)+.223(r7+r23)-.623(r11+r19)-r15		; 28-30
	vmovapd	ymm12, [srcreg+d2+d1]		;; r4

	vsubpd	ymm11, ymm11, ymm2		;; .623(r3+r27)-.901(r7+r23)-.223(r11+r19)+r15		; 29-31
	vmovapd	ymm2, [srcreg+5*d2+d1]		;; r12

	vaddpd	ymm6, ymm6, ymm9		;; .223(r3+r27)-.623(r7+r23)+.901(r11+r19)-r15		; 30-32

	yloop_optional_early_prefetch

	vsubpd	ymm9, ymm4, ymm1		;; Real odd-cols row #8 (final real #8)			; 31-33
	vaddpd	ymm4, ymm4, ymm1		;; Real odd-cols row #1 (final real #1A)		; 32-34
	L1prefetchw srcreg+d2+d1+L1pd, L1pt

	vsubpd	ymm1, ymm5, ymm0		;; Real odd-cols row #7					; 33-35
	vaddpd	ymm5, ymm5, ymm0		;; Real odd-cols row #2					; 34-36
	ystore	YMM_TMPS[6*32], ymm9		;; Real #8						; 34
	vmovapd	ymm9, [srcreg+2*d2+d1]		;; r6

	vsubpd	ymm0, ymm7, ymm11		;; Real odd-cols row #6					; 35-37
	ystore	[srcreg], ymm4			;; Final real #1A					; 35
	vmovapd	ymm4, [srcreg+4*d2+d1]		;; r10

	vaddpd	ymm7, ymm7, ymm11		;; Real odd-cols row #3					; 36-38
	ystore	YMM_TMPS[5*32], ymm1		;; Real odd-cols row #7					; 36

	vsubpd	ymm11, ymm3, ymm6		;; Real odd-cols row #5					; 37-39
	ystore	YMM_TMPS[0*32], ymm5		;; Real odd-cols row #2					; 37

	vaddpd	ymm3, ymm3, ymm6		;; Real odd-cols row #4					; 38-40
	ystore	YMM_TMPS[4*32], ymm0		;; Real odd-cols row #6					; 38

	;; Do the even columns for the real results

	vaddpd	ymm10, ymm10, [srcreg+6*d2+d1+32] ;; r2+r28									; 39-41
	ystore	YMM_TMPS[1*32], ymm7		;; Real odd-cols row #3					; 39

	vaddpd	ymm8, ymm8, [srcreg+d1+32]	;; r14+r16									; 40-42
	ystore	YMM_TMPS[3*32], ymm11		;; Real odd-cols row #5					; 40

	vaddpd	ymm12, ymm12, [srcreg+5*d2+d1+32] ;; r4+r26									; 41-43
	ystore	YMM_TMPS[2*32], ymm3		;; Real odd-cols row #4					; 41

	vaddpd	ymm2, ymm2, [srcreg+d2+d1+32]	;; r12+r18									; 42-44

	vaddpd	ymm3, ymm10, ymm8		;; (r2+r28)+(r14+r16)								; 43-45
	L1prefetchw srcreg+2*d2+L1pd, L1pt

	vaddpd	ymm11, ymm12, ymm2		;; (r4+r26)+(r12+r18)								; 44-46

	vaddpd	ymm9, ymm9, [srcreg+4*d2+d1+32]	;; r6+r24									; 45-47

	vaddpd	ymm4, ymm4, [srcreg+2*d2+d1+32]	;; r10+r20									; 46-48
	vmulpd	ymm7, ymm15, ymm3		;; .901((r2+r28)+(r14+r16))							;	46-50

	vmovapd	ymm1, [srcreg+3*d2+d1]		;; r8
	vaddpd	ymm1, ymm1, [srcreg+3*d2+d1+32] ;; r8+r22									; 47-49
	vmulpd	ymm0, ymm14, ymm3		;; .623((r2+r28)+(r14+r16))							;	47-51

	vsubpd	ymm10, ymm10, ymm8		;; (r2+r28)-(r14+r16)								; 48-50
	vmulpd	ymm5, ymm13, ymm3		;; .223((r2+r28)+(r14+r16))							;	48-52
	L1prefetchw srcreg+2*d2+d1+L1pd, L1pt

	vaddpd	ymm8, ymm9, ymm4		;; (r6+r24)+(r10+r20)								; 49-51
	vmulpd	ymm6, ymm13, ymm11		;; .223((r4+r26)+(r12+r18))							;	49-53

	yloop_optional_early_prefetch

	vaddpd	ymm3, ymm3, ymm1		;; ((r2+r28)+(r14+r16))+(r8+r22)						; 50-52

	vsubpd	ymm12, ymm12, ymm2		;; (r4+r26)-(r12+r18)								; 51-53
	vmulpd	ymm2, ymm15, ymm11		;; .901((r4+r26)+(r12+r18))							;	50-54

	vsubpd	ymm9, ymm9, ymm4		;; (r6+r24)-(r10+r20)								; 52-54
	vmulpd	ymm4, ymm14, ymm11		;; .623((r4+r26)+(r12+r18))							;	51-55

	vaddpd	ymm3, ymm3, ymm11		;; ((r2+r28)+(r14+r16))+((r4+r26)+(r12+r18))+(r8+r22)				; 53-55
	vmulpd	ymm11, ymm14, ymm8		;; .623((r6+r24)+(r10+r20))							;	52-56
	L1prefetchw srcreg+3*d2+L1pd, L1pt

	vsubpd	ymm7, ymm7, ymm1		;; .901((r2+r28)+(r14+r16))-(r8+r22)						; 54-56
	vaddpd	ymm0, ymm0, ymm1		;; .623((r2+r28)+(r14+r16))+(r8+r22)						; 55-57
	vsubpd	ymm5, ymm5, ymm1		;; .223((r2+r28)+(r14+r16))-(r8+r22)						; 56-58
	vmulpd	ymm1, ymm13, ymm8		;; .223((r6+r24)+(r10+r20))							;	53-57

	vaddpd	ymm3, ymm3, ymm8		;; ((r2+r28)+(r14+r16))+((r4+r26)+(r12+r18))+((r6+r24)+(r10+r20))+(r8+r22)	; 57-59
	vmulpd	ymm8, ymm15, ymm8		;; .901((r6+r24)+(r10+r20))							;	54-58

	vaddpd	ymm7, ymm7, ymm6		;; .901((r2+r28)+(r14+r16))+.223((r4+r26)+(r12+r18))-(r8+r22)			; 58-60
	vbroadcastsd ymm15, Q YMM_P975
	vmulpd	ymm6, ymm15, ymm10		;; .975((r2+r28)-(r14+r16))							;	55-59

	vsubpd	ymm0, ymm0, ymm2		;; .623((r2+r28)+(r14+r16))-.901((r4+r26)+(r12+r18))+(r8+r22)			; 59-61
	vbroadcastsd ymm14, Q YMM_P782
	vmulpd	ymm2, ymm14, ymm10		;; .782((r2+r28)-(r14+r16))							;	56-60
	vbroadcastsd ymm13, Q YMM_P434
	vmulpd	ymm10, ymm13, ymm10		;; .434((r2+r28)-(r14+r16))							;	57-61

	vsubpd	ymm5, ymm5, ymm4		;; .223((r2+r28)+(r14+r16))-.623((r4+r26)+(r12+r18))-(r8+r22)			; 60-62
	vmulpd	ymm4, ymm14, ymm12		;; .782((r4+r26)-(r12+r18))							;	58-62
	ystore	[srcreg+32], ymm3		;; Save final real #1B (real even-cols row #1)					; 60
	vmulpd	ymm3, ymm13, ymm12		;; .434((r4+r26)-(r12+r18))							;	59-63
	vmulpd	ymm12, ymm15, ymm12		;; .975((r4+r26)-(r12+r18))							;	60-64

	vsubpd	ymm7, ymm7, ymm11		;; .901((r2+r28)+(r14+r16))+.223((r4+r26)+(r12+r18))-.623((r6+r24)+(r10+r20))-(r8+r22) ; 61-63
	vmulpd	ymm11, ymm13, ymm9		;; .434((r6+r24)-(r10+r20))							;	61-65

	vsubpd	ymm0, ymm0, ymm1		;; .623((r2+r28)+(r14+r16))-.901((r4+r26)+(r12+r18))-.223((r6+r24)+(r10+r20))+(r8+r22) ; 62-64
	vmulpd	ymm1, ymm15, ymm9		;; .975((r6+r24)-(r10+r20))							;	62-66
	L1prefetchw srcreg+3*d2+d1+L1pd, L1pt

	vaddpd	ymm5, ymm5, ymm8		;; .223((r2+r28)+(r14+r16))-.623((r4+r26)+(r12+r18))+.901((r6+r24)+(r10+r20))-(r8+r22) ; 63-65
	vmulpd	ymm9, ymm14, ymm9		;; .782((r6+r24)-(r10+r20))							;	63-67

	vmovapd	ymm8, [srcreg+2*d2]		;; r5
	vsubpd	ymm8, ymm8, [srcreg+5*d2+32]	;; r5-r25						; 64-66
	ystore	YMM_TMPS[10*32], ymm7		;; Save real even-cols row #3							; 64
	vmovapd	ymm7, [srcreg+4*d2]		;; r9

	vaddpd	ymm6, ymm6, ymm4		;; .975((r2+r28)-(r14+r16))+.782((r4+r26)-(r12+r18))				; 65-67
	ystore	YMM_TMPS[11*32], ymm0		;; Save real even-cols row #5							; 65
	vmovapd	ymm4, [srcreg+6*d2]		;; r13

	vsubpd	ymm2, ymm2, ymm3		;; .782((r2+r28)-(r14+r16))-.434((r4+r26)-(r12+r18))				; 66-68
	ystore	YMM_TMPS[12*32], ymm5		;; Save real even-cols row #7							; 66

	vsubpd	ymm7, ymm7, [srcreg+3*d2+32]	;; r9-r21						; 67-69
	vmulpd	ymm5, ymm14, ymm8		;; .782(r5-r25)						;	67-71

	vsubpd	ymm10, ymm10, ymm12		;; .434((r2+r28)-(r14+r16))-.975((r4+r26)-(r12+r18))				; 68-70
	vmulpd	ymm12, ymm15, ymm8		;; .975(r5-r25)						;	68-72

	vaddpd	ymm6, ymm6, ymm11		;; .975((r2+r28)-(r14+r16))+.782((r4+r26)-(r12+r18))+.434((r6+r24)-(r10+r20))	; 69-71
	vmulpd	ymm8, ymm13, ymm8		;; .434(r5-r25)						;	69-73

	vsubpd	ymm4, ymm4, [srcreg+d2+32]	;; r13-r17						; 70-72
	vmulpd	ymm11, ymm15, ymm7		;; .975(r9-r21)						;	70-74
	vmovapd	ymm0, [srcreg+d2]		;; r3

	vsubpd	ymm2, ymm2, ymm1		;; .782((r2+r28)-(r14+r16))-.434((r4+r26)-(r12+r18))-.975((r6+r24)-(r10+r20))	; 71-73
	vmulpd	ymm1, ymm13, ymm7		;; .434(r9-r21)						;	71-75
	vmovapd	ymm3, [srcreg+3*d2]		;; r7

	vaddpd	ymm10, ymm10, ymm9		;; .434((r2+r28)-(r14+r16))-.975((r4+r26)-(r12+r18))+.782((r6+r24)-(r10+r20))	; 72-74
	vmulpd	ymm7, ymm14, ymm7		;; .782(r9-r21)						;	72-76
	ystore	YMM_TMPS[7*32], ymm6		;; Save real even-cols row #2							; 72

	vsubpd	ymm0, ymm0, [srcreg+6*d2+32]	;; r3-r27						; 73-75
	vmulpd	ymm6, ymm13, ymm4		;; .434(r13-r17)					;	73-77

	vsubpd	ymm3, ymm3, [srcreg+4*d2+32]	;; r7-r23						; 74-76
	vmulpd	ymm9, ymm14, ymm4		;; .782(r13-r17)					;	74-78
	ystore	YMM_TMPS[8*32], ymm2		;; Save real even-cols row #4							; 74

	vmovapd	ymm2, [srcreg+5*d2]		;; r11
	vsubpd	ymm2, ymm2, [srcreg+2*d2+32]	;; r11-r19						; 75-77
	vmulpd	ymm4, ymm15, ymm4		;; .975(r13-r17)					;	75-79
	ystore	YMM_TMPS[9*32], ymm10		;; Save real even-cols row #6							; 75

	vaddpd	ymm5, ymm5, ymm11		;; .782(r5-r25)+.975(r9-r21)				; 76-78
	vmulpd	ymm11, ymm13, ymm0		;; .434(r3-r27)						;	76-80
	vmovapd	ymm10, [srcreg+d1]		;; r2

	vsubpd	ymm12, ymm12, ymm1		;; .975(r5-r25)-.434(r9-r21)				; 77-79
	vmulpd	ymm1, ymm14, ymm0		;; .782(r3-r27)						;	77-81

	vsubpd	ymm8, ymm8, ymm7		;; .434(r5-r25)-.782(r9-r21)				; 78-80
	vmulpd	ymm0, ymm15, ymm0		;; .975(r3-r27)						;	78-82
	vmovapd	ymm7, [srcreg+6*d2+d1]		;; r14

	vaddpd	ymm5, ymm5, ymm6		;; .782(r5-r25)+.975(r9-r21)+.434(r13-r17)		; 79-81
	vmulpd	ymm6, ymm15, ymm3		;; .975(r7-r23)						;	79-83

	vsubpd	ymm12, ymm12, ymm9		;; .975(r5-r25)-.434(r9-r21)-.782(r13-r17)		; 80-82
	vmulpd	ymm9, ymm13, ymm3		;; .434(r7-r23)						;	80-84
	L1prefetchw srcreg+4*d2+L1pd, L1pt

	vaddpd	ymm8, ymm8, ymm4		;; .434(r5-r25)-.782(r9-r21)+.975(r13-r17)		; 81-83
	vmulpd	ymm3, ymm14, ymm3		;; .782(r7-r23)						;	81-85

	vsubpd	ymm10, ymm10, [srcreg+6*d2+d1+32] ;; r2-r28									; 82-84
	vmulpd	ymm4, ymm14, ymm2		;; .782(r11-r19)					;	82-86

	yloop_optional_early_prefetch

	vsubpd	ymm7, ymm7, [srcreg+d1+32]	;; r14-r16									; 83-85

	vaddpd	ymm11, ymm11, ymm6		;; .434(r3-r27)+.975(r7-r23)				; 84-86
	vmulpd	ymm6, ymm15, ymm2		;; .975(r11-r19)					;	83-87
	vmulpd	ymm2, ymm13, ymm2		;; .434(r11-r19)					;	84-88

	vaddpd	ymm1, ymm1, ymm9		;; .782(r3-r27)+.434(r7-r23)				; 85-87
	vmovapd	ymm9, [srcreg+d2+d1]		;; r4

	vsubpd	ymm0, ymm0, ymm3		;; .975(r3-r27)-.782(r7-r23)				; 86-88
	vmovapd	ymm3, [srcreg+5*d2+d1]		;; r12

	vaddpd	ymm11, ymm11, ymm4		;; .434(r3-r27)+.975(r7-r23)+.782(r11-r19)		; 87-89
	vmovapd	ymm4, [srcreg+3*d2+d1]		;; r8

	vsubpd	ymm1, ymm1, ymm6		;; .782(r3-r27)+.434(r7-r23)-.975(r11-r19)		; 88-90
	vmovapd	ymm6, [srcreg+2*d2+d1]		;; r6

	vaddpd	ymm0, ymm0, ymm2		;; .975(r3-r27)-.782(r7-r23)+.434(r11-r19)		; 89-91

	vaddpd	ymm2, ymm11, ymm5		;; Imag odd-cols row #2					; 90-92
	L1prefetchw srcreg+4*d2+d1+L1pd, L1pt

	vsubpd	ymm11, ymm11, ymm5		;; Imag odd-cols row #7					; 91-93

	vaddpd	ymm5, ymm1, ymm12		;; Imag odd-cols row #3					; 92-94

	vsubpd	ymm1, ymm1, ymm12		;; Imag odd-cols row #6					; 93-95
	ystore	YMM_TMPS[20*32], ymm2		;; Imag odd-cols row #2					; 93

	vsubpd	ymm9, ymm9, [srcreg+5*d2+d1+32]	;; r4-r26									; 94-96
	ystore	YMM_TMPS[17*32], ymm11		;; Imag odd-cols row #7					; 94

	vsubpd	ymm3, ymm3, [srcreg+d2+d1+32]	;; r12-r18									; 95-97
	ystore	YMM_TMPS[21*32], ymm5		;; Imag odd-cols row #3					; 95

	vsubpd	ymm5, ymm10, ymm7		;; (r2-r28)-(r14-r16)								; 96-98
	ystore	YMM_TMPS[13*32], ymm1		;; Imag odd-cols row #6					; 96

	;; Do the even columns for the imaginary results

	vsubpd	ymm4, ymm4, [srcreg+3*d2+d1+32] ;; r8-r22									; 97-99
	L1prefetchw srcreg+5*d2+L1pd, L1pt

	vaddpd	ymm10, ymm10, ymm7		;; (r2-r28)+(r14-r16)								; 98-100

	vsubpd	ymm6, ymm6, [srcreg+4*d2+d1+32]	;; r6-r24									; 99-101

	vmovapd	ymm12, [srcreg+4*d2+d1]		;; r10
	vsubpd	ymm12, ymm12, [srcreg+2*d2+d1+32] ;; r10-r20									; 100-102

	vsubpd	ymm7, ymm9, ymm3		;; (r4-r26)-(r12-r18)								; 101-104

	vaddpd	ymm9, ymm9, ymm3		;; (r4-r26)+(r12-r18)								; 102-104
	vmulpd	ymm3, ymm13, ymm5		;; .434((r2-r28)-(r14-r16))							;	102-106
	L1prefetchw srcreg+5*d2+d1+L1pd, L1pt

	vsubpd	ymm1, ymm10, ymm4		;; ((r2-r28)+(r14-r16))-(r8-r22)						; 103-105
	vmulpd	ymm11, ymm14, ymm5		;; .782((r2-r28)-(r14-r16))							;	103-107

	vsubpd	ymm2, ymm6, ymm12		;; (r6-r24)-(r10-r20)								; 104-106
	vmulpd	ymm5, ymm15, ymm5		;; .975((r2-r28)-(r14-r16))							;	104-108

	vaddpd	ymm6, ymm6, ymm12		;; (r6-r24)+(r10-r20)								; 105-107

	vsubpd	ymm1, ymm1, ymm9		;; ((r2-r28)+(r14-r16))-((r4-r26)+(r12-r18))-(r8-r22)				; 106-108
	L1prefetchw srcreg+6*d2+L1pd, L1pt

	vaddpd	ymm12, ymm0, ymm8		;; Imag odd-cols row #4								; 107-109

	vsubpd	ymm0, ymm0, ymm8		;; Imag odd-cols row #5								; 108-110
	vmulpd	ymm8, ymm15, ymm7		;; .975((r4-r26)-(r12-r18))							;	105-109
	ystore	YMM_TMPS[22*32], ymm12		;; Imag odd-cols row #4								; 110
	vmulpd	ymm12, ymm13, ymm7		;; .434((r4-r26)-(r12-r18))							;	106-110
	vmulpd	ymm7, ymm14, ymm7 		;; .782((r4-r26)-(r12-r18))							;	107-111
	ystore	YMM_TMPS[14*32], ymm0		;; Imag odd-cols row #5								; 111
	vmulpd	ymm0, ymm14, ymm2		;; .782((r6-r24)-(r10-r20))							;	108-112

	vaddpd	ymm1, ymm1, ymm6		;; ((r2-r28)+(r14-r16))-((r4-r26)+(r12-r18))+((r6-r24)+(r10-r20))-(r8-r22)	; 109-111
	L1prefetchw srcreg+6*d2+d1+L1pd, L1pt

	vaddpd	ymm3, ymm3, ymm8		;; .434((r2-r28)-(r14-r16))+.975((r4-r26)-(r12-r18))				; 110-112
	vmulpd	ymm8, ymm15, ymm2		;; .975((r6-r24)-(r10-r20))							;	109-113
	vmulpd	ymm2, ymm13, ymm2		;; .434((r6-r24)-(r10-r20))							;	110-114

	vaddpd	ymm11, ymm11, ymm12		;; .782((r2-r28)-(r14-r16))+.434((r4-r26)-(r12-r18))				; 111-113
	vbroadcastsd ymm13, Q YMM_P223
	vmulpd	ymm12, ymm13, ymm10		;; .223((r2-r28)+(r14-r16))							;	111-115

	vsubpd	ymm5, ymm5, ymm7		;; .975((r2-r28)-(r14-r16))-.782((r4-r26)-(r12-r18))				; 112-114
	vbroadcastsd ymm14, Q YMM_P623
	vmulpd	ymm7, ymm14, ymm10		;; .623((r2-r28)+(r14-r16))							;	112-116

	vaddpd	ymm3, ymm3, ymm0		;; .434((r2-r28)-(r14-r16))+.975((r4-r26)-(r12-r18))+.782((r6-r24)-(r10-r20))	; 113-115
	vbroadcastsd ymm15, Q YMM_P901
	vmulpd	ymm10, ymm15, ymm10		;; .901((r2-r28)+(r14-r16))							;	113-117
	vmovapd	ymm0, [screg+6*64+32]		;; cosine/sine

	vsubpd	ymm11, ymm11, ymm8		;; .782((r2-r28)-(r14-r16))+.434((r4-r26)-(r12-r18))-.975((r6-r24)-(r10-r20))	; 114-116
	vmulpd	ymm8, ymm1, ymm0		;; B8 = I8 * cosine/sine							;	114-118

	vaddpd	ymm5, ymm5, ymm2		;; .975((r2-r28)-(r14-r16))-.782((r4-r26)-(r12-r18))+.434((r6-r24)-(r10-r20))	; 115-117
	vmovapd	ymm2, YMM_TMPS[6*32]		;; Real #8
	vmulpd	ymm0, ymm2, ymm0		;; A8 = R8 * cosine/sine							;	115-119

	vaddpd	ymm12, ymm12, ymm4		;; .223((r2-r28)+(r14-r16))+(r8-r22)						; 116-118
	ystore	YMM_TMPS[18*32], ymm11		;; Save imag even-cols row #5							; 117
	vmulpd	ymm11, ymm14, ymm9		;; .623((r4-r26)+(r12-r18))							;	116-120

	vsubpd	ymm7, ymm7, ymm4		;; .623((r2-r28)+(r14-r16))-(r8-r22)						; 117-119
	ystore	YMM_TMPS[19*32], ymm5		;; Save imag even-cols row #7							; 118
	vmulpd	ymm5, ymm15, ymm9		;; .901((r4-r26)+(r12-r18))							;	117-121

	vaddpd	ymm10, ymm10, ymm4		;; .901((r2-r28)+(r14-r16))+(r8-r22)						; 118-120
	vmulpd	ymm9, ymm13, ymm9		;; .223((r4-r26)+(r12-r18))							;	118-122
	vmovapd	ymm4, [screg+6*64]		;; sine

	vaddpd	ymm8, ymm8, ymm2		;; B8 = B8 + R8									; 119-121
	vmulpd	ymm2, ymm15, ymm6		;; .901((r6-r24)+(r10-r20))							;	119-123

	vsubpd	ymm0, ymm0, ymm1		;; A8 = A8 - I8									; 120-122
	vmulpd	ymm1, ymm13, ymm6		;; .223((r6-r24)+(r10-r20))							;	120-124

	vaddpd	ymm12, ymm12, ymm11		;; .223((r2-r28)+(r14-r16))+.623((r4-r26)+(r12-r18))+(r8-r22)			; 121-123
	vmulpd	ymm6, ymm14, ymm6		;; .623((r6-r24)+(r10-r20))							;	121-125
	vmovapd	ymm11, YMM_TMPS[0*32]		;; Real odd-cols row #2

	vaddpd	ymm7, ymm7, ymm5		;; .623((r2-r28)+(r14-r16))+.901((r4-r26)+(r12-r18))-(r8-r22)			; 122-124
	vmulpd	ymm8, ymm8, ymm4		;; B8 = B8 * sine (final I8)							;	122-126
	vmovapd	ymm5, YMM_TMPS[7*32]		;; Real even-cols row #2

	vsubpd	ymm10, ymm10, ymm9		;; .901((r2-r28)+(r14-r16))-.223((r4-r26)+(r12-r18))+(r8-r22)			; 123-125
	vmulpd	ymm0, ymm0, ymm4		;; A8 = A8 * sine (final R8)							;	123-127
	vmovapd	ymm9, YMM_TMPS[20*32]		;; Imag odd-cols row #2

	vaddpd	ymm12, ymm12, ymm2		;; .223((r2-r28)+(r14-r16))+.623((r4-r26)+(r12-r18))+.901((r6-r24)+(r10-r20))+(r8-r22)	; 124-126
	vmovapd	ymm4, [screg+12*64+32]		;; cosine/sine

	vsubpd	ymm7, ymm7, ymm1		;; .623((r2-r28)+(r14-r16))+.901((r4-r26)+(r12-r18))-.223((r6-r24)+(r10-r20))-(r8-r22)	; 125-127
	vmovapd	ymm2, YMM_TMPS[1*32]		;; Real odd-cols row #3

	vsubpd	ymm10, ymm10, ymm6		;; .901((r2-r28)+(r14-r16))-.223((r4-r26)+(r12-r18))-.623((r6-r24)+(r10-r20))+(r8-r22)	; 126-128
	vmovapd	ymm1, YMM_TMPS[10*32]		;; Real even-cols row #3

	;; Now combine the even and odd columns then do the post-multiply by twiddle factors.

	vsubpd	ymm6, ymm11, ymm5		;; Real #14						; 127-129
	ystore	[srcreg+3*d2+d1+32], ymm8	;; Save final I8								; 127
	vmovapd	ymm8, [screg+32]		;; cosine/sine

	vaddpd	ymm11, ymm11, ymm5		;; Real #2						; 128-130
	ystore	YMM_TMPS[15*32], ymm7		;; Save imag even-cols row #4							; 128
	vmovapd	ymm5, YMM_TMPS[21*32]		;; Imag odd-cols row #3

	vsubpd	ymm7, ymm12, ymm9		;; Imag #14						; 129-131
	ystore	[srcreg+3*d2+d1], ymm0		;; Save final R8								; 128

	vaddpd	ymm12, ymm12, ymm9		;; Imag #2						; 130-132
	vmulpd	ymm9, ymm6, ymm4		;; A14 = R14 * cosine/sine				;	130-134
	ystore	YMM_TMPS[16*32], ymm10		;; Save imag even-cols row #6							; 129

	vsubpd	ymm10, ymm2, ymm1		;; Real #13						; 131-133
	vmulpd	ymm0, ymm11, ymm8		;; A2 = R2 * cosine/sine				;	131-135

	vaddpd	ymm2, ymm2, ymm1		;; Real #3						; 132-134
	vmulpd	ymm4, ymm7, ymm4		;; B14 = I14 * cosine/sine				;	132-136

	vsubpd	ymm1, ymm3, ymm5		;; Imag #13						; 133-135
	vmulpd	ymm8, ymm12, ymm8		;; B2 = I2 * cosine/sine				;	133-137

	vaddpd	ymm3, ymm3, ymm5		;; Imag #3						; 134-136
	vmovapd	ymm5, [screg+11*64+32]		;; cosine/sine

	vsubpd	ymm9, ymm9, ymm7		;; A14 = A14 - I14					; 135-137
	vmulpd	ymm7, ymm10, ymm5		;; A13 = R13 * cosine/sine				;	134-138

	vsubpd	ymm0, ymm0, ymm12		;; A2 = A2 - I2						; 136-138
	vaddpd	ymm4, ymm4, ymm6		;; B14 = B14 + R14					; 137-139
	vmovapd	ymm6, [screg+64+32]		;; cosine/sine
	vmulpd	ymm12, ymm2, ymm6		;; A3 = R3 * cosine/sine				;	135-139
	vmulpd	ymm5, ymm1, ymm5		;; B13 = I13 * cosine/sine				;	136-140
	vmulpd	ymm6, ymm3, ymm6		;; B3 = I3 * cosine/sine				;	137-141

	vaddpd	ymm8, ymm8, ymm11		;; B2 = B2 + R2						; 138-140
	vmovapd	ymm11, [screg+12*64]		;; sine
	vmulpd	ymm9, ymm9, ymm11		;; A14 = A14 * sine (final R14)				;	138-142

	vsubpd	ymm7, ymm7, ymm1		;; A13 = A13 - I13					; 139-141
	vmovapd	ymm1, [screg]			;; sine
	vmulpd	ymm0, ymm0, ymm1		;; A2 = A2 * sine (final R2)				;	139-143

	vsubpd	ymm12, ymm12, ymm3		;; A3 = A3 - I3						; 140-142
	vmulpd	ymm4, ymm4, ymm11		;; B14 = B14 * sine (final I14)				;	140-144
	vmovapd	ymm3, [screg+11*64]		;; sine

	vaddpd	ymm5, ymm5, ymm10		;; B13 = B13 + R13					; 141-143
	vmulpd	ymm8, ymm8, ymm1		;; B2 = B2 * sine (final I2)				;	141-145
	vmovapd	ymm11, YMM_TMPS[2*32]		;; Real odd-cols row #4
	vmovapd	ymm10, YMM_TMPS[8*32]		;; Real even-cols row #4

	vaddpd	ymm6, ymm6, ymm2		;; B3 = B3 + R3						; 142-144
	vmulpd	ymm7, ymm7, ymm3		;; A13 = A13 * sine (final R13)				;	142-146
	vmovapd	ymm1, [screg+64]		;; sine

	vsubpd	ymm2, ymm11, ymm10		;; Real #12						; 143-145
	vmulpd	ymm12, ymm12, ymm1		;; A3 = A3 * sine (final R3)				;	143-147
	ystore	[srcreg+6*d2+d1], ymm9		;; Save final R14					; 143
	vmovapd	ymm9, YMM_TMPS[15*32]		;; Imag even-cols row #4

	vaddpd	ymm11, ymm11, ymm10		;; Real #4						; 144-146
	vmulpd	ymm5, ymm5, ymm3		;; B13 = B13 * sine (final I13)				;	144-148
	vmovapd	ymm10, YMM_TMPS[22*32]		;; Imag odd-cols row #4
	ystore	[srcreg+d1], ymm0		;; Save final R2					; 144

	vsubpd	ymm3, ymm9, ymm10		;; Imag #12						; 145-147
	vmulpd	ymm6, ymm6, ymm1		;; B3 = B3 * sine (final I3)				;	145-149
	vmovapd	ymm0, [screg+10*64+32]		;; cosine/sine
	ystore	[srcreg+6*d2+d1+32], ymm4	;; Save final I14					; 145

	vaddpd	ymm9, ymm9, ymm10		;; Imag #4						; 146-148
	vmulpd	ymm1, ymm2, ymm0		;; A12 = R12 * cosine/sine				;	146-150
	vmovapd	ymm4, YMM_TMPS[3*32]		;; Real odd-cols row #5
	vmovapd	ymm10, YMM_TMPS[11*32]		;; Real even-cols row #5
	ystore	[srcreg+d1+32], ymm8		;; Save final I2					; 146

	vsubpd	ymm8, ymm4, ymm10		;; Real #11						; 147-149
	ystore	[srcreg+6*d2], ymm7		;; Save final R13					; 147
	vmovapd	ymm7, [screg+2*64+32]		;; cosine/sine
	ystore	[srcreg+d2], ymm12		;; Save final R3					; 148
	vmulpd	ymm12, ymm11, ymm7		;; A4 = R4 * cosine/sine				;	147-151

	vaddpd	ymm4, ymm4, ymm10		;; Real #5						; 148-150
	vmulpd	ymm0, ymm3, ymm0		;; B12 = I12 * cosine/sine				;	148-152
	vmovapd	ymm10, YMM_TMPS[18*32]		;; Imag even-cols row #5

	ystore	[srcreg+6*d2+32], ymm5		;; Save final I13					; 149
	vmovapd	ymm5, YMM_TMPS[14*32]		;; Imag odd-cols row #5
	ystore	[srcreg+d2+32], ymm6		;; Save final I3					; 150
	vsubpd	ymm6, ymm10, ymm5		;; Imag #11						; 149-151
	vmulpd	ymm7, ymm9, ymm7		;; B4 = I4 * cosine/sine				;	149-153

	vaddpd	ymm10, ymm10, ymm5		;; Imag #5						; 150-152
	vmovapd	ymm5, [screg+9*64+32]		;; cosine/sine

	vsubpd	ymm1, ymm1, ymm3		;; A12 = A12 - I12					; 151-153
	vmulpd	ymm3, ymm8, ymm5		;; A11 = R11 * cosine/sine				;	150-154

	vsubpd	ymm12, ymm12, ymm9		;; A4 = A4 - I4						; 152-154
	vmovapd	ymm9, [screg+3*64+32]		;; cosine/sine

	vaddpd	ymm0, ymm0, ymm2		;; B12 = B12 + R12					; 153-155
	vmulpd	ymm2, ymm4, ymm9		;; A5 = R5 * cosine/sine				;	151-155
	vmulpd	ymm5, ymm6, ymm5		;; B11 = I11 * cosine/sine				;	152-156
	vmulpd	ymm9, ymm10, ymm9		;; B5 = I5 * cosine/sine				;	153-157

	vaddpd	ymm7, ymm7, ymm11		;; B4 = B4 + R4						; 154-156
	vmovapd ymm11, [screg+10*64]		;; sine
	vmulpd	ymm1, ymm1, ymm11		;; A12 = A12 * sine (final R12)				;	154-158

	vsubpd	ymm3, ymm3, ymm6		;; A11 = A11 - I11					; 155-157
	vmovapd	ymm6, [screg+2*64]		;; sine
	vmulpd	ymm12, ymm12, ymm6		;; A4 = A4 * sine (final R4)				;	155-159

	vsubpd	ymm2, ymm2, ymm10		;; A5 = A5 - I5						; 156-158
	vmulpd	ymm0, ymm0, ymm11		;; B12 = B12 * sine (final I12)				;	156-160
	vmovapd	ymm10, [screg+9*64]		;; sine

	vaddpd	ymm5, ymm5, ymm8		;; B11 = B11 + R11					; 157-159
	vmulpd	ymm7, ymm7, ymm6		;; B4 = B4 * sine (final I4)				;	157-161
	vmovapd	ymm11, YMM_TMPS[4*32]		;; Real odd-cols row #6
	vmovapd	ymm8, YMM_TMPS[9*32]		;; Real even-cols row #6

	vaddpd	ymm9, ymm9, ymm4		;; B5 = B5 + R5						; 158-160
	vmulpd	ymm3, ymm3, ymm10		;; A11 = A11 * sine (final R11)				;	158-162
	vmovapd	ymm6, [screg+3*64]		;; sine

	vsubpd	ymm4, ymm11, ymm8		;; Real #10						; 159-161
	vmulpd	ymm2, ymm2, ymm6		;; A5 = A5 * sine (final R5)				;	159-163
	ystore	[srcreg+5*d2+d1], ymm1		;; Save final R12					; 159
	vmovapd	ymm1, YMM_TMPS[16*32]		;; Imag even-cols row #6

	vaddpd	ymm11, ymm11, ymm8		;; Real #6						; 160-162
	vmulpd	ymm5, ymm5, ymm10		;; B11 = B11 * sine (final I11)				;	160-164
	vmovapd	ymm8, YMM_TMPS[13*32]		;; Imag odd-cols row #6
	ystore	[srcreg+d2+d1], ymm12		;; Save final R4					; 160

	vsubpd	ymm12, ymm1, ymm8		;; Imag #10						; 161-163
	vmulpd	ymm9, ymm9, ymm6		;; B5 = B5 * sine (final I5)				;	161-165
	vmovapd	ymm10, [screg+8*64+32]		;; cosine/sine
	ystore	[srcreg+5*d2+d1+32], ymm0	;; Save final I12					; 161

	vaddpd	ymm1, ymm1, ymm8		;; Imag #6						; 162-164
	vmulpd	ymm8, ymm4, ymm10		;; A10 = R10 * cosine/sine				;	162-166
	vmovapd	ymm6, YMM_TMPS[5*32]		;; Real odd-cols row #7
	vmovapd	ymm0, YMM_TMPS[12*32]		;; Real even-cols row #7
	ystore	[srcreg+d2+d1+32], ymm7		;; Save final I4					; 162

	vsubpd	ymm7, ymm6, ymm0		;; Real #9						; 163-165
	ystore	[srcreg+5*d2], ymm3		;; Save final R11					; 163
	vmovapd	ymm3, [screg+4*64+32]		;; cosine/sine
	ystore	[srcreg+2*d2], ymm2		;; Save final R5					; 164
	vmulpd	ymm2, ymm11, ymm3		;; A6 = R6 * cosine/sine				;	163-167

	vaddpd	ymm6, ymm6, ymm0		;; Real #7						; 164-166
	vmulpd	ymm10, ymm12, ymm10		;; B10 = I10 * cosine/sine				;	164-168

	vmovapd	ymm0, YMM_TMPS[19*32]		;; Imag even-cols row #7
	ystore	[srcreg+5*d2+32], ymm5		;; Save final I11					; 165
	vmovapd	ymm5, YMM_TMPS[17*32]		;; Imag odd-cols row #7
	ystore	[srcreg+2*d2+32], ymm9		;; Save final I5					; 166
	vsubpd	ymm9, ymm0, ymm5		;; Imag #9						; 165-167
	vmulpd	ymm3, ymm1, ymm3		;; B6 = I6 * cosine/sine				;	165-169

	vaddpd	ymm0, ymm0, ymm5		;; Imag #7						; 166-168
	vmovapd	ymm5, [screg+7*64+32]		;; cosine/sine

	vsubpd	ymm8, ymm8, ymm12		;; A10 = A10 - I10					; 167-169
	vmulpd	ymm12, ymm7, ymm5		;; A9 = R9 * cosine/sine				;	166-170

	vsubpd	ymm2, ymm2, ymm1		;; A6 = A6 - I6						; 168-170

	vaddpd	ymm10, ymm10, ymm4		;; B10 = B10 + R10					; 169-171
	vmovapd	ymm4, [screg+5*64+32]		;; cosine/sine
	vmulpd	ymm1, ymm6, ymm4		;; A7 = R7 * cosine/sine				;	167-171
	vmulpd	ymm5, ymm9, ymm5		;; B9 = I9 * cosine/sine				;	168-172
	vmulpd	ymm4, ymm0, ymm4		;; B7 = I7 * cosine/sine				;	169-173

	vaddpd	ymm3, ymm3, ymm11		;; B6 = B6 + R6						; 170-172
	vmovapd	ymm11, [screg+8*64]		;; sine
	vmulpd	ymm8, ymm8, ymm11		;; A10 = A10 * sine (final R10)				;	170-174

	vsubpd	ymm12, ymm12, ymm9		;; A9 = A9 - I9						; 171-173
	vmovapd	ymm9, [screg+4*64]		;; sine
	vmulpd	ymm2, ymm2, ymm9		;; A6 = A6 * sine (final R6)				;	171-175

	vsubpd	ymm1, ymm1, ymm0		;; A7 = A7 - I7						; 172-174
	vmulpd	ymm10, ymm10, ymm11		;; B10 = B10 * sine (final I10)				;	172-176
	vmovapd	ymm0, [screg+7*64]		;; sine

	vaddpd	ymm5, ymm5, ymm7		;; B9 = B9 + R9						; 173-175
	vmulpd	ymm3, ymm3, ymm9		;; B6 = B6 * sine (final I6)				;	173-177
	vmovapd	ymm11, [screg+5*64]		;; sine

	vaddpd	ymm4, ymm4, ymm6		;; B7 = B7 + R7						; 174-176
	vmulpd	ymm12, ymm12, ymm0		;; A9 = A9 * sine (final R9)				;	174-178

	vmulpd	ymm1, ymm1, ymm11		;; A7 = A7 * sine (final R7)				;	175-179
	ystore	[srcreg+4*d2+d1], ymm8		;; Save final R10					; 175

	vmulpd	ymm5, ymm5, ymm0		;; B9 = B9 * sine (final I9)				;	176-180
	ystore	[srcreg+2*d2+d1], ymm2		;; Save final R6					; 176

	vmulpd	ymm4, ymm4, ymm11		;; B7 = B7 * sine (final I7)				;	177-181
	ystore	[srcreg+4*d2+d1+32], ymm10	;; Save final I10					; 177

	ystore	[srcreg+2*d2+d1+32], ymm3	;; Save final I6					; 178
	ystore	[srcreg+4*d2], ymm12		;; Save final R9					; 179
	ystore	[srcreg+3*d2], ymm1		;; Save final R7					; 180
	ystore	[srcreg+4*d2+32], ymm5		;; Save final I9					; 181
	ystore	[srcreg+3*d2+32], ymm4		;; Save final I7					; 182

	bump	srcreg, srcinc
	bump	screg, scinc
	ENDM

;; Haswell FMA3 version

IF (@INSTR(,%yarch,<FMA3>) NE 0)

yr7_14cl_28_reals_fft_preload MACRO
	vmovapd	ymm15, YMM_ONE
	ENDM

;; This has been timed at 112 clocks.  Further attempts at replacing adds and subs with FMA3 by letting out-of-order
;; execution free up a register did not prove successful.
yr7_14cl_28_reals_fft MACRO srcreg,srcinc,d1,d2,screg,scinc,maxrpt,L1pt,L1pd

	;; Do the odd columns for the real results

	vmovapd	ymm1, [srcreg+d1]		;; r2
	vmovapd	ymm2, [srcreg+6*d2+d1+32]	;; r28
	yfmsubpd ymm0, ymm1, ymm15, ymm2	;; r2-r28								; 1-5		n 8
	yfmaddpd ymm1, ymm1, ymm15, ymm2	;; r2+r28								; 1-5		n 9

	vmovapd	ymm3, [srcreg+6*d2+d1]		;; r14
	vmovapd	ymm4, [srcreg+d1+32]		;; r16
	yfmsubpd ymm2, ymm3, ymm15, ymm4	;; r14-r16								; 2-6		n 8
	yfmaddpd ymm3, ymm3, ymm15, ymm4	;; r14+r16								; 2-6		n 9

	vmovapd	ymm5, [srcreg+d2+d1]		;; r4
	vmovapd	ymm6, [srcreg+5*d2+d1+32]	;; r26
	yfmsubpd ymm4, ymm5, ymm15, ymm6	;; r4-r26								; 3-7		n 10
	yfmaddpd ymm5, ymm5, ymm15, ymm6	;; r4+r26								; 3-7		n 11

	vmovapd	ymm7, [srcreg+5*d2+d1]		;; r12
	vmovapd	ymm8, [srcreg+d2+d1+32]		;; r18
	yfmsubpd ymm6, ymm7, ymm15, ymm8	;; r12-r18								; 4-8		n 10
	yfmaddpd ymm7, ymm7, ymm15, ymm8	;; r12+r18								; 4-8		n 11

	vmovapd	ymm9, [srcreg+2*d2+d1]		;; r6
	vmovapd	ymm10, [srcreg+4*d2+d1+32]	;; r24
	yfmsubpd ymm8, ymm9, ymm15, ymm10	;; r6-r24								; 5-9		n 12
	yfmaddpd ymm9, ymm9, ymm15, ymm10	;; r6+r24								; 5-9		n 15

	vmovapd	ymm11, [srcreg+4*d2+d1]		;; r10
	vmovapd	ymm12, [srcreg+2*d2+d1+32]	;; r20
	yfmsubpd ymm10, ymm11, ymm15, ymm12	;; r10-r20								; 6-10		n 12
	yfmaddpd ymm11, ymm11, ymm15, ymm12	;; r10+r20								; 6-10		n 15

	vmovapd	ymm13, [srcreg+3*d2+d1]		;; r8
	vmovapd	ymm14, [srcreg+3*d2+d1+32]	;; r22
	yfmsubpd ymm12, ymm13, ymm15, ymm14	;; r8-r22								; 7-11		n 13
	yfmaddpd ymm13, ymm13, ymm15, ymm14	;; r8+r22								; 7-11		n 16

	yfmaddpd ymm14, ymm0, ymm15, ymm2	;; (r2-r28)+(r14-r16)							; 8-12		n 13
	yfmsubpd ymm0, ymm0, ymm15, ymm2	;; (r2-r28)-(r14-r16)							; 8-12
	L1prefetchw srcreg+L1pd, L1pt

	yfmaddpd ymm2, ymm1, ymm15, ymm3	;; (r2+r28)+(r14+r16)							; 9-13		n 16
	yfmsubpd ymm1, ymm1, ymm15, ymm3	;; (r2+r28)-(r14+r16)							; 9-13
	L1prefetchw srcreg+d1+L1pd, L1pt

	yfmaddpd ymm3, ymm4, ymm15, ymm6	;; (r4-r26)+(r12-r18)							; 10-14		n 18
	yfmsubpd ymm4, ymm4, ymm15, ymm6	;; (r4-r26)-(r12-r18)							; 10-14
	L1prefetchw srcreg+d2+L1pd, L1pt

	yfmaddpd ymm6, ymm5, ymm15, ymm7	;; (r4+r26)+(r12+r18)							; 11-15		n 21
	yfmsubpd ymm5, ymm5, ymm15, ymm7	;; (r4+r26)-(r12+r18)							; 11-15
	L1prefetchw srcreg+d2+d1+L1pd, L1pt

	yfmaddpd ymm7, ymm8, ymm15, ymm10	;; (r6-r24)+(r10-r20)							; 12-16		n 23
	yfmsubpd ymm8, ymm8, ymm15, ymm10	;; (r6-r24)-(r10-r20)							; 12-16

	vmovapd ymm10, YMM_P223
	ystore	YMM_TMPS[0*32], ymm0		;; Save (r2-r28)-(r14-r16)						; 13
	yfmsubpd ymm0, ymm14, ymm15, ymm12	;; ((r2-r28)+(r14-r16))-(r8-r22)	 				; 13-17		n 18
	ystore	YMM_TMPS[1*32], ymm1		;; Save (r2+r28)-(r14+r16)						; 14
	yfmaddpd ymm1, ymm10, ymm14, ymm12	;; .223((r2-r28)+(r14-r16))+(r8-r22)					; 13-17		n 18

	ystore	YMM_TMPS[2*32], ymm4		;; Save (r4-r26)-(r12-r18)						; 15
	vmovapd ymm4, YMM_P623
	ystore	YMM_TMPS[3*32], ymm5		;; Save (r4+r26)-(r12+r18)						; 16
	yfmsubpd ymm5, ymm4, ymm14, ymm12	;; .623((r2-r28)+(r14-r16))-(r8-r22)					; 14-18		n 19
	ystore	YMM_TMPS[4*32], ymm8		;; Save (r6-r24)-(r10-r20)						; 17
	vmovapd ymm8, YMM_P901
	yfmaddpd ymm14, ymm8, ymm14, ymm12	;; .901((r2-r28)+(r14-r16))+(r8-r22)					; 14-18		n 19

	yloop_optional_early_prefetch

	yfmaddpd ymm12, ymm9, ymm15, ymm11	;; (r6+r24)+(r10+r20)							; 15-19		n 26
	yfmsubpd ymm9, ymm9, ymm15, ymm11	;; (r6+r24)-(r10+r20)							; 15-19
	L1prefetchw srcreg+2*d2+L1pd, L1pt

	yfmaddpd ymm15, ymm2, ymm15, ymm13	;; ((r2+r28)+(r14+r16))+(r8+r22)					; 16-20		n 21
	yfmsubpd ymm11, ymm8, ymm2, ymm13	;; .901((r2+r28)+(r14+r16))-(r8+r22)					; 16-20		n 21
	L1prefetchw srcreg+2*d2+d1+L1pd, L1pt

	ystore	YMM_TMPS[5*32], ymm9		;; Save (r6+r24)-(r10+r20)						; 20
	yfmaddpd ymm9, ymm4, ymm2, ymm13	;; .623((r2+r28)+(r14+r16))+(r8+r22)					; 17-21		n 22
	yfmsubpd ymm2, ymm10, ymm2, ymm13	;; .223((r2+r28)+(r14+r16))-(r8+r22)					; 17-21		n 22
	vmovapd	ymm13, YMM_ONE

	yfmsubpd ymm0, ymm0, ymm13, ymm3	;; ((r2-r28)+(r14-r16))-((r4-r26)+(r12-r18))-(r8-r22)			; 18-22		n 23
	yfmaddpd ymm1, ymm4, ymm3, ymm1		;; .223((r2-r28)+(r14-r16))+.623((r4-r26)+(r12-r18))+(r8-r22)		; 18-22		n 23
	L1prefetchw srcreg+3*d2+L1pd, L1pt

	yfmaddpd ymm5, ymm8, ymm3, ymm5		;; .623((r2-r28)+(r14-r16))+.901((r4-r26)+(r12-r18))-(r8-r22)		; 19-23		n 24
	yfnmaddpd ymm14, ymm10, ymm3, ymm14	;; .901((r2-r28)+(r14-r16))-.223((r4-r26)+(r12-r18))+(r8-r22)		; 19-23		n 24

	vmovapd	ymm3, [srcreg+2*d2]		;; r5
	yfmaddpd ymm3, ymm3, ymm13, [srcreg+5*d2+32] ;; r5+r25								; 20-24		n 29

	yfmaddpd ymm15, ymm15, ymm13, ymm6	;; ((r2+r28)+(r14+r16))+((r4+r26)+(r12+r18))+(r8+r22)			; 21-25		n 26
	yfmaddpd ymm11, ymm10, ymm6, ymm11	;; .901((r2+r28)+(r14+r16))+.223((r4+r26)+(r12+r18))-(r8+r22)		; 21-25		n 26
	L1prefetchw srcreg+3*d2+d1+L1pd, L1pt

	yfnmaddpd ymm9, ymm8, ymm6, ymm9	;; .623((r2+r28)+(r14+r16))-.901((r4+r26)+(r12+r18))+(r8+r22)		; 22-26		n 27
	yfnmaddpd ymm2, ymm4, ymm6, ymm2	;; .223((r2+r28)+(r14+r16))-.623((r4+r26)+(r12+r18))-(r8+r22)		; 22-26		n 27

	vmovapd	ymm6, [srcreg+d2]		;; r3
	yfmaddpd ymm6, ymm6, ymm13, [srcreg+6*d2+32] ;; r3+r27								; 20-24		n 31

	yfmaddpd ymm0, ymm0, ymm13, ymm7	;; ((r2-r28)+(r14-r16))-((r4-r26)+(r12-r18))+((r6-r24)+(r10-r20))-(r8-r22) (i8)		     ; 23-27
	yfmaddpd ymm1, ymm8, ymm7, ymm1		;; .223((r2-r28)+(r14-r16))+.623((r4-r26)+(r12-r18))+.901((r6-r24)+(r10-r20))+(r8-r22) (ie2) ; 23-27
	L1prefetchw srcreg+4*d2+L1pd, L1pt

	yfnmaddpd ymm5, ymm10, ymm7, ymm5	;; .623((r2-r28)+(r14-r16))+.901((r4-r26)+(r12-r18))-.223((r6-r24)+(r10-r20))-(r8-r22) (ie4) ; 24-28
	yfnmaddpd ymm14, ymm4, ymm7, ymm14	;; .901((r2-r28)+(r14-r16))-.223((r4-r26)+(r12-r18))-.623((r6-r24)+(r10-r20))+(r8-r22) (ie6) ; 24-28

	vmovapd	ymm7, [srcreg+4*d2]		;; r9
	ystore	YMM_TMPS[6*32], ymm0		;; Save imag #8								; 28
	vmovapd	ymm0, [srcreg+3*d2+32]		;; r21
	ystore	YMM_TMPS[7*32], ymm1		;; Save imag even-cols row #2						; 28+1
	yfmaddpd ymm1, ymm7, ymm13, ymm0	;; r9+r21								; 25-29		n 34
	yfmsubpd ymm7, ymm7, ymm13, ymm0	;; r9-r21								; 25-29

	yfmaddpd ymm15, ymm15, ymm13, ymm12	;; ((r2+r28)+(r14+r16))+((r4+r26)+(r12+r18))+((r6+r24)+(r10+r20))+(r8+r22) (re1)	     ; 26-30
	yfnmaddpd ymm11, ymm4, ymm12, ymm11	;; .901((r2+r28)+(r14+r16))+.223((r4+r26)+(r12+r18))-.623((r6+r24)+(r10+r20))-(r8+r22) (re3) ; 26-30
	L1prefetchw srcreg+4*d2+d1+L1pd, L1pt

	yfnmaddpd ymm9, ymm10, ymm12, ymm9	;; .623((r2+r28)+(r14+r16))-.901((r4+r26)+(r12+r18))-.223((r6+r24)+(r10+r20))+(r8+r22) (re5) ; 27-31
	yfmaddpd ymm2, ymm8, ymm12, ymm2	;; .223((r2+r28)+(r14+r16))-.623((r4+r26)+(r12+r18))+.901((r6+r24)+(r10+r20))-(r8+r22) (re7) ; 27-31

	vmovapd	ymm0, [srcreg+3*d2]		;; r7
	vmovapd	ymm12, [srcreg+4*d2+32]		;; r23
	ystore	YMM_TMPS[8*32], ymm5		;; Save imag even-cols row #4						; 29+1
	yfmaddpd ymm5, ymm0, ymm13, ymm12	;; r7+r23								; 28-32		n 37
	yfmsubpd ymm0, ymm0, ymm13, ymm12	;; r7-r23								; 28-32

	vmovapd	ymm12, [srcreg]			;; r1
	ystore	YMM_TMPS[9*32], ymm14		;; Save imag even-cols row #6						; 29+2
	yfmaddpd ymm14, ymm12, ymm13, ymm3	;; r1+(r5+r25)								; 29-33		n 34
	ystore	YMM_TMPS[10*32], ymm7		;; Save r9-r21								; 30+2
	yfmaddpd ymm7, ymm4, ymm3, ymm12	;; r1+.623(r5+r25)							; 29-33		n 34

	ystore	YMM_TMPS[11*32], ymm11		;; Save real even-cols row #3						; 31+2
	yfnmaddpd ymm11, ymm10, ymm3, ymm12	;; r1-.223(r5+r25)							; 30-34		n 35
	yfnmaddpd ymm3, ymm8, ymm3, ymm12	;; r1-.901(r5+r25)							; 30-34		n 35

	vmovapd	ymm12, [srcreg+32]		;; r15
	ystore	[srcreg+32], ymm15		;; Save final real #1B (real even-cols row #1)				; 31+3
	yfmaddpd ymm15, ymm6, ymm13, ymm12	;; (r3+r27)+r15								; 31-35		n 37
	ystore	YMM_TMPS[12*32], ymm9		;; Save real even-cols row #5						; 32+3
	yfmsubpd ymm9, ymm8, ymm6, ymm12	;; .901(r3+r27)-r15							; 31-35		n 37

	ystore	YMM_TMPS[13*32], ymm2		;; Save real even-cols row #7						; 32+4
	yfmaddpd ymm2, ymm4, ymm6, ymm12	;; .623(r3+r27)+r15							; 32-36		n 38
	yfmsubpd ymm6, ymm10, ymm6, ymm12	;; .223(r3+r27)-r15							; 32-36		n 38

	vmovapd	ymm12, [srcreg+6*d2]		;; r13
	ystore	YMM_TMPS[14*32], ymm0		;; Save r7-r23								; 33+4
	vmovapd	ymm0, [srcreg+d2+32]		;; r17
	yfmaddpd ymm13, ymm12, ymm13, ymm0	;; r13+r17								; 33-37		n 40
	vsubpd ymm12, ymm12, ymm0		;; r13-r17								; 33-37

	yloop_optional_early_prefetch

	vaddpd	ymm14, ymm14, ymm1		;; r1+(r5+r25)+(r9+r21)							; 34-38		n 40
	yfnmaddpd ymm7, ymm10, ymm1, ymm7	;; r1+.623(r5+r25)-.223(r9+r21)						; 34-38		n 40
	vmovapd	ymm0, [srcreg+5*d2]		;; r11

	yfnmaddpd ymm11, ymm8, ymm1, ymm11	;; r1-.223(r5+r25)-.901(r9+r21)						; 35-39		n 41
	yfmaddpd ymm3, ymm4, ymm1, ymm3		;; r1-.901(r5+r25)+.623(r9+r21)						; 35-39		n 41

	vmovapd	ymm1, [srcreg+2*d2+32]		;; r19
	ystore	YMM_TMPS[15*32], ymm12		;; Save r13-r17								; 38
	vaddpd	ymm12, ymm0, ymm1		;; r11+r19								; 36-40		n 42
	yfmsubpd ymm0, ymm0, YMM_ONE, ymm1	;; r11-r19								; 36-40

	vaddpd	ymm15, ymm15, ymm5		;; (r3+r27)+(r7+r23)+r15						; 37-41		n 42
	yfmaddpd ymm9, ymm10, ymm5, ymm9	;; .901(r3+r27)+.223(r7+r23)-r15					; 37-41		n 42
	vmovapd	ymm1, [srcreg+2*d2]		;; r5

	yfnmaddpd ymm2, ymm8, ymm5, ymm2	;; .623(r3+r27)-.901(r7+r23)+r15					; 38-42		n 43
	yfnmaddpd ymm6, ymm4, ymm5, ymm6	;; .223(r3+r27)-.623(r7+r23)-r15					; 38-42		n 43

	vmovapd	ymm5, YMM_ONE
	yfmsubpd ymm1, ymm1, ymm5, [srcreg+5*d2+32] ;; r5-r25								; 39-43		n 44
	vmovapd	ymm5, [srcreg+d2]		;; r3
	vsubpd ymm5, ymm5, [srcreg+6*d2+32]	;; r3-r27								; 39-43		n 45

	vaddpd	ymm14, ymm14, ymm13		;; r1+(r5+r25)+(r9+r21)+(r13+r17) (ro18a)				; 40-44		n 47
	yfnmaddpd ymm7, ymm8, ymm13, ymm7	;; r1+.623(r5+r25)-.223(r9+r21)-.901(r13+r17) (ro27a)			; 40-44		n 48
	L1prefetchw srcreg+5*d2+L1pd, L1pt

	yfmaddpd ymm11, ymm4, ymm13, ymm11	;; r1-.223(r5+r25)-.901(r9+r21)+.623(r13+r17) (ro36a)			; 41-45
	yfnmaddpd ymm3, ymm10, ymm13, ymm3	;; r1-.901(r5+r25)+.623(r9+r21)-.223(r13+r17) (ro45a)			; 41-45
	vmovapd ymm13, YMM_P975_P434

	vaddpd	ymm15, ymm15, ymm12		;; (r3+r27)+(r7+r23)+(r11+r19)+r15 (ro18b)				; 42-46		n 47
	yfnmaddpd ymm9, ymm4, ymm12, ymm9	;; .901(r3+r27)+.223(r7+r23)-.623(r11+r19)-r15 (ro27b)			; 42-46		n 48
	vmovapd ymm4, YMM_P782_P434

	yfnmaddpd ymm2, ymm10, ymm12, ymm2	;; .623(r3+r27)-.901(r7+r23)-.223(r11+r19)+r15 (ro36b)			; 43-47
	yfmaddpd ymm6, ymm8, ymm12, ymm6	;; .223(r3+r27)-.623(r7+r23)+.901(r11+r19)-r15 (ro45b)			; 43-47
	vmovapd	ymm10, YMM_TMPS[10*32]		;; Reload r9-r21

	vmovapd	ymm8, YMM_TMPS[15*32]		;; Reload r13-r17
	yfmsubpd ymm12, ymm13, ymm1, ymm10	;; .975/.434(r5-r25)-(r9-r21)						; 44-48		n 51
	ystore	YMM_TMPS[10*32], ymm11		;; Real odd-cols row #3/6a						; 46
	yfmaddpd ymm11, ymm4, ymm1, ymm8	;; .782/.434(r5-r25)+(r13-r17)						; 44-48		n 51

	ystore	YMM_TMPS[15*32], ymm3		;; Real odd-cols row #4/5a						; 46+1
	yfnmaddpd ymm3, ymm4, ymm10, ymm1	;; (r5-r25)-.782/.434(r9-r21)						; 45-49		n 52
	vmovapd	ymm1, YMM_TMPS[14*32]		;; Reload r7-r23
	ystore	YMM_TMPS[14*32], ymm2		;; Real odd-cols row #3/6b						; 48
	yfmaddpd ymm2, ymm13, ymm1, ymm5		;; (r3-r27)+.975/.434(r7-r23)						; 45-49		n 52

	ystore	YMM_TMPS[16*32], ymm6		;; Real odd-cols row #4/5b						; 48+1
	vsubpd	ymm6, ymm14, ymm15		;; Real odd-cols row #8 (final real #8)					; 47-51
	yfmaddpd ymm14, ymm14, YMM_ONE, ymm15	;; Real odd-cols row #1 (final real #1A)				; 47-51

	yfmaddpd ymm15, ymm4, ymm5, ymm1	;; .782/.434(r3-r27)+(r7-r23)						; 46-50		n 53
	yfmaddpd ymm5, ymm13, ymm5, ymm0		;; .975/.434(r3-r27)+(r11-r19)						; 46-50		n 53
	L1prefetchw srcreg+5*d2+d1+L1pd, L1pt

	ystore	YMM_TMPS[17*32], ymm6		;; Real #8								; 52
	vsubpd	ymm6, ymm7, ymm9		;; Real odd-cols row #7							; 48-52
	yfmaddpd ymm7, ymm7, YMM_ONE, ymm9	;; Real odd-cols row #2							; 48-52

	vmovapd	ymm9, YMM_P434
	ystore	[srcreg], ymm14			;; Final real #1A							; 52+1
	vmulpd	ymm14, ymm9, YMM_TMPS[0*32]	;; .434 * (r2-r28)-(r14-r16)						; 49-53		n 55
	ystore	YMM_TMPS[0*32], ymm6		;; Real odd-cols row #7							; 53+1
	vmulpd	ymm6, ymm9, YMM_TMPS[2*32]	;; .434 * (r4-r26)-(r12-r18)						; 49-53		n 55

	vmulpd	ymm9, ymm9, YMM_TMPS[4*32]	;; .434 * (r6-r24)-(r10-r20)						; 50-54		n 56
	L1prefetchw srcreg+6*d2+L1pd, L1pt

	yfmaddpd ymm11, ymm13, ymm10, ymm11	;; .782/.434(r5-r25)+.975/.434(r9-r21)+(r13-r17) (io27b/.434)		; 51-55		n 57
	yfnmaddpd ymm12, ymm4, ymm8, ymm12	;; .975/.434(r5-r25)-(r9-r21)-.782/.434(r13-r17) (io36b/.434)		; 51-55
	vmovapd	ymm10, YMM_TMPS[1*32]		;; Reload (r2+r28)-(r14+r16)

	yfmaddpd ymm3, ymm13, ymm8, ymm3	;; (r5-r25)-.782/.434(r9-r21)+.975/.434(r13-r17) (io45b/.434)		; 52-56		n 58
	yfmaddpd ymm2, ymm4, ymm0, ymm2		;; (r3-r27)+.975/.434(r7-r23)+.782/.434(r11-r19) (io27a/.434)		; 52-56		n 57
	vmovapd	ymm8, YMM_TMPS[3*32]		;; Reload (r4+r26)-(r12+r18)

	yloop_optional_early_prefetch

	yfnmaddpd ymm15, ymm13, ymm0, ymm15	;; .782/.434(r3-r27)+(r7-r23)-.975/.434(r11-r19) (io36a/.434)		; 53-57
	yfnmaddpd ymm5, ymm4, ymm1, ymm5	;; .975/.434(r3-r27)-.782/.434(r7-r23)+(r11-r19) (io45a/.434)		; 53-57		n 58
	vmovapd	ymm0, YMM_TMPS[5*32]		;; Reload (r6+r24)-(r10+r20)

	yfmaddpd ymm1, ymm13, ymm10, ymm0	;; .975/.434((r2+r28)-(r14+r16))+((r6+r24)-(r10+r20))			; 54-58		n 59
	ystore	YMM_TMPS[1*32], ymm7		;; Real odd-cols row #2							; 53+2
	yfmsubpd ymm7, ymm4, ymm10, ymm8	;; .782/.434((r2+r28)-(r14+r16))-((r4+r26)-(r12+r18))			; 54-58		n 59

	yfnmaddpd ymm10, ymm13, ymm8, ymm10	;; ((r2+r28)-(r14+r16))-.975/.434((r4+r26)-(r12+r18))			; 55-59		n 60
	ystore	YMM_TMPS[2*32], ymm12		;; Imag odd-cols row #36b / .434					; 56
	yfmaddpd ymm12, ymm13, ymm6, ymm14	;; ((r2-r28)-(r14-r16))*.434+.975/.434((r4-r26)-(r12-r18))*.434		; 55-59		n 60

	ystore	YMM_TMPS[3*32], ymm15		;; Imag odd-cols row #36a / .434					; 58
	vaddpd	ymm15, ymm2, ymm11		;; Imag odd-cols row #2 / .434 (io27a + io27b)				; 57-61		n 66
	yfmsubpd ymm2, ymm2, YMM_ONE, ymm11	;; Imag odd-cols row #7 / .434 (io27a - io27b)				; 57-61

	yfmaddpd ymm11, ymm4, ymm14, ymm6	;; .782/.434((r2-r28)-(r14-r16))*.434+((r4-r26)-(r12-r18))*.434		; 56-60		n 61
	yfmaddpd ymm14, ymm13, ymm14, ymm9	;; .975/.434((r2-r28)-(r14-r16))*.434+((r6-r24)-(r10-r20))*.434		; 56-60		n 61
	L1prefetchw srcreg+6*d2+d1+L1pd, L1pt

	ystore	YMM_TMPS[4*32], ymm2		;; Imag odd-cols row #7 / .434						; 62
	vaddpd	ymm2, ymm5, ymm3		;; Imag odd-cols row #4 / .434 (io45a + io45b)				; 58-62
	yfmsubpd ymm5, ymm5, YMM_ONE, ymm3	;; Imag odd-cols row #5 / .434 (io45a - io45b)				; 58-62
	vmovapd	ymm3, YMM_ONE

	yfmaddpd ymm1, ymm4, ymm8, ymm1		;; .975/.434((r2+r28)-(r14+r16))+.782/.434((r4+r26)-(r12+r18))+((r6+r24)-(r10+r20)) (re2/.434) ; 59-63 n 65
	yfnmaddpd ymm7, ymm13, ymm0, ymm7	;; .782/.434((r2+r28)-(r14+r16))-((r4+r26)-(r12+r18))-.975/.434((r6+r24)-(r10+r20)) (re4/.434) ; 59-63
	vmovapd	ymm8, YMM_TMPS[3*32]		;; Imag odd-cols row #36a / .434

	yfmaddpd ymm10, ymm4, ymm0, ymm10	;; ((r2+r28)-(r14+r16))-.975/.434((r4+r26)-(r12+r18))+.782/.434((r6+r24)-(r10+r20)) (re6/.434) ; 60-64 n 70
	yfmaddpd ymm12, ymm4, ymm9, ymm12	;; .434*((r2-r28)-(r14-r16))+.975((r4-r26)-(r12-r18))+.782/.434((r6-r24)-(r10-r20))*.434 (ie3) ; 60-64 n 67
	vmovapd	ymm0, YMM_TMPS[2*32]		;; Imag odd-cols row #36b / .434

	yfnmaddpd ymm11, ymm13, ymm9, ymm11	;; .782((r2-r28)-(r14-r16))+.434*((r4-r26)-(r12-r18))-.975/.434((r6-r24)-(r10-r20))*.434 (ie5) ; 61-65
	yfnmaddpd ymm14, ymm4, ymm6, ymm14	;; .975((r2-r28)-(r14-r16))+.434*((r6-r24)-(r10-r20))-.782/.434((r4-r26)-(r12-r18))*.434 (ie7) ; 61-65
	vmovapd	ymm9, YMM_TMPS[10*32]		;; Real odd-cols row #3/6a

	yfmaddpd ymm4, ymm8, ymm3, ymm0		;; Imag odd-cols row #3 / .434 (io36a + io36b)				; 62-66		n 67
	yfmsubpd ymm8, ymm8, ymm3, ymm0		;; Imag odd-cols row #6 / .434 (io36a - io36b)				; 62-66		n 71
	vmovapd	ymm6, YMM_TMPS[14*32]		;; Real odd-cols row #3/6b

	yfmaddpd ymm0, ymm9, ymm3, ymm6		;; Real odd-cols row #3							; 63-67		n 68
	yfmsubpd ymm9, ymm9, ymm3, ymm6		;; Real odd-cols row #6							; 63-67		n 70

	vmovapd	ymm13, [screg+6*64+32]		;; cosine/sine for R8/I8
	vmovapd	ymm6, YMM_TMPS[17*32]		;; Real #8
	ystore	YMM_TMPS[2*32], ymm2		;; Imag odd-cols row #4 / .434						; 63
	vmovapd	ymm2, YMM_TMPS[6*32]		;; Imag #8
	ystore	YMM_TMPS[3*32], ymm5		;; Imag odd-cols row #5 / .434						; 63+1
	yfmaddpd ymm5, ymm2, ymm13, ymm6	;; B8 = I8 * cosine/sine + R8						; 64-68		n 69
	yfmsubpd ymm6, ymm6, ymm13, ymm2	;; A8 = R8 * cosine/sine - I8						; 64-68		n 69

	vmovapd	ymm13, YMM_TMPS[1*32]		;; Real odd-cols row #2
	vmovapd	ymm2, YMM_P434
	ystore	YMM_TMPS[1*32], ymm7		;; Save real even-cols row #4 / .434					; 64+1
	yfnmaddpd ymm7, ymm1, ymm2, ymm13	;; Real #14 (ro2 - re2 * .434)						; 65-69		n 72
	yfmaddpd ymm1, ymm1, ymm2, ymm13		;; Real #2 (ro2 + re2 * .434)						; 65-69		n 73

	vmovapd	ymm13, YMM_TMPS[7*32]		;; Imag even-cols row #2
	ystore	YMM_TMPS[5*32], ymm11		;; Save imag even-cols row #5						; 66
	yfnmaddpd ymm11, ymm15, ymm2, ymm13	;; Imag #14 (ie2 - io2 * .434)						; 66-70		n 72
	yfmaddpd ymm15, ymm15, ymm2, ymm13	;; Imag #2 (ie2 + io2 * .434)						; 66-70		n 73

	yloop_optional_early_prefetch

	yfnmaddpd ymm13, ymm4, ymm2, ymm12	;; Imag #13 (ie3 - io3 * .434)						; 67-71		n 74
	yfmaddpd ymm4, ymm4, ymm2, ymm12	;; Imag #3 (ie3 + io3 * .434)						; 67-71		n 75

	vmovapd	ymm12, YMM_TMPS[11*32]		;; Real even-cols row #3
	ystore	YMM_TMPS[6*32], ymm14		;; Save imag even-cols row #7						; 66+1
	yfmsubpd ymm14, ymm0, ymm3, ymm12	;; Real #13 (ro3 - re3)							; 68-72		n 74
	yfmaddpd ymm0, ymm0, ymm3, ymm12	;; Real #3 (ro3 + re3)							; 68-72		n 75

	vmovapd	ymm12, [screg+6*64]		;; sine for R8/I8
	vmulpd	ymm5, ymm5, ymm12		;; B8 = B8 * sine (final I8)						; 69-73
	vmulpd	ymm6, ymm6, ymm12		;; A8 = A8 * sine (final R8)						; 69-73

	yfnmaddpd ymm12, ymm10, ymm2, ymm9	;; Real #10 (ro6 - re6 * .434)						; 70-74		n 76
	yfmaddpd ymm10, ymm10, ymm2, ymm9	;; Real #6 (ro6 + re6 * .434)						; 70-74		n 77

	vmovapd	ymm9, YMM_TMPS[9*32]		;; Imag even-cols row #6
	ystore	[srcreg+3*d2+d1+32], ymm5	;; Save final I8							; 74
	yfnmaddpd ymm5, ymm8, ymm2, ymm9	;; Imag #10 (ie6 - io6 * .434)						; 71-75		n 76
	yfmaddpd ymm8, ymm8, ymm2, ymm9		;; Imag #6 (ie6 + io6 * .434)						; 71-75		n 77

	vmovapd	ymm9, [screg+12*64+32]		;; cosine/sine for R14/I14
	ystore	[srcreg+3*d2+d1], ymm6		;; Save final R8							; 74+1
	yfmsubpd ymm6, ymm7, ymm9, ymm11	;; A14 = R14 * cosine/sine - I14					; 72-76		n 78
	yfmaddpd ymm11, ymm11, ymm9, ymm7	;; B14 = I14 * cosine/sine + R14					; 72-76		n 78

	vmovapd	ymm9, [screg+32]		;; cosine/sine for R2/I2
	yfmsubpd ymm7, ymm1, ymm9, ymm15	;; A2 = R2 * cosine/sine - I2						; 73-77		n 79
	yfmaddpd ymm15, ymm15, ymm9, ymm1	;; B2 = I2 * cosine/sine + R2						; 73-77		n 79

	vmovapd	ymm9, [screg+11*64+32]		;; cosine/sine for R13/I13
	yfmsubpd ymm1, ymm14, ymm9, ymm13	;; A13 = R13 * cosine/sine - I13					; 74-78		n 80
	yfmaddpd ymm13, ymm13, ymm9, ymm14	;; B13 = I13 * cosine/sine + R13					; 74-78		n 80

	vmovapd	ymm9, [screg+64+32]		;; cosine/sine for R13/I13
	yfmsubpd ymm14, ymm0, ymm9, ymm4	;; A3 = R3 * cosine/sine - I3						; 75-79		n 81
	yfmaddpd ymm4, ymm4, ymm9, ymm0		;; B3 = I3 * cosine/sine + R3						; 75-79		n 81

	vmovapd	ymm9, [screg+8*64+32]		;; cosine/sine for R10/I10
	yfmsubpd ymm0, ymm12, ymm9, ymm5	;; A10 = R10 * cosine/sine - I10					; 76-80		n 83
	yfmaddpd ymm5, ymm5, ymm9, ymm12	;; B10 = I10 * cosine/sine + R10					; 76-80		n 83

	vmovapd	ymm9, [screg+4*64+32]		;; cosine/sine for R6/I6
	yfmsubpd ymm12, ymm10, ymm9, ymm8	;; A6 = R6 * cosine/sine - I6						; 77-81		n 84
	yfmaddpd ymm8, ymm8, ymm9, ymm10	;; B6 = I6 * cosine/sine + R6						; 77-81		n 84

	vmovapd	ymm9, [screg+12*64]		;; sine for R14/I14
	vmulpd	ymm6, ymm6, ymm9		;; A14 = A14 * sine (final R14)						; 78-82
	vmulpd	ymm11, ymm11, ymm9		;; B14 = B14 * sine (final I14)						; 78-82

	vmovapd	ymm9, [screg]			;; sine for R2/I2
	vmulpd	ymm7, ymm7, ymm9		;; A2 = A2 * sine (final R2)						; 79-83
	vmulpd	ymm15, ymm15, ymm9		;; B2 = B2 * sine (final I2)						; 79-83

	vmovapd	ymm9, [screg+11*64]		;; sine for R13/I13
	vmulpd	ymm1, ymm1, ymm9		;; A13 = A13 * sine (final R13)						; 80-84
	vmulpd	ymm13, ymm13, ymm9		;; B13 = B13 * sine (final I13)						; 80-84

	vmovapd	ymm9, [screg+64]		;; sine for R3/I3
	vmulpd	ymm14, ymm14, ymm9		;; A3 = A3 * sine (final R3)						; 81-85
	vmulpd	ymm4, ymm4, ymm9		;; B3 = B3 * sine (final I3)						; 81-85

	vmovapd	ymm9, YMM_TMPS[15*32]		;; Real odd-cols row #4/5a
	vmovapd	ymm10, YMM_TMPS[16*32]		;; Real odd-cols row #4/5b
	ystore	[srcreg+6*d2+d1], ymm6		;; Save final R14							; 83
	yfmaddpd ymm6, ymm9, ymm3, ymm10	;; Real odd-cols row #4							; 82-86		n 87
	yfmsubpd ymm9, ymm9, ymm3, ymm10	;; Real odd-cols row #5							; 82-86		n 89

	vmovapd	ymm10, [screg+8*64]		;; sine for R10/I10
	vmulpd	ymm0, ymm0, ymm10		;; A10 = A10 * sine (final R10)						; 83-87
	vmulpd	ymm5, ymm5, ymm10		;; B10 = B10 * sine (final I10)						; 83-87

	vmovapd	ymm10, [screg+4*64]		;; sine for R6/I6
	vmulpd	ymm12, ymm12, ymm10		;; A6 = A6 * sine (final R6)						; 84-88
	vmulpd	ymm8, ymm8, ymm10		;; B6 = B6 * sine (final I6)						; 84-88

	vmovapd	ymm10, YMM_TMPS[4*32]		;; Imag odd-cols row #7 / .434
	ystore	[srcreg+6*d2+d1+32], ymm11	;; Save final I14							; 83+1
	vmovapd	ymm11, YMM_TMPS[6*32]		;; Imag even-cols row #7
	ystore	[srcreg+d1], ymm7		;; Save final R2							; 84+1
	yfnmaddpd ymm7, ymm10, ymm2, ymm11	;; Imag #9 (ie7 - io7 * .434)						; 85-89		n 91
	yfmaddpd ymm10, ymm10, ymm2, ymm11	;; Imag #7 (ie7 + io7 * .434)						; 85-89		n 92

	vmovapd	ymm11, YMM_TMPS[0*32]		;; Real odd-cols row #7
	ystore	[srcreg+d1+32], ymm15		;; Save final I2							; 84+2
	vmovapd	ymm15, YMM_TMPS[13*32]		;; Real even-cols row #7
	ystore	[srcreg+6*d2], ymm1		;; Save final R13							; 85+2
	yfmsubpd ymm1, ymm11, ymm3, ymm15	;; Real #9 (ro7 - re7)							; 86-90		n 91
	yfmaddpd ymm11, ymm11, ymm3, ymm15	;; Real #7 (ro7 + re7)							; 86-90		n 92

	vmovapd	ymm15, YMM_TMPS[1*32]		;; Real even-cols row #4 / .434
	ystore	[srcreg+6*d2+32], ymm13		;; Save final I13							; 85+3
	yfnmaddpd ymm13, ymm15, ymm2, ymm6	;; Real #12 (ro4 - re4 * .434)						; 87-91		n 93
	yfmaddpd ymm15, ymm15, ymm2, ymm6	;; Real #4 (ro4 + re4 * .434)						; 87-91		n 94

	vmovapd	ymm6, YMM_TMPS[8*32]		;; Imag even-cols row #4
	ystore	[srcreg+d2], ymm14		;; Save final R3							; 86+3
	vmovapd	ymm14, YMM_TMPS[2*32]		;; Imag odd-cols row #4 / .434
	ystore	[srcreg+d2+32], ymm4		;; Save final I3							; 86+4
	yfnmaddpd ymm4, ymm14, ymm2, ymm6	;; Imag #12 (ie4 - io4 * .434)						; 88-92		n 93
	yfmaddpd ymm14, ymm14, ymm2, ymm6	;; Imag #4 (ie4 + io4 * .434)						; 88-92		n 94

	vmovapd	ymm6, YMM_TMPS[12*32]		;; Real even-cols row #5
	ystore	[srcreg+4*d2+d1], ymm0		;; Save final R10							; 88+3
	yfmsubpd ymm0, ymm9, ymm3, ymm6		;; Real #11 (ro5 - re5)							; 89-93		n 95
	yfmaddpd ymm9, ymm9, ymm3, ymm6		;; Real #5 (ro5 + re5)							; 89-93		n 96

	vmovapd	ymm6, YMM_TMPS[5*32]		;; Imag even-cols row #5
	ystore	[srcreg+4*d2+d1+32], ymm5	;; Save final I10							; 88+4
	vmovapd	ymm5, YMM_TMPS[3*32]		;; Imag odd-cols row #5 / .434
	ystore	[srcreg+2*d2+d1], ymm12		;; Save final R6							; 89+4
	yfnmaddpd ymm12, ymm5, ymm2, ymm6	;; Imag #11 (ie5 - io5 * .434)						; 90-94		n 95
	yfmaddpd ymm5, ymm5, ymm2, ymm6		;; Imag #5 (ie5 + io5 * .434)						; 90-94		n 96

	vmovapd	ymm6, [screg+7*64+32]		;; cosine/sine for R9/I9
	yfmsubpd ymm2, ymm1, ymm6, ymm7		;; A9 = R9 * cosine/sine - I9						; 91-95		n 97
	yfmaddpd ymm7, ymm7, ymm6, ymm1		;; B9 = I9 * cosine/sine + R9						; 91-95		n 97

	vmovapd	ymm6, [screg+5*64+32]		;; cosine/sine for R7/I7
	yfmsubpd ymm1, ymm11, ymm6, ymm10	;; A7 = R7 * cosine/sine - I7						; 92-96		n 98
	yfmaddpd ymm10, ymm10, ymm6, ymm11	;; B7 = I7 * cosine/sine + R7						; 92-96		n 98

	vmovapd	ymm6, [screg+10*64+32]		;; cosine/sine for R12/I12
	yfmsubpd ymm11, ymm13, ymm6, ymm4	;; A12 = R12 * cosine/sine - I12					; 93-97		n 99
	yfmaddpd ymm4, ymm4, ymm6, ymm13	;; B12 = I12 * cosine/sine + R12					; 93-97		n 99

	vmovapd	ymm6, [screg+2*64+32]		;; cosine/sine for R4/I4
	yfmsubpd ymm13, ymm15, ymm6, ymm14	;; A4 = R4 * cosine/sine - I4						; 94-98		n 100
	yfmaddpd ymm14, ymm14, ymm6, ymm15	;; B4 = I4 * cosine/sine + R4						; 94-98		n 100
	ystore	[srcreg+2*d2+d1+32], ymm8	;; Save final I6							; 89+5

	vmovapd	ymm6, [screg+9*64+32]		;; cosine/sine for R11/I11
	yfmsubpd ymm15, ymm0, ymm6, ymm12	;; A11 = R11 * cosine/sine - I11					; 95-99		n 101
	yfmaddpd ymm12, ymm12, ymm6, ymm0	;; B11 = I11 * cosine/sine + R11					; 95-99		n 101

	vmovapd	ymm6, [screg+3*64+32]		;; cosine/sine for R5/I5
	yfmsubpd ymm0, ymm9, ymm6, ymm5		;; A5 = R5 * cosine/sine - I5						; 96-100	n 102
	yfmaddpd ymm5, ymm5, ymm6, ymm9		;; B5 = I5 * cosine/sine + R5						; 96-100	n 102

	vmovapd	ymm6, [screg+7*64]		;; sine for R9/I9
	vmulpd	ymm2, ymm2, ymm6		;; A9 = A9 * sine (final R9)						; 97-101
	vmulpd	ymm7, ymm7, ymm6		;; B9 = B9 * sine (final I9)						; 97-101

	vmovapd	ymm6, [screg+5*64]		;; sine for R7/I7
	vmulpd	ymm1, ymm1, ymm6		;; A7 = A7 * sine (final R7)						; 98-102
	vmulpd	ymm10, ymm10, ymm6		;; B7 = B7 * sine (final I7)						; 98-102

	vmovapd ymm6, [screg+10*64]		;; sine for R12/I12
	vmulpd	ymm11, ymm11, ymm6		;; A12 = A12 * sine (final R12)						; 99-103
	vmulpd	ymm4, ymm4, ymm6		;; B12 = B12 * sine (final I12)						; 99-103

	vmovapd	ymm6, [screg+2*64]		;; sine for R4/I4
	vmulpd	ymm13, ymm13, ymm6		;; A4 = A4 * sine (final R4)						; 100-104
	vmulpd	ymm14, ymm14, ymm6		;; B4 = B4 * sine (final I4)						; 100-104

	vmovapd	ymm6, [screg+9*64]		;; sine for R11/I11
	vmulpd	ymm15, ymm15, ymm6		;; A11 = A11 * sine (final R11)						; 101-105
	vmulpd	ymm12, ymm12, ymm6		;; B11 = B11 * sine (final I11)						; 101-105

	vmovapd	ymm6, [screg+3*64]		;; sine for R5/I5
	vmulpd	ymm0, ymm0, ymm6		;; A5 = A5 * sine (final R5)						; 102-106
	vmulpd	ymm5, ymm5, ymm6		;; B5 = B5 * sine (final I5)						; 102-106

	ystore	[srcreg+4*d2], ymm2		;; Save final R9							; 102
	ystore	[srcreg+4*d2+32], ymm7		;; Save final I9							; 102+1
	ystore	[srcreg+3*d2], ymm1		;; Save final R7							; 103+1
	ystore	[srcreg+3*d2+32], ymm10		;; Save final I7							; 103+2
	ystore	[srcreg+5*d2+d1], ymm11		;; Save final R12							; 104+2
	ystore	[srcreg+5*d2+d1+32], ymm4	;; Save final I12							; 104+3
	ystore	[srcreg+d2+d1], ymm13		;; Save final R4							; 105+3
	ystore	[srcreg+d2+d1+32], ymm14	;; Save final I4							; 105+4
	ystore	[srcreg+5*d2], ymm15		;; Save final R11							; 106+4
	vmovapd ymm15, ymm3			;; Move YMM_ONE to starting register
	ystore	[srcreg+5*d2+32], ymm12		;; Save final I11							; 106+5
	ystore	[srcreg+2*d2], ymm0		;; Save final R5							; 107+5
	ystore	[srcreg+2*d2+32], ymm5		;; Save final I5							; 107+6

	bump	srcreg, srcinc
	bump	screg, scinc
	ENDM

ENDIF

ENDIF

;;
;; ************************************* 28-reals-last-unfft variants ******************************************
;;

;; These macros produce 28 reals after doing 4.807 levels of the inverse FFT applying
;; the sin/cos multipliers beforehand.  The input is 2 real and 13 complex numbers.

;; To calculate a 28-reals inverse FFT, we calculate 28 real values from 28 complex inputs in a brute force way.
;; First we note that the 28 complex values are computed from the 13 complex and 2 real inputs using Hermetian symmetry, thusly:
;; c1 = r1A + 0*i
;; c2 = r2 + i2*i
;; ...
;; c14 = r14 + i14*i
;; c15 = r1B + 0*i
;; c16 = r14 - i14*i
;; ...
;; c28 = r2 - i2*i 
;;
;; The brute force calculations are:
;;
;; c1 + c2 + ... + c28	*  w^-0000000000...
;; c1 + c2 + ... + c28	*  w^-0123456789A...
;; c1 + c2 + ... + c28	*  w^-02468ACE....
;;    ...
;; c1 + c2 + ... + c28	*  w^-...A987654321
;;
;; The sin/cos values (w = 28th root of unity) are:
;; w^-1 = .975 - .223i
;; w^-2 = .901 - .434i
;; w^-3 = .782 - .623i
;; w^-4 = .623 - .782i
;; w^-5 = .434 - .901i
;; w^-6 = .223 - .975i
;; w^-7 = 0 - 1i
;; w^-8 = -.223 - .975i
;; w^-9 = -.434 - .901i
;; w^-10 = -.623 - .782i
;; w^-11 = -.782 - .623i
;; w^-12 = -.901 - .434i
;; w^-13 = -.975 - .223i
;; w^-14 = -1
;;
;; Applying the sin/cos values above, taking advantage of symmetry, and ignoring a lot of multiplies by 2:
;; r1     +(r2+r14)     +(r3+r13)     +(r4+r12)     +(r5+r11)     +(r6+r10)     +(r7+r9) + r8 + r15
;; r1 +.975(r2-r14) +.901(r3-r13) +.782(r4-r12) +.623(r5-r11) +.434(r6-r10) +.223(r7-r9)      - r15 +.223(i2+i14) +.434*(i3+i13) +.623(i4+i12) +.782(i5+i11) +.901(i6+i10) +.975(i7+i9) + i8
;; r1 +.901(r2+r14) +.623(r3+r13) +.223(r4+r12) -.223(r5+r11) -.623(r6+r10) -.901(r7+r9) - r8 + r15 +.434(i2-i14) +.782*(i3-i13) +.975(i4-i12) +.975(i5-i11) +.782(i6-i10) +.434(i7-i9)
;; r1 +.782(r2-r14) +.223(r3-r13) -.434(r4-r12) -.901(r5-r11) -.975(r6-r10) -.623(r7-r9)      - r15 +.623(i2+i14) +.975*(i3+i13) +.901(i4+i12) +.434(i5+i11) -.223(i6+i10) -.782(i7+i9) - i8
;; r1 +.623(r2+r14) -.223(r3+r13) -.901(r4+r12) -.901(r5+r11) -.223(r6+r10) +.623(r7+r9) + r8 + r15 +.782(i2-i14) +.975*(i3-i13) +.434(i4-i12) -.434(i5-i11) -.975(i6-i10) -.782(i7-i9)
;; r1 +.434(r2-r14) -.623(r3-r13) -.975(r4-r12) -.223(r5-r11) +.782(r6-r10) +.901(r7-r9)      - r15 +.901(i2+i14) +.782*(i3+i13) -.223(i4+i12) -.975(i5+i11) -.623(i6+i10) +.434(i7+i9) + i8
;; r1 +.223(r2+r14) -.901(r3+r13) -.623(r4+r12) +.623(r5+r11) +.901(r6+r10) -.223(r7+r9) - r8 + r15 +.975(i2-i14) +.434*(i3-i13) -.782(i4-i12) -.782(i5-i11) +.434(i6-i10) +.975(i7-i9)
;; r1                   -(r3-r13)                   +(r5-r11)                   -(r7-r9)      - r15     +(i2+i14)                    -(i4+i12)                   +(i6+i10)              - i8
;; r1 -.223(r2+r14) -.901(r3+r13) +.623(r4+r12) +.623(r5+r11) -.901(r6+r10) -.223(r7+r9) + r8 + r15 +.975(i2-i14) -.434*(i3-i13) -.782(i4-i12) +.782(i5-i11) +.434(i6-i10) -.975(i7-i9)
;; r1 -.434(r2-r14) -.623(r3-r13) +.975(r4-r12) -.223(r5-r11) -.782(r6-r10) +.901(r7-r9)      - r15 +.901(i2+i14) -.782*(i3+i13) -.223(i4+i12) +.975(i5+i11) -.623(i6+i10) -.434(i7+i9) + i8
;; r1 -.623(r2+r14) -.223(r3+r13) +.901(r4+r12) -.901(r5+r11) +.223(r6+r10) +.623(r7+r9) - r8 + r15 +.782(i2-i14) -.975*(i3-i13) +.434(i4-i12) +.434(i5-i11) -.975(i6-i10) +.782(i7-i9)
;; r1 -.782(r2-r14) +.223(r3-r13) +.434(r4-r12) -.901(r5-r11) +.975(r6-r10) -.623(r7-r9)      - r15 +.623(i2+i14) -.975*(i3+i13) +.901(i4+i12) -.434(i5+i11) -.223(i6+i10) +.782(i7+i9) - i8
;; r1 -.901(r2+r14) +.623(r3+r13) -.223(r4+r12) -.223(r5+r11) +.623(r6+r10) -.901(r7+r9) + r8 + r15 +.434(i2-i14) -.782*(i3-i13) +.975(i4-i12) -.975(i5-i11) +.782(i6-i10) -.434(i7-i9)
;; r1 -.975(r2-r14) +.901(r3-r13) -.782(r4-r12) +.623(r5-r11) -.434(r6-r10) +.223(r7-r9)      - r15 +.223(i2+i14) -.434*(i3+i13) +.623(i4+i12) -.782(i5+i11) +.901(i6+i10) -.975(i7+i9) + i8
;; r1     -(r2+r14)     +(r3+r13)     -(r4+r12)     +(r5+r11)     -(r6+r10)     +(r7+r9) - r8 + r15
;; r1 -.975(r2-r14) +.901(r3-r13) -.782(r4-r12) +.623(r5-r11) -.434(r6-r10) +.223(r7-r9)      - r15 -.223(i2-i14) +.434*(i3-i13) -.623(i4-i12) +.782(i5-i11) -.901(i6-i10) +.975(i7-i9) - i8
;; ... r17 thru r28 are the same as r12 through r1 but with the sign of the imaginary component changed.
;;
;; Also remember that due to the funny way we do things reals input r1A = r1+r15 and r1B = r1-15

;; Store intermediate results in YMM_TMPS

yr7_14cl_28_reals_unfft_preload MACRO
	ENDM

yr7_14cl_28_reals_unfft MACRO srcreg,srcinc,d1,d2,screg,scinc,maxrpt,L1pt,L1pd

;; Apply the 13 twiddle factors.  Also do the first +/- on the reals so that
;; we have a perfect balance of adds and multiplies.

	vmovapd	ymm0, [screg+32]		;; cosine/sine
	vmovapd	ymm1, [srcreg+d1]		;; R2
	vmulpd	ymm2, ymm1, ymm0		;; A2 = R2 * cosine/sine
	vmovapd	ymm4, [screg+12*64+32]		;; cosine/sine
	vmovapd	ymm5, [srcreg+6*d2+d1]		;; R14
	vmulpd	ymm6, ymm5, ymm4		;; A14 = R14 * cosine/sine
	vmovapd	ymm3, [srcreg+d1+32]		;; I2
	vaddpd	ymm2, ymm2, ymm3		;; A2 = A2 + I2
	vmulpd	ymm3, ymm3, ymm0		;; B2 = I2 * cosine/sine
	vmovapd	ymm7, [srcreg+6*d2+d1+32]	;; I14
	vaddpd	ymm6, ymm6, ymm7		;; A14 = A14 + I14
	vmulpd	ymm7, ymm7, ymm4		;; B14 = I14 * cosine/sine
	vsubpd	ymm3, ymm3, ymm1		;; B2 = B2 - R2
	vmovapd	ymm0, [screg]			;; sine
	vmulpd	ymm2, ymm2, ymm0		;; R2 = A2 * sine
	vsubpd	ymm7, ymm7, ymm5		;; B14 = B14 - R14
	vmovapd	ymm4, [screg+12*64]		;; sine
	vmulpd	ymm6, ymm6, ymm4		;; R14 = A14 * sine
	vmulpd	ymm3, ymm3, ymm0		;; I2 = B2 * sine
	vmulpd	ymm7, ymm7, ymm4		;; I14 = B14 * sine
	vaddpd	ymm0, ymm2, ymm6		;; R2+R14
	vsubpd	ymm2, ymm2, ymm6		;; R2-R14
	L1prefetchw srcreg+L1pd, L1pt
	vaddpd	ymm1, ymm3, ymm7		;; I2+I14
	vsubpd	ymm3, ymm3, ymm7		;; I2-I14
	ystore	YMM_TMPS[24*32], ymm0		;; Save R2+R14
	ystore	YMM_TMPS[0*32], ymm2		;; Save R2-R14
	ystore	YMM_TMPS[25*32], ymm1		;; Save I2+I14
	ystore	YMM_TMPS[1*32], ymm3		;; Save I2-I14

	vmovapd	ymm0, [screg+64+32]		;; cosine/sine
	vmovapd	ymm1, [srcreg+d2]		;; R3
	vmulpd	ymm2, ymm1, ymm0		;; A3 = R3 * cosine/sine
	vmovapd	ymm4, [screg+11*64+32]		;; cosine/sine
	vmovapd	ymm5, [srcreg+6*d2]		;; R13
	vmulpd	ymm6, ymm5, ymm4		;; A13 = R13 * cosine/sine
	vmovapd	ymm3, [srcreg+d2+32]		;; I3
	vaddpd	ymm2, ymm2, ymm3		;; A3 = A3 + I3
	vmulpd	ymm3, ymm3, ymm0		;; B3 = I3 * cosine/sine
	vmovapd	ymm7, [srcreg+6*d2+32]		;; I13
	vaddpd	ymm6, ymm6, ymm7		;; A13 = A13 + I13
	vmulpd	ymm7, ymm7, ymm4		;; B13 = I13 * cosine/sine
	vsubpd	ymm3, ymm3, ymm1		;; B3 = B3 - R3
	vmovapd	ymm0, [screg+64]		;; sine
	vmulpd	ymm2, ymm2, ymm0		;; R3 = A3 * sine
	vsubpd	ymm7, ymm7, ymm5		;; B13 = B13 - R13
	vmovapd	ymm4, [screg+11*64]		;; sine
	vmulpd	ymm6, ymm6, ymm4		;; R13 = A13 * sine
	vmulpd	ymm3, ymm3, ymm0		;; I3 = B3 * sine
	vmulpd	ymm7, ymm7, ymm4		;; I13 = B13 * sine
	vaddpd	ymm0, ymm2, ymm6		;; R3+R13
	vsubpd	ymm2, ymm2, ymm6		;; R3-R13
	L1prefetchw srcreg+d1+L1pd, L1pt
	vaddpd	ymm1, ymm3, ymm7		;; I3+I13
	vsubpd	ymm3, ymm3, ymm7		;; I3-I13
	ystore	YMM_TMPS[22*32], ymm0		;; Save R3+R13
	ystore	YMM_TMPS[2*32], ymm2		;; Save R3-R13
	ystore	YMM_TMPS[23*32], ymm1		;; Save I3+I13
	ystore	YMM_TMPS[3*32], ymm3		;; Save I3-I13

	vmovapd	ymm0, [screg+2*64+32]		;; cosine/sine
	vmovapd	ymm1, [srcreg+d2+d1]		;; R4
	vmulpd	ymm2, ymm1, ymm0		;; A4 = R4 * cosine/sine
	vmovapd	ymm4, [screg+10*64+32]		;; cosine/sine
	vmovapd	ymm5, [srcreg+5*d2+d1]		;; R12
	vmulpd	ymm6, ymm5, ymm4		;; A12 = R12 * cosine/sine
	vmovapd	ymm3, [srcreg+d2+d1+32]		;; I4
	vaddpd	ymm2, ymm2, ymm3		;; A4 = A4 + I4
	vmulpd	ymm3, ymm3, ymm0		;; B4 = I4 * cosine/sine
	vmovapd	ymm7, [srcreg+5*d2+d1+32]	;; I12
	vaddpd	ymm6, ymm6, ymm7		;; A12 = A12 + I12
	vmulpd	ymm7, ymm7, ymm4		;; B12 = I12 * cosine/sine
	vsubpd	ymm3, ymm3, ymm1		;; B4 = B4 - R4
	vmovapd	ymm0, [screg+2*64]		;; sine
	vmulpd	ymm2, ymm2, ymm0		;; R4 = A4 * sine
	vsubpd	ymm7, ymm7, ymm5		;; B12 = B12 - R12
	vmovapd	ymm4, [screg+10*64]		;; sine
	vmulpd	ymm6, ymm6, ymm4		;; R12 = A12 * sine
	vmulpd	ymm3, ymm3, ymm0		;; I4 = B4 * sine
	vmulpd	ymm7, ymm7, ymm4		;; I12 = B12 * sine
	vaddpd	ymm0, ymm2, ymm6		;; R4+R12
	vsubpd	ymm2, ymm2, ymm6		;; R4-R12
	L1prefetchw srcreg+d2+L1pd, L1pt
	vaddpd	ymm1, ymm3, ymm7		;; I4+I12
	vsubpd	ymm3, ymm3, ymm7		;; I4-I12
	ystore	YMM_TMPS[20*32], ymm0		;; Save R4+R12
	ystore	YMM_TMPS[4*32], ymm2		;; Save R4-R12
	ystore	YMM_TMPS[21*32], ymm1		;; Save I4+I12
	ystore	YMM_TMPS[5*32], ymm3		;; Save I4-I12

	vmovapd	ymm0, [screg+3*64+32]		;; cosine/sine
	vmovapd	ymm1, [srcreg+2*d2]		;; R5
	vmulpd	ymm2, ymm1, ymm0		;; A5 = R5 * cosine/sine
	vmovapd	ymm4, [screg+9*64+32]		;; cosine/sine
	vmovapd	ymm5, [srcreg+5*d2]		;; R11
	vmulpd	ymm6, ymm5, ymm4		;; A11 = R11 * cosine/sine
	vmovapd	ymm3, [srcreg+2*d2+32]		;; I5
	vaddpd	ymm2, ymm2, ymm3		;; A5 = A5 + I5
	vmulpd	ymm3, ymm3, ymm0		;; B5 = I5 * cosine/sine
	vmovapd	ymm7, [srcreg+5*d2+32]		;; I11
	vaddpd	ymm6, ymm6, ymm7		;; A11 = A11 + I11
	vmulpd	ymm7, ymm7, ymm4		;; B11 = I11 * cosine/sine
	vsubpd	ymm3, ymm3, ymm1		;; B5 = B5 - R5
	vmovapd	ymm0, [screg+3*64]		;; sine
	vmulpd	ymm2, ymm2, ymm0		;; R5 = A5 * sine
	vsubpd	ymm7, ymm7, ymm5		;; B11 = B11 - R11
	vmovapd	ymm4, [screg+9*64]		;; sine
	vmulpd	ymm6, ymm6, ymm4		;; R11 = A11 * sine
	vmulpd	ymm3, ymm3, ymm0		;; I5 = B5 * sine
	vmulpd	ymm7, ymm7, ymm4		;; I11 = B11 * sine
	vaddpd	ymm0, ymm2, ymm6		;; R5+R11
	vsubpd	ymm2, ymm2, ymm6		;; R5-R11
	vaddpd	ymm1, ymm3, ymm7		;; I5+I11
	vsubpd	ymm3, ymm3, ymm7		;; I5-I11
	ystore	YMM_TMPS[18*32], ymm0		;; Save R5+R11
	ystore	YMM_TMPS[6*32], ymm2		;; Save R5-R11
	ystore	YMM_TMPS[19*32], ymm1		;; Save I5+I11
	ystore	YMM_TMPS[7*32], ymm3		;; Save I5-I11

	vmovapd	ymm0, [screg+4*64+32]		;; cosine/sine
	vmovapd	ymm1, [srcreg+2*d2+d1]		;; R6
	vmulpd	ymm2, ymm1, ymm0		;; A6 = R6 * cosine/sine
	vmovapd	ymm4, [screg+8*64+32]		;; cosine/sine
	vmovapd	ymm5, [srcreg+4*d2+d1]		;; R10
	vmulpd	ymm6, ymm5, ymm4		;; A10 = R10 * cosine/sine
	vmovapd	ymm3, [srcreg+2*d2+d1+32]	;; I6
	vaddpd	ymm2, ymm2, ymm3		;; A6 = A6 + I6
	vmulpd	ymm3, ymm3, ymm0		;; B6 = I6 * cosine/sine
	vmovapd	ymm7, [srcreg+4*d2+d1+32]	;; I10
	vaddpd	ymm6, ymm6, ymm7		;; A10 = A10 + I10
	vmulpd	ymm7, ymm7, ymm4		;; B10 = I10 * cosine/sine
	vsubpd	ymm3, ymm3, ymm1		;; B6 = B6 - R6
	vmovapd	ymm0, [screg+4*64]		;; sine
	vmulpd	ymm2, ymm2, ymm0		;; R6 = A6 * sine
	vsubpd	ymm7, ymm7, ymm5		;; B10 = B10 - R10
	vmovapd	ymm4, [screg+8*64]		;; sine
	vmulpd	ymm6, ymm6, ymm4		;; R10 = A10 * sine
	vmulpd	ymm3, ymm3, ymm0		;; I6 = B6 * sine
	vmulpd	ymm7, ymm7, ymm4		;; I10 = B10 * sine
	vaddpd	ymm0, ymm2, ymm6		;; R6+R10
	vsubpd	ymm2, ymm2, ymm6		;; R6-R10
	L1prefetchw srcreg+d2+d1+L1pd, L1pt
	vaddpd	ymm1, ymm3, ymm7		;; I6+I10
	vsubpd	ymm3, ymm3, ymm7		;; I6-I10
	ystore	YMM_TMPS[16*32], ymm0		;; Save R6+R10
	ystore	YMM_TMPS[8*32], ymm2		;; Save R6-R10
	ystore	YMM_TMPS[17*32], ymm1		;; Save I6+I10
	ystore	YMM_TMPS[9*32], ymm3		;; Save I6-I10

	vmovapd	ymm0, [screg+5*64+32]		;; cosine/sine
	vmovapd	ymm1, [srcreg+3*d2]		;; R7
	vmulpd	ymm2, ymm1, ymm0		;; A7 = R7 * cosine/sine
	vmovapd	ymm4, [screg+7*64+32]		;; cosine/sine
	vmovapd	ymm5, [srcreg+4*d2]		;; R9
	vmulpd	ymm6, ymm5, ymm4		;; A9 = R9 * cosine/sine
	vmovapd	ymm3, [srcreg+3*d2+32]		;; I7
	vaddpd	ymm2, ymm2, ymm3		;; A7 = A7 + I7
	vmulpd	ymm3, ymm3, ymm0		;; B7 = I7 * cosine/sine
	vmovapd	ymm7, [srcreg+4*d2+32]		;; I9
	vaddpd	ymm6, ymm6, ymm7		;; A9 = A9 + I9
	vmulpd	ymm7, ymm7, ymm4		;; B9 = I9 * cosine/sine
	vsubpd	ymm3, ymm3, ymm1		;; B7 = B7 - R7
	vmovapd	ymm0, [screg+5*64]		;; sine
	vmulpd	ymm2, ymm2, ymm0		;; R7 = A7 * sine
	vsubpd	ymm7, ymm7, ymm5		;; B9 = B9 - R9
	vmovapd	ymm4, [screg+7*64]		;; sine
	vmulpd	ymm6, ymm6, ymm4		;; R9 = A9 * sine
	vmulpd	ymm3, ymm3, ymm0		;; I7 = B7 * sine
	vmulpd	ymm7, ymm7, ymm4		;; I9 = B9 * sine
	vaddpd	ymm0, ymm2, ymm6		;; R7+R9
	vsubpd	ymm2, ymm2, ymm6		;; R7-R9
	L1prefetchw srcreg+2*d2+L1pd, L1pt
	vaddpd	ymm1, ymm3, ymm7		;; I7+I9
	vsubpd	ymm3, ymm3, ymm7		;; I7-I9
	ystore	YMM_TMPS[14*32], ymm0		;; Save R7+R9
	ystore	YMM_TMPS[10*32], ymm2		;; Save R7-R9
	ystore	YMM_TMPS[15*32], ymm1		;; Save I7+I9
	ystore	YMM_TMPS[11*32], ymm3		;; Save I7-I9

	vmovapd	ymm0, [screg+6*64+32]		;; cosine/sine
	vmovapd	ymm1, [srcreg+3*d2+d1]		;; R8
	vmulpd	ymm2, ymm1, ymm0		;; A8 = R8 * cosine/sine
	vmovapd	ymm3, [srcreg+3*d2+d1+32]	;; I8
	vaddpd	ymm2, ymm2, ymm3		;; A8 = A8 + I8
	vmulpd	ymm3, ymm3, ymm0		;; B8 = I8 * cosine/sine
	vsubpd	ymm3, ymm3, ymm1		;; B8 = B8 - R8
	vmovapd	ymm0, [screg+6*64]		;; sine
	vmulpd	ymm2, ymm2, ymm0		;; R8 = A8 * sine
	vmulpd	ymm3, ymm3, ymm0		;; I8 = B8 * sine
	ystore	YMM_TMPS[12*32], ymm2		;; Save R8
	ystore	YMM_TMPS[13*32], ymm3		;; Save I8

	;; Do the 28 reals inverse FFT

	;; Calculate odd columns derived from real inputs (even rows)

	vmovapd	ymm0, YMM_TMPS[2*32]		;; r3-r13
	vmovapd	ymm2, YMM_P901
	vmulpd	ymm4, ymm2, ymm0		;; .901(r3-r13)
	vmovapd	ymm7, [srcreg+32]		;; r1-r15
	vaddpd	ymm4, ymm7, ymm4		;; r1+.901(r3-r13)-r15
	vmovapd	ymm3, YMM_P223
	vmulpd	ymm5, ymm3, ymm0		;; .223(r3-r13)
	vaddpd	ymm5, ymm7, ymm5		;; r1+.223(r3-r13)-r15
	vmovapd	ymm1, YMM_P623
	vmulpd	ymm6, ymm1, ymm0		;; .623(r3-r13)
	vsubpd	ymm6, ymm7, ymm6		;; r1-.623(r3-r13)-r15
	vsubpd	ymm7, ymm7, ymm0		;; r1-(r3-r13)-r15

	vmovapd	ymm0, YMM_TMPS[6*32]		;; r5-r11
	vmulpd	ymm1, ymm1, ymm0		;; .623(r5-r11)
	vaddpd	ymm4, ymm4, ymm1		;; r1+.901(r3-r13)+.623(r5-r11)-r15
	vmulpd	ymm1, ymm2, ymm0		;; .901(r5-r11)
	vsubpd	ymm5, ymm5, ymm1		;; r1+.223(r3-r13)-.901(r5-r11)-r15
	L1prefetchw srcreg+2*d2+d1+L1pd, L1pt
	vmulpd	ymm1, ymm3, ymm0		;; .223(r5-r11)
	vsubpd	ymm6, ymm6, ymm1		;; r1-.623(r3-r13)-.223(r5-r11)-r15
	vaddpd	ymm7, ymm7, ymm0		;; r1-(r3-r13)+(r5-r11)-r15

	vmovapd	ymm0, YMM_TMPS[10*32]		;; r7-r9
	vmulpd	ymm1, ymm3, ymm0		;; .223(r7-r9)
	vaddpd	ymm4, ymm4, ymm1		;; r1+.901(r3-r13)+.623(r5-r11)+.223(r7-r9)-r15
	vmulpd	ymm1, ymm0, YMM_P623		;; .623(r7-r9)
	vsubpd	ymm5, ymm5, ymm1		;; r1+.223(r3-r13)-.901(r5-r11)-.623(r7-r9)-r15
	vmulpd	ymm1, ymm2, ymm0		;; .901(r7-r9)
	L1prefetchw srcreg+3*d2+L1pd, L1pt
	vaddpd	ymm6, ymm6, ymm1		;; r1-.623(r3-r13)-.223(r5-r11)+.901(r7-r9)-r15
	vsubpd	ymm7, ymm7, ymm0		;; r1-(r3-r13)+(r5-r11)-(r7-r9)-r15

	ystore	[srcreg+d1], ymm7		;; Save odd-real-cols row #8 (also is real-cols row #8)

	;; Calculate even columns derived from real inputs (even rows)
	;; From above, odd-real-col rols rows #2,4,6 are in ymm4, ymm5, ymm6

	vmovapd	ymm3, YMM_TMPS[0*32]		;; r2-r14
	vmulpd	ymm1, ymm3, YMM_P975		;; .975(r2-r14)
	vmovapd	ymm7, YMM_P782
	vmulpd	ymm2, ymm7, ymm3		;; .782(r2-r14)
	vmulpd	ymm3, ymm3, YMM_P434		;; .434(r2-r14)

	vmovapd	ymm0, YMM_TMPS[4*32]		;; r4-r12
	vmulpd	ymm7, ymm7, ymm0		;; .782(r4-r12)
	vaddpd	ymm1, ymm1, ymm7		;; .975(r2-r14)+.782(r4-r12)
	vmulpd	ymm7, ymm0, YMM_P434		;; .434(r4-r12)
	vsubpd	ymm2, ymm2, ymm7		;; .782(r2-r14)-.434(r4-r12)
	L1prefetchw srcreg+3*d2+d1+L1pd, L1pt
	vmulpd	ymm0, ymm0, YMM_P975		;; .975(r4-r12)
	vsubpd	ymm3, ymm3, ymm0		;; .434(r2-r14)-.975(r4-r12)

	vmovapd	ymm0, YMM_TMPS[8*32]		;; r6-r10
	vmulpd	ymm7, ymm0, YMM_P434		;; .434(r6-r10)
	vaddpd	ymm1, ymm1, ymm7		;; .975(r2-r14)+.782(r4-r12)+.434(r6-r10)
	vmulpd	ymm7, ymm0, YMM_P975		;; .975(r6-r10)
	vsubpd	ymm2, ymm2, ymm7		;; .782(r2-r14)-.434(r4-r12)-.975(r6-r10)
	L1prefetchw srcreg+4*d2+L1pd, L1pt
	vmulpd	ymm0, ymm0, YMM_P782		;; .782(r6-r10)
	vaddpd	ymm3, ymm3, ymm0		;; .434(r2-r14)-.975(r4-r12)+.782(r6-r10)

	;; Combine even and odd columns (even rows)

	vsubpd	ymm0, ymm4, ymm1		;; real-cols row #14 (odd#2 - even#2)
	vaddpd	ymm4, ymm4, ymm1		;; real-cols row #2 (odd#2 + even#2)

	vsubpd	ymm1, ymm5, ymm2		;; real-cols row #12 (odd#4 - even#4)
	vaddpd	ymm5, ymm5, ymm2		;; real-cols row #4 (odd#4 + even#4)
	L1prefetchw srcreg+4*d2+d1+L1pd, L1pt

	vsubpd	ymm2, ymm6, ymm3		;; real-cols row #10 (odd#6 - even#6)
	vaddpd	ymm6, ymm6, ymm3		;; real-cols row #6 (odd#6 + even#6)

	ystore	YMM_TMPS[2*32], ymm0		;; Save real-cols row #14
	ystore	YMM_TMPS[0*32], ymm4		;; Save real-cols row #2
	ystore	YMM_TMPS[6*32], ymm1		;; Save real-cols row #12
	ystore	YMM_TMPS[4*32], ymm5		;; Save real-cols row #4
	ystore	YMM_TMPS[10*32], ymm2		;; Save real-cols row #10
	ystore	YMM_TMPS[8*32], ymm6		;; Save real-cols row #6

	;; Calculate even columns derived from real inputs (odd rows)

	vmovapd	ymm0, YMM_TMPS[24*32]		;; r2+r14
	vmovapd	ymm1, YMM_P901
	vmulpd	ymm5, ymm1, ymm0		;; .901(r2+r14)
	vmovapd	ymm2, YMM_P623
	vmulpd	ymm6, ymm2, ymm0		;; .623(r2+r14)
	vmovapd	ymm3, YMM_P223
	vmulpd	ymm7, ymm3, ymm0		;; .223(r2+r14)

	vmovapd	ymm4, YMM_TMPS[20*32]		;; r4+r12
	vaddpd	ymm0, ymm0, ymm4		;; (r2+r14)+(r4+r12)
	vmulpd	ymm3, ymm3, ymm4		;; .223(r4+r12)
	vaddpd	ymm5, ymm5, ymm3		;; .901(r2+r14)+.223(r4+r12)
	vmulpd	ymm3, ymm1, ymm4		;; .901(r4+r12)
	vsubpd	ymm6, ymm6, ymm3		;; .623(r2+r14)-.901(r4+r12)
	L1prefetchw srcreg+5*d2+L1pd, L1pt
	vmulpd	ymm3, ymm2, ymm4		;; .623(r4+r12)
	vsubpd	ymm7, ymm7, ymm3		;; .223(r2+r14)-.623(r4+r12)

	vmovapd	ymm4, YMM_TMPS[16*32]		;; r6+r10
	vaddpd	ymm0, ymm0, ymm4		;; (r2+r14)+(r4+r12)+(r6+r10)
	vmulpd	ymm3, ymm2, ymm4		;; .623(r6+r10)
	vsubpd	ymm5, ymm5, ymm3		;; .901(r2+r14)+.223(r4+r12)-.623(r6+r10)
	vmulpd	ymm3, ymm4, YMM_P223		;; .223(r6+r10)
	vsubpd	ymm6, ymm6, ymm3		;; .623(r2+r14)-.901(r4+r12)-.223(r6+r10)
	L1prefetchw srcreg+5*d2+d1+L1pd, L1pt
	vmulpd	ymm3, ymm1, ymm4		;; .901(r6+r10)
	vaddpd	ymm7, ymm7, ymm3		;; .223(r2+r14)-.623(r4+r12)+.901(r6+r10)

	vmovapd	ymm4, YMM_TMPS[12*32]		;; r8
	vaddpd	ymm0, ymm0, ymm4		;; (r2+r14)+(r4+r12)+(r6+r10)+r8
	vsubpd	ymm5, ymm5, ymm4		;; .901(r2+r14)+.223(r4+r12)-.623(r6+r10)-r8
	vaddpd	ymm6, ymm6, ymm4		;; .623(r2+r14)-.901(r4+r12)-.223(r6+r10)+r8
	L1prefetchw srcreg+6*d2+L1pd, L1pt
	vsubpd	ymm7, ymm7, ymm4		;; .223(r2+r14)-.623(r4+r12)+.901(r6+r10)-r8

	ystore	YMM_TMPS[12*32], ymm0		;; Save even-real-cols row #1	;; We could save a few loads and stores
	ystore	YMM_TMPS[16*32], ymm5		;; Save even-real-cols row #3	;; if two of these registers were left
	ystore	YMM_TMPS[20*32], ymm6		;; Save even-real-cols row #5	;; unchanged through the next section
	ystore	YMM_TMPS[24*32], ymm7		;; Save even-real-cols row #7

	;; Calculate odd columns derived from real inputs (odd rows)

	vmovapd	ymm0, YMM_TMPS[22*32]		;; r3+r13
	vmulpd	ymm5, ymm2, ymm0		;; .623(r3+r13)
	vmovapd	ymm3, YMM_P223
	vmulpd	ymm6, ymm3, ymm0		;; .223(r3+r13)
	vmulpd	ymm7, ymm1, ymm0		;; .901(r3+r13)
	vmovapd	ymm4, [srcreg]			;; r1+r15
	vaddpd	ymm0, ymm4, ymm0		;; r1+(r3+r13)+r15
	vaddpd	ymm5, ymm4, ymm5		;; r1+.623(r3+r13)+r15
	L1prefetchw srcreg+6*d2+d1+L1pd, L1pt
	vsubpd	ymm6, ymm4, ymm6		;; r1-.223(r3+r13)+r15
	vsubpd	ymm7, ymm4, ymm7		;; r1-.901(r3+r13)+r15

	vmovapd	ymm4, YMM_TMPS[18*32]		;; r5+r11
	vaddpd	ymm0, ymm0, ymm4		;; r1+(r3+r13)+(r5+r11)+r15
	vmulpd	ymm3, ymm3, ymm4		;; .223(r5+r11)
	vsubpd	ymm5, ymm5, ymm3		;; r1+.623(r3+r13)-.223(r5+r11)+r15
	vmulpd	ymm3, ymm1, ymm4		;; .901(r5+r11)
	vsubpd	ymm6, ymm6, ymm3		;; r1-.223(r3+r13)-.901(r5+r11)+r15
	yloop_optional_early_prefetch
	vmulpd	ymm3, ymm2, ymm4		;; .623(r5+r11)
	vaddpd	ymm7, ymm7, ymm3		;; r1-.901(r3+r13)+.623(r5+r11)+r15

	vmovapd	ymm4, YMM_TMPS[14*32]		;; r7+r9
	vaddpd	ymm0, ymm0, ymm4		;; r1+(r3+r13)+(r5+r11)+(r7+r9)+r15
	vmulpd	ymm3, ymm1, ymm4		;; .901(r7+r9)
	vsubpd	ymm5, ymm5, ymm3		;; r1+.623(r3+r13)-.223(r5+r11)-.901(r7+r9)+r15
	vmulpd	ymm3, ymm2, ymm4		;; .623(r7+r9)
	vaddpd	ymm6, ymm6, ymm3		;; r1-.223(r3+r13)-.901(r5+r11)+.623(r7+r9)+r15
	vmulpd	ymm3, ymm4, YMM_P223		;; .223(r7+r9)
	vsubpd	ymm7, ymm7, ymm3		;; r1-.901(r3+r13)+.623(r5+r11)-.223(r7+r9)+r15

	;; Combine even and odd columns (odd rows)

	vmovapd	ymm1, YMM_TMPS[12*32]		;; even-real-cols row #1
	vaddpd	ymm2, ymm0, ymm1		;; real-cols row #1 (and final R1)
	vsubpd	ymm3, ymm0, ymm1		;; real-cols row #15 (and final R15)
	ystore	[srcreg], ymm2			;; Save final R1
	ystore	[srcreg+32], ymm3		;; Save final R15

	vmovapd	ymm1, YMM_TMPS[16*32]		;; even-real-cols row #3
	vaddpd	ymm2, ymm5, ymm1		;; real-cols row #3
	vsubpd	ymm3, ymm5, ymm1		;; real-cols row #13
	ystore	YMM_TMPS[12*32], ymm2		;; Save real-cols row #3
	ystore	YMM_TMPS[14*32], ymm3		;; Save real-cols row #13

	vmovapd	ymm1, YMM_TMPS[20*32]		;; even-real-cols row #5
	vaddpd	ymm2, ymm6, ymm1		;; real-cols row #5
	vsubpd	ymm3, ymm6, ymm1 		;; real-cols row #11
	ystore	YMM_TMPS[16*32], ymm2		;; Save real-cols row #5
	ystore	YMM_TMPS[18*32], ymm3		;; Save real-cols row #11

	vmovapd	ymm1, YMM_TMPS[24*32]		;; even-real-cols row #7
	vaddpd	ymm2, ymm7, ymm1		;; real-cols row #7
	vsubpd	ymm3, ymm7, ymm1		;; real-cols row #9
	ystore	YMM_TMPS[20*32], ymm2		;; Save real-cols row #7
	ystore	YMM_TMPS[24*32], ymm3		;; Save real-cols row #9

	;; Calculate even columns derived from imaginary inputs (even rows)

	vmovapd	ymm0, YMM_TMPS[25*32]		;; i2+i14
	vmovapd	ymm1, YMM_P223
	vmulpd	ymm5, ymm1, ymm0		;; .223(i2+i14)
	vmovapd	ymm2, YMM_P623
	vmulpd	ymm6, ymm2, ymm0		;; .623(i2+i14)
	vmovapd	ymm3, YMM_P901
	vmulpd	ymm7, ymm3, ymm0		;; .901(i2+i14)

	vmovapd	ymm4, YMM_TMPS[21*32]		;; i4+i12
	vsubpd	ymm0, ymm0, ymm4		;; (i2+i14)-(i4+i12)
	vmulpd	ymm2, ymm2, ymm4		;; .623(i4+i12)
	vaddpd	ymm5, ymm5, ymm2		;; .223(i2+i14)+.623(i4+i12)
	vmulpd	ymm2, ymm3, ymm4		;; .901(i4+i12)
	yloop_optional_early_prefetch
	vaddpd	ymm6, ymm6, ymm2		;; .623(i2+i14)+.901(i4+i12)
	vmulpd	ymm2, ymm1, ymm4		;; .223(i4+i12)
	vsubpd	ymm7, ymm7, ymm2		;; .901(i2+i14)-.223(i4+i12)

	vmovapd	ymm4, YMM_TMPS[17*32]		;; i6+i10
	vaddpd	ymm0, ymm0, ymm4		;; (i2+i14)-(i4+i12)+(i6+i10)
	vmulpd	ymm2, ymm3, ymm4		;; .901(i6+i10)
	vaddpd	ymm5, ymm5, ymm2		;; .223(i2+i14)+.623(i4+i12)+.901(i6+i10)
	vmulpd	ymm2, ymm1, ymm4		;; .223(i6+i10)
	vsubpd	ymm6, ymm6, ymm2		;; .623(i2+i14)+.901(i4+i12)-.223(i6+i10)
	vmulpd	ymm2, ymm4, YMM_P623		;; .623(i6+i10)
	vsubpd	ymm7, ymm7, ymm2		;; .901(i2+i14)-.223(i4+i12)-.623(i6+i10)

	vmovapd	ymm4, YMM_TMPS[13*32]		;; i8
	vsubpd	ymm0, ymm0, ymm4		;; (i2+i14)-(i4+i12)+(i6+i10)-i8
	vaddpd	ymm5, ymm5, ymm4		;; .223(i2+i14)+.623(i4+i12)+.901(i6+i10)+i8
	vsubpd	ymm6, ymm6, ymm4		;; .623(i2+i14)+.901(i4+i12)-.223(i6+i10)-i8
	vaddpd	ymm7, ymm7, ymm4		;; .901(i2+i14)-.223(i4+i12)-.623(i6+i10)+i8

	;; Combine real and imaginary data for row #8

	vmovapd	ymm4, [srcreg+d1]		;; Load real-cols row #8
	vsubpd	ymm1, ymm4, ymm0		;; final R22
	vaddpd	ymm2, ymm4, ymm0		;; final R8
	ystore	[srcreg+3*d2+d1+32], ymm1	;; Save R22
	ystore	[srcreg+3*d2+d1], ymm2		;; Save R8

	;; Calculate odd columns derived from imaginary inputs (even rows)
	;; From above, even-imag-cols row #2,4,6 are in ymm5, ymm6, ymm7

	vmovapd	ymm4, YMM_TMPS[23*32]		;; i3+i13
	vmulpd	ymm2, ymm4, YMM_P434		;; .434(i3+i13)
	vmulpd	ymm3, ymm4, YMM_P975		;; .975(i3+i13)
	vmovapd	ymm1, YMM_P782
	vmulpd	ymm4, ymm1, ymm4		;; .782(i3+i13)

	vmovapd	ymm0, YMM_TMPS[19*32]		;; i5+i11
	vmulpd	ymm1, ymm1, ymm0		;; .782(i5+i11)
	vaddpd	ymm2, ymm2, ymm1		;; .434(i3+i13)+.782(i5+i11)
	vmulpd	ymm1, ymm0, YMM_P434		;; .434(i5+i11)
	vaddpd	ymm3, ymm3, ymm1		;; .975(i3+i13)+.434(i5+i11)
	vmulpd	ymm1, ymm0, YMM_P975		;; .975(i5+i11)
	vsubpd	ymm4, ymm4, ymm1		;; .782(i3+i13)-.975(i5+i11)

	vmovapd	ymm0, YMM_TMPS[15*32]		;; i7+i9
	vmulpd	ymm1, ymm0, YMM_P975		;; .975(i7+i9)
	vaddpd	ymm2, ymm2, ymm1		;; .434(i3+i13)+.782(i5+i11)+.975(i7+i9)
	vmulpd	ymm1, ymm0, YMM_P782		;; .782(i7+i9)
	vsubpd	ymm3, ymm3, ymm1		;; .975(i3+i13)+.434(i5+i11)-.782(i7+i9)
	vmulpd	ymm1, ymm0, YMM_P434		;; .434(i7+i9)
	vaddpd	ymm4, ymm4, ymm1		;; .782(i3+i13)-.975(i5+i11)+.434(i7+i9)

	;; Combine even and odd columns, then real and imag data (even rows)

	vsubpd	ymm0, ymm5, ymm2		;; imag-cols row #14 (even#2 - odd#2)
	vaddpd	ymm5, ymm5, ymm2		;; imag-cols row #2 (even#2 + odd#2)
	vsubpd	ymm1, ymm6, ymm3		;; imag-cols row #12 (even#4 - odd#4)
	vaddpd	ymm6, ymm6, ymm3		;; imag-cols row #4 (even#4 + odd#4)
	yloop_optional_early_prefetch
	vsubpd	ymm2, ymm7, ymm4		;; imag-cols row #10 (even#6 - odd#6)
	vaddpd	ymm7, ymm7, ymm4		;; imag-cols row #6 (even#6 + odd#6)

	vmovapd	ymm3, YMM_TMPS[2*32]		;; Load real-cols row #14
	vsubpd	ymm4, ymm3, ymm0		;; final R16
	vaddpd	ymm3, ymm3, ymm0		;; final R14
	ystore	[srcreg+d1+32], ymm4		;; Save R16
	ystore	[srcreg+6*d2+d1], ymm3		;; Save R14

	vmovapd	ymm3, YMM_TMPS[0*32]		;; Load real-cols row #2
	vsubpd	ymm4, ymm3, ymm5		;; final R28
	vaddpd	ymm3, ymm3, ymm5		;; final R2
	ystore	[srcreg+6*d2+d1+32], ymm4	;; Save R28
	ystore	[srcreg+d1], ymm3		;; Save R2

	vmovapd	ymm3, YMM_TMPS[6*32]		;; Load real-cols row #12
	vsubpd	ymm4, ymm3, ymm1		;; final R18
	vaddpd	ymm3, ymm3, ymm1		;; final R12
	ystore	[srcreg+d2+d1+32], ymm4		;; Save R18
	ystore	[srcreg+5*d2+d1], ymm3		;; Save R12

	vmovapd	ymm3, YMM_TMPS[4*32]		;; Load real-cols row #4
	vsubpd	ymm4, ymm3, ymm6		;; final R26
	vaddpd	ymm3, ymm3, ymm6		;; final R4
	ystore	[srcreg+5*d2+d1+32], ymm4	;; Save R26
	ystore	[srcreg+d2+d1], ymm3		;; Save R4

	vmovapd	ymm3, YMM_TMPS[10*32]		;; Load real-cols row #10
	vsubpd	ymm4, ymm3, ymm2		;; final R20
	vaddpd	ymm3, ymm3, ymm2		;; final R10
	ystore	[srcreg+2*d2+d1+32], ymm4	;; Save R20
	ystore	[srcreg+4*d2+d1], ymm3		;; Save R10

	vmovapd	ymm3, YMM_TMPS[8*32]		;; Load real-cols row #6
	vsubpd	ymm4, ymm3, ymm7		;; final R24
	vaddpd	ymm3, ymm3, ymm7		;; final R6
	ystore	[srcreg+4*d2+d1+32], ymm4	;; Save R24
	ystore	[srcreg+2*d2+d1], ymm3		;; Save R6

	;; Calculate even columns derived from imaginary inputs (odd rows)

	vmovapd	ymm4, YMM_TMPS[1*32]		;; i2-i14
	vmovapd	ymm1, YMM_P434
	vmulpd	ymm5, ymm1, ymm4		;; .434(i2-i14)
	vmovapd	ymm2, YMM_P782
	vmulpd	ymm6, ymm2, ymm4		;; .782(i2-i14)
	vmovapd	ymm3, YMM_P975
	vmulpd	ymm7, ymm3, ymm4		;; .975(i2-i14)

	vmovapd	ymm4, YMM_TMPS[5*32]		;; i4-i12
	vmulpd	ymm0, ymm3, ymm4		;; .975(i4-i12)
	vaddpd	ymm5, ymm5, ymm0		;; .434(i2-i14)+.975(i4-i12)
	vmulpd	ymm0, ymm1, ymm4		;; .434(i4-i12)
	vaddpd	ymm6, ymm6, ymm0		;; .782(i2-i14)+.434(i4-i12)
	vmulpd	ymm0, ymm2, ymm4		;; .782(i4-i12)
	vsubpd	ymm7, ymm7, ymm0		;; .975(i2-i14)-.782(i4-i12)

	vmovapd	ymm4, YMM_TMPS[9*32]		;; i6-i10
	vmulpd	ymm0, ymm2, ymm4		;; .782(i6-i10)
	vaddpd	ymm5, ymm5, ymm0		;; .434(i2-i14)+.975(i4-i12)+.782(i6-i10)
	vmulpd	ymm0, ymm3, ymm4		;; .975(i6-i10)
	vsubpd	ymm6, ymm6, ymm0		;; .782(i2-i14)+.434(i4-i12)-.975(i6-i10)
	vmulpd	ymm0, ymm1, ymm4		;; .434(i6-i10)
	vaddpd	ymm7, ymm7, ymm0		;; .975(i2-i14)-.782(i4-i12)+.434(i6-i10)

	;; Calculate odd columns derived from imaginary inputs (odd rows)
	;; From above, even-imag-cols row #3,5,7 are in ymm5,ymm6,ymm7

	vmovapd	ymm4, YMM_TMPS[3*32]		;; i3-i13
	vmulpd	ymm2, ymm2, ymm4		;; .782(i3-i13)
	vmulpd	ymm3, ymm3, ymm4		;; .975(i3-i13)
	vmulpd	ymm4, ymm1, ymm4		;; .434(i3-i13)

	vmovapd	ymm0, YMM_TMPS[7*32]		;; i5-i11
	vmulpd	ymm1, ymm1, ymm0		;; .434(i5-i11)
	vsubpd	ymm3, ymm3, ymm1		;; .975(i3-i13)-.434(i5-i11)
	vmulpd	ymm1, ymm0, YMM_P975		;; .975(i5-i11)
	vaddpd	ymm2, ymm2, ymm1		;; .782(i3-i13)+.975(i5-i11)
	vmulpd	ymm1, ymm0, YMM_P782		;; .782(i5-i11)
	vsubpd	ymm4, ymm4, ymm1		;; .434(i3-i13)-.782(i5-i11)

	vmovapd	ymm0, YMM_TMPS[11*32]		;; i7-i9
	vmulpd	ymm1, ymm0, YMM_P434		;; .434(i7-i9)
	vaddpd	ymm2, ymm2, ymm1		;; .782(i3-i13)+.975(i5-i11)+.434(i7-i9)
	vmulpd	ymm1, ymm0, YMM_P782		;; .782(i7-i9)
	vsubpd	ymm3, ymm3, ymm1		;; .975(i3-i13)-.434(i5-i11)-.782(i7-i9)
	vmulpd	ymm1, ymm0, YMM_P975		;; .975(i7-i9)
	vaddpd	ymm4, ymm4, ymm1		;; .434(i3-i13)-.782(i5-i11)+.975(i7-i9)

	;; Combine even and odd columns, then real and imag data (odd rows)

	yloop_optional_early_prefetch

	vsubpd	ymm0, ymm5, ymm2		;; imag-cols row #13 (even#3 - odd#3)
	vaddpd	ymm5, ymm5, ymm2		;; imag-cols row #3 (even#3 + odd#3)
	vsubpd	ymm1, ymm6, ymm3		;; imag-cols row #11 (even#5 - odd#5)
	vaddpd	ymm6, ymm6, ymm3		;; imag-cols row #5 (even#5 + odd#5)
	vsubpd	ymm2, ymm7, ymm4		;; imag-cols row #9 (even#7 - odd#7)
	vaddpd	ymm7, ymm7, ymm4		;; imag-cols row #7 (even#7 + odd#7)

	vmovapd	ymm3, YMM_TMPS[14*32]		;; Load real-cols row #13
	vsubpd	ymm4, ymm3, ymm0		;; final R17
	vaddpd	ymm3, ymm3, ymm0		;; final R13
	ystore	[srcreg+d2+32], ymm4		;; Save R17
	ystore	[srcreg+6*d2], ymm3		;; Save R13

	vmovapd	ymm3, YMM_TMPS[12*32]		;; Load real-cols row #3
	vsubpd	ymm4, ymm3, ymm5		;; final R27
	vaddpd	ymm3, ymm3, ymm5		;; final R3
	ystore	[srcreg+6*d2+32], ymm4		;; Save R27
	ystore	[srcreg+d2], ymm3		;; Save R3

	vmovapd	ymm3, YMM_TMPS[18*32]		;; Load real-cols row #11
	vsubpd	ymm4, ymm3, ymm1		;; final R19
	vaddpd	ymm3, ymm3, ymm1		;; final R11
	ystore	[srcreg+2*d2+32], ymm4		;; Save R19
	ystore	[srcreg+5*d2], ymm3		;; Save R11

	vmovapd	ymm3, YMM_TMPS[16*32]		;; Load real-cols row #5
	vsubpd	ymm4, ymm3, ymm6		;; final R25
	vaddpd	ymm3, ymm3, ymm6		;; final R5
	ystore	[srcreg+5*d2+32], ymm4		;; Save R25
	ystore	[srcreg+2*d2], ymm3		;; Save R5

	vmovapd	ymm3, YMM_TMPS[24*32]		;; Load real-cols row #9
	vsubpd	ymm4, ymm3, ymm2		;; final R21
	vaddpd	ymm3, ymm3, ymm2		;; final R9
	ystore	[srcreg+3*d2+32], ymm4		;; Save R21
	ystore	[srcreg+4*d2], ymm3		;; Save R9

	vmovapd	ymm3, YMM_TMPS[20*32]		;; Load real-cols row #7
	vsubpd	ymm4, ymm3, ymm7		;; final R23
	vaddpd	ymm3, ymm3, ymm7		;; final R7
	ystore	[srcreg+4*d2+32], ymm4		;; Save R23
	ystore	[srcreg+3*d2], ymm3		;; Save R7

	bump	srcreg, srcinc
	bump	screg, scinc
	ENDM

;; 64-bit version

IFDEF X86_64

yr7_14cl_28_reals_unfft_preload MACRO
	vbroadcastsd ymm13, Q YMM_P223
	vbroadcastsd ymm14, Q YMM_P623
	vbroadcastsd ymm15, Q YMM_P901
	ENDM

yr7_14cl_28_reals_unfft MACRO srcreg,srcinc,d1,d2,screg,scinc,maxrpt,L1pt,L1pd
	vmovapd	ymm0, [screg+32]		;; cosine/sine
	vmovapd	ymm1, [srcreg+d1]		;; R2
	vmulpd	ymm2, ymm1, ymm0		;; A2 = R2 * cosine/sine				;	1-5

	vmovapd	ymm3, [screg+12*64+32]		;; cosine/sine
	vmovapd	ymm4, [srcreg+6*d2+d1]		;; R14
	vmulpd	ymm5, ymm4, ymm3		;; A14 = R14 * cosine/sine				;	2-6

	vmovapd	ymm6, [srcreg+d1+32]		;; I2
	vmulpd	ymm0, ymm6, ymm0		;; B2 = I2 * cosine/sine				;	3-7

	vmovapd	ymm7, [srcreg+6*d2+d1+32]	;; I14
	vmulpd	ymm3, ymm7, ymm3		;; B14 = I14 * cosine/sine				;	4-8

	vmovapd	ymm8, [screg+64+32]		;; cosine/sine
	vmovapd	ymm9, [srcreg+d2]		;; R3
	vmulpd	ymm10, ymm9, ymm8		;; A3 = R3 * cosine/sine				;	5-9

	vaddpd	ymm2, ymm2, ymm6		;; A2 = A2 + I2						; 6-8
	vmovapd	ymm11, [screg+11*64+32]		;; cosine/sine
	vmovapd	ymm12, [srcreg+6*d2]		;; R13
	vmulpd	ymm6, ymm12, ymm11		;; A13 = R13 * cosine/sine				;	6-10

	vaddpd	ymm5, ymm5, ymm7		;; A14 = A14 + I14					; 7-9
	vmovapd	ymm7, [srcreg+d2+32]		;; I3
	vmulpd	ymm8, ymm7, ymm8		;; B3 = I3 * cosine/sine				;	7-11

	vsubpd	ymm0, ymm0, ymm1		;; B2 = B2 - R2						; 8-10
	vmovapd	ymm1, [srcreg+6*d2+32]		;; I13
	vmulpd	ymm11, ymm1, ymm11		;; B13 = I13 * cosine/sine				;	8-12

	vsubpd	ymm3, ymm3, ymm4		;; B14 = B14 - R14					; 9-11
	vmovapd	ymm4, [screg]			;; sine
	vmulpd	ymm2, ymm2, ymm4		;; R2 = A2 * sine					;	9-13

	vaddpd	ymm10, ymm10, ymm7		;; A3 = A3 + I3						; 10-12
	vmovapd	ymm7, [screg+12*64]		;; sine
	vmulpd	ymm5, ymm5, ymm7		;; R14 = A14 * sine					;	10-14

	vaddpd	ymm6, ymm6, ymm1		;; A13 = A13 + I13					; 11-13
	vmulpd	ymm0, ymm0, ymm4		;; I2 = B2 * sine					;	11-15
	vmovapd	ymm1, [screg+64]		;; sine

	vsubpd	ymm8, ymm8, ymm9		;; B3 = B3 - R3						; 12-14
	vmulpd	ymm3, ymm3, ymm7		;; I14 = B14 * sine					;	12-16
	vmovapd	ymm4, [screg+11*64]		;; sine

	vsubpd	ymm11, ymm11, ymm12		;; B13 = B13 - R13					; 13-15
	vmulpd	ymm10, ymm10, ymm1		;; R3 = A3 * sine					;	13-17
	vmovapd	ymm9, [screg+2*64+32]		;; cosine/sine

	vmulpd	ymm6, ymm6, ymm4		;; R13 = A13 * sine					;	14-18
	vmovapd	ymm7, [srcreg+d2+d1]		;; R4

	vaddpd	ymm12, ymm2, ymm5		;; R2+R14						; 15-17
	vmulpd	ymm8, ymm8, ymm1		;; I3 = B3 * sine					;	15-19
	vmovapd	ymm1, [screg+10*64+32]		;; cosine/sine

	vsubpd	ymm2, ymm2, ymm5		;; R2-R14						; 16-18
	vmulpd	ymm11, ymm11, ymm4		;; I13 = B13 * sine					;	16-20

	yloop_optional_early_prefetch

	vaddpd	ymm4, ymm0, ymm3		;; I2+I14						; 17-19
	vmulpd	ymm5, ymm7, ymm9		;; A4 = R4 * cosine/sine				;	17-21

	vsubpd	ymm0, ymm0, ymm3		;; I2-I14						; 18-20
	vmovapd	ymm3, [srcreg+5*d2+d1]		;; R12
	ystore	YMM_TMPS[17*32], ymm12		;; Save R2+R14						; 18
	vmulpd	ymm12, ymm3, ymm1		;; A12 = R12 * cosine/sine				;	18-22

	ystore	YMM_TMPS[0*32], ymm2		;; Save R2-R14						; 19
	vaddpd	ymm2, ymm10, ymm6		;; R3+R13						; 19-21
	ystore	YMM_TMPS[12*32], ymm4		;; Save I2+I14						; 20
	vmovapd	ymm4, [srcreg+d2+d1+32]		;; I4
	vmulpd	ymm9, ymm4, ymm9		;; B4 = I4 * cosine/sine				;	19-23

	vsubpd	ymm10, ymm10, ymm6		;; R3-R13						; 20-22
	vmovapd	ymm6, [srcreg+5*d2+d1+32]	;; I12
	vmulpd	ymm1, ymm6, ymm1		;; B12 = I12 * cosine/sine				;	20-24

	ystore	YMM_TMPS[1*32], ymm0		;; Save I2-I14						; 21
	vaddpd	ymm0, ymm8, ymm11		;; I3+I13						; 21-23
	ystore	YMM_TMPS[10*32], ymm2		;; Save R3+R13						; 22
	vmovapd	ymm2, [screg+3*64+32]		;; cosine/sine
	ystore	YMM_TMPS[2*32], ymm10		;; Save R3-R13						; 23
	vmovapd	ymm10, [srcreg+2*d2]		;; R5
	ystore	YMM_TMPS[13*32], ymm0		;; Save I3+I13						; 24
	vmulpd	ymm0, ymm10, ymm2		;; A5 = R5 * cosine/sine				;	21-25

	vaddpd	ymm5, ymm5, ymm4		;; A4 = A4 + I4						; 22-24
	vmovapd	ymm4, [screg+9*64+32]		;; cosine/sine

	vaddpd	ymm12, ymm12, ymm6		;; A12 = A12 + I12					; 23-25
	vmovapd	ymm6, [srcreg+5*d2]		;; R11

	vsubpd	ymm9, ymm9, ymm7		;; B4 = B4 - R4						; 24-26
	vmulpd	ymm7, ymm6, ymm4		;; A11 = R11 * cosine/sine				;	22-26

	vsubpd	ymm1, ymm1, ymm3		;; B12 = B12 - R12					; 25-27
	vmovapd	ymm3, [srcreg+2*d2+32]		;; I5
	vmulpd	ymm2, ymm3, ymm2		;; B5 = I5 * cosine/sine				;	23-27

	vaddpd	ymm0, ymm0, ymm3		;; A5 = A5 + I5						; 26-28
	vmovapd	ymm3, [srcreg+5*d2+32]		;; I11
	vmulpd	ymm4, ymm3, ymm4		;; B11 = I11 * cosine/sine				;	24-28

	vaddpd	ymm7, ymm7, ymm3		;; A11 = A11 + I11					; 27-29
	vmovapd	ymm3, [screg+2*64]		;; sine
	vmulpd	ymm5, ymm5, ymm3		;; R4 = A4 * sine					;	25-29

	vsubpd	ymm2, ymm2, ymm10		;; B5 = B5 - R5						; 28-30
	vmovapd	ymm10, [screg+10*64]		;; sine
	vmulpd	ymm12, ymm12, ymm10		;; R12 = A12 * sine					;	26-30
	vmulpd	ymm9, ymm9, ymm3		;; I4 = B4 * sine					;	27-31
	vmulpd	ymm1, ymm1, ymm10		;; I12 = B12 * sine					;	28-32

	vsubpd	ymm4, ymm4, ymm6		;; B11 = B11 - R11					; 29-31
	vmovapd	ymm3, [screg+3*64]		;; sine
	vmulpd	ymm0, ymm0, ymm3		;; R5 = A5 * sine					;	29-33

	vsubpd	ymm8, ymm8, ymm11		;; I3-I13						; 30-32
	vmovapd	ymm10, [screg+9*64]		;; sine
	vmulpd	ymm7, ymm7, ymm10		;; R11 = A11 * sine					;	30-34

	vaddpd	ymm11, ymm5, ymm12		;; R4+R12						; 31-33
	vmulpd	ymm2, ymm2, ymm3		;; I5 = B5 * sine					;	31-35
	vmovapd	ymm6, [screg+4*64+32]		;; cosine/sine

	vsubpd	ymm5, ymm5, ymm12		;; R4-R12						; 32-34
	vmulpd	ymm4, ymm4, ymm10		;; I11 = B11 * sine					;	32-36
	vmovapd	ymm3, [srcreg+2*d2+d1]		;; R6

	yloop_optional_early_prefetch

	vaddpd	ymm10, ymm9, ymm1		;; I4+I12						; 33-35
	vmulpd	ymm12, ymm3, ymm6		;; A6 = R6 * cosine/sine				;	33-37
	ystore	YMM_TMPS[3*32], ymm8		;; Save I3-I13						; 33

	vsubpd	ymm9, ymm9, ymm1		;; I4-I12						; 34-36
	vmovapd	ymm1, [screg+8*64+32]		;; cosine/sine
	vmovapd	ymm8, [srcreg+4*d2+d1]		;; R10
	ystore	YMM_TMPS[20*32], ymm11		;; Save R4+R12						; 34
	vmulpd	ymm11, ymm8, ymm1		;; A10 = R10 * cosine/sine				;	34-38

	ystore	YMM_TMPS[4*32], ymm5		;; Save R4-R12						; 35
	vaddpd	ymm5, ymm0, ymm7		;; R5+R11						; 35-37
	ystore	YMM_TMPS[21*32], ymm10		;; Save I4+I12						; 36
	vmovapd	ymm10, [srcreg+2*d2+d1+32]	;; I6
	vmulpd	ymm6, ymm10, ymm6		;; B6 = I6 * cosine/sine				;	35-39

	vsubpd	ymm0, ymm0, ymm7		;; R5-R11						; 36-38
	vmovapd	ymm7, [srcreg+4*d2+d1+32]	;; I10
	vmulpd	ymm1, ymm7, ymm1		;; B10 = I10 * cosine/sine				;	36-40

	ystore	YMM_TMPS[5*32], ymm9		;; Save I4-I12						; 37
	vaddpd	ymm9, ymm2, ymm4		;; I5+I11						; 37-39
	ystore	YMM_TMPS[18*32], ymm5		;; Save R5+R11						; 38
	vmovapd	ymm5, [screg+5*64+32]		;; cosine/sine
	ystore	YMM_TMPS[6*32], ymm0		;; Save R5-R11						; 39
	vmovapd	ymm0, [srcreg+3*d2]		;; R7
	ystore	YMM_TMPS[19*32], ymm9		;; Save I5+I11						; 40
	vmulpd	ymm9, ymm0, ymm5		;; A7 = R7 * cosine/sine				;	37-41

	vaddpd	ymm12, ymm12, ymm10		;; A6 = A6 + I6						; 38-40
	vmovapd	ymm10, [screg+7*64+32]		;; cosine/sine

	vaddpd	ymm11, ymm11, ymm7		;; A10 = A10 + I10					; 39-41
	vmovapd	ymm7, [srcreg+4*d2]		;; R9

	vsubpd	ymm6, ymm6, ymm3		;; B6 = B6 - R6						; 40-42
	vmulpd	ymm3, ymm7, ymm10		;; A9 = R9 * cosine/sine				;	38-42

	vsubpd	ymm1, ymm1, ymm8		;; B10 = B10 - R10					; 41-43
	vmovapd	ymm8, [srcreg+3*d2+32]		;; I7
	vmulpd	ymm5, ymm8, ymm5		;; B7 = I7 * cosine/sine				;	39-43

	vaddpd	ymm9, ymm9, ymm8		;; A7 = A7 + I7						; 42-44
	vmovapd	ymm8, [srcreg+4*d2+32]		;; I9
	vmulpd	ymm10, ymm8, ymm10		;; B9 = I9 * cosine/sine				;	40-44

	vaddpd	ymm3, ymm3, ymm8		;; A9 = A9 + I9						; 43-45
	vmovapd	ymm8, [screg+4*64]		;; sine
	vmulpd	ymm12, ymm12, ymm8		;; R6 = A6 * sine					;	41-45

	vsubpd	ymm5, ymm5, ymm0		;; B7 = B7 - R7						; 44-46
	vmovapd	ymm0, [screg+8*64]		;; sine
	vmulpd	ymm11, ymm11, ymm0		;; R10 = A10 * sine					;	42-46
	vmulpd	ymm6, ymm6, ymm8		;; I6 = B6 * sine					;	43-47
	vmulpd	ymm1, ymm1, ymm0		;; I10 = B10 * sine					;	44-48

	vsubpd	ymm10, ymm10, ymm7		;; B9 = B9 - R9						; 45-47
	vmovapd	ymm8, [screg+5*64]		;; sine
	vmulpd	ymm9, ymm9, ymm8		;; R7 = A7 * sine					;	45-49

	vsubpd	ymm2, ymm2, ymm4		;; I5-I11						; 46-48
	vmovapd	ymm0, [screg+7*64]		;; sine
	vmulpd	ymm3, ymm3, ymm0		;; R9 = A9 * sine					;	46-50

	vaddpd	ymm4, ymm12, ymm11		;; R6+R10						; 47-50
	vmulpd	ymm5, ymm5, ymm8		;; I7 = B7 * sine					;	47-51
	vmovapd	ymm7, [screg+6*64+32]		;; cosine/sine

	vsubpd	ymm12, ymm12, ymm11		;; R6-R10						; 48-51
	vmulpd	ymm10, ymm10, ymm0		;; I9 = B9 * sine					;	48-52
	vmovapd	ymm8, [srcreg+3*d2+d1+32]	;; I8

	vaddpd	ymm0, ymm6, ymm1		;; I6+I10						; 49-52
	vmovapd	ymm11, [srcreg+3*d2+d1]		;; R8
	ystore	YMM_TMPS[7*32], ymm2		;; Save I5-I11						; 49

	vsubpd	ymm6, ymm6, ymm1		;; I6-I10						; 50-53
	vmovapd	ymm2, YMM_TMPS[2*32]		;; r3-r13
	ystore	YMM_TMPS[16*32], ymm4		;; Save R6+R10						; 50

	vaddpd	ymm1, ymm9, ymm3		;; R7+R9						; 51-53
	vmulpd	ymm4, ymm8, ymm7		;; B8 = I8 * cosine/sine				;	51-55
	ystore	YMM_TMPS[8*32], ymm12		;; Save R6-R10						; 51

	vsubpd	ymm9, ymm9, ymm3		;; R7-R9						; 52-54
	vmulpd	ymm7, ymm11, ymm7		;; A8 = R8 * cosine/sine				;	52-56

	yloop_optional_early_prefetch

	vaddpd	ymm3, ymm5, ymm10		;; I7+I9						; 53-55
	vmulpd	ymm12, ymm15, ymm2		;; .901(r3-r13)						;	53-57
	ystore	YMM_TMPS[9*32], ymm6		;; Save I6-I10						; 53

	vsubpd	ymm5, ymm5, ymm10		;; I7-I9						; 54-56
	vmulpd	ymm10, ymm13, ymm2		;; .223(r3-r13)						;	54-58
	vmovapd	ymm6, [srcreg+32]		;; r1-r15
	ystore	YMM_TMPS[14*32], ymm1		;; Save R7+R9						; 54

	;; Do the 28 reals inverse FFT

	;; Calculate odd columns derived from real inputs (even rows)

	vsubpd	ymm1, ymm6, ymm2		;; r1-(r3-r13)-r15					; 55-57
	vmulpd	ymm2, ymm14, ymm2		;; .623(r3-r13)						;	55-59

	vsubpd	ymm4, ymm4, ymm11		;; B8 = B8 - R8						; 56-58
	vmovapd	ymm11, YMM_TMPS[6*32]		;; r5-r11
	ystore	YMM_TMPS[15*32], ymm3		;; Save I7+I9						; 56
	vmulpd	ymm3, ymm14, ymm11		;; .623(r5-r11)						;	56-60

	vaddpd	ymm7, ymm7, ymm8		;; A8 = A8 + I8						; 57-59
	vmulpd	ymm8, ymm15, ymm11		;; .901(r5-r11)						;	57-61
	ystore	YMM_TMPS[11*32], ymm5		;; Save I7-I9						; 57

	vaddpd	ymm12, ymm6, ymm12		;; r1+.901(r3-r13)-r15					; 58-60
	vmulpd	ymm5, ymm13, ymm11		;; .223(r5-r11)						;	58-62
	L1prefetchw srcreg+L1pd, L1pt

	vaddpd	ymm1, ymm1, ymm11		;; r1-(r3-r13)+(r5-r11)-r15				; 59-61
	vmovapd	ymm11, [screg+6*64]		;; sine
	vmulpd	ymm4, ymm4, ymm11		;; I8 = B8 * sine					;	59-63

	vaddpd	ymm10, ymm6, ymm10		;; r1+.223(r3-r13)-r15					; 60-62
	vmulpd	ymm7, ymm7, ymm11		;; R8 = A8 * sine					;	60-64

	vsubpd	ymm6, ymm6, ymm2		;; r1-.623(r3-r13)-r15					; 61-63
	vmulpd	ymm2, ymm13, ymm9		;; .223(r7-r9)						;	61-65

	vsubpd	ymm1, ymm1, ymm9		;; r1-(r3-r13)+(r5-r11)-(r7-r9)-r15			; 62-66
	vmulpd	ymm11, ymm14, ymm9		;; .623(r7-r9)						;	62-66
	L1prefetchw srcreg+d1+L1pd, L1pt

	vaddpd	ymm12, ymm12, ymm3		;; r1+.901(r3-r13)+.623(r5-r11)-r15			; 63-65
	vmulpd	ymm9, ymm15, ymm9		;; .901(r7-r9)						;	63-67
	vmovapd	ymm3, YMM_TMPS[12*32]		;; i2+i14

	vsubpd	ymm10, ymm10, ymm8		;; r1+.223(r3-r13)-.901(r5-r11)-r15			; 64-66
	vmovapd	ymm8, YMM_TMPS[21*32]		;; i4+i12

	vsubpd	ymm6, ymm6, ymm5		;; r1-.623(r3-r13)-.223(r5-r11)-r15			; 65-67
	vmulpd	ymm5, ymm13, ymm3		;; .223(i2+i14)						;	65-69
	ystore	YMM_TMPS[12*32], ymm7		;; Save R8						; 65

	vaddpd	ymm12, ymm12, ymm2		;; r1+.901(r3-r13)+.623(r5-r11)+.223(r7-r9)-r15		; 66-68
	vmulpd	ymm2, ymm14, ymm3		;; .623(i2+i14)						;	66-70
	vmovapd	ymm7, YMM_TMPS[13*32]		;; i3+i13

	vsubpd	ymm10, ymm10, ymm11		;; r1+.223(r3-r13)-.901(r5-r11)-.623(r7-r9)-r15		; 67-69

	yloop_optional_early_prefetch

	vaddpd	ymm6, ymm6, ymm9		;; r1-.623(r3-r13)-.223(r5-r11)+.901(r7-r9)-r15		; 68-70
	vmulpd	ymm9, ymm15, ymm3		;; .901(i2+i14)						;	68-72

	;; Calculate even columns derived from imaginary inputs (even rows)

	vsubpd	ymm3, ymm3, ymm4		;; (i2+i14)-i8						; 69-71
	vmulpd	ymm11, ymm14, ymm8		;; .623(i4+i12)						;	69-73
	ystore	YMM_TMPS[2*32], ymm12		;; Save odd-real-#2					; 69

	vaddpd	ymm5, ymm5, ymm4		;; .223(i2+i14)+i8					; 70-74
	vmulpd	ymm12, ymm15, ymm8		;; .901(i4+i12)						;	70-74
	ystore	YMM_TMPS[6*32], ymm10		;; Save odd-real-#4					; 70

	vsubpd	ymm2, ymm2, ymm4		;; .623(i2+i14)-i8					; 71-73
	ystore	YMM_TMPS[21*32], ymm6		;; Save odd-real-#6					; 71
	vmulpd	ymm6, ymm13, ymm8		;; .223(i4+i12)						;	71-75

	vsubpd	ymm3, ymm3, ymm8		;; (i2+i14)-(i4+i12)-i8					; 72-74
	vmulpd	ymm8, ymm15, ymm0		;; .901(i6+i10)						;	72-76

	vaddpd	ymm9, ymm9, ymm4		;; .901(i2+i14)+i8					; 73-75
	vmulpd	ymm4, ymm13, ymm0		;; .223(i6+i10)						;	73-77
	vbroadcastsd ymm13, Q YMM_P434

	vaddpd	ymm5, ymm5, ymm11		;; .223(i2+i14)+.623(i4+i12)+i8				; 74-76
	vmulpd	ymm11, ymm14, ymm0		;; .623(i6+i10)						;	74-78
	vbroadcastsd ymm15, Q YMM_P975

	vaddpd	ymm3, ymm3, ymm0		;; (i2+i14)-(i4+i12)+(i6+i10)-i8			; 75-77
	vmulpd	ymm0, ymm13, ymm7		;; .434(i3+i13)						;	75-79
	vbroadcastsd ymm14, Q YMM_P782

	vaddpd	ymm2, ymm2, ymm12		;; .623(i2+i14)+.901(i4+i12)-i8				; 76-78
	vmulpd	ymm12, ymm15, ymm7		;; .975(i3+i13)						;	76-80
	vmovapd	ymm10, YMM_TMPS[19*32]		;; i5+i11

	vsubpd	ymm9, ymm9, ymm6		;; .901(i2+i14)-.223(i4+i12)+i8				; 77-79
	vmulpd	ymm7, ymm14, ymm7		;; .782(i3+i13)						;	77-81
	vmovapd	ymm6, YMM_TMPS[15*32]		;; i7+i9

	vaddpd	ymm5, ymm5, ymm8		;; .223(i2+i14)+.623(i4+i12)+.901(i6+i10)+i8		; 78-80
	vmulpd	ymm8, ymm14, ymm10		;; .782(i5+i11)						;	78-82

	vsubpd	ymm2, ymm2, ymm4		;; .623(i2+i14)+.901(i4+i12)-.223(i6+i10)-i8		; 79-81
	vmulpd	ymm4, ymm13, ymm10		;; .434(i5+i11)						;	79-83
	L1prefetchw srcreg+d2+L1pd, L1pt

	vsubpd	ymm9, ymm9, ymm11		;; .901(i2+i14)-.223(i4+i12)-.623(i6+i10)+i8		; 80-82
	vmulpd	ymm10, ymm15, ymm10		;; .975(i5+i11)						;	80-84

	;; Combine real and imaginary data for row #8

	vsubpd	ymm11, ymm1, ymm3		;; final R22						; 81-83
	ystore	[srcreg+3*d2+d1+32], ymm11	;; Save R22						; 84
	vmulpd	ymm11, ymm15, ymm6		;; .975(i7+i9)						;	81-85

	vaddpd	ymm1, ymm1, ymm3		;; final R8						; 82-84
	vmulpd	ymm3, ymm14, ymm6		;; .782(i7+i9)						;	82-86

	;; Calculate odd columns derived from imaginary inputs (even rows)

	vaddpd	ymm0, ymm0, ymm8		;; .434(i3+i13)+.782(i5+i11)				; 83-85
	vmulpd	ymm6, ymm13, ymm6		;; .434(i7+i9)						;	83-87
	vmovapd	ymm8, YMM_TMPS[0*32]		;; r2-r14

	vaddpd	ymm12, ymm12, ymm4		;; .975(i3+i13)+.434(i5+i11)				; 84-86
	vmovapd	ymm4, YMM_TMPS[4*32]		;; r4-r12

	vsubpd	ymm7, ymm7, ymm10		;; .782(i3+i13)-.975(i5+i11)				; 85-87
	ystore	[srcreg+3*d2+d1], ymm1		;; Save R8						; 85

	vaddpd	ymm0, ymm0, ymm11		;; .434(i3+i13)+.782(i5+i11)+.975(i7+i9)		; 86-88
	L1prefetchw srcreg+d2+d1+L1pd, L1pt

	vsubpd	ymm12, ymm12, ymm3		;; .975(i3+i13)+.434(i5+i11)-.782(i7+i9)		; 87-89
	vmulpd	ymm11, ymm15, ymm8		;; .975(r2-r14)						;	87-91

	vaddpd	ymm7, ymm7, ymm6		;; .782(i3+i13)-.975(i5+i11)+.434(i7+i9)		; 88-90
	vmulpd	ymm6, ymm14, ymm8		;; .782(r2-r14)						;	88-92

	;; Combine even and odd columns, then real and imag data (even rows)

	vsubpd	ymm3, ymm5, ymm0		;; imag-cols row #14 (even#2 - odd#2)			; 89-91
	vmulpd	ymm8, ymm13, ymm8		;; .434(r2-r14)						;	89-93

	vaddpd	ymm5, ymm5, ymm0		;; imag-cols row #2 (even#2 + odd#2)			; 90-92
	vmulpd	ymm1, ymm14, ymm4		;; .782(r4-r12)						;	90-94
	L1prefetchw srcreg+2*d2+L1pd, L1pt

	vsubpd	ymm0, ymm2, ymm12		;; imag-cols row #12 (even#4 - odd#4)			; 91-93
	vmulpd	ymm10, ymm13, ymm4		;; .434(r4-r12)						;	91-95

	vaddpd	ymm2, ymm2, ymm12		;; imag-cols row #4 (even#4 + odd#4)			; 92-94
	vmulpd	ymm4, ymm15, ymm4		;; .975(r4-r12)						;	92-96

	vsubpd	ymm12, ymm9, ymm7		;; imag-cols row #10 (even#6 - odd#6)			; 93-95

	vaddpd	ymm9, ymm9, ymm7		;; imag-cols row #6 (even#6 + odd#6)			; 94-96
	vmovapd	ymm7, YMM_TMPS[8*32]		;; r6-r10

	;; Calculate even columns derived from real inputs (even rows)

	vaddpd	ymm11, ymm11, ymm1		;; .975(r2-r14)+.782(r4-r12)				; 95-97
	vmulpd	ymm1, ymm13, ymm7		;; .434(r6-r10)						;	93-97

	vsubpd	ymm6, ymm6, ymm10		;; .782(r2-r14)-.434(r4-r12)				; 96-98
	vmulpd	ymm10, ymm15, ymm7		;; .975(r6-r10)						;	94-98
	vmulpd	ymm7, ymm14, ymm7		;; .782(r6-r10)						;	95-99

	vsubpd	ymm8, ymm8, ymm4		;; .434(r2-r14)-.975(r4-r12)				; 97-99
	vmovapd	ymm4, YMM_TMPS[2*32]		;; Load odd-real-#2

	vaddpd	ymm11, ymm11, ymm1		;; .975(r2-r14)+.782(r4-r12)+.434(r6-r10)		; 98-100
	vmovapd	ymm1, YMM_TMPS[6*32]		;; Load odd-real-#4

	vsubpd	ymm6, ymm6, ymm10		;; .782(r2-r14)-.434(r4-r12)-.975(r6-r10)		; 99-101
	vmovapd	ymm10, YMM_TMPS[21*32]		;; Load odd-real-#6

	vaddpd	ymm8, ymm8, ymm7		;; .434(r2-r14)-.975(r4-r12)+.782(r6-r10)		; 100-102
	L1prefetchw srcreg+2*d2+d1+L1pd, L1pt

	;; Combine even and odd columns (even rows)

	vsubpd	ymm7, ymm4, ymm11		;; real-cols row #14 (odd#2 - even#2)			; 101-103

	vaddpd	ymm4, ymm4, ymm11		;; real-cols row #2 (odd#2 + even#2)			; 102-104

	vsubpd	ymm11, ymm1, ymm6		;; real-cols row #12 (odd#4 - even#4)			; 103-105

	vaddpd	ymm1, ymm1, ymm6		;; real-cols row #4 (odd#4 + even#4)			; 104-106

	vsubpd	ymm6, ymm10, ymm8		;; real-cols row #10 (odd#6 - even#6)			; 105-107
	L1prefetchw srcreg+3*d2+L1pd, L1pt

	vaddpd	ymm10, ymm10, ymm8		;; real-cols row #6 (odd#6 + even#6)			; 106-108

	;; Combine real and imag data (even rows)

	vsubpd	ymm8, ymm7, ymm3		;; final R16						; 107-109

	vaddpd	ymm7, ymm7, ymm3		;; final R14						; 108-110
	vmovapd	ymm3, YMM_TMPS[1*32]		;; i2-i14
	ystore	[srcreg+d1+32], ymm8		;; Save R16						; 110
	vmulpd	ymm8, ymm13, ymm3		;; .434(i2-i14)						;	108-112

	ystore	[srcreg+6*d2+d1], ymm7		;; Save R14						; 111
	vsubpd	ymm7, ymm4, ymm5		;; final R28						; 109-111
	ystore	[srcreg+6*d2+d1+32], ymm7	;; Save R28						; 112
	vmulpd	ymm7, ymm14, ymm3		;; .782(i2-i14)						;	109-113

	vaddpd	ymm4, ymm4, ymm5		;; final R2						; 110-112
	vmulpd	ymm3, ymm15, ymm3		;; .975(i2-i14)						;	110-114

	vsubpd	ymm5, ymm11, ymm0		;; final R18						; 111-113
	ystore	[srcreg+d1], ymm4		;; Save R2						; 113
	vmovapd	ymm4, YMM_TMPS[5*32]		;; i4-i12
	ystore	[srcreg+d2+d1+32], ymm5		;; Save R18						; 114
	vmulpd	ymm5, ymm15, ymm4		;; .975(i4-i12)						;	111-115

	vaddpd	ymm11, ymm11, ymm0		;; final R12						; 112-114
	vmulpd	ymm0, ymm13, ymm4		;; .434(i4-i12)						;	112-116

	ystore	[srcreg+5*d2+d1], ymm11		;; Save R12						; 115
	vsubpd	ymm11, ymm1, ymm2		;; final R26						; 113-115
	vmulpd	ymm4, ymm14, ymm4		;; .782(i4-i12)						;	113-117

	vaddpd	ymm1, ymm1, ymm2		;; final R4						; 114-116
	vmovapd	ymm2, YMM_TMPS[9*32]		;; i6-i10
	ystore	[srcreg+5*d2+d1+32], ymm11	;; Save R26						; 116
	vmulpd	ymm11, ymm14, ymm2		;; .782(i6-i10)						;	114-118

	ystore	[srcreg+d2+d1], ymm1		;; Save R4						; 117
	vsubpd	ymm1, ymm6, ymm12		;; final R20						; 115-117
	ystore	[srcreg+2*d2+d1+32], ymm1	;; Save R20						; 118
	vmulpd	ymm1, ymm15, ymm2		;; .975(i6-i10)						;	115-119

	vaddpd	ymm6, ymm6, ymm12		;; final R10						; 116-118
	vmulpd	ymm2, ymm13, ymm2		;; .434(i6-i10)						;	116-120

	vsubpd	ymm12, ymm10, ymm9		;; final R24						; 117-119
	ystore	[srcreg+4*d2+d1], ymm6		;; Save R10						; 119
	vmovapd	ymm6, YMM_TMPS[3*32]		;; i3-i13
	ystore	[srcreg+4*d2+d1+32], ymm12	;; Save R24						; 120
	vmulpd	ymm12, ymm14, ymm6		;; .782(i3-i13)						;	117-121

	vaddpd	ymm10, ymm10, ymm9		;; final R6						; 118-120
	vmulpd	ymm9, ymm15, ymm6		;; .975(i3-i13)						;	118-122

	;; Calculate even columns derived from imaginary inputs (odd rows)

	vaddpd	ymm8, ymm8, ymm5		;; .434(i2-i14)+.975(i4-i12)				; 119-121
	vmulpd	ymm6, ymm13, ymm6		;; .434(i3-i13)						;	119-123
	vmovapd	ymm5, YMM_TMPS[7*32]		;; i5-i11

	vaddpd	ymm7, ymm7, ymm0		;; .782(i2-i14)+.434(i4-i12)				; 120-122
	vmulpd	ymm0, ymm15, ymm5		;; .975(i5-i11)						;	120-124

	vsubpd	ymm3, ymm3, ymm4		;; .975(i2-i14)-.782(i4-i12)				; 121-123
	vmulpd	ymm4, ymm13, ymm5		;; .434(i5-i11)						;	121-125
	ystore	[srcreg+2*d2+d1], ymm10		;; Save R6						; 121
	vmovapd	ymm10, YMM_TMPS[11*32]		;; i7-i9

	vaddpd	ymm8, ymm8, ymm11		;; .434(i2-i14)+.975(i4-i12)+.782(i6-i10)		; 122-124
	vmulpd	ymm5, ymm14, ymm5		;; .782(i5-i11)						;	122-126
	vmovapd	ymm11, YMM_TMPS[10*32]		;; r3+r13

	vsubpd	ymm7, ymm7, ymm1		;; .782(i2-i14)+.434(i4-i12)-.975(i6-i10)		; 123-125
	vmulpd	ymm1, ymm13, ymm10		;; .434(i7-i9)						;	123-127

	vaddpd	ymm3, ymm3, ymm2		;; .975(i2-i14)-.782(i4-i12)+.434(i6-i10)		; 124-126
	vmulpd	ymm2, ymm14, ymm10		;; .782(i7-i9)						;	124-128
	vbroadcastsd ymm14, Q YMM_P623

	;; Calculate odd columns derived from imaginary inputs (odd rows)

	vaddpd	ymm12, ymm12, ymm0		;; .782(i3-i13)+.975(i5-i11)				; 125-127
	vmulpd	ymm10, ymm15, ymm10		;; .975(i7-i9)						;	125-129
	vbroadcastsd ymm13, Q YMM_P223

	vsubpd	ymm9, ymm9, ymm4		;; .975(i3-i13)-.434(i5-i11)				; 126-128
	vbroadcastsd ymm15, Q YMM_P901

	vsubpd	ymm6, ymm6, ymm5		;; .434(i3-i13)-.782(i5-i11)				; 127-129
	vmovapd	ymm0, [srcreg]			;; r1+r15

	vaddpd	ymm12, ymm12, ymm1		;; .782(i3-i13)+.975(i5-i11)+.434(i7-i9)		; 128-130
	vmovapd	ymm4, YMM_TMPS[18*32]		;; r5+r11

	vsubpd	ymm9, ymm9, ymm2		;; .975(i3-i13)-.434(i5-i11)-.782(i7-i9)		; 129-131
	vmovapd	ymm5, YMM_TMPS[14*32]		;; r7+r9

	vaddpd	ymm6, ymm6, ymm10		;; .434(i3-i13)-.782(i5-i11)+.975(i7-i9)		; 130-132

	;; Combine even and odd columns, then real and imag data (odd rows)

	vsubpd	ymm10, ymm8, ymm12		;; imag-cols row #13 (even#3 - odd#3)			; 131-133
	L1prefetchw srcreg+3*d2+d1+L1pd, L1pt

	vaddpd	ymm8, ymm8, ymm12		;; imag-cols row #3 (even#3 + odd#3)			; 132-134

	vsubpd	ymm12, ymm7, ymm9		;; imag-cols row #11 (even#5 - odd#5)			; 133-135
	vmulpd	ymm2, ymm14, ymm11		;; .623(r3+r13)						;	133-137

	vaddpd	ymm7, ymm7, ymm9		;; imag-cols row #5 (even#5 + odd#5)			; 134-136
	vmulpd	ymm9, ymm13, ymm11		;; .223(r3+r13)						;	134-138
	ystore	YMM_TMPS[11*32], ymm10		;; Save imag-cols row #13				; 134

	vsubpd	ymm1, ymm3, ymm6		;; imag-cols row #9 (even#7 - odd#7)			; 135-137
	vmovapd	ymm10, YMM_TMPS[17*32]		;; r2+r14
	ystore	YMM_TMPS[1*32], ymm8		;; Save imag-cols row #3				; 135

	vaddpd	ymm3, ymm3, ymm6		;; imag-cols row #7 (even#7 + odd#7)			; 136-138
	vmulpd	ymm6, ymm15, ymm11		;; .901(r3+r13)						;	136-140
	vmovapd	ymm8, YMM_TMPS[12*32]		;; r8
	ystore	YMM_TMPS[9*32], ymm12		;; Save imag-cols row #11				; 136

	;; Calculate odd columns derived from real inputs (odd rows)

	vaddpd	ymm11, ymm0, ymm11		;; r1+(r3+r13)+r15					; 137-139
	vmulpd	ymm12, ymm13, ymm4		;; .223(r5+r11)						;	137-141
	ystore	YMM_TMPS[3*32], ymm7		;; Save imag-cols row #5				; 137

	vaddpd	ymm2, ymm0, ymm2		;; r1+.623(r3+r13)+r15					; 138-140
	L1prefetchw srcreg+4*d2+L1pd, L1pt

	vsubpd	ymm9, ymm0, ymm9		;; r1-.223(r3+r13)+r15					; 139-141
	vmulpd	ymm7, ymm15, ymm4		;; .901(r5+r11)						;	139-143

	vaddpd	ymm11, ymm11, ymm4		;; r1+(r3+r13)+(r5+r11)+r15				; 140-142
	vmulpd	ymm4, ymm14, ymm4		;; .623(r5+r11)						;	140-144

	vsubpd	ymm0, ymm0, ymm6		;; r1-.901(r3+r13)+r15					; 141-143
	vmulpd	ymm6, ymm15, ymm5		;; .901(r7+r9)						;	141-145

	vsubpd	ymm2, ymm2, ymm12		;; r1+.623(r3+r13)-.223(r5+r11)+r15			; 142-144
	vmulpd	ymm12, ymm14, ymm5		;; .623(r7+r9)						;	142-146
	L1prefetchw srcreg+4*d2+d1+L1pd, L1pt

	vaddpd	ymm11, ymm11, ymm5		;; r1+(r3+r13)+(r5+r11)+(r7+r9)+r15			; 143-145
	vmulpd	ymm5, ymm13, ymm5		;; .223(r7+r9)						;	143-147

	vsubpd	ymm9, ymm9, ymm7		;; r1-.223(r3+r13)-.901(r5+r11)+r15			; 144-146
	vmovapd	ymm7, YMM_TMPS[20*32]		;; r4+r12

	vaddpd	ymm0, ymm0, ymm4		;; r1-.901(r3+r13)+.623(r5+r11)+r15			; 145-147
	vmulpd	ymm4, ymm15, ymm10		;; .901(r2+r14)						;	145-149

	vsubpd	ymm2, ymm2, ymm6		;; r1+.623(r3+r13)-.223(r5+r11)-.901(r7+r9)+r15		; 146-148
	vmulpd	ymm6, ymm14, ymm10		;; .623(r2+r14)						;	146-150

	vaddpd	ymm9, ymm9, ymm12		;; r1-.223(r3+r13)-.901(r5+r11)+.623(r7+r9)+r15		; 147-149
	L1prefetchw srcreg+5*d2+L1pd, L1pt

	vsubpd	ymm0, ymm0, ymm5		;; r1-.901(r3+r13)+.623(r5+r11)-.223(r7+r9)+r15		; 148-150
	vmulpd	ymm5, ymm13, ymm10		;; .223(r2+r14)						;	148-152

	;; Calculate even columns derived from real inputs (odd rows)

	vaddpd	ymm10, ymm10, ymm8		;; (r2+r14)+r8						; 149-151
	vmulpd	ymm12, ymm13, ymm7		;; .223(r4+r12)						;	149-153

	vsubpd	ymm4, ymm4, ymm8		;; .901(r2+r14)-r8					; 150-152

	vaddpd	ymm6, ymm6, ymm8		;; .623(r2+r14)+r8					; 151-153

	vaddpd	ymm10, ymm10, ymm7		;; (r2+r14)+(r4+r12)+r8					; 152-154
	L1prefetchw srcreg+5*d2+d1+L1pd, L1pt

	vsubpd	ymm5, ymm5, ymm8		;; .223(r2+r14)-r8					; 153-155
	vmulpd	ymm8, ymm15, ymm7		;; .901(r4+r12)						;	151-155
	vmulpd	ymm7, ymm14, ymm7		;; .623(r4+r12)						;	152-156

	vaddpd	ymm4, ymm4, ymm12		;; .901(r2+r14)+.223(r4+r12)-r8				; 154-156
	vmovapd	ymm12, YMM_TMPS[16*32]		;; r6+r10

	vaddpd	ymm10, ymm10, ymm12		;; (r2+r14)+(r4+r12)+(r6+r10)+r8			; 155-157
	vsubpd	ymm6, ymm6, ymm8		;; .623(r2+r14)-.901(r4+r12)+r8				; 156-158
	vmulpd	ymm8, ymm14, ymm12		;; .623(r6+r10)						;	153-157

	vsubpd	ymm5, ymm5, ymm7		;; .223(r2+r14)-.623(r4+r12)-r8				; 157-159
	vmulpd	ymm7, ymm13, ymm12		;; .223(r6+r10)						;	154-158
	vmulpd	ymm12, ymm15, ymm12		;; .901(r6+r10)						;	155-159

	vsubpd	ymm4, ymm4, ymm8		;; .901(r2+r14)+.223(r4+r12)-.623(r6+r10)-r8		; 158-160
	vmovapd	ymm8, YMM_TMPS[1*32]		;; Load imag-cols row #3

	vsubpd	ymm6, ymm6, ymm7		;; .623(r2+r14)-.901(r4+r12)-.223(r6+r10)+r8		; 159-161
	vmovapd	ymm7, YMM_TMPS[11*32]		;; Load imag-cols row #13

	vaddpd	ymm5, ymm5, ymm12		;; .223(r2+r14)-.623(r4+r12)+.901(r6+r10)-r8		; 160-162
	L1prefetchw srcreg+6*d2+L1pd, L1pt

	;; Combine even and odd columns (odd rows)

	vaddpd	ymm12, ymm11, ymm10		;; real-cols row #1 (and final R1)			; 161-163
	vsubpd	ymm11, ymm11, ymm10		;; real-cols row #15 (and final R15)			; 162-164

	vaddpd	ymm10, ymm2, ymm4		;; real-cols row #3					; 163-165
	vsubpd	ymm2, ymm2, ymm4		;; real-cols row #13					; 164-166
	ystore	[srcreg], ymm12			;; Save final R1					; 164
	vmovapd	ymm12, YMM_TMPS[3*32]		;; Load imag-cols row #5

	vaddpd	ymm4, ymm9, ymm6		;; real-cols row #5					; 165-167
	ystore	[srcreg+32], ymm11		;; Save final R15					; 165
	vmovapd	ymm11, YMM_TMPS[9*32]		;; Load imag-cols row #11
	vsubpd	ymm9, ymm9, ymm6 		;; real-cols row #11					; 166-168

	vaddpd	ymm6, ymm0, ymm5		;; real-cols row #7					; 167-169
	vsubpd	ymm0, ymm0, ymm5		;; real-cols row #9					; 168-170
	L1prefetchw srcreg+6*d2+d1+L1pd, L1pt

	;; Combine real and imag data (odd rows)

	vsubpd	ymm5, ymm10, ymm8		;; final R27						; 169-171
	vaddpd	ymm10, ymm10, ymm8		;; final R3						; 170-172

	vsubpd	ymm8, ymm2, ymm7		;; final R17						; 171-173
	vaddpd	ymm2, ymm2, ymm7		;; final R13						; 172-174
	ystore	[srcreg+6*d2+32], ymm5		;; Save R27						; 172
	ystore	[srcreg+d2], ymm10		;; Save R3						; 173

	vsubpd	ymm7, ymm4, ymm12		;; final R25						; 173-175
	vaddpd	ymm4, ymm4, ymm12		;; final R5						; 174-176
	ystore	[srcreg+d2+32], ymm8		;; Save R17						; 174
	ystore	[srcreg+6*d2], ymm2		;; Save R13						; 175

	vsubpd	ymm12, ymm9, ymm11		;; final R19						; 175-177
	vaddpd	ymm9, ymm9, ymm11		;; final R11						; 176-178
	ystore	[srcreg+5*d2+32], ymm7		;; Save R25						; 176
	ystore	[srcreg+2*d2], ymm4		;; Save R5						; 177

	vsubpd	ymm11, ymm6, ymm3		;; final R23						; 177-179
	vaddpd	ymm6, ymm6, ymm3		;; final R7						; 178-180
	ystore	[srcreg+2*d2+32], ymm12		;; Save R19						; 178
	ystore	[srcreg+5*d2], ymm9		;; Save R11						; 179

	vsubpd	ymm3, ymm0, ymm1		;; final R21						; 179-181
	vaddpd	ymm0, ymm0, ymm1		;; final R9						; 180-182
	ystore	[srcreg+4*d2+32], ymm11		;; Save R23						; 180
	ystore	[srcreg+3*d2], ymm6		;; Save R7						; 181
	ystore	[srcreg+3*d2+32], ymm3		;; Save R21						; 182
	ystore	[srcreg+4*d2], ymm0		;; Save R9						; 183

	bump	srcreg, srcinc
	bump	screg, scinc
	ENDM

;; Haswell FMA3 version

IF (@INSTR(,%yarch,<FMA3>) NE 0)

yr7_14cl_28_reals_unfft_preload MACRO
	vmovapd ymm15, YMM_ONE
	ENDM

;; Timed at 114.5 clocks.  Converting more adds and subs to FMA3 might help.
yr7_14cl_28_reals_unfft MACRO srcreg,srcinc,d1,d2,screg,scinc,maxrpt,L1pt,L1pd

	vmovapd	ymm3, [screg+3*64+32]		;; cosine/sine for R5/I5
	vmovapd	ymm2, [srcreg+2*d2]		;; R5
	vmovapd	ymm1, [srcreg+2*d2+32]		;; I5
	yfmaddpd ymm0, ymm2, ymm3, ymm1		;; A5 = R5 * cosine/sine + I5				; 1-5		n 7
	yfmsubpd ymm1, ymm1, ymm3, ymm2		;; B5 = I5 * cosine/sine - R5				; 1-5		n 7

	vmovapd	ymm5, [screg+9*64+32]		;; cosine/sine for R11/I11
	vmovapd	ymm4, [srcreg+5*d2]		;; R11
	vmovapd	ymm3, [srcreg+5*d2+32]		;; I11
	yfmaddpd ymm2, ymm4, ymm5, ymm3		;; A11 = R11 * cosine/sine + I11			; 2-6		n 8
	yfmsubpd ymm3, ymm3, ymm5, ymm4		;; B11 = I11 * cosine/sine - R11			; 2-6		n 8

	vmovapd	ymm7, [screg+4*64+32]		;; cosine/sine for R6/I6
	vmovapd	ymm6, [srcreg+2*d2+d1]		;; R6
	vmovapd	ymm5, [srcreg+2*d2+d1+32]	;; I6
	yfmaddpd ymm4, ymm6, ymm7, ymm5		;; A6 = R6 * cosine/sine + I6				; 3-7		n 9
	yfmsubpd ymm5, ymm5, ymm7, ymm6		;; B6 = I6 * cosine/sine - R6				; 3-7		n 9

	vmovapd	ymm9, [screg+8*64+32]		;; cosine/sine for R10/I10
	vmovapd	ymm8, [srcreg+4*d2+d1]		;; R10
	vmovapd	ymm7, [srcreg+4*d2+d1+32]	;; I10
	yfmaddpd ymm6, ymm8, ymm9, ymm7		;; A10 = R10 * cosine/sine + I10			; 4-8		n 10
	yfmsubpd ymm7, ymm7, ymm9, ymm8		;; B10 = I10 * cosine/sine - R10			; 4-8		n 10

	vmovapd	ymm11, [screg+5*64+32]		;; cosine/sine for R7/I7
	vmovapd	ymm10, [srcreg+3*d2]		;; R7
	vmovapd	ymm9, [srcreg+3*d2+32]		;; I7
	yfmaddpd ymm8, ymm10, ymm11, ymm9	;; A7 = R7 * cosine/sine + I7				; 5-9		n 11
	yfmsubpd ymm9, ymm9, ymm11, ymm10	;; B7 = I7 * cosine/sine - R7				; 5-9		n 11

	vmovapd	ymm13, [screg+7*64+32]		;; cosine/sine for R9/I9
	vmovapd	ymm12, [srcreg+4*d2]		;; R9
	vmovapd	ymm11, [srcreg+4*d2+32]		;; I9
	yfmaddpd ymm10, ymm12, ymm13, ymm11	;; A9 = R9 * cosine/sine + I9				; 6-10		n 12
	yfmsubpd ymm11, ymm11, ymm13, ymm12	;; B9 = I9 * cosine/sine - R9				; 6-10		n 12

	vmovapd	ymm12, [screg+3*64]		;; sine for R5/I5
	vmulpd	ymm0, ymm0, ymm12		;; R5 = A5 * sine					; 7-11		n 13
	vmulpd	ymm1, ymm1, ymm12		;; I5 = B5 * sine					; 7-11		n 14

	vmovapd	ymm12, [screg+9*64]		;; sine for R11/I11
	vmulpd	ymm2, ymm2, ymm12		;; R11 = A11 * sine					; 8-12		n 13
	vmulpd	ymm3, ymm3, ymm12		;; I11 = B11 * sine					; 8-12		n 14

	vmovapd	ymm12, [screg+4*64]		;; sine for R6/I6
	vmulpd	ymm4, ymm4, ymm12		;; R6 = A6 * sine					; 9-13		n 15
	vmulpd	ymm5, ymm5, ymm12		;; I6 = B6 * sine					; 9-13		n 16

	vmovapd	ymm12, [screg+8*64]		;; sine for R10/I10
	vmulpd	ymm6, ymm6, ymm12		;; R10 = A10 * sine					; 10-14		n 15
	vmulpd	ymm7, ymm7, ymm12		;; I10 = B10 * sine					; 10-14		n 16

	vmovapd	ymm12, [screg+5*64]		;; sine for R7/I7
	vmulpd	ymm8, ymm8, ymm12		;; R7 = A7 * sine					; 11-15		n 17
	vmulpd	ymm9, ymm9, ymm12		;; I7 = B7 * sine					; 11-15		n 18

	vmovapd	ymm12, [screg+7*64]		;; sine for R9/I9
	vmulpd	ymm10, ymm10, ymm12		;; R9 = A9 * sine					; 12-16		n 17
	vmulpd	ymm11, ymm11, ymm12		;; I9 = B9 * sine					; 12-16		n 18

	yfmaddpd ymm12, ymm0, ymm15, ymm2	;; R5+R11						; 13-17
	yfmsubpd ymm0, ymm0, ymm15, ymm2	;; R5-R11						; 13-17
	vmovapd	ymm13, [screg+6*64+32]		;; cosine/sine for R8/I8

	yfmaddpd ymm2, ymm1, ymm15, ymm3	;; I5+I11						; 14-18
	yfmsubpd ymm1, ymm1, ymm15, ymm3	;; I5-I11						; 14-18
	vmovapd	ymm14, [srcreg+3*d2+d1]		;; R8

	yfmaddpd ymm3, ymm4, ymm15, ymm6	;; R6+R10						; 15-19
	yfmsubpd ymm4, ymm4, ymm15, ymm6	;; R6-R10						; 15-19
	L1prefetchw srcreg+L1pd, L1pt

	yfmaddpd ymm6, ymm5, ymm15, ymm7	;; I6+I10						; 16-20
	yfmsubpd ymm5, ymm5, ymm15, ymm7	;; I6-I10						; 16-20
	L1prefetchw srcreg+d1+L1pd, L1pt

	yfmaddpd ymm7, ymm8, ymm15, ymm10	;; R7+R9						; 17-21
	yfmsubpd ymm8, ymm8, ymm15, ymm10	;; R7-R9						; 17-21
	L1prefetchw srcreg+d2+L1pd, L1pt

	yfmaddpd ymm10, ymm9, ymm15, ymm11	;; I7+I9						; 18-22
	yfmsubpd ymm9, ymm9, ymm15, ymm11	;; I7-I9						; 18-22

	vmovapd	ymm11, [srcreg+3*d2+d1+32]	;; I8
	ystore	YMM_TMPS[0*32], ymm12		;; Save R5+R11						; 18
	yfmaddpd ymm12, ymm14, ymm13, ymm11	;; A8 = R8 * cosine/sine + I8				; 19-23		n 24
	yfmsubpd ymm11, ymm11, ymm13, ymm14	;; B8 = I8 * cosine/sine - R8				; 19-23		n 24

	vmovapd	ymm13, [screg+64+32]		;; cosine/sine for R3/I3
	vmovapd	ymm14, [srcreg+d2+32]		;; I3
	ystore	YMM_TMPS[1*32], ymm0		;; Save R5-R11						; 18+1
	vmovapd	ymm0, [srcreg+d2]		;; R3
	ystore	YMM_TMPS[2*32], ymm2		;; Save I5+I11						; 19+1
	yfmsubpd ymm2, ymm14, ymm13, ymm0	;; B3 = I3 * cosine/sine - R3				; 20-24		n 27
	yfmaddpd ymm0, ymm0, ymm13, ymm14	;; A3 = R3 * cosine/sine + I3				; 20-24		n 27

	vmovapd	ymm13, [screg+11*64+32]		;; cosine/sine for R13/I13
	vmovapd	ymm14, [srcreg+6*d2+32]		;; I13
	ystore	YMM_TMPS[3*32], ymm1		;; Save I5-I11						; 19+2
	vmovapd	ymm1, [srcreg+6*d2]		;; R13
	ystore	YMM_TMPS[4*32], ymm3		;; Save R6+R10						; 20+2
	yfmsubpd ymm3, ymm14, ymm13, ymm1	;; B13 = I13 * cosine/sine - R13			; 21-25		n 28
	yfmaddpd ymm1, ymm1, ymm13, ymm14	;; A13 = R13 * cosine/sine + I13			; 21-25		n 28

	vmovapd	ymm13, [screg+32]		;; cosine/sine for R2/I2
	vmovapd	ymm14, [srcreg+d1+32]		;; I2
	ystore	YMM_TMPS[5*32], ymm4		;; Save R6-R10						; 20+3
	vmovapd	ymm4, [srcreg+d1]		;; R2
	ystore	YMM_TMPS[6*32], ymm6		;; Save I6+I10						; 21+3
	yfmsubpd ymm6, ymm14, ymm13, ymm4	;; B2 = I2 * cosine/sine - R2				; 22-26		n 29
	yfmaddpd ymm4, ymm4, ymm13, ymm14	;; A2 = R2 * cosine/sine + I2				; 22-26		n 29

	vmovapd	ymm13, [screg+12*64+32]		;; cosine/sine for R14/I14
	vmovapd	ymm14, [srcreg+6*d2+d1+32]	;; I14
	ystore	YMM_TMPS[7*32], ymm5		;; Save I6-I10						; 21+4
	vmovapd	ymm5, [srcreg+6*d2+d1]		;; R14
	ystore	YMM_TMPS[8*32], ymm7		;; Save R7+R9						; 22+4
	yfmsubpd ymm7, ymm14, ymm13, ymm5	;; B14 = I14 * cosine/sine - R14			; 23-27		n 30
	yfmaddpd ymm5, ymm5, ymm13, ymm14	;; A14 = R14 * cosine/sine + I14			; 23-27		n 30

	vmovapd	ymm13, [screg+6*64]		;; sine for R8/I8
	vmulpd	ymm12, ymm12, ymm13		;; R8 = A8 * sine					; 24-28
	vmulpd	ymm11, ymm11, ymm13		;; I8 = B8 * sine					; 24-28

	vmovapd	ymm13, [screg+2*64+32]		;; cosine/sine for R4/I4
	vmovapd	ymm14, [srcreg+d2+d1+32]	;; I4
	ystore	YMM_TMPS[9*32], ymm8		;; Save R7-R9						; 22+5
	vmovapd	ymm8, [srcreg+d2+d1]		;; R4
	ystore	YMM_TMPS[10*32], ymm10		;; Save I7+I9						; 23+5
	yfmsubpd ymm10, ymm14, ymm13, ymm8	;; B4 = I4 * cosine/sine - R4				; 25-29		n 31
	yfmaddpd ymm8, ymm8, ymm13, ymm14	;; A4 = R4 * cosine/sine + I4				; 25-29		n 31

	vmovapd	ymm13, [screg+10*64+32]		;; cosine/sine for R12/I12
	vmovapd	ymm14, [srcreg+5*d2+d1+32]	;; I12
	ystore	YMM_TMPS[11*32], ymm9		;; Save I7-I9						; 23+6
	vmovapd	ymm9, [srcreg+5*d2+d1]		;; R12
	ystore	YMM_TMPS[12*32], ymm12		;; Save R8						; 29+1
	yfmsubpd ymm12, ymm14, ymm13, ymm9	;; B12 = I12 * cosine/sine - R12			; 26-30		n 32
	yfmaddpd ymm9, ymm9, ymm13, ymm14	;; A12 = R12 * cosine/sine + I12			; 26-30		n 32

	vmovapd	ymm13, [screg+64]		;; sine for R3/I3
	vmulpd	ymm2, ymm2, ymm13		;; I3 = B3 * sine					; 27-31		n 33
	vmulpd	ymm0, ymm0, ymm13		;; R3 = A3 * sine					; 27-31		n 34

	vmovapd	ymm13, [screg+11*64]		;; sine for R13/I13
	vmulpd	ymm3, ymm3, ymm13		;; I13 = B13 * sine					; 28-32		n 33
	vmulpd	ymm1, ymm1, ymm13		;; R13 = A13 * sine					; 28-32		n 34

	vmovapd	ymm13, [screg]			;; sine for R2/I2
	vmulpd	ymm6, ymm6, ymm13		;; I2 = B2 * sine					; 29-33		n 35
	vmulpd	ymm4, ymm4, ymm13		;; R2 = A2 * sine					; 29-33		n 36

	vmovapd	ymm13, [screg+12*64]		;; sine for R14/I14
	vmulpd	ymm7, ymm7, ymm13		;; I14 = B14 * sine					; 30-34		n 35
	vmulpd	ymm5, ymm5, ymm13		;; R14 = A14 * sine					; 30-34		n 36

	vmovapd	ymm13, [screg+2*64]		;; sine for R4/I4
	vmulpd	ymm10, ymm10, ymm13		;; I4 = B4 * sine					; 31-35		n 37
	vmulpd	ymm8, ymm8, ymm13		;; R4 = A4 * sine					; 31-35		n 39
	ystore	YMM_TMPS[13*32], ymm11		;; Save I8						; 29+2

	vmovapd	ymm13, [screg+10*64]		;; sine for R12/I12
	vmulpd	ymm12, ymm12, ymm13		;; I12 = B12 * sine					; 32-36		n 37
	vmulpd	ymm9, ymm9, ymm13		;; R12 = A12 * sine					; 32-36		n 39
	vmovapd ymm11, YMM_P975_P434

	yfmsubpd ymm13, ymm2, ymm15, ymm3	;; I3-I13						; 33-37		n 38
	yfmaddpd ymm2, ymm2, ymm15, ymm3	;; I3+I13						; 33-37		n 40
	vmovapd ymm14, YMM_P782_P434

	yfmaddpd ymm3, ymm0, ymm15, ymm1	;; R3+R13						; 34-38
	yfmsubpd ymm0, ymm0, ymm15, ymm1	;; R3-R13						; 34-38
	L1prefetchw srcreg+d2+d1+L1pd, L1pt

	yfmsubpd ymm1, ymm6, ymm15, ymm7	;; I2-I14						; 35-39		n 42
	yfmaddpd ymm6, ymm6, ymm15, ymm7	;; I2+I14						; 35-39

	yloop_optional_early_prefetch

	yfmsubpd ymm7, ymm4, ymm15, ymm5	;; R2-R14						; 36-40		n 45
	yfmaddpd ymm4, ymm4, ymm15, ymm5	;; R2+R14						; 36-40
	L1prefetchw srcreg+2*d2+L1pd, L1pt

	yfmsubpd ymm5, ymm10, ymm15, ymm12	;; I4-I12						; 37-41		n 42
	yfmaddpd ymm10, ymm10, ymm15, ymm12	;; I4+I12						; 37-41

	;; Do the 28 reals inverse FFT

	vmovapd	ymm12, YMM_TMPS[3*32]		;; i5-i11
	ystore	YMM_TMPS[3*32], ymm3		;; Save R3+R13						; 39
	yfmsubpd ymm3, ymm11, ymm13, ymm12	;; .975/.434(i3-i13)-(i5-i11)				; 38-42		n 43
	ystore	YMM_TMPS[14*32], ymm0		;; Save R3-R13						; 39+1
	yfnmaddpd ymm0, ymm14, ymm12, ymm13	;; (i3-i13)-.782/.434(i5-i11)				; 38-42		n 43

	ystore	YMM_TMPS[15*32], ymm6		;; Save I2+I14						; 40+1
	vmovapd	ymm6, YMM_TMPS[11*32]		;; i7-i9
	yfmaddpd ymm13, ymm14, ymm13, ymm6	;; .782/.434(i3-i13)+(i7-i9)				; 39-43		n 44
	ystore	YMM_TMPS[11*32], ymm4		;; Save R2+R14						; 41+1
	yfmsubpd ymm4, ymm8, ymm15, ymm9	;; R4-R12						; 39-43		n 47

	yfmaddpd ymm8, ymm8, ymm15, ymm9	;; R4+R12						; 40-44
	vmovapd	ymm15, YMM_TMPS[2*32]		;; i5+i11
	yfmaddpd ymm9, ymm14, ymm15, ymm2	;; (i3+i13)+.782/.434(i5+i11)				; 40-44		n 45

	ystore	YMM_TMPS[2*32], ymm10		;; Save I4+I12						; 42+1
	yfmaddpd ymm10, ymm11, ymm2, ymm15	;; .975/.434(i3+i13)+(i5+i11)				; 41-45		n 46
	ystore	YMM_TMPS[16*32], ymm8		;; Save R4+R12						; 45
	vmovapd	ymm8, YMM_TMPS[10*32]		;; i7+i9
	yfmaddpd ymm2, ymm14, ymm2, ymm8	;; .782/.434(i3+i13)+(i7+i9)				; 41-45		n 46

	yfnmaddpd ymm3, ymm14, ymm6, ymm3	;; .975/.434(i3-i13)-(i5-i11)-.782/.434(i7-i9) (io5/.434) ; 43-47
	yfmaddpd ymm0, ymm11, ymm6, ymm0	;; (i3-i13)-.782/.434(i5-i11)+.975/.434(i7-i9) (io7/.434) ; 43-47

	yfmaddpd ymm6, ymm11, ymm5, ymm1	;; (i2-i14)+.975/.434(i4-i12)				; 42-46		n 48
	ystore	YMM_TMPS[10*32], ymm3		;; Save imag-odd-#5 / .434				; 48
	yfmaddpd ymm3, ymm14, ymm1, ymm5	;; .782/.434(i2-i14)+(i4-i12)				; 42-46		n 48

	yfmaddpd ymm13, ymm11, ymm12, ymm13	;; .782/.434(i3-i13)+.975/.434(i5-i11)+(i7-i9) (io3/.434) ; 44-48
	vmovapd	ymm12, YMM_TMPS[7*32]		;; i6-i10
	yfmaddpd ymm1, ymm11, ymm1, ymm12	;; .975/.434(i2-i14)+(i6-i10)				; 44-48		n 50

	yfmaddpd ymm9, ymm11, ymm8, ymm9	;; (i3+i13)+.782/.434(i5+i11)+.975/.434(i7+i9) (io2/.434) ; 45-49
	ystore	YMM_TMPS[7*32], ymm0		;; Save imag-odd-#7 / .434				; 48+1
	vmovapd	ymm0, YMM_TMPS[5*32]		;; r6-r10
	ystore	YMM_TMPS[5*32], ymm13		;; Save imag-odd-#3 / .434				; 49+1
	yfmaddpd ymm13, ymm11, ymm7, ymm0	;; .975/.434(r2-r14)+(r6-r10)				; 45-49		n 50

	yfnmaddpd ymm10, ymm14, ymm8, ymm10	;; .975/.434(i3+i13)+(i5+i11)-.782/.434(i7+i9) (io4/.434) ; 46-50
	yfnmaddpd ymm2, ymm11, ymm15, ymm2	;; .782/.434(i3+i13)-.975/.434(i5+i11)+(i7+i9) (io6/.434) ; 46-50
	vmovapd ymm15, YMM_P901

	yfmsubpd ymm8, ymm14, ymm7, ymm4	;; .782/.434(r2-r14)-(r4-r12)				; 47-51		n 55
	yfnmaddpd ymm7, ymm11, ymm4, ymm7	;; (r2-r14)-.975/.434(r4-r12)				; 47-51		n 55
	L1prefetchw srcreg+2*d2+d1+L1pd, L1pt

	yfmaddpd ymm6, ymm14, ymm12, ymm6	;; (i2-i14)+.975/.434(i4-i12)+.782/.434(i6-i10) (ie3/.434) ; 48-52
	yfnmaddpd ymm3, ymm11, ymm12, ymm3	;; .782/.434(i2-i14)+(i4-i12)-.975/.434(i6-i10) (ie5/.434) ; 48-52

	vmovapd	ymm12, [srcreg+32]		;; r1-r15
	vmovapd	ymm11, YMM_TMPS[14*32]		;; r3-r13
	ystore	YMM_TMPS[14*32], ymm9		;; Save imag-odd-#2 / .434				; 50+1
	yfmaddpd ymm9, ymm15, ymm11, ymm12	;; r1+.901(r3-r13)-r15					; 49-53		n 54
	ystore	YMM_TMPS[17*32], ymm10		;; Save imag-odd-#4 / .434				; 51+1
	vmovapd ymm10, YMM_P223
	ystore	YMM_TMPS[18*32], ymm2		;; Save imag-odd-#6 / .434				; 51+2
	yfmaddpd ymm2, ymm10, ymm11, ymm12	;; r1+.223(r3-r13)-r15					; 49-53		n 54

	yfnmaddpd ymm1, ymm14, ymm5, ymm1	;; .975/.434(i2-i14)-.782/.434(i4-i12)+(i6-i10) (ie7/.434) ; 50-54
	yfmaddpd ymm13, ymm14, ymm4, ymm13	;; .975/.434(r2-r14)+.782/.434(r4-r12)+(r6-r10) (re2/.434) ; 50-54

	vmovapd	ymm5, YMM_TMPS[13*32]		;; i8
	vmovapd	ymm4, YMM_TMPS[15*32]		;; i2+i14
	ystore	YMM_TMPS[13*32], ymm6		;; Save imag-even-#3 / .434				; 53+1
	yfmaddpd ymm6, ymm10, ymm4, ymm5	;; .223(i2+i14)+i8					; 51-55		n 56
	ystore	YMM_TMPS[15*32], ymm3		;; Save imag-even-#5 / .434				; 53+2
	vsubpd	ymm3, ymm4, ymm5		;; (i2+i14)-i8						; 51-55		n 56

	ystore	YMM_TMPS[19*32], ymm1		;; Save imag-even-#7 / .434				; 55+1
	vsubpd	ymm1, ymm12, ymm11		;; r1-(r3-r13)-r15					; 52-56		n 57
	ystore	YMM_TMPS[20*32], ymm13		;; Save real-even-#2 / .434				; 55+2
	vmovapd ymm13, YMM_P623
	yfnmaddpd ymm11, ymm13, ymm11, ymm12	;; r1-.623(r3-r13)-r15					; 52-56		n 57

	yloop_optional_early_prefetch

	yfmsubpd ymm12, ymm13, ymm4, ymm5	;; .623(i2+i14)-i8					; 53-57		n 58
	yfmaddpd ymm4, ymm15, ymm4, ymm5	;; .901(i2+i14)+i8					; 53-57		n 58

	vmovapd	ymm5, YMM_TMPS[1*32]		;; r5-r11
	yfmaddpd ymm9, ymm13, ymm5, ymm9	;; r1+.901(r3-r13)+.623(r5-r11)-r15			; 54-58		n 59
	yfnmaddpd ymm2, ymm15, ymm5, ymm2	;; r1+.223(r3-r13)-.901(r5-r11)-r15			; 54-58		n 59

	yfnmaddpd ymm8, ymm0, YMM_P975_P434, ymm8 ;; .782/.434(r2-r14)-(r4-r12)-.975/.434(r6-r10) (re4/.434) ; 55-59	n 65
	yfmaddpd ymm7, ymm14, ymm0, ymm7	;; (r2-r14)-.975/.434(r4-r12)+.782/.434(r6-r10) (re6/.434) ; 55-59	n 68

	vmovapd	ymm14, YMM_TMPS[2*32]		;; i4+i12
	yfmaddpd ymm6, ymm13, ymm14, ymm6	;; .223(i2+i14)+.623(i4+i12)+i8				; 56-60		n 61
	vsubpd	ymm3, ymm3, ymm14		;; (i2+i14)-(i4+i12)-i8					; 56-60		n 61

	vaddpd	ymm1, ymm1, ymm5		;; r1-(r3-r13)+(r5-r11)-r15				; 57-61		n 62
	yfnmaddpd ymm11, ymm10, ymm5, ymm11	;; r1-.623(r3-r13)-.223(r5-r11)-r15			; 57-61		n 62
	vmovapd	ymm0, YMM_TMPS[9*32]		;; r7-r9

	yfmaddpd ymm12, ymm15, ymm14, ymm12	;; .623(i2+i14)+.901(i4+i12)-i8				; 58-62		n 63
	yfnmaddpd ymm4, ymm10, ymm14, ymm4	;; .901(i2+i14)-.223(i4+i12)+i8				; 58-62		n 63
	vmovapd	ymm5, YMM_TMPS[6*32]		;; i6+i10

	yfmaddpd ymm9, ymm10, ymm0, ymm9	;; r1+.901(r3-r13)+.623(r5-r11)+.223(r7-r9)-r15 (ro2)	; 59-63		n 64
	yfnmaddpd ymm2, ymm13, ymm0, ymm2	;; r1+.223(r3-r13)-.901(r5-r11)-.623(r7-r9)-r15	(ro4)	; 59-63		n 65
	vmovapd ymm14, YMM_P434

	yloop_optional_early_prefetch

													; 60 stall

	yfmaddpd ymm6, ymm15, ymm5, ymm6	;; .223(i2+i14)+.623(i4+i12)+.901(i6+i10)+i8 (ie2)	; 61-65		n 66
	vaddpd	ymm3, ymm3, ymm5		;; (i2+i14)-(i4+i12)+(i6+i10)-i8 (r8/22b)		; 61-65		n 67
	L1prefetchw srcreg+3*d2+L1pd, L1pt

	vsubpd	ymm1, ymm1, ymm0		;; r1-(r3-r13)+(r5-r11)-(r7-r9)-r15 (r8/22a)		; 62-66		n 67
	yfmaddpd ymm11, ymm15, ymm0, ymm11	;; r1-.623(r3-r13)-.223(r5-r11)+.901(r7-r9)-r15	(ro6)	; 62-66		n 69
	vmovapd	ymm0, YMM_TMPS[20*32]		;; Load real-even-#2 / .434

	yfnmaddpd ymm12, ymm10, ymm5, ymm12	;; .623(i2+i14)+.901(i4+i12)-.223(i6+i10)-i8 (ie4)	; 63-67		n 68
	yfnmaddpd ymm4, ymm13, ymm5, ymm4	;; .901(i2+i14)-.223(i4+i12)-.623(i6+i10)+i8 (ie6)	; 63-67		n 70
	vmovapd	ymm5, YMM_TMPS[14*32]		;; Load imag-odd-#2 / .434

	yfnmaddpd ymm10, ymm0, ymm14, ymm9	;; real-cols row #14 (odd#2 - even#2 * .434)		; 64-68		n 71
	yfmaddpd ymm0, ymm0, ymm14, ymm9	;; real-cols row #2 (odd#2 + even#2 * .434)		; 64-68		n 72
	vmovapd ymm15, YMM_ONE

	yfnmaddpd ymm9, ymm8, ymm14, ymm2	;; real-cols row #12 (odd#4 - even#4 * .434)		; 65-69		n 73
	yfmaddpd ymm8, ymm8, ymm14, ymm2	;; real-cols row #4 (odd#4 + even#4 * .434)		; 65-69		n 74
	L1prefetchw srcreg+3*d2+d1+L1pd, L1pt

	yfnmaddpd ymm2, ymm5, ymm14, ymm6	;; imag-cols row #14 (even#2 - odd#2 * .434)		; 66-70		n 71
	yfmaddpd ymm5, ymm5, ymm14, ymm6	;; imag-cols row #2 (even#2 + odd#2 * .434)		; 66-70		n 72
	L1prefetchw srcreg+4*d2+L1pd, L1pt

	yfmsubpd ymm6, ymm1, ymm15, ymm3	;; final R22						; 67-71
	yfmaddpd ymm1, ymm1, ymm15, ymm3	;; final R8						; 67-71

	vmovapd	ymm3, YMM_TMPS[17*32]		;; Load imag-odd-#4 / .434
	ystore	[srcreg+3*d2+d1+32], ymm6	;; Save R22						; 72
	yfnmaddpd ymm6, ymm3, ymm14, ymm12	;; imag-cols row #12 (even#4 - odd#4 * .434)		; 68-72		n 73
	yfmaddpd ymm3, ymm3, ymm14, ymm12	;; imag-cols row #4 (even#4 + odd#4 * .434)		; 68-72		n 74
 
	yfnmaddpd ymm12, ymm7, ymm14, ymm11	;; real-cols row #10 (odd#6 - even#6 * .434)		; 69-73		n 75
	yfmaddpd ymm7, ymm7, ymm14, ymm11	;; real-cols row #6 (odd#6 + even#6 * .434)		; 69-73		n 80

	vmovapd	ymm11, YMM_TMPS[18*32]		;; Load imag-odd-#6 / .434
	ystore	[srcreg+3*d2+d1], ymm1		;; Save R8						; 72+1
	yfnmaddpd ymm1, ymm11, ymm14, ymm4	;; imag-cols row #10 (even#6 - odd#6 * .434)		; 70-74		n 75
	yfmaddpd ymm11, ymm11, ymm14, ymm4	;; imag-cols row #6 (even#6 + odd#6 * .434)		; 70-74		n 80

	yfmsubpd ymm4, ymm10, ymm15, ymm2	;; real #14 - imag #14 (final R16)			; 71-75
	yfmaddpd ymm10, ymm10, ymm15, ymm2	;; real #14 + imag #14 (final R14)			; 71-75
	vmovapd	ymm14, [srcreg]			;; r1+r15

	yfmsubpd ymm2, ymm0, ymm15, ymm5	;; real #2 - imag #2 (final R28)			; 72-76
	yfmaddpd ymm0, ymm0, ymm15, ymm5	;; real #2 + imag #2 (final R2)				; 72-76
	L1prefetchw srcreg+4*d2+d1+L1pd, L1pt

	yfmsubpd ymm5, ymm9, ymm15, ymm6	;; real #12 - imag #12 (final R18)			; 73-77
	yfmaddpd ymm9, ymm9, ymm15, ymm6	;; real #12 + imag #12 (final R12)			; 73-77
	L1prefetchw srcreg+5*d2+L1pd, L1pt

	yfmsubpd ymm6, ymm8, ymm15, ymm3	;; real #4 - imag #4 (final R26)			; 74-78
	yfmaddpd ymm8, ymm8, ymm15, ymm3	;; real #4 + imag #4 (final R4)				; 74-78

	yfmsubpd ymm3, ymm12, ymm15, ymm1	;; real #10 - imag #10 (final R20)			; 75-79
	yfmaddpd ymm12, ymm12, ymm15, ymm1	;; real #10 + imag #10 (final R10)			; 75-79

	vmovapd	ymm1, YMM_TMPS[3*32]		;; r3+r13
	ystore	[srcreg+d1+32], ymm4		;; Save R16						; 76
	yfmaddpd ymm4, ymm13, ymm1, ymm14	;; r1+.623(r3+r13)+r15					; 76-80		n 81
	ystore	[srcreg+6*d2+d1], ymm10		;; Save R14						; 76+1
	vmovapd	ymm10, YMM_TMPS[11*32]		;; r2+r14
	ystore	[srcreg+6*d2+d1+32], ymm2	;; Save R28						; 77+1
	vmovapd	ymm2, YMM_TMPS[12*32]		;; r8
	ystore	[srcreg+d1], ymm0		;; Save R2						; 77+2
	vmovapd ymm0, YMM_P901
	ystore	[srcreg+d2+d1+32], ymm5		;; Save R18						; 78+2
	yfmsubpd ymm5, ymm0, ymm10, ymm2	;; .901(r2+r14)-r8					; 76-80		n 81

	ystore	[srcreg+5*d2+d1], ymm9		;; Save R12						; 78+3
	vmovapd ymm9, YMM_P223
	ystore	[srcreg+5*d2+d1+32], ymm6	;; Save R26						; 79+3
	yfnmaddpd ymm6, ymm9, ymm1, ymm14	;; r1-.223(r3+r13)+r15					; 77-81		n 82
	ystore	[srcreg+d2+d1], ymm8		;; Save R4						; 79+4
	yfmaddpd ymm8, ymm13, ymm10, ymm2	;; .623(r2+r14)+r8					; 77-81		n 82

	ystore	[srcreg+2*d2+d1+32], ymm3	;; Save R20						; 80+4
	yfmaddpd ymm3, ymm14, ymm15, ymm1	;; r1+(r3+r13)+r15					; 78-82		n 83
	yfnmaddpd ymm1, ymm0, ymm1, ymm14	;; r1-.901(r3+r13)+r15					; 78-82		n 83

	yfmaddpd ymm14, ymm10, ymm15, ymm2	;; (r2+r14)+r8						; 79-83		n 84
	yfmsubpd ymm10, ymm9, ymm10, ymm2	;; .223(r2+r14)-r8					; 79-83		n 84

	yloop_optional_early_prefetch

	yfmsubpd ymm2, ymm7, ymm15, ymm11	;; real #6 - imag #6 (final R24)			; 80-84
	yfmaddpd ymm7, ymm7, ymm15, ymm11	;; real #6 + imag #6 (final R6)				; 80-84

	vmovapd	ymm11, YMM_TMPS[0*32]		;; r5+r11
	yfnmaddpd ymm4, ymm9, ymm11, ymm4	;; r1+.623(r3+r13)-.223(r5+r11)+r15			; 81-85		n 86
	ystore	[srcreg+4*d2+d1], ymm12		;; Save R10						; 80+5
	vmovapd	ymm12, YMM_TMPS[16*32]		;; r4+r12
	yfmaddpd ymm5, ymm9, ymm12, ymm5	;; .901(r2+r14)+.223(r4+r12)-r8				; 81-85		n 86

	yfnmaddpd ymm6, ymm0, ymm11, ymm6	;; r1-.223(r3+r13)-.901(r5+r11)+r15			; 82-86		n 87
	yfnmaddpd ymm8, ymm0, ymm12, ymm8	;; .623(r2+r14)-.901(r4+r12)+r8				; 82-86		n 87

	yfmaddpd ymm3, ymm3, ymm15, ymm11	;; r1+(r3+r13)+(r5+r11)+r15				; 83-87		n 88
	yfmaddpd ymm1, ymm13, ymm11, ymm1	;; r1-.901(r3+r13)+.623(r5+r11)+r15			; 83-87		n 88
	vmovapd	ymm11, YMM_TMPS[13*32]		;; Load imag-even row #3 / .434

	yfmaddpd ymm14, ymm14, ymm15, ymm12	;; (r2+r14)+(r4+r12)+r8					; 84-88		n 89
	yfnmaddpd ymm10, ymm13, ymm12, ymm10	;; .223(r2+r14)-.623(r4+r12)-r8				; 84-88		n 89

	vmovapd	ymm12, YMM_TMPS[5*32]		;; Load imag-odd row #3 / .434
	ystore	[srcreg+4*d2+d1+32], ymm2	;; Save R24						; 85+1
	yfmsubpd ymm2, ymm11, ymm15, ymm12	;; imag-cols row #13 / .434 (even#3 - odd#3)		; 85-89		n 97
	yfmaddpd ymm11, ymm11, ymm15, ymm12	;; imag-cols row #3 / .434 (even#3 + odd#3)		; 85-89		n 96

	vmovapd	ymm12, YMM_TMPS[8*32]		;; r7+r9
	yfnmaddpd ymm4, ymm0, ymm12, ymm4	;; r1+.623(r3+r13)-.223(r5+r11)-.901(r7+r9)+r15 (ro3)	; 86-90		n 91
	ystore	[srcreg+2*d2+d1], ymm7		;; Save R6						; 85+2
	vmovapd	ymm7, YMM_TMPS[4*32]		;; r6+r10
	yfnmaddpd ymm5, ymm13, ymm7, ymm5	;; .901(r2+r14)+.223(r4+r12)-.623(r6+r10)-r8 (re3)	; 86-90		n 91

	yfmaddpd ymm6, ymm13, ymm12, ymm6	;; r1-.223(r3+r13)-.901(r5+r11)+.623(r7+r9)+r15 (ro5)	; 87-91		n 93
	yfnmaddpd ymm8, ymm9, ymm7, ymm8	;; .623(r2+r14)-.901(r4+r12)-.223(r6+r10)+r8 (re5)	; 87-91		n 93
	vmovapd	ymm13, YMM_TMPS[15*32]		;; Load imag-even row #5 / .434

	yfmaddpd ymm3, ymm3, ymm15, ymm12	;; r1+(r3+r13)+(r5+r11)+(r7+r9)+r15 (ro1)		; 88-92		n 94
	yfnmaddpd ymm1, ymm9, ymm12, ymm1	;; r1-.901(r3+r13)+.623(r5+r11)-.223(r7+r9)+r15 (ro7)	; 88-92		n 95
	vmovapd	ymm12, YMM_TMPS[10*32]		;; Load imag-odd row #5 / .434

	yfmaddpd ymm14, ymm14, ymm15, ymm7	;; (r2+r14)+(r4+r12)+(r6+r10)+r8 (re1)			; 89-93		n 94
	yfmaddpd ymm10, ymm0, ymm7, ymm10	;; .223(r2+r14)-.623(r4+r12)+.901(r6+r10)-r8 (re7)	; 89-93		n 95
	vmovapd	ymm7, YMM_TMPS[19*32]		;; Load imag-even row #7 / .434

	yfmsubpd ymm9, ymm13, ymm15, ymm12	;; imag-cols row #11 / .434 (even#5 - odd#5)		; 90-94		n 99
	yfmaddpd ymm13, ymm13, ymm15, ymm12	;; imag-cols row #5 / .434 (even#5 + odd#5)		; 90-94		n 98
	vmovapd	ymm0, YMM_TMPS[7*32]		;; Load imag-odd row #7 / .434

	yfmaddpd ymm12, ymm4, ymm15, ymm5	;; real-cols row #3 (odd#3 + even#3)			; 91-95		n 96
	yfmsubpd ymm4, ymm4, ymm15, ymm5	;; real-cols row #13 (odd#3 - even#3)			; 91-95		n 97
	L1prefetchw srcreg+5*d2+d1+L1pd, L1pt

	yfmsubpd ymm5, ymm7, ymm15, ymm0	;; imag-cols row #9 / .434 (even#7 - odd#7)		; 92-96		n 101
	yfmaddpd ymm7, ymm7, ymm15, ymm0	;; imag-cols row #7 / .434 (even#7 + odd#7)		; 92-96		n 100

	yfmaddpd ymm0, ymm6, ymm15, ymm8	;; real-cols row #5 (odd#5 + even#5)			; 93-97		n 98
	yfmsubpd ymm6, ymm6, ymm15, ymm8	;; real-cols row #11 (odd#5 - even#5)			; 93-97		n 99
	L1prefetchw srcreg+6*d2+L1pd, L1pt

	yfmaddpd ymm8, ymm3, ymm15, ymm14	;; real-cols row #1 (odd#1 + even#1) (final R1)		; 94-98
	yfmsubpd ymm3, ymm3, ymm15, ymm14	;; real-cols row #15 (odd#1 - even#1) (final R15)	; 94-98
	L1prefetchw srcreg+6*d2+d1+L1pd, L1pt

	yfmaddpd ymm14, ymm1, ymm15, ymm10	;; real-cols row #7 (odd#7 + even#7)			; 95-99		n 100
	yfmsubpd ymm1, ymm1, ymm15, ymm10	;; real-cols row #9 (odd#7 - even#7)			; 95-99		n 101

	vmovapd ymm10, YMM_P434
	ystore	[srcreg], ymm8			;; Save final R1					; 99
	yfnmaddpd ymm8, ymm11, ymm10, ymm12	;; real #3 - imag #3 * .434 (final R27)			; 96-100
	yfmaddpd ymm11, ymm11, ymm10, ymm12	;; real #3 + imag #3 * .434 (final R3)			; 96-100

	yfnmaddpd ymm12, ymm2, ymm10, ymm4	;; real #13 - imag #13 * .434 (final R17)		; 97-101
	yfmaddpd ymm2, ymm2, ymm10, ymm4	;; real #13 + imag #13 * .434 (final R13)		; 97-101

	yfnmaddpd ymm4, ymm13, ymm10, ymm0	;; real #5 - imag #5 * .434 (final R25)			; 98-102
	yfmaddpd ymm13, ymm13, ymm10, ymm0	;; real #5 + imag #5 * .434 (final R5)			; 98-102

	yfnmaddpd ymm0, ymm9, ymm10, ymm6	;; real #11 - imag #11 * .434 (final R19)		; 99-103
	yfmaddpd ymm9, ymm9, ymm10, ymm6	;; real #11 + imag #11 * .434 (final R11)		; 99-103

	yfnmaddpd ymm6, ymm7, ymm10, ymm14	;; real #7 - imag #7 * .434 (final R23)			; 100-104
	yfmaddpd ymm7, ymm7, ymm10, ymm14	;; real #7 + imag #7 * .434 (final R7)			; 100-104
	ystore	[srcreg+32], ymm3		;; Save final R15					; 99+1

	yfnmaddpd ymm14, ymm5, ymm10, ymm1	;; real #9 - imag #9 * .434 (final R21)			; 101-105
	yfmaddpd ymm5, ymm5, ymm10, ymm1	;; real #9 + imag #9 * .434 (final R9)			; 101-105
	ystore	[srcreg+6*d2+32], ymm8		;; Save R27						; 101

	ystore	[srcreg+d2], ymm11		;; Save R3						; 101+1
	ystore	[srcreg+d2+32], ymm12		;; Save R17						; 102+1
	ystore	[srcreg+6*d2], ymm2		;; Save R13						; 102+2
	ystore	[srcreg+5*d2+32], ymm4		;; Save R25						; 103+2
	ystore	[srcreg+2*d2], ymm13		;; Save R5						; 103+3
	ystore	[srcreg+2*d2+32], ymm0		;; Save R19						; 104+3
	ystore	[srcreg+5*d2], ymm9		;; Save R11						; 104+4
	ystore	[srcreg+4*d2+32], ymm6		;; Save R23						; 105+4
	ystore	[srcreg+3*d2], ymm7		;; Save R7						; 105+5
	ystore	[srcreg+3*d2+32], ymm14		;; Save R21						; 106+5
	ystore	[srcreg+4*d2], ymm5		;; Save R9						; 106+6

	bump	srcreg, srcinc
	bump	screg, scinc
	ENDM

ENDIF

ENDIF

