; Copyright 2011-2024 - Mersenne Research, Inc.  All rights reserved.
; Author:  George Woltman
; Email: woltman@alum.mit.edu
;
; These macros efficiently implement the normalization to integers
; and multiplication by two-to-phi powers using SSE2 instructions.
;

; Utility macros used in normalization macros

; Compute absolute value of a ymm register.  In 64-bit mode we assume
; ymm15 has been preloaded with the necessary constant.
 
absval MACRO ymmreg
	IFDEF X86_64
	vandpd	ymmreg, ymmreg, ymm15		;; Compute absolute value
	ELSE
	vandpd	ymmreg, ymmreg, YMM_ABSVAL	;; Compute absolute value
	ENDIF
	ENDM

; These macros implement the variants of the normalization routines
; in a non-pipelined way.  It is simply too much work to hand optimize
; all normalization variants.

; Compute the convolution error and if greater than MAXERR, set MAXERR

error_check MACRO ymmreg, tmpreg, errreg
	vroundpd tmpreg, ymmreg, 0		;; Convert to an integer
	vsubpd	tmpreg, tmpreg, ymmreg		;; This is the convolution error
	absval	tmpreg				;; Compute absolute value
	vmaxpd	errreg, errreg, tmpreg		;; Compute maximum error
	ENDM

error_check_interleaved MACRO ymmreg1, tmpreg1, ymmreg2, tmpreg2, errreg
	vroundpd tmpreg1, ymmreg1, 0		;; Convert to an integer
	vroundpd tmpreg2, ymmreg2, 0		;; Convert to an integer
	vsubpd	tmpreg1, tmpreg1, ymmreg1	;; This is the convolution error
	vsubpd	tmpreg2, tmpreg2, ymmreg2	;; This is the convolution error
	absval	tmpreg1				;; Compute absolute value
	absval	tmpreg2				;; Compute absolute value
	vmaxpd	errreg, errreg, tmpreg1		;; Compute maximum error
	vmaxpd	errreg, errreg, tmpreg2		;; Compute maximum error
	ENDM

; In general, normalization routines calculate:
;		newFFTvalue = (FFTvalue * const + carry) % base
;		carry = (FFTvalue * const + carry) / base
; Since FFTvalue * const can exceed 51 bits, we instead split FFTvalue into:
;		hi = FFTvalue / base
;		lo = FFTvalue % base
; and then calculate:
;		newFFTvalue = (lo * const + carry) % base
;		carry = hi * const + (lo * const + carry) / base
;
; For the b = 2 case, we can split hi and lo using any power of 2 larger
; than the FFT base, this allows for some simpler code in this case.



; These routines split an FFT value and then multiplies it by the small constant.
; The previous carry is added in and a new carry is generated from the high
; bits of the FFT value.

mul_by_const MACRO ttp, base2, echk, ymmval, ymmcarry, ymmtmp1, ymmtmp2, biglitreg, errreg
base2	 base2_mul_by_const echk, ymmval, ymmcarry, ymmtmp1, ymmtmp2, errreg
no base2 ttp nobase2_mul_by_const echk, ymmval, ymmcarry, ymmtmp1, ymmtmp2, biglitreg, errreg
no base2 no ttp nobase2_mul_by_const echk, ymmval, ymmcarry, ymmtmp1, ymmtmp2, 0, errreg
	ENDM

base2_mul_by_const MACRO echk, ymmval, ymmcarry, ymmtmp1, ymmtmp2, errreg
IFNDEF X86_64
	vmovapd	ymmtmp2, YMM_BIGBIGVAL		;; Round to nearest multiple of 2^25
	vaddpd	ymmtmp1, ymmval, ymmtmp2
	vsubpd	ymmtmp1, ymmtmp1, ymmtmp2	;; ymmtmp1 is the high bits of FFT value
ELSE
	vaddpd	ymmtmp1, ymmval, ymm14		;; Round to nearest multiple of 2^25
	vsubpd	ymmtmp1, ymmtmp1, ymm14		;; ymmtmp1 is the high bits of FFT value
ENDIF
	vroundpd ymmtmp2, ymmval, 0		;; Round FFT value to an integer
echk	vsubpd	ymmval, ymmval, ymmtmp2		;; This is the convolution error
echk	absval	ymmval				;; Compute absolute value
echk	vmaxpd	errreg, errreg, ymmval		;; Compute maximum error
	vsubpd	ymmval, ymmtmp2, ymmtmp1	;; ymmval now contains low 25 bits of FFT value
IFNDEF X86_64
	vmovapd	ymmtmp2, YMM_MULCONST
	vmulpd	ymmval, ymmval, ymmtmp2		;; Multiply low bits of FFT value by the small constant
	vaddpd	ymmval, ymmval, ymmcarry	;; Add in the previous carry
	vmulpd	ymmcarry, ymmtmp1, ymmtmp2	;; Next carry = high bits of FFT value times the small constant
ELSE
	vmulpd	ymmval, ymmval, ymm13		;; Multiply low bits of FFT value by the small constant
	vaddpd	ymmval, ymmval, ymmcarry	;; Add in the previous carry
	vmulpd	ymmcarry, ymmtmp1, ymm13	;; Next carry = high bits of FFT value times the small constant
ENDIF
	ENDM

nobase2_mul_by_const MACRO echk, ymmval, ymmcarry, ymmtmp1, ymmtmp2, basereg, errreg
	vmulpd	ymmtmp1, ymmval, YMM_LIMIT_INVERSE[basereg]	;; Compute FFT value / base
	vroundpd ymmtmp2, ymmval, 0				;; Round FFT value to an integer
	vroundpd ymmtmp1, ymmtmp1, 0				;; Round FFT value / base to an integer
echk	vsubpd	ymmval, ymmval, ymmtmp2				;; This is the convolution error
echk	absval	ymmval						;; Compute absolute value
echk	vmaxpd	errreg, errreg, ymmval				;; Compute maximum error
	vmulpd	ymmval, ymmtmp1, YMM_LIMIT_BIGMAX[basereg]	;; round (FFT value / base) * base
	vsubpd	ymmval, ymmtmp2, ymmval				;; This is FFTvalue % base
IFNDEF X86_64
	vmovapd	ymmtmp2, YMM_MULCONST
	vmulpd	ymmval, ymmval, ymmtmp2				;; Multiply FFTvalue % base by the small constant
	vaddpd	ymmval, ymmval, ymmcarry			;; Add in the previous carry
	vmulpd	ymmcarry, ymmtmp1, ymmtmp2			;; Next carry = round (FFTvalue / base) * the small constant
ELSE
	vmulpd	ymmval, ymmval, ymm13				;; Multiply FFTvalue % base by the small constant
	vaddpd	ymmval, ymmval, ymmcarry			;; Add in the previous carry
	vmulpd	ymmcarry, ymmtmp1, ymm13			;; Next carry = round (FFTvalue / base) * the small constant
ENDIF
	ENDM


	;; BUG BUG, I think the code below will do the mul by const with one register.  Perfect for interleaving!!
IFDEF  TRY_THIS_CODE
base2
	vmovapd	ymmtmp, ymmval			;; Copy value (can't be avoided without using a 2nd temp reg)
	vroundpd ymmval, ymmval, 0		;; Round FFT value to an integer
echk	vsubpd	ymmtmp, ymmval, ymmtmp		;; This is the convolution error
echk	absval	ymmtmp				;; Compute absolute value
echk	vmaxpd	errreg, errreg, ymmtmp		;; Compute maximum error
	vaddpd	ymmtmp, ymmval, YMM_BIGBIGVAL	;; Round to nearest multiple of 2^25
	vsubpd	ymmtmp, ymmtmp, YMM_BIGBIGVAL	;; ymmtmp is the high bits of FFT value
	vsubpd	ymmval, ymmval, ymmtmp		;; ymmval now contains low 25 bits of FFT value
	vmulpd	ymmval, ymmval, YMM_MULCONST	;; Multiply low bits of FFT value by the small constant
	vaddpd	ymmval, ymmval, ymmcarry	;; Add in the previous carry
	vmulpd	ymmcarry, ymmtmp, YMM_MULCONST	;; Next carry = high bits of FFT value times the small constant

nobase2:
	vmovapd	ymmtmp, ymmval					;; Copy value (can't be avoided without using a 2nd temp reg)
	vroundpd ymmval, ymmval, 0				;; Round FFT value to an integer
echk	vsubpd	ymmtmp, ymmval, ymmtmp				;; This is the convolution error
echk	absval	ymmtmp						;; Compute absolute value
echk	vmaxpd	errreg, errreg, ymmtmp				;; Compute maximum error
	vmulpd	ymmtmp, ymmval, YMM_LIMIT_INVERSE[basereg]	;; Compute FFT value / base
	vroundpd ymmtmp, ymmtmp, 0				;; Round FFT value / base to an integer
	vsubpd	ymmval, ymmval, ymmtmp				;; Compute fractional part of FFT value / base
	vmulpd	ymmval, ymmval, YMM_LIMIT_BIGMAX[basereg]	;; This is FFT value %
	vmulpd	ymmval, ymmval, YMM_MULCONST			;; Multiply FFTvalue % base by the small constant
	vaddpd	ymmval, ymmval, ymmcarry			;; Add in the previous carry
	vmulpd	ymmcarry, ymmtmp, YMM_MULCONST			;; Next carry = round (FFTvalue / base) * the small constant
ENDIF



mul_by_const_interleaved MACRO ttp, base2, echk, ymmval, ymmcarry, ymmtmp, basereg, ymmval2, ymmcarry2, ymmtmp2, basereg2, errreg
base2	 base2_mul_by_const_interleaved echk, ymmval, ymmcarry, ymmtmp, ymmval2, ymmcarry2, ymmtmp2, errreg
no base2 ttp nobase2_mul_by_const_interleaved echk, ymmval, ymmcarry, ymmtmp, basereg, ymmval2, ymmcarry2, ymmtmp2, basereg2, errreg
no base2 no ttp nobase2_mul_by_const_interleaved echk, ymmval, ymmcarry, ymmtmp, 0, ymmval2, ymmcarry2, ymmtmp2, 0, errreg
	ENDM

base2_mul_by_const_interleaved MACRO echk, ymmval, ymmcarry, ymmtmp, ymmval2, ymmcarry2, ymmtmp2, errreg
	base2_mul_by_const echk, ymmval, ymmcarry, ymmtmp, ymmtmp2, errreg
	base2_mul_by_const echk, ymmval2, ymmcarry2, ymmtmp, ymmtmp2, errreg
;; BUG -  we may need a third temporary register to do this
;	vmovapd	ymmtmp, YMM_BIGBIGVAL
;	vaddpd	ymmcarry, ymmval, ymmtmp	;; Round to nearest multiple of 2^25
;	vaddpd	ymmcarry2, ymmval2, ymmtmp	;; Round to nearest multiple of 2^25
;	vsubpd	ymmcarry, ymmcarry, ymmtmp
;	vsubpd	ymmcarry2, ymmcarry2, ymmtmp
;	vroundpd ymmtmp, ymmval, 0		;; Round to an integer
;	vroundpd ymmtmp2, ymmval2, 0		;; Round to an integer
;echk	vsubpd	ymmval, ymmval, ymmtmp		;; This is the convolution error
;echk	vsubpd	ymmval2, ymmval2, ymmtmp2	;; This is the convolution error
;echk	absval	ymmval				;; Compute absolute value
;echk	absval	ymmval2				;; Compute absolute value
;echk	vmaxpd	errreg, errreg, ymmval		;; Compute maximum error
;echk	vmaxpd	errreg, errreg, ymmval2		;; Compute maximum error
;	vsubpd	ymmval, ymmtmp, ymmcarry	;; ymmreg now contains low 25 bits
;	vsubpd	ymmval2, ymmtmp2, ymmcarry2	;; ymmreg2 now contains low 25 bits
;	vmovapd	ymmtmp, YMM_MULCONST
;	vmulpd	ymmcarry, ymmcarry, ymmtmp	;; Multiply by the small constant
;	vmulpd	ymmcarry2, ymmcarry2, ymmtmp	;; Multiply by the small constant
;	vmulpd	ymmval, ymmval, ymmtmp
;	vmulpd	ymmval2, ymmval2, ymmtmp
	ENDM

nobase2_mul_by_const_interleaved MACRO echk, ymmval, ymmcarry, ymmtmp, basereg, ymmval2, ymmcarry2, ymmtmp2, basereg2, errreg
	nobase2_mul_by_const echk, ymmval, ymmcarry, ymmtmp, ymmtmp2, basereg, errreg
	nobase2_mul_by_const echk, ymmval2, ymmcarry2, ymmtmp, ymmtmp2, basereg2, errreg
;;;;; bugs!!!!
;;;xxx	vmulpd	ymmcarry, ymmval, YMM_LIMIT_INVERSE[basereg]	;; Compute FFTvalue / base
;;	vmulpd	ymmcarry2, ymmval2, YMM_LIMIT_INVERSE[basereg2] ;; Compute FFTvalue / base
;;	vroundpd ymmtmp, ymmval, 0				;; Round to an integer
;;	vroundpd ymmtmp2, ymmval2, 0				;; Round to an integer
;;	vroundpd ymmcarry, ymmcarry, 0				;; Round to an integer				;;BUG - isn't reghi already an integer???
;;	vroundpd ymmcarry2, ymmcarry2, 0			;; Round to an integer
;;echk	vsubpd	ymmval, ymmval, ymmtmp				;; This is the convolution error
;;echk	vsubpd	ymmval2, ymmval2, ymmtmp2			;; This is the convolution error
;;echk	absval	ymmval						;; Compute absolute value
;;echk	absval	ymmval2						;; Compute absolute value
;;echk	vmaxpd	errreg, errreg, ymmval				;; Compute maximum error
;;echk	vmaxpd	errreg, errreg, ymmval2				;; Compute maximum error
;;	vmulpd	ymmval, ymmcarry, YMM_LIMIT_BIGMAX[basereg]
;;	vmulpd	ymmval2, ymmcarry2, YMM_LIMIT_BIGMAX[basereg2]
;;	vsubpd	ymmval, ymmtmp, ymmval				;; This is FFTvalue % base
;;	vsubpd	ymmval2, ymmtmp2, ymmval2			;; This is FFTvalue % base
;;	vmovapd	ymmval, YMM_MULCONST
;;	vmulpd	ymmcarry, ymmcarry, ymmtmp			;; Multiply by the small constant
;;	vmulpd	ymmcarry2, ymmcarry2, ymmtmp			;; Multiply by the small constant
;;	vmulpd	ymmval, ymmval, ymmtmp
;;	vmulpd	ymmval2, ymmval2, ymmtmp
	ENDM

;
; These macros do the base2 and nobase2 roundings
; const - set to exec if mul_by_const has already computed part of the next carry.
;	  One can also assume ymmval has been rounded if const is set.
; ymmval - input: number to round, output: value to store in the FFT
; ymmcarry - input: part of the next carry if mulbyconst set, output: the next carry
; ymmtmp - a temporary register
;

rounding MACRO ttp, base2, const, ymmval, ymmcarry, ymmtmp, basereg
base2 const ttp		base2_const_rounding ymmval, ymmcarry, ymmtmp, basereg
base2 const no ttp	base2_const_rounding ymmval, ymmcarry, ymmtmp, 0
base2 no const ttp	base2_noconst_rounding ymmval, ymmcarry, ymmtmp, basereg
base2 no const no ttp	base2_noconst_rounding ymmval, ymmcarry, ymmtmp, 0
no base2 const ttp	nobase2_const_rounding ymmval, ymmcarry, ymmtmp, basereg
no base2 const no ttp	nobase2_const_rounding ymmval, ymmcarry, ymmtmp, 0
no base2 no const ttp	nobase2_noconst_rounding ymmval, ymmcarry, ymmtmp, basereg
no base2 no const no ttp nobase2_noconst_rounding ymmval, ymmcarry, ymmtmp, 0
	ENDM

base2_noconst_rounding MACRO ymmval, ymmcarry, ymmtmp, basereg
	vmovapd	ymmtmp, YMM_LIMIT_BIGMAX[basereg]		;; Load maximum * BIGVAL - BIGVAL
	vaddpd	ymmcarry, ymmval, ymmtmp			;; y = top bits of val
	vsubpd	ymmtmp, ymmcarry, ymmtmp			;; z = y - (maximum * BIGVAL - BIGVAL)
	vsubpd	ymmval, ymmval, ymmtmp				;; rounded val = x - z
	vmulpd	ymmcarry, ymmcarry, YMM_LIMIT_INVERSE[basereg]	;; carry = shifted y
	ENDM

base2_const_rounding MACRO ymmval, ymmcarry, ymmtmp, basereg
	vaddpd	ymmtmp, ymmval, YMM_LIMIT_BIGMAX[basereg]	;; y = top bits of x
	vaddpd	ymmcarry, ymmcarry, ymmtmp			;; next carry = upper mul-by-const bits + y
	vsubpd	ymmtmp, ymmtmp, YMM_LIMIT_BIGMAX[basereg]	;; z = y - (maximum*BIGVAL-BIGVAL)
	vmulpd	ymmcarry, ymmcarry, YMM_LIMIT_INVERSE[basereg]	;; shift next carry appropriately
	vsubpd	ymmval, ymmval, ymmtmp				;; rounded value = x - z
	ENDM

nobase2_noconst_rounding MACRO ymmval, ymmcarry, ymmtmp, basereg
	vmulpd	ymmcarry, ymmval, YMM_LIMIT_INVERSE[basereg]	;; val / base
	vroundpd ymmval, ymmval, 0				;; Round val
	vroundpd ymmcarry, ymmcarry, 0				;; next carry = round (val / base)
	vmulpd	ymmtmp, ymmcarry, YMM_LIMIT_BIGMAX[basereg]	;; z = round (val / base) * base
	vsubpd	ymmval, ymmval, ymmtmp				;; new value = val - z
	ENDM

nobase2_const_rounding MACRO ymmval, ymmcarry, ymmtmp, basereg
	vmulpd	ymmtmp, ymmval, YMM_LIMIT_INVERSE[basereg]	;; val / base
	vroundpd ymmtmp, ymmtmp, 0				;; round (val / base)
	vaddpd	ymmcarry, ymmcarry, ymmtmp			;; next carry = partially computed next carry + round (val / base)
	vmulpd	ymmtmp, ymmtmp, YMM_LIMIT_BIGMAX[basereg]	;; z = round (val / base) * base
	vsubpd	ymmval, ymmval, ymmtmp				;; new value = val - z
	ENDM

; Same as above except interleaved for better scheduling

rounding_interleaved MACRO ttp, base2, const, ymmval, ymmcarry, ymmtmp, basereg, ymmval2, ymmcarry2, ymmtmp2, basereg2
base2 const ttp		base2_const_rounding_interleaved ymmval, ymmcarry, ymmtmp, basereg, ymmval2, ymmcarry2, ymmtmp2, basereg2
base2 const no ttp	base2_const_rounding_interleaved ymmval, ymmcarry, ymmtmp, 0, ymmval2, ymmcarry2, ymmtmp2, 0
base2 no const ttp	base2_noconst_rounding_interleaved ymmval, ymmcarry, ymmtmp, basereg, ymmval2, ymmcarry2, ymmtmp2, basereg2
base2 no const no ttp	base2_noconst_rounding_interleaved ymmval, ymmcarry, ymmtmp, 0, ymmval2, ymmcarry2, ymmtmp2, 0
no base2 const ttp	nobase2_const_rounding_interleaved ymmval, ymmcarry, ymmtmp, basereg, ymmval2, ymmcarry2, ymmtmp2, basereg2
no base2 const no ttp	nobase2_const_rounding_interleaved ymmval, ymmcarry, ymmtmp, 0, ymmval2, ymmcarry2, ymmtmp2, 0
no base2 no const ttp	nobase2_noconst_rounding_interleaved ymmval, ymmcarry, ymmtmp, basereg, ymmval2, ymmcarry2, ymmtmp2, basereg2
no base2 no const no ttp nobase2_noconst_rounding_interleaved ymmval, ymmcarry, ymmtmp, 0, ymmval2, ymmcarry2, ymmtmp2, 0
	ENDM

base2_noconst_rounding_interleaved MACRO ymmval, ymmcarry, ymmtmp, basereg, ymmval2, ymmcarry2, ymmtmp2, basereg2
	vmovapd	ymmtmp, YMM_LIMIT_BIGMAX[basereg]	;; Load maximum * BIGVAL - BIGVAL
	vaddpd	ymmcarry, ymmval, ymmtmp		;; y = top bits of val
	vmovapd	ymmtmp2, YMM_LIMIT_BIGMAX[basereg2]	;; Load maximum * BIGVAL - BIGVAL
	vaddpd	ymmcarry2, ymmval2, ymmtmp2		;; y = top bits of val
	vsubpd	ymmtmp, ymmcarry, ymmtmp		;; z = y - (maximum * BIGVAL - BIGVAL)
	vsubpd	ymmtmp2, ymmcarry2, ymmtmp2		;; z = y - (maximum * BIGVAL - BIGVAL)
	vsubpd	ymmval, ymmval, ymmtmp			;; rounded val = x - z
	vsubpd	ymmval2, ymmval2, ymmtmp2		;; rounded val = x - z
	vmulpd	ymmcarry, ymmcarry, YMM_LIMIT_INVERSE[basereg] ;; carry = shifted y
	vmulpd	ymmcarry2, ymmcarry2, YMM_LIMIT_INVERSE[basereg2] ;; carry = shifted y
	ENDM

base2_const_rounding_interleaved MACRO ymmval, ymmcarry, ymmtmp, basereg, ymmval2, ymmcarry2, ymmtmp2, basereg2
	vaddpd	ymmtmp, ymmval, YMM_LIMIT_BIGMAX[basereg]	;; y = top bits of x
	vaddpd	ymmtmp2, ymmval2, YMM_LIMIT_BIGMAX[basereg2]	;; y = top bits of x
	vaddpd	ymmcarry, ymmcarry, ymmtmp			;; next carry = y + upper mul-by-const bits
	vaddpd	ymmcarry2, ymmcarry2, ymmtmp2			;; next carry = y + upper mul-by-const bits
	vsubpd	ymmtmp, ymmtmp, YMM_LIMIT_BIGMAX[basereg]	;; z = y - (maximum*BIGVAL-BIGVAL)
	vsubpd	ymmtmp2, ymmtmp2, YMM_LIMIT_BIGMAX[basereg2]	;; z = y - (maximum*BIGVAL-BIGVAL)
	vmulpd	ymmcarry, ymmcarry, YMM_LIMIT_INVERSE[basereg]	;; shift next carry appropriately
	vmulpd	ymmcarry2, ymmcarry2, YMM_LIMIT_INVERSE[basereg2] ;; shift next carry appropriately
	vsubpd	ymmval, ymmval, ymmtmp				;; rounded value = x - z
	vsubpd	ymmval2, ymmval2, ymmtmp2			;; rounded value = x - z
	ENDM

nobase2_noconst_rounding_interleaved MACRO ymmval, ymmcarry, ymmtmp, basereg, ymmval2, ymmcarry2, ymmtmp2, basereg2
	vmulpd	ymmcarry, ymmval, YMM_LIMIT_INVERSE[basereg]	;; val / base
	vmulpd	ymmcarry2, ymmval2, YMM_LIMIT_INVERSE[basereg2] ;; val / base
	vroundpd ymmval, ymmval, 0				;; Round val
	vroundpd ymmval2, ymmval2, 0				;; Round val
	vroundpd ymmcarry, ymmcarry, 0				;; next carry = round (val / base)
	vroundpd ymmcarry2, ymmcarry2, 0			;; next carry = round (val / base)
	vmulpd	ymmtmp, ymmcarry, YMM_LIMIT_BIGMAX[basereg]	;; z = round (val / base) * base
	vmulpd	ymmtmp2, ymmcarry2, YMM_LIMIT_BIGMAX[basereg2]	;; z = round (val / base) * base
	vsubpd	ymmval, ymmval, ymmtmp				;; new value = val - z
	vsubpd	ymmval2, ymmval2, ymmtmp2			;; new value = val - z
	ENDM

nobase2_const_rounding_interleaved MACRO ymmval, ymmcarry, ymmtmp, basereg, ymmval2, ymmcarry2, ymmtmp2, basereg2
	vmulpd	ymmtmp, ymmval, YMM_LIMIT_INVERSE[basereg]	;; val / base
	vmulpd	ymmtmp2, ymmval2, YMM_LIMIT_INVERSE[basereg2]	;; val / base
	vroundpd ymmtmp, ymmtmp, 0				;; round (val / base)
	vroundpd ymmtmp2, ymmtmp2, 0				;; round (val / base)
	vaddpd	ymmcarry, ymmcarry, ymmtmp			;; next carry = partially computed next carry + round (val / base)
	vaddpd	ymmcarry2, ymmcarry2, ymmtmp2			;; next carry = partially computed next carry + round (val / base)
	vmulpd	ymmtmp, ymmtmp, YMM_LIMIT_BIGMAX[basereg]	;; z = round (val / base) * base
	vmulpd	ymmtmp2, ymmtmp2, YMM_LIMIT_BIGMAX[basereg2]	;; z = round (val / base) * base
	vsubpd	ymmval, ymmval, ymmtmp				;; new value = val - z
	vsubpd	ymmval2, ymmval2, ymmtmp2			;; new value = val - z
	ENDM

;
; These macros round just one value in an YMM register.  This is done
; as part of the cleanup process where the final carry must be added
; back into the results.
;
			;; this is the preferred interface. use of single_rounding without ttp is discouraged
new_single_rounding MACRO ttp, base2, xmmval, xmmcarry, xmmtmp, basereg
base2 ttp	base2_single_rounding xmmval, xmmcarry, xmmtmp, basereg
base2 no ttp	base2_single_rounding xmmval, xmmcarry, xmmtmp, 0
no base2 ttp	nobase2_single_rounding xmmval, xmmcarry, xmmtmp, basereg
no base2 no ttp	nobase2_single_rounding xmmval, xmmcarry, xmmtmp, 0
	ENDM

single_rounding MACRO base2, xmmval, xmmcarry, xmmtmp, basereg
base2		base2_single_rounding xmmval, xmmcarry, xmmtmp, basereg
no base2	nobase2_single_rounding xmmval, xmmcarry, xmmtmp, basereg
	ENDM

base2_single_rounding MACRO xmmval, xmmcarry, xmmtmp, basereg
	vmovsd	xmmtmp, Q YMM_LIMIT_BIGMAX[basereg]		;; Load maximum * BIGVAL - BIGVAL
	vaddsd	xmmcarry, xmmval, xmmtmp			;; y1 = top bits of x
	vsubsd	xmmtmp, xmmcarry, xmmtmp			;; z1 = y1-(maximum * BIGVAL - BIGVAL)
	vsubsd	xmmval, xmmval, xmmtmp				;; rounded value = x1 - z1
	vmulsd	xmmcarry, xmmcarry, Q YMM_LIMIT_INVERSE[basereg];; next carry = shifted y1
	ENDM

nobase2_single_rounding MACRO xmmval, xmmcarry, xmmtmp, basereg
	vmulsd	xmmcarry, xmmval, Q YMM_LIMIT_INVERSE[basereg]	;; val / base
	vroundsd xmmval, xmmval, xmmval, 0			;; Round val
	vroundsd xmmcarry, xmmcarry, xmmcarry, 0		;; next carry = round (val / base)
	vmulsd	xmmtmp, xmmcarry, Q YMM_LIMIT_BIGMAX[basereg]	;; z = round (val / base) * base
	vsubsd	xmmval, xmmval, xmmtmp				;; new value = val - z
	ENDM

			;; this is the preferred interface. use of single_rounding_interleaved without ttp is discouraged
new_single_rounding_interleaved MACRO ttp, base2, xmmval, xmmcarry, xmmtmp, basereg, xmmval2, xmmcarry2, xmmtmp2, basereg2
base2 ttp	base2_single_rounding_interleaved xmmval, xmmcarry, xmmtmp, basereg, xmmval2, xmmcarry2, xmmtmp2, basereg2
base2 no ttp	base2_single_rounding_interleaved xmmval, xmmcarry, xmmtmp, 0, xmmval2, xmmcarry2, xmmtmp2, 0
no base2 ttp	nobase2_single_rounding_interleaved xmmval, xmmcarry, xmmtmp, basereg, xmmval2, xmmcarry2, xmmtmp2, basereg2
no base2 no ttp	nobase2_single_rounding_interleaved xmmval, xmmcarry, xmmtmp, 0, xmmval2, xmmcarry2, xmmtmp2, 0
	ENDM

single_rounding_interleaved MACRO base2, xmmval, xmmcarry, xmmtmp, basereg, xmmval2, xmmcarry2, xmmtmp2, basereg2
base2		base2_single_rounding_interleaved xmmval, xmmcarry, xmmtmp, basereg, xmmval2, xmmcarry2, xmmtmp2, basereg2
no base2	nobase2_single_rounding_interleaved xmmval, xmmcarry, xmmtmp, basereg, xmmval2, xmmcarry2, xmmtmp2, basereg2
	ENDM

base2_single_rounding_interleaved MACRO xmmval, xmmcarry, xmmtmp, basereg, xmmval2, xmmcarry2, xmmtmp2, basereg2
	vmovsd	xmmtmp, Q YMM_LIMIT_BIGMAX[basereg]		;; Load maximum * BIGVAL - BIGVAL
	vmovsd	xmmtmp2, Q YMM_LIMIT_BIGMAX[basereg2]		;; Load maximum * BIGVAL - BIGVAL
	vaddsd	xmmcarry, xmmval, xmmtmp			;; y1 = top bits of x
	vaddsd	xmmcarry2, xmmval2, xmmtmp2			;; y1 = top bits of x
	vsubsd	xmmtmp, xmmcarry, xmmtmp			;; z1 = y1-(maximum * BIGVAL - BIGVAL)
	vsubsd	xmmtmp2, xmmcarry2, xmmtmp2			;; z1 = y1-(maximum * BIGVAL - BIGVAL)
	vsubsd	xmmval, xmmval, xmmtmp				;; rounded value = x1 - z1
	vsubsd	xmmval2, xmmval2, xmmtmp2			;; rounded value = x1 - z1
	vmulsd	xmmcarry, xmmcarry, Q YMM_LIMIT_INVERSE[basereg];; next carry = shifted y1
	vmulsd	xmmcarry2, xmmcarry2, Q YMM_LIMIT_INVERSE[basereg2] ;; next carry = shifted y1
	ENDM

nobase2_single_rounding_interleaved MACRO xmmval, xmmcarry, xmmtmp, basereg, xmmval2, xmmcarry2, xmmtmp2, basereg2
	vmulsd	xmmcarry, xmmval, Q YMM_LIMIT_INVERSE[basereg]	;; val / base
	vmulsd	xmmcarry2, xmmval2, Q YMM_LIMIT_INVERSE[basereg2] ;; val / base
	vroundsd xmmval, xmmval, xmmval, 0			;; Round val
	vroundsd xmmval2, xmmval2, xmmval2, 0			;; Round val
	vroundsd xmmcarry, xmmcarry, xmmcarry, 0		;; next carry = round (val / base)
	vroundsd xmmcarry2, xmmcarry2, xmmcarry2, 0		;; next carry = round (val / base)
	vmulsd	xmmtmp, xmmcarry, Q YMM_LIMIT_BIGMAX[basereg]	;; z = round (val / base) * base
	vmulsd	xmmtmp2, xmmcarry2, Q YMM_LIMIT_BIGMAX[basereg2] ;; z = round (val / base) * base
	vsubsd	xmmval, xmmval, xmmtmp				;; new value = val - z
	vsubsd	xmmval2, xmmval2, xmmtmp2			;; new value = val - z
	ENDM

;
; These macros process zero-padded FFT result words.  These FFT results must
; be split into high and low parts with the high part used as a carry into
; the splitting the next FFT result word.
;

split_lower_zpad_word MACRO ttp, base2, echk, ymmvalin, ymmcarry, ymmvalout, basereg
base2 ttp	base2_split_lower_zpad_word echk, ymmvalin, ymmcarry, ymmvalout, basereg
base2 no ttp	base2_split_lower_zpad_word echk, ymmvalin, ymmcarry, ymmvalout, 0
no base2 ttp	nobase2_split_lower_zpad_word echk, ymmvalin, ymmcarry, ymmvalout, basereg
no base2 no ttp	nobase2_split_lower_zpad_word echk, ymmvalin, ymmcarry, ymmvalout, 0
	ENDM

base2_split_lower_zpad_word MACRO echk, ymmvalin, ymmcarry, ymmvalout, basereg
	vaddpd	ymmvalin, ymmvalin, ymmcarry			;; Add in previous high FFT data
IFNDEF X86_64
	vmovapd	ymmvalout, YMM_BIGBIGVAL			;; Big word rounding constant
	vaddpd	ymmcarry, ymmvalin, ymmvalout			;; Round to multiple of big word
	vsubpd	ymmcarry, ymmcarry, ymmvalout
ELSE
	vaddpd	ymmcarry, ymmvalin, ymm14			;; Round to multiple of big word
	vsubpd	ymmcarry, ymmcarry, ymm14
ENDIF
	vroundpd ymmvalout, ymmvalin, 0				;; Round to an integer
echk	vsubpd	ymmvalin, ymmvalin, ymmvalout			;; This is the convolution error
echk	absval	ymmvalin					;; Compute absolute value
echk	vmaxpd	ymm6, ymm6, ymmvalin				;; Compute maximum error
	vsubpd	ymmvalout, ymmvalout, ymmcarry			;; ymmvalout now contains low bigword bits
	vmulpd	ymmcarry, ymmcarry, YMM_LIMIT_INVERSE[basereg]	;; Saved shifted FFT hi data
	ENDM

nobase2_split_lower_zpad_word MACRO echk, ymmvalin, ymmcarry, ymmvalout, basereg
	vaddpd	ymmvalin, ymmvalin, ymmcarry			;; Add in previous high FFT data
	vmulpd	ymmcarry, ymmvalin,YMM_LIMIT_INVERSE[basereg]	;; Compute FFTvalue / base
	vroundpd ymmvalout, ymmvalin, 0				;; Round input to an integer
	vroundpd ymmcarry, ymmcarry, 0				;; Round FFTvalue / base to integer
echk	vsubpd	ymmvalin, ymmvalin, ymmvalout			;; This is the convolution error
echk	absval	ymmvalin					;; Compute absolute value
echk	vmaxpd	ymm6, ymm6, ymmvalin				;; Compute maximum error
	vmulpd	ymmvalin, ymmcarry, YMM_LIMIT_BIGMAX[basereg]
	vsubpd	ymmvalout, ymmvalout, ymmvalin			;; ymmvalout now contains FFTvalue % base
	ENDM

; Split upper is like split lower except that previous carry is not added in and
; result carry is not shifted down.

split_upper_zpad_word MACRO ttp, base2, echk, ymmvalin, ymmcarry, ymmvalout, basereg
base2 ttp	base2_split_upper_zpad_word echk, ymmvalin, ymmcarry, ymmvalout, basereg
base2 no ttp	base2_split_upper_zpad_word echk, ymmvalin, ymmcarry, ymmvalout, 0
no base2 ttp	nobase2_split_upper_zpad_word echk, ymmvalin, ymmcarry, ymmvalout, basereg
no base2 no ttp	nobase2_split_upper_zpad_word echk, ymmvalin, ymmcarry, ymmvalout, 0
	ENDM

base2_split_upper_zpad_word MACRO echk, ymmvalin, ymmcarry, ymmvalout, basereg
IFNDEF X86_64
	vmovapd	ymmvalout, YMM_BIGBIGVAL		;; Big word rounding constant
	vaddpd	ymmcarry, ymmvalin, ymmvalout		;; Round to multiple of big word
	vsubpd	ymmcarry, ymmcarry, ymmvalout
ELSE
	vaddpd	ymmcarry, ymmvalin, ymm14		;; Round to multiple of big word
	vsubpd	ymmcarry, ymmcarry, ymm14
ENDIF
	vroundpd ymmvalout, ymmvalin, 0			;; Round input to an integer
echk	vsubpd	ymmvalin, ymmvalin, ymmvalout		;; This is the convolution error
echk	absval	ymmvalin				;; Compute absolute value
echk	vmaxpd	ymm6, ymm6, ymmvalin			;; Compute maximum error
	vsubpd	ymmvalout, ymmvalout, ymmcarry		;; ymmvalout now contains low bigword bits
	ENDM

nobase2_split_upper_zpad_word MACRO echk, ymmvalin, ymmcarry, ymmvalout, basereg
	vmulpd	ymmcarry, ymmvalin, YMM_LIMIT_INVERSE[basereg]	;; Compute FFTvalue / base
	vroundpd ymmvalout, ymmvalin, 0				;; Round input to an integer
	vroundpd ymmcarry, ymmcarry, 0				;; Round FFTvalue / base to integer
echk	vsubpd	ymmvalin, ymmvalin, ymmvalout			;; This is the convolution error
echk	absval	ymmvalin					;; Compute absolute value
echk	vmaxpd	ymm6, ymm6, ymmvalin				;; Compute maximum error
	vmulpd	ymmvalin, ymmcarry, YMM_LIMIT_BIGMAX[basereg]
	vsubpd	ymmvalout, ymmvalout, ymmvalin			;; ymmvalout now contains FFTvalue % base
	ENDM

; The single word version

new_single_split_lower_zpad_word MACRO ttp, base2, xmmval, xmmcarry, xmmtmp, basereg
base2 ttp	base2_single_split_lower_zpad_word xmmval, xmmcarry, xmmtmp, basereg
base2 no ttp	base2_single_split_lower_zpad_word xmmval, xmmcarry, xmmtmp, 0
no base2 ttp	nobase2_single_split_lower_zpad_word xmmval, xmmcarry, xmmtmp, basereg
no base2 no ttp	nobase2_single_split_lower_zpad_word xmmval, xmmcarry, xmmtmp, 0
	ENDM

single_split_lower_zpad_word MACRO base2, xmmval, xmmcarry, xmmtmp, basereg
base2		base2_single_split_lower_zpad_word xmmval, xmmcarry, xmmtmp, basereg
no base2	nobase2_single_split_lower_zpad_word xmmval, xmmcarry, xmmtmp, basereg
	ENDM

base2_single_split_lower_zpad_word MACRO xmmval, xmmcarry, xmmtmp, basereg
	vaddsd	xmmval, xmmval, xmmcarry			;; Add in previous high FFT data
	vmovsd	xmmtmp, Q YMM_BIGBIGVAL				;; Big word rounding constant
	vaddsd	xmmcarry, xmmval, xmmtmp			;; Round to multiple of big word
	vsubsd	xmmcarry, xmmcarry, xmmtmp
	vroundsd xmmval, xmmval, xmmval, 0			;; Round to an integer
	vsubsd	xmmval, xmmval, xmmcarry			;; xmmval now contains low bigword bits
	vmulsd	xmmcarry, xmmcarry, Q YMM_LIMIT_INVERSE[basereg] ;; Next carry = shifted FFT hi data
	ENDM

nobase2_single_split_lower_zpad_word MACRO xmmval, xmmcarry, xmmtmp, basereg
	vaddsd	xmmval, xmmval, xmmcarry			;; Add in previous high FFT data
	vmulsd	xmmcarry, xmmval, Q YMM_LIMIT_INVERSE[basereg]	;; Compute FFTvalue / base
	vroundsd xmmval, xmmval, xmmval, 0			;; Round input to an integer
	vroundsd xmmcarry, xmmcarry, xmmcarry, 0		;; Next carry = round ( FFTvalue / base )
	vmulsd	xmmtmp, xmmcarry, Q YMM_LIMIT_BIGMAX[basereg]
	vsubsd	xmmval, xmmval, xmmtmp				;; xmmval now contains FFTvalue % base
	ENDM

; The version for splitting the high FFT carry.  The high carry input has already
; been rounded to an integer.

split_upper_carry_zpad_word MACRO ttp, base2, ymmcarryin, ymmcarryout, ymmtmp, basereg
base2 ttp	base2_split_upper_carry_zpad_word ymmcarryin, ymmcarryout, ymmtmp, basereg
base2 no ttp	base2_split_upper_carry_zpad_word ymmcarryin, ymmcarryout, ymmtmp, 0
no base2 ttp	nobase2_split_upper_carry_zpad_word ymmcarryin, ymmcarryout, ymmtmp, basereg
no base2 no ttp	nobase2_split_upper_carry_zpad_word ymmcarryin, ymmcarryout, ymmtmp, 0
	ENDM

base2_split_upper_carry_zpad_word MACRO ymmcarryin, ymmcarryout, ymmtmp, basereg
	vmovapd	ymmtmp, YMM_BIGBIGVAL			;; Big word rounding constant
	vaddpd	ymmcarryout, ymmcarryin, ymmtmp		;; Round to multiple of big word
	vsubpd	ymmcarryout, ymmcarryout, ymmtmp
	vsubpd	ymmcarryin, ymmcarryin, ymmcarryout	;; ymmcarryin now contains low bigword bits
	vmulpd	ymmcarryout, ymmcarryout, YMM_LIMIT_INVERSE[basereg] ;; Next carry = shifted carry
	ENDM

nobase2_split_upper_carry_zpad_word MACRO ymmcarryin, ymmcarryout, ymmtmp, basereg
	vmulpd	ymmcarryout, ymmcarryin, YMM_LIMIT_INVERSE[basereg]	;; Compute carry / base
	vroundpd ymmcarryout, ymmcarryout, 0				;; Next carry = round(carry / base)
	vmulpd	ymmtmp, ymmcarryout, YMM_LIMIT_BIGMAX[basereg]
	vsubpd	ymmcarryin, ymmcarryin, ymmtmp				;; ymmcarryin now contains carry % base
	ENDM

; Round the ZPAD0 - ZPAD6 values.  Simpler than other rounding macros
; in that we always round to a big word (and input value and output
; carry do not have YMM_BIGVAL added in).

new_round_zpad7_word MACRO ttp, base2, xmmvalin, xmmcarry, xmmvalout, basereg
base2 ttp	base2_round_zpad7_word xmmvalin, xmmcarry, xmmvalout, basereg
base2 no ttp	base2_round_zpad7_word xmmvalin, xmmcarry, xmmvalout, 0
no base2 ttp	nobase2_round_zpad7_word xmmvalin, xmmcarry, xmmvalout, basereg
no base2 no ttp	nobase2_round_zpad7_word xmmvalin, xmmcarry, xmmvalout, 0
	ENDM

round_zpad7_word MACRO base2, xmmvalin, xmmcarry, xmmvalout, basereg
base2		base2_round_zpad7_word xmmvalin, xmmcarry, xmmvalout, basereg
no base2	nobase2_round_zpad7_word xmmvalin, xmmcarry, xmmvalout, basereg
	ENDM

base2_round_zpad7_word MACRO xmmval, xmmcarry, xmmtmp, basereg
	vaddsd	xmmval, xmmval, xmmcarry		;; Add in high part of last calculation
	vmovsd	xmmtmp, Q YMM_BIGBIGVAL			;; Big word rounding constant
	vaddsd	xmmcarry, xmmval, xmmtmp		;; Round to multiple of big word
	vsubsd	xmmcarry, xmmcarry, xmmtmp
	vsubsd	xmmval, xmmval, xmmcarry		;; xmmval now contains low bigword bits
	vmulsd	xmmcarry, xmmcarry, Q YMM_LIMIT_INVERSE[basereg] ;; Shift high ZPAD data
	ENDM

nobase2_round_zpad7_word MACRO xmmval, xmmcarry, xmmtmp, basereg
	vaddsd	xmmval, xmmval, xmmcarry			;; Add in high part of last calculation
	vmulsd	xmmcarry, xmmval, Q YMM_LIMIT_INVERSE[basereg]	;; val / base
	vroundsd xmmcarry, xmmcarry, xmmcarry, 0		;; next carry = round (val / base)
	vmulsd	xmmtmp, xmmcarry, Q YMM_LIMIT_BIGMAX[basereg]	;; z = round (val / base) * base
	vsubsd	xmmval, xmmval, xmmtmp				;; new value = val - z
	ENDM


;; Rotate the YMM registers right by one double.
;; For example in section 1 of a length 32 FFT:
;; On input ymmreg1 is MSW (1 2 3 4) LSW
;; On output ymmreg1 is (x 1 2 3) and ymmreg2 is (4 x x x)
;; where x is either BIGVAL or zero.

rotate_carries MACRO base2, ymmreg1, ymmreg2, ymmtmp, ymmtmp2
base2	vmovapd	ymmreg2, YMM_BIGVAL
no base2 vxorpd	ymmreg2, ymmreg2, ymmreg2
	vperm2f128 ymmtmp, ymmreg1, ymmreg2, 03h	;; create (x x 1 2)
	vperm2f128 ymmtmp2, ymmreg1, ymmreg2, 21h	;; create (3 4 x x)
	vshufpd ymmreg1, ymmtmp, ymmreg1, 0100b		;; create (x 1 2 3)
	vshufpd	ymmreg2, ymmtmp2, ymmreg2, 0001b	;; create (4 x x x)
	ENDM

rotate_carries_interleaved MACRO base2, ymmreg1, ymmreg2, ymmreg3, ymmreg4, ymmtmp, ymmtmp2
base2	vmovapd	ymmreg4, YMM_BIGVAL
no base2 vxorpd	ymmreg4, ymmreg4, ymmreg4
	vperm2f128 ymmtmp, ymmreg1, ymmreg4, 03h	;; create (x x 1 2)
	vperm2f128 ymmreg2, ymmreg1, ymmreg4, 21h	;; create (3 4 x x)
	vperm2f128 ymmtmp2, ymmreg3, ymmreg4, 21h	;; create (3 4 x x)
	vshufpd ymmreg1, ymmtmp, ymmreg1, 0100b		;; create (x 1 2 3)
	vperm2f128 ymmtmp, ymmreg3, ymmreg4, 03h	;; create (x x 1 2)
	vshufpd	ymmreg2, ymmreg2, ymmreg4, 0001b	;; create (4 x x x)
	vshufpd	ymmreg4, ymmtmp2, ymmreg4, 0001b	;; create (4 x x x)
	vshufpd ymmreg3, ymmtmp, ymmreg3, 0100b		;; create (x 1 2 3)
	ENDM

;; Alternative rotate of YMM registers right by one double.
;; Used in two-pass FFT carry propagation
;; ymmreg1 is MSW (4 3 2 1) LSW
;; ymmreg2 is MSW (x x x 5) LSW
;; output is:
;; ymmreg1 is (3 2 1 5)
;; ymmreg2 is (x x x 4)
rotate_five_carries MACRO ymmreg1, ymmreg2, ymmtmp
	vperm2f128 ymmtmp, ymmreg1, ymmreg2, 02h	;; create (2 1 x 5)
	vperm2f128 ymmreg2, ymmreg1, ymmreg2, 31h	;; create (x x 4 3)
	vshufpd ymmreg1, ymmtmp, ymmreg1, 0100b		;; create (3 2 1 5)
	vshufpd	ymmreg2, ymmreg2, ymmreg2, 0001b	;; create (x x x 4)
	ENDM
;
; Now for the actual normalization macros!
;

; For 1D macros, these registers are set on input:
; ymm6 = MAXERR
; rsi = pointer to the FFT data values
; rbp = pointer two-to-phi multipliers
; rdi = pointer to array of big vs. little flags
; ecx = big vs. little word flag #2
; eax = big vs. little word flag #1
; ymm2 = carry #1
; ymm3 = carry #2


; *************** 1D macro ******************
; A pipelined version of this code:
;	movzx	rax, BYTE PTR [rdi]	;; Load big vs. little flags
;	vmovapd	ymm0, [rsi+0*dist1]	;; Load values
;	vmulpd	ymm4, ymm0, [rbp+0]	;; Mul values1 by two-to-minus-phi
;	vaddpd	ymm4, ymm4, ymm2	;; x = values + carry
;	vmovpad	ymm0, YMM_LIMIT_BIGMAX[rax*2];; Load maximum * BIGVAL - BIGVAL
;	vaddpd	ymm2, ymm4, ymm0	;; y = top bits of x
;	vaddpd	ymm0, ymm2, ymm0	;; z = y - (maximum * BIGVAL - BIGVAL)
;	vsubpd	ymm4, ymm4, ymm0	;; rounded value = x - z
;	vmulpd	ymm2, ymm2, YMM_LIMIT_INVERSE[rax*2];; next carry = shifted y
;	vmulpd	ymm4, ymm4, [rbp+32]	;; new value = val * two-to-phi
;	ystore	[rsi+0*dist1], ymm4	;; Save new value

ynorm_1d_preload MACRO ttp, base2, echk, const
	ENDM

ynorm_1d MACRO ttp, base2, echk, const
ttp		movzx	rax, BYTE PTR [rdi]	;; Load big vs. little flags
ttp		movzx	rcx, BYTE PTR [rdi+1]
		vmovapd	ymm0, [rsi]		;; Load value1
		vmulpd	ymm4, ymm0, [rbp]	;; Mul value1 by two-to-minus-phi
		vmovapd	ymm1, [rsi+32]		;; Load value2
		vmulpd	ymm5, ymm1, [rbp+64]	;; Mul value2 by two-to-minus-phi
const		mul_by_const_interleaved ttp, base2, echk, ymm4, ymm2, ymm0, rax*2, ymm5, ymm3, ymm1, rcx*2, ymm6
no const echk	error_check_interleaved ymm4, ymm0, ymm5, ymm1, ymm6
no const	vaddpd	ymm4, ymm4, ymm2	;; x1 = value + carry
no const	vaddpd	ymm5, ymm5, ymm3	;; x2 = value + carry
		rounding_interleaved ttp, base2, const, ymm4, ymm2, ymm0, rax*2, ymm5, ymm3, ymm1, rcx*2
ttp		vmulpd	ymm4, ymm4, [rbp+32]	;; new value1 = val * two-to-phi
ttp		vmulpd	ymm5, ymm5, [rbp+96]	;; new value2 = val * two-to-phi
		ystore	[rsi], ymm4		;; Save value1
		ystore	[rsi+32], ymm5		;; Save value2
	ENDM


IFDEF X86_64

ynorm_1d_preload MACRO ttp, base2, echk, const
echk		vmovapd	ymm15, YMM_ABSVAL
base2 const	vmovapd	ymm14, YMM_BIGBIGVAL
const		vmovapd	ymm13, YMM_MULCONST
IF (@INSTR(,%yarch,<FMA3>) NE 0)
base2 no const	vmovapd ymm14, YMM_ONE
ENDIF
	ENDM

ynorm_1d MACRO ttp, base2, echk, const
ttp		movzx	rax, BYTE PTR [rdi]	;; Load big vs. little flags
ttp		movzx	rcx, BYTE PTR [rdi+1]

base2 no const ttp	ynorm_1d_base2_noconst	echk, rax*2, rcx*2
base2 no const no ttp	ynorm_1d_base2_noconst	echk, 0, 0
base2 const ttp		ynorm_1d_base2_const	echk, rax*2, rcx*2
base2 const no ttp	ynorm_1d_base2_const	echk, 0, 0
no base2 no const ttp	ynorm_1d_nobase2_noconst echk, rax*2, rcx*2
no base2 no const no ttp ynorm_1d_nobase2_noconst echk, 0, 0
no base2 const ttp	ynorm_1d_nobase2_const	echk, rax*2, rcx*2
no base2 const no ttp	ynorm_1d_nobase2_const	echk, 0, 0

ttp		vmulpd	ymm4, ymm4, [rbp+32]	;; new value1 = val * two-to-phi
ttp		vmulpd	ymm5, ymm5, [rbp+96]	;; new value2 = val * two-to-phi
		ystore	[rsi], ymm4		;; Save value1
		ystore	[rsi+32], ymm5		;; Save value2
	ENDM


ynorm_1d_base2_noconst MACRO echk, basereg, basereg2
	vmovapd	ymm0, [rsi]				;; Load value1
	vmulpd	ymm8, ymm0, [rbp]			;; Mul value1 by two-to-minus-phi		; 1-5
	vmovapd	ymm1, [rsi+32]				;; Load value2
	vmulpd	ymm9, ymm1, [rbp+64]			;; Mul value2 by two-to-minus-phi		; 2-6
	vaddpd	ymm0, ymm8, ymm2			;; x = value1 + carry				; 6-8	First dependence on previous iteration
	vaddpd	ymm1, ymm9, ymm3			;; x = value2 + carry				; 7-9
echk	vroundpd ymm10, ymm8, 0				;; Convert to an integer			;	(8-10)	(and 11-13 on Haswell)
echk	vroundpd ymm11, ymm9, 0				;; Convert to an integer			;	(9-11)	(and 12-14 on Haswell)
	vmovapd	ymm4, YMM_LIMIT_BIGMAX[basereg]		;; Load maximum * BIGVAL - BIGVAL	
	vaddpd	ymm2, ymm0, ymm4			;; y = top bits of val				; 9-11	(10-12)	(10-12)
	vmovapd	ymm5, YMM_LIMIT_BIGMAX[basereg2]	;; Load maximum * BIGVAL - BIGVAL	
	vaddpd	ymm3, ymm1, ymm5			;; y = top bits of val				; 10-12	(11-13) (13-15)
echk	vsubpd	ymm10, ymm10, ymm8			;; This is the convolution error		;	(12-14)	(14-16)
echk	vsubpd	ymm11, ymm11, ymm9			;; This is the convolution error		;	(13-15)	(15-17)
	vsubpd	ymm4, ymm2, ymm4			;; z = y - (maximum * BIGVAL - BIGVAL)		; 12-14	(14-16) (16-18)
	vmulpd	ymm2, ymm2, YMM_LIMIT_INVERSE[basereg]	;; carry = shifted y				; 12-16	(13-17)	(13-17)
	vsubpd	ymm5, ymm3, ymm5			;; z = y - (maximum * BIGVAL - BIGVAL)		; 13-15	(15-17)	(17-19)
	vmulpd	ymm3, ymm3, YMM_LIMIT_INVERSE[basereg2]	;; carry = shifted y				; 13-17	(14-18)	(16-20)
echk	absval	ymm10					;; Compute absolute value			;	(15)	(17)
echk	absval	ymm11					;; Compute absolute value			;	(16)	(18)
	vsubpd	ymm4, ymm0, ymm4			;; rounded val = x - z				; 15-17	(17-19)	(19-21)
	vsubpd	ymm5, ymm1, ymm5			;; rounded val = x - z				; 16-18	(18-20)	(20-22)
echk	vmaxpd	ymm6, ymm6, ymm10			;; Compute maximum error			;	(19-21)	(18-20)
echk	vmaxpd	ymm6, ymm6, ymm11			;; Compute maximum error			;	(20-22)	(21-23)
ENDM

;; I haven't found a way to use FMA3 to make ynorm_1d_base2_noconst faster.  It is slower to use FMA to
;; calculate "x = value * two-to-minus-phi + carry" because "value * two-to-minus-phi" can be
;; calculated for free in the previous iteration, using FMA introduces a premature dependence on the
;; previous carry.
;; 
;; We can, however, avoid using the vroundpd instruction (2 uops, 6 clocks on Haswell) when error checking.
;; This is done by computing "convolution error = value - (x - carry)

IF (@INSTR(,%yarch,<FMA3>) NE 0)
ynorm_1d_base2_noconst MACRO echk, basereg, basereg2
	vmovapd	ymm0, [rsi]				;; Load value1
	vmulpd	ymm8, ymm0, [rbp]			;; Mul value1 by two-to-minus-phi		; presumably free (can execute in previous iteration)
	vmovapd	ymm1, [rsi+32]				;; Load value2
	vmulpd	ymm9, ymm1, [rbp+64]			;; Mul value2 by two-to-minus-phi		; presumably free
	vaddpd	ymm0, ymm8, ymm2			;; x = value1 + carry				; 6-8	First dependence on previous iteration
	vaddpd	ymm1, ymm9, ymm3			;; x = value2 + carry				; 7-9
	vmovapd	ymm4, YMM_LIMIT_BIGMAX[basereg]		;; Load maximum * BIGVAL - BIGVAL
	vaddpd	ymm12, ymm0, ymm4			;; y = top bits of val				; 9-11
	vmovapd	ymm5, YMM_LIMIT_BIGMAX[basereg2]	;; Load maximum * BIGVAL - BIGVAL
	vaddpd	ymm13, ymm1, ymm5			;; y = top bits of val				; 10-12
echk	vsubpd	ymm10, ymm0, ymm2			;; Start error calc (x - carry)			;	(11-13)
echk	vsubpd	ymm11, ymm1, ymm3			;; Start error calc (x - carry)			;	(12-14)
	vsubpd	ymm4, ymm12, ymm4			;; z = y - (maximum * BIGVAL - BIGVAL)		; 12-14	(13-15)
	vmulpd	ymm2, ymm12, YMM_LIMIT_INVERSE[basereg]	;; carry = shifted y				; 12-16	(13-17)
	vsubpd	ymm5, ymm13, ymm5			;; z = y - (maximum * BIGVAL - BIGVAL)		; 13-15	(14-16)
	vmulpd	ymm3, ymm13, YMM_LIMIT_INVERSE[basereg2];; carry = shifted y				; 13-17	(14-18)
echk	vsubpd	ymm10, ymm8, ymm10			;; Convolution error = value - (x - carry)	;	(15-17)
echk	vsubpd	ymm11, ymm9, ymm11			;; Convolution error = value - (x - carry)	;	(16-18)
	vsubpd	ymm4, ymm0, ymm4			;; rounded val = x - z				; 15-17	(17-19)
	vsubpd	ymm5, ymm1, ymm5			;; rounded val = x - z				; 16-18	(18-20)
echk	absval	ymm10					;; Compute absolute value			;	(18)
echk	absval	ymm11					;; Compute absolute value			;	(19)
echk	vmaxpd	ymm6, ymm6, ymm10			;; Compute maximum error			;	(19-21)
echk	vmaxpd	ymm6, ymm6, ymm11			;; Compute maximum error			;	(20-22)
	ENDM
ENDIF

ynorm_1d_base2_const MACRO echk, basereg, basereg2
	vmovapd	ymm0, [rsi]				;; Load value1
	vmulpd	ymm10, ymm0, [rbp]			;; Mul value1 by two-to-minus-phi		; 1-5
	vmovapd	ymm1, [rsi+32]				;; Load value2
	vmulpd	ymm11, ymm1, [rbp+64]			;; Mul value2 by two-to-minus-phi		; 2-6
	vaddpd	ymm0, ymm10, ymm14			;; Round to nearest multiple of 2^25		; 6-8
	vaddpd	ymm1, ymm11, ymm14			;; Round to nearest multiple of 2^25		; 7-9
	vroundpd ymm8, ymm10, 0				;; Round FFT value to an integer		; 8-10
	vsubpd	ymm0, ymm0, ymm14			;; Compute high bits of FFT value		; 9-11
	vsubpd	ymm1, ymm1, ymm14			;; Compute high bits of FFT value		; 10-12
	vroundpd ymm9, ymm11, 0				;; Round FFT value to an integer		; 11-13
	vsubpd	ymm4, ymm8, ymm0			;; Compute low 25 bits of FFT value		; 12-14
echk	vsubpd	ymm10, ymm8, ymm10			;; This is the convolution error		;  13-15
	vsubpd	ymm5, ymm9, ymm1			;; Compute low 25 bits of FFT value		; 14-16
	vmulpd	ymm4, ymm4, ymm13			;; Mul low bits of FFT value by the constant	; 15-19
echk	vsubpd	ymm11, ymm9, ymm11			;; This is the convolution error		;  15-17
echk	absval	ymm10					;; Compute absolute value			;  16
	vmulpd	ymm5, ymm5, ymm13			;; Mul low bits of FFT value by the constant	; 17-21
echk	vmaxpd	ymm6, ymm6, ymm10			;; Compute maximum error			;  17-19
echk	absval	ymm11					;; Compute absolute value			;  18
	vmulpd	ymm0, ymm0, ymm13			;; next carry = high bits of value times const	; 12-16 (result not needed until clock 27)
echk	vmaxpd	ymm6, ymm6, ymm11			;; Compute maximum error			;  19-21
	vmulpd	ymm1, ymm1, ymm13			;; next carry = high bits of value times const	; 13-17 (result not needed until clock 28)
	vaddpd	ymm4, ymm4, ymm2			;; Add in the previous carry			; 20-22
	vaddpd	ymm5, ymm5, ymm3			;; Add in the previous carry			; 22-24
	vmovapd	ymm8, YMM_LIMIT_BIGMAX[basereg]		;; Load maximum * BIGVAL - BIGVAL
	vaddpd	ymm2, ymm4, ymm8			;; y = top bits of val				; 23-25
	vmovapd	ymm9, YMM_LIMIT_BIGMAX[basereg2]	;; Load maximum * BIGVAL - BIGVAL
	vaddpd	ymm3, ymm5, ymm9			;; y = top bits of val				; 25-27
	vsubpd	ymm8, ymm2, ymm8			;; z = y - (maximum * BIGVAL - BIGVAL)		; 26-28
	vaddpd	ymm0, ymm0, ymm2			;; next carry += y				; 27-29
	vsubpd	ymm9, ymm3, ymm9			;; z = y - (maximum * BIGVAL - BIGVAL)		; 28-30
	vaddpd	ymm1, ymm1, ymm3			;; next carry += y				; 29-31
	vsubpd	ymm4, ymm4, ymm8			;; rounded val = x - z				; 30-32
	vmulpd	ymm2, ymm0, YMM_LIMIT_INVERSE[basereg]	;; shifted carry				; 30-34
	vsubpd	ymm5, ymm5, ymm9			;; rounded val = x - z				; 31-33
	vmulpd	ymm3, ymm1, YMM_LIMIT_INVERSE[basereg2]	;; shifted carry				; 32-36
ENDM

IF (@INSTR(,%yarch,<FMA3>) NE 0)
ynorm_1d_base2_const MACRO echk, basereg, basereg2
	vmovapd	ymm0, [rsi]				;; Load value1
	vmulpd	ymm8, ymm0, [rbp]			;; Mul value1 by two-to-minus-phi		; 1-5
	vmovapd	ymm1, [rsi+32]				;; Load value2
	vmulpd	ymm9, ymm1, [rbp+64]			;; Mul value2 by two-to-minus-phi		; 2-6
	vaddpd	ymm0, ymm8, ymm14			;; Round to nearest multiple of 2^25		; 6-8
	vaddpd	ymm1, ymm9, ymm14			;; Round to nearest multiple of 2^25		; 7-9
	vroundpd ymm10, ymm8, 0				;; Round FFT value to an integer		; 8-10
	vsubpd	ymm0, ymm0, ymm14			;; Compute high bits of FFT value		; 9-11
	vsubpd	ymm1, ymm1, ymm14			;; Compute high bits of FFT value		; 10-12
	vroundpd ymm11, ymm9, 0				;; Round FFT value to an integer		; 11-13
	vsubpd	ymm4, ymm10, ymm0			;; Compute low 25 bits of FFT value		; 12-14
echk	vsubpd	ymm10, ymm10, ymm8			;; This is the convolution error		;  13-15
	vsubpd	ymm5, ymm11, ymm1			;; Compute low 25 bits of FFT value		; 14-16
	yfmaddpd ymm4, ymm4, ymm13, ymm2		;; Low bits of value * constant + carry		; 15-19
echk	vsubpd	ymm11, ymm11, ymm9			;; This is the convolution error		;  15-17
echk	absval	ymm10					;; Compute absolute value			;  16
	yfmaddpd ymm5, ymm5, ymm13, ymm3		;; Low bits of value * constant + carry		; 17-21
echk	vmaxpd	ymm6, ymm6, ymm10			;; Compute maximum error			;  17-19
echk	absval	ymm11					;; Compute absolute value			;  18
	vmulpd	ymm0, ymm0, ymm13			;; next carry = high bits of value times const	; 12-16
echk	vmaxpd	ymm6, ymm6, ymm11			;; Compute maximum error			;  19-21
	vmulpd	ymm1, ymm1, ymm13			;; next carry = high bits of value times const	; 13-17
	vmovapd	ymm8, YMM_LIMIT_BIGMAX[basereg]		;; Load maximum * BIGVAL - BIGVAL
	vaddpd	ymm2, ymm4, ymm8			;; y = top bits of val				; 20-22
	vmovapd	ymm9, YMM_LIMIT_BIGMAX[basereg2]	;; Load maximum * BIGVAL - BIGVAL
	vaddpd	ymm3, ymm5, ymm9			;; y = top bits of val				; 22-24
	vsubpd	ymm8, ymm2, ymm8			;; z = y - (maximum * BIGVAL - BIGVAL)		; 23-25
	;;;yfmaddpd ymm0, ymm0, ymm13, ymm2		;; next carry = high bits of value * const + y	; 23-27		this fma would increase latency but ease add scheduling
	vaddpd	ymm0, ymm0, ymm2			;; next carry += y				; 24-26
	vsubpd	ymm9, ymm3, ymm9			;; z = y - (maximum * BIGVAL - BIGVAL)		; 25-27
	;;;yfmaddpd ymm1, ymm1, ymm13, ymm3		;; next carry = high bits of value * const + y	; 25-29		this fma would increase latency but ease add scheduling
	vaddpd	ymm1, ymm1, ymm3			;; next carry += y				; 26-28
	vsubpd	ymm4, ymm4, ymm8			;; rounded val = x - z				; 27-29		could be done 26-28, but
	vmulpd	ymm2, ymm0, YMM_LIMIT_INVERSE[basereg]	;; shifted carry				; 27-31
	vsubpd	ymm5, ymm5, ymm9			;; rounded val = x - z				; 28-30
	vmulpd	ymm3, ymm1, YMM_LIMIT_INVERSE[basereg2]	;; shifted carry				; 29-33		but this would be 30-34
ENDM
ENDIF


ynorm_1d_nobase2_noconst MACRO echk, basereg, basereg2
	vmovapd	ymm0, [rsi]				;; Load value1
	vmulpd	ymm4, ymm0, [rbp]			;; Mul value1 by two-to-minus-phi		; 1-5
	vmovapd	ymm1, [rsi+32]				;; Load value2
	vmulpd	ymm5, ymm1, [rbp+64]			;; Mul value2 by two-to-minus-phi		; 2-6
	vaddpd	ymm4, ymm4, ymm2			;; x = value + carry				; 6-8	First dependence on previous iteration
	vaddpd	ymm5, ymm5, ymm3			;; x = value + carry				; 7-9
	vmulpd	ymm2, ymm4, YMM_LIMIT_INVERSE[basereg]	;; val / base					; 9-13
	vroundpd ymm8, ymm4, 0				;; Round val					; 9-11
	vmulpd	ymm3, ymm5, YMM_LIMIT_INVERSE[basereg2]	;; val / base					; 10-14
	vroundpd ymm9, ymm5, 0				;; Round val					; 10-12
echk	vsubpd	ymm10, ymm4, ymm8			;; This is the convolution error		;  12-14
echk	vsubpd	ymm11, ymm5, ymm9			;; This is the convolution error		;  13-15
	vroundpd ymm2, ymm2, 0				;; next carry = round (val / base)		; 14-16
	vroundpd ymm3, ymm3, 0				;; next carry = round (val / base)		; 15-17
echk	absval	ymm10					;; Compute absolute value			;  15
echk	vmaxpd	ymm6, ymm6, ymm10			;; Compute maximum error			;  16-18
echk	absval	ymm11					;; Compute absolute value			;  16
	vmulpd	ymm4, ymm2, YMM_LIMIT_BIGMAX[basereg]	;; z = round (val / base) * base		; 17-21
echk	vmaxpd	ymm6, ymm6, ymm11			;; Compute maximum error			;  17-19
	vmulpd	ymm5, ymm3, YMM_LIMIT_BIGMAX[basereg2]	;; z = round (val / base) * base		; 18-22
	vsubpd	ymm4, ymm8, ymm4			;; new value = val - z				; 22-24
	vsubpd	ymm5, ymm9, ymm5			;; new value = val - z				; 23-25
ENDM

IF (@INSTR(,%yarch,<FMA3>) NE 0)
ynorm_1d_nobase2_noconst MACRO echk, basereg, basereg2
	vmovapd	ymm0, [rsi]				;; Load value1
	vmulpd	ymm4, ymm0, [rbp]			;; x = value1 * two-to-minus-phi		; presumably free (can execute in previous iteration)
	vmovapd	ymm1, [rsi+32]				;; Load value2
	vmulpd	ymm5, ymm1, [rbp+64]			;; x = value2 * two-to-minus-phi		; presumably free
	vaddpd	ymm4, ymm4, ymm2			;; x += carry					; 3-5	First dependence on previous iteration
	vaddpd	ymm5, ymm5, ymm3			;; x += carry					; 4-6
	vmulpd	ymm2, ymm4, YMM_LIMIT_INVERSE[basereg]	;; val / base					; 6-10
	vroundpd ymm8, ymm4, 0				;; Round val					; 6-8 and 9-11
	vmulpd	ymm3, ymm5, YMM_LIMIT_INVERSE[basereg2]	;; val / base					; 7-11
	vroundpd ymm9, ymm5, 0				;; Round val					; 7-9 and 10-12
echk	vsubpd	ymm10, ymm4, ymm8			;; This is the convolution error		;  12-14
echk	vsubpd	ymm11, ymm5, ymm9			;; This is the convolution error		;  13-15
	vroundpd ymm2, ymm2, 0				;; next carry = round (val / base)		; 11-13 and 14-16
	vroundpd ymm3, ymm3, 0				;; next carry = round (val / base)		; 12-14 and 15-17
echk	absval	ymm10					;; Compute absolute value			;  15
echk	vmaxpd	ymm6, ymm6, ymm10			;; Compute maximum error			;  16-18
echk	absval	ymm11					;; Compute absolute value			;  16
	yfnmaddpd ymm4, ymm2, YMM_LIMIT_BIGMAX[basereg], ymm8 ;; new value = val - round (val / base) * base ; 17-21
echk	vmaxpd	ymm6, ymm6, ymm11			;; Compute maximum error			;  17-19
	yfnmaddpd ymm5, ymm3, YMM_LIMIT_BIGMAX[basereg2], ymm9 ;; new value = val - round (val / base) * base ; 18-22
ENDM
ENDIF


ynorm_1d_nobase2_const MACRO echk, basereg, basereg2
	vmovapd	ymm0, [rsi]				;; Load value1
	vmulpd	ymm4, ymm0, [rbp]			;; Mul value1 by two-to-minus-phi		; presumably free
	vmovapd	ymm1, [rsi+32]				;; Load value2
	vmulpd	ymm5, ymm1, [rbp+64]			;; Mul value2 by two-to-minus-phi		; presumably free
	vmulpd	ymm0, ymm4, YMM_LIMIT_INVERSE[basereg]	;; HiFFTval = FFT value / base			; 6-10
	vroundpd ymm8, ymm4, 0				;; Round FFT value				; 6-8
	vmulpd	ymm1, ymm5, YMM_LIMIT_INVERSE[basereg2]	;; HiFFTval = FFT value / base			; 7-11
	vroundpd ymm9, ymm5, 0				;; Round FFT value				; 7-9
echk	vsubpd	ymm10, ymm4, ymm8			;; This is the convolution error		;  9-11
echk	vsubpd	ymm11, ymm5, ymm9			;; This is the convolution error		;  10-12
	vroundpd ymm0, ymm0, 0				;; Round (HiFFTval)				; 11-13
	vroundpd ymm1, ymm1, 0				;; Round (HiFFTval)				; 12-14
echk	absval	ymm10					;; Compute absolute value			;  12
echk	vmaxpd	ymm6, ymm6, ymm10			;; Compute maximum error			;  13-15
echk	absval	ymm11					;; Compute absolute value			;  13
	vmulpd	ymm4, ymm0, YMM_LIMIT_BIGMAX[basereg]	;; HiFFTval * base				; 14-18
echk	vmaxpd	ymm6, ymm6, ymm11			;; Compute maximum error			;  14-16
	vmulpd	ymm5, ymm1, YMM_LIMIT_BIGMAX[basereg2]	;; HiFFTval * base				; 15-19
	vmulpd	ymm0, ymm0, ymm13			;; carryB = HiFFTval * constant			;  16-20
	vmulpd	ymm1, ymm1, ymm13			;; carryB = HiFFTval * constant			;  17-21
	vsubpd	ymm4, ymm8, ymm4			;; LoFFTval = FFTval - HiFFTval * base		; 19-21
	vsubpd	ymm5, ymm9, ymm5			;; LoFFTval = FFTval - HiFFTval * base		; 20-22
	vmulpd	ymm4, ymm4, ymm13			;; val = LoFFTval * constant			; 22-26
	vmulpd	ymm5, ymm5, ymm13			;; val = LoFFTval * constant			; 23-27
	vaddpd	ymm4, ymm4, ymm2			;; val += previous carry			; 27-29	First dependence on previous iteration
	vaddpd	ymm5, ymm5, ymm3			;; val += previous carry			; 28-30
	vmulpd	ymm8, ymm4, YMM_LIMIT_INVERSE[basereg]	;; val / base					; 30-34
	vmulpd	ymm9, ymm5, YMM_LIMIT_INVERSE[basereg2]	;; val / base					; 31-35
	vroundpd ymm8, ymm8, 0				;; carryA = round (val / base)			; 35-37
	vroundpd ymm9, ymm9, 0				;; carryA = round (val / base)			; 36-38
	vaddpd	ymm2, ymm0, ymm8			;; carry = carryA + carryB			; 38-40
	vmulpd	ymm10, ymm8, YMM_LIMIT_BIGMAX[basereg]	;; z = round (val / base) * base		; 38-42
	vaddpd	ymm3, ymm1, ymm9			;; carry = carryA + carryB			; 39-41
	vmulpd	ymm11, ymm9, YMM_LIMIT_BIGMAX[basereg2]	;; z = round (val / base) * base		; 39-43
	vsubpd	ymm4, ymm4, ymm10			;; new value = val - z				; 43-45
	vsubpd	ymm5, ymm5, ymm11			;; new value = val - z				; 44-46
ENDM

IF (@INSTR(,%yarch,<FMA3>) NE 0)
ynorm_1d_nobase2_const MACRO echk, basereg, basereg2
	vmovapd	ymm0, [rsi]				;; Load value1
	vmulpd	ymm4, ymm0, [rbp]			;; Mul value1 by two-to-minus-phi		; presumably free
	vmovapd	ymm1, [rsi+32]				;; Load value2
	vmulpd	ymm5, ymm1, [rbp+64]			;; Mul value2 by two-to-minus-phi		; presumably free
	vmulpd	ymm0, ymm4, YMM_LIMIT_INVERSE[basereg]	;; HiFFTval = FFT value / base			; 6-10
	vroundpd ymm8, ymm4, 0				;; Round FFT value				; 6-8,9-11
	vmulpd	ymm1, ymm5, YMM_LIMIT_INVERSE[basereg2]	;; HiFFTval = FFT value / base			; 7-11
	vroundpd ymm9, ymm5, 0				;; Round FFT value				; 7-9,10-12
echk	vsubpd	ymm10, ymm4, ymm8			;; This is the convolution error		;  12-14*
echk	vsubpd	ymm11, ymm5, ymm9			;; This is the convolution error		;  13-15
	vroundpd ymm0, ymm0, 0				;; Round (HiFFTval)				; 11-13,14-16
	vroundpd ymm1, ymm1, 0				;; Round (HiFFTval)				; 12-14,15-17
echk	absval	ymm10					;; Compute absolute value			;  15
echk	vmaxpd	ymm6, ymm6, ymm10			;; Compute maximum error			;  16-18
echk	absval	ymm11					;; Compute absolute value			;  16
	yfnmaddpd ymm4, ymm0, YMM_LIMIT_BIGMAX[basereg], ymm8 ;; LoFFTval = FFTval - HiFFTval * base	; 17-21
echk	vmaxpd	ymm6, ymm6, ymm11			;; Compute maximum error			;  17-19*
	yfnmaddpd ymm5, ymm1, YMM_LIMIT_BIGMAX[basereg2], ymm9 ;; LoFFTval = FFTval - HiFFTval * base	; 18-22
	vmulpd	ymm0, ymm0, ymm13			;; carryB = HiFFTval * constant			; 17-21	  FMA-able but may make latency worse
	vmulpd	ymm1, ymm1, ymm13			;; carryB = HiFFTval * constant			; 18-22	  FMA-able but may make latency worse
	yfmaddpd ymm4, ymm4, ymm13, ymm2		;; val = LoFFTval * constant + previous carry	; 22-26	First dependence on previous iteration
	yfmaddpd ymm5, ymm5, ymm13, ymm3		;; val = LoFFTval * constant + previous carry	; 23-27
	vmulpd	ymm8, ymm4, YMM_LIMIT_INVERSE[basereg]	;; val / base					; 27-31
	vmulpd	ymm9, ymm5, YMM_LIMIT_INVERSE[basereg2]	;; val / base					; 28-32
	vroundpd ymm8, ymm8, 0				;; carryA = round (val / base)			; 32-34,35-37
	vroundpd ymm9, ymm9, 0				;; carryA = round (val / base)			; 33-35,36-38
	vaddpd	ymm2, ymm0, ymm8			;; carry = carryA + carryB			; 38-40	  FMA-able but may make latency worse
	yfnmaddpd ymm4, ymm8, YMM_LIMIT_BIGMAX[basereg], ymm4 ;; newval = val - round (val/base) * base	; 38-42
	vaddpd	ymm3, ymm1, ymm9			;; carry = carryA + carryB			; 39-41   FMA-able but may make latency worse
	yfnmaddpd ymm5, ymm9, YMM_LIMIT_BIGMAX[basereg2], ymm5 ;; newval = val - round (val/base) * base; 39-43
ENDM
ENDIF


;; Proposed new non-base-2 code which gens next carry in 8 clocks
;;
;; In this code, the carry includes the integer rounding constant (BIGVAL = 3*2^52).
;; The key is that the next carry can be computed in a single FMA instruction.
;; Given x = value + BIGVAL, the next carry will be (x - BIGVAL) / base + BIGVAL.
;; This simplifies to x / base + (BIGVAL - BIGVAL / base).  This works as long as we define
;; the BIGVAL constant as the first value above 3*2^52 that is divisible by the base.
;;
;; To calculate value, value = (x - BIGVAL) - (next carry - BIGVAL) * base
;; Rearranging:  value = x - next carry * base + BIGVAL * base - BIGVAL
;; And:  value = x - (next_carry - (BIGVAL - BIGVAL / base)) * base

;;; BUG - need 3rd basereg indexed set of constants

IF (@INSTR(,%yarch,<FMA3>) NE 0)
new_ynorm_1d_nobase2_noconst MACRO echk, basereg, basereg2
	vmovapd	ymm0, [rsi]				;; Load value1
	vmulpd	ymm4, ymm0, [rbp]			;; value1 * two-to-minus-phi			; presumably free
	vmovapd	ymm1, [rsi+32]				;; Load value2
	vmulpd	ymm5, ymm1, [rbp+64]			;; value2 * two-to-minus-phi			; presumably free
	vaddpd	ymm0, ymm4, ymm2			;; x = value + carry				; 3-5	First dependence on previous iteration
	vaddpd	ymm1, ymm5, ymm3			;; x = value + carry				; 4-6
	vmovapd ymm8, YMM_LIMIT_BIGMAX[basereg]		;; Load constant for creating next carry
echk	vsubpd	ymm10, ymm0, ymm2			;; Start error calc (x - carry)			;  6-8
	yfmaddpd ymm2, ymm0, YMM_LIMIT_INVERSE[basereg], ymm8	;; next carry = x / base + BIGVAL	; 6-10
	vmovapd ymm9, YMM_LIMIT_BIGMAX[basereg2]	;; Load constant for creating next carry
echk	vsubpd	ymm11, ymm1, ymm3			;; Start error calc (x - carry)			;  7-9
	yfmaddpd ymm3, ymm1, YMM_LIMIT_INVERSE[basereg2], ymm9	;; next carry = x / base + BIGVAL	; 7-11
echk	vsubpd	ymm10, ymm4, ymm10			;; Convolution error = value - (x - carry)	;  9-11
echk	vsubpd	ymm11, ymm5, ymm11			;; Convolution error = value - (x - carry)	;  10-12
	vsubpd ymm4, ymm2, ymm8				;; y = next carry - BIGVAL			; 11-13		
	vsubpd ymm5, ymm3, ymm9				;; y = next carry - BIGVAL			; 12-14
echk	absval	ymm10					;; Compute absolute value			;  12
echk	absval	ymm11					;; Compute absolute value			;  13
echk	vmaxpd	ymm6, ymm6, ymm10			;; Compute maximum error			;  13-15
echk	vmaxpd	ymm6, ymm6, ymm11			;; Compute maximum error			;  14-16
;; BUG - s.b. BASE not BIGMAX
	yfnmaddpd ymm4, ymm4, YMM_LIMIT_BIGMAX[basereg], ymm0 ;; new value = x - y * base			; 14-18
	yfnmaddpd ymm5, ymm5, YMM_LIMIT_BIGMAX[basereg2], ymm1 ;; new value = x - y * base		; 15-19
ENDM
ENDIF

;; Proposed new non-base-2 code to handle const
;;
;; Like the non-const code, we do 3 operations on x = value + BIGVAL to generate 2 needed values:
;;	x / base + (BIGVAL - BIGVAL / base)	gives us value / base + BIGVAL.
;; Then, given y = value / base + BIGVAL,
;;	tmp = y - (BIGVAL - BIGVAL / base)
;;	value = y - tmp * base			gives us value % base

;;BUG  -- set registers to:
;ymm14 = YMM_BIGVAL
;ymm13 = YMM_BIGVAL * const
;ymm12 = const

IF (@INSTR(,%yarch,<FMA3>) NE 0)
new_ynorm_1d_nobase2_const MACRO echk, basereg, basereg2
	vmovapd	ymm0, [rsi]				;; Load value1
	yfmaddpd ymm4, ymm0, [rbp], ymm14		;; Mul value1 by two-to-minus-phi + BIGVAL	; presumably free
	vmovapd	ymm1, [rsi+32]				;; Load value2
	yfmaddpd ymm5, ymm1, [rbp+64], ymm14		;; Mul value2 by two-to-minus-phi + BIGVAL	; presumably free

	vmovapd ymm8, YMM_LIMIT_BIGMAX[basereg]		;; Load constant for creating next carry
	vmovapd ymm9, YMM_LIMIT_BIGMAX[basereg2]	;; Load constant for creating next carry

echk	vsubpd	ymm10, ymm4, ymm14			;; Start error calc, subtract BIGVAL		;  6-8
echk	vsubpd	ymm11, ymm5, ymm14			;; Start error calc, subtract BIGVAL		;  7-9

echk	yfmsubpd ymm10, ymm0, [rbp], ymm10		;; Convolution error, subtract val * ttmp	;  9-13
echk	yfmsubpd ymm11, ymm1, [rbp+64], ymm11		;; Convolution error, subtract val * ttmp	;  10-14

	yfmaddpd ymm0, ymm4, YMM_LIMIT_INVERSE[basereg], ymm8 ;; HiFFTval = FFT value / base + BIGVAL	; 6-10
	yfmaddpd ymm1, ymm5, YMM_LIMIT_INVERSE[basereg2], ymm9 ;; HiFFTval = FFT value / base + BIGVAL	; 7-11

echk	absval	ymm10					;; Compute absolute value			;  14
echk	absval	ymm11					;; Compute absolute value			;  15
echk	vmaxpd	ymm6, ymm6, ymm10			;; Compute maximum error			;  15-17
echk	vmaxpd	ymm6, ymm6, ymm11			;; Compute maximum error			;  16-18

	vsubpd	ymm10, ymm0, ymm8			;; Rounded HiFFTval				; 11-13
	vsubpd	ymm11, ymm1, ymm9			;; Rounded HiFFTval				; 12-14

	yfmaddpd ymm0, ymm0, ymm12, ymm13		;; carryB = HiFFTval * constant			; 11-15
	yfmaddpd ymm1, ymm1, ymm12, ymm13		;; carryB = HiFFTval * constant			; 12-16

	;; s.b. BASE
	yfnmaddpd ymm10, ymm10, YMM_LIMIT_BIGMAX[basereg], ymm4 ;; LoFFTval = FFTval - HiFFTval * base	; 14-18
	yfnmaddpd ymm11, ymm11, YMM_LIMIT_BIGMAX[basereg2], ymm5 ;; LoFFTval = FFTval - HiFFTval * base	; 15-19

	yfmaddpd ymm4, ymm10, ymm12, ymm2		;; val = LoFFTval * constant + previous carry	; 19-23	First dependence on previous iteration
	yfmaddpd ymm5, ymm11, ymm12, ymm3		;; val = LoFFTval * constant + previous carry	; 20-24

	yfmaddpd ymm2, ymm4, YMM_LIMIT_INVERSE[basereg], ymm8 ;; val / base + BIGVAL			; 24-28
	yfmaddpd ymm3, ymm5, YMM_LIMIT_INVERSE[basereg2], ymm9	;; val / base + BIGVAL			; 25-29

	vsubpd	ymm10, ymm2, ymm8			;; Rounded val / base				; 29-31
	vsubpd	ymm11, ymm3, ymm9			;; Rounded val / base				; 30-32

	vaddpd	ymm2, ymm2, ymm0			;; next carry = carryA + carryB			; 31-33
	vaddpd	ymm3, ymm3, ymm1			;; next carry = carryA + carryB			; 32-34

	;; s.b. BASE
	yfnmaddpd ymm4, ymm10, YMM_LIMIT_BIGMAX[basereg], ymm4 ;; newval = val - round (val/base) * base; 32-36
	yfnmaddpd ymm5, ymm11, YMM_LIMIT_BIGMAX[basereg2], ymm5 ;; newval = val - round (val/base) * base; 33-37
ENDM
ENDIF

ENDIF


; This is the normalization routine when we are computing modulo k*b^n+c
; with a zero-padded b^2n FFT.  We do this by multiplying the lower FFT
; word by k and adding in the upper word times -c.  Of course, this is made
; very tedious because we have to carefully avoid any loss of precision.
;
; ymm6 = MAXERR
; ymm3 = carry #2 (previous high FFT data - not yet mul'ed by K)
; ymm2 = carry #1 (traditional carry)
; rsi = pointer to the FFT data values
; rbp = pointer two-to-phi multipliers
; rdi = pointer to array of big vs. little flags
; eax = big vs. little word flag #1

ynorm_1d_zpad_preload MACRO ttp, base2, echk, const, khi, c1, cm1
	ENDM

ynorm_1d_zpad MACRO ttp, base2, echk, const, khi, c1, cm1
ttp		movzx	rax, BYTE PTR [rdi]	;; Load big vs. little flags
		vmovapd	ymm0, [rsi]		;; Load values1
		vmovapd	ymm1, [rsi+32]		;; Load values2
		vmulpd	ymm0, ymm0, [rbp]	;; Mul values1 by two-to-minus-phi
		vmulpd	ymm1, ymm1, [rbp]	;; Mul values2 by two-to-minus-phi

		split_lower_zpad_word ttp, base2, echk, ymm0, ymm3, ymm4, rax*2

no const	vmulpd	ymm0, ymm4, YMM_K_LO
const		vmulpd	ymm0, ymm4, YMM_K_TIMES_MULCONST_LO

khi base2 no const	vmulpd	ymm5, ymm4, YMM_K_HI
khi base2 const		vmulpd	ymm5, ymm4, YMM_K_TIMES_MULCONST_HI
khi no base2 no const	vmovapd	ymm5, YMM_K_HI
khi no base2 const	vmovapd	ymm5, YMM_K_TIMES_MULCONST_HI
khi no base2 ttp	vmulpd	ymm5, ymm5, YMM_LIMIT_INVERSE[rax*2] ;; Non-base2 rounding needs shifted carry
khi no base2 no ttp	vmulpd	ymm5, ymm5, YMM_LIMIT_INVERSE[0] ;; Non-base2 rounding needs shifted carry
khi no base2		vroundpd ymm5, ymm5, 0			;; THIS IS WASTEFUL.  The mul and round should be precomputed!
khi no base2		vmulpd	ymm5, ymm5, ymm4

		vaddpd	ymm0, ymm0, ymm2	;; x1 = values + carry

c1		vmulpd	ymm1, ymm1, YMM_MINUS_C	;; Do one mul before split rather than two after split

		split_upper_zpad_word ttp, base2, echk, ymm1, ymm2, ymm4, rax*2

no const no c1 no cm1	vmulpd	ymm4, ymm4, YMM_MINUS_C
no const no c1 no cm1	vmulpd	ymm2, ymm2, YMM_MINUS_C
const			vmulpd	ymm4, ymm4, YMM_MINUS_C_TIMES_MULCONST
const			vmulpd	ymm2, ymm2, YMM_MINUS_C_TIMES_MULCONST

		vaddpd	ymm0, ymm0, ymm4	;; Add upper FFT word to lower FFT word
khi		vaddpd	ymm2, ymm2, ymm5	;; Add upper FFT word to lower FFT word

		rounding ttp, base2, exec, ymm0, ymm2, ymm4, rax*2

ttp		vmulpd	ymm0, ymm0, [rbp+32]	;; new value1 = val * two-to-phi
		ystore	[rsi], ymm0		;; Save value1
		vxorpd	ymm1, ymm1, ymm1	;; new value2 = zero
		ystore	[rsi+32], ymm1		;; Zero value2
	ENDM

;; 64-bit version

IFDEF X86_64

ynorm_1d_zpad_preload MACRO ttp, base2, echk, const, khi, c1, cm1
echk			vmovapd	ymm15, YMM_ABSVAL
base2			vmovapd ymm14, YMM_BIGBIGVAL
no const		vmovapd	ymm13, YMM_K_LO
const			vmovapd	ymm13, YMM_K_TIMES_MULCONST_LO
khi no const		vmovapd	ymm12, YMM_K_HI
khi const		vmovapd	ymm12, YMM_K_TIMES_MULCONST_HI
no const no c1 no cm1	vmovapd	ymm11, YMM_MINUS_C
const			vmovapd	ymm11, YMM_MINUS_C_TIMES_MULCONST
	ENDM

ynorm_1d_zpad MACRO ttp, base2, echk, const, khi, c1, cm1
ttp	movzx	rax, BYTE PTR [rdi]		;; Load big vs. little flags
	vmovapd	ymm0, [rsi]			;; Load value1
	vmovapd	ymm1, [rsi+32]			;; Load value2
	vmulpd	ymm0, ymm0, [rbp]		;; Mul value1 by two-to-minus-phi
	vmulpd	ymm1, ymm1, [rbp]		;; Mul value2 by two-to-minus-phi
	vaddpd	ymm0, ymm0, ymm3		;; Add in previous high FFT data

base2 no const ttp	ynorm_1d_zpad_base2	echk, khi, c1, cm1, rax*2
base2 no const no ttp	ynorm_1d_zpad_base2	echk, khi, c1, cm1, 0
base2 const ttp		ynorm_1d_zpad_base2	echk, khi, noexec, noexec, rax*2
base2 const no ttp	ynorm_1d_zpad_base2	echk, khi, noexec, noexec, 0
no base2 no const ttp	ynorm_1d_zpad_nobase2	echk, khi, c1, cm1, rax*2
no base2 no const no ttp ynorm_1d_zpad_nobase2	echk, khi, c1, cm1, 0
no base2 const ttp	ynorm_1d_zpad_nobase2	echk, khi, noexec, noexec, rax*2
no base2 const no ttp	ynorm_1d_zpad_nobase2	echk, khi, noexec, noexec, 0

ttp	vmulpd	ymm0, ymm0, [rbp+32]		;; new value1 = val * two-to-phi
	ystore	[rsi], ymm0			;; Save value1
	vxorpd	ymm1, ymm1, ymm1		;; new value2 = zero
	ystore	[rsi+32], ymm1			;; Zero value2
	ENDM

IF (@INSTR(,%yarch,<FMA3>) NE 0)
ynorm_1d_zpad MACRO ttp, base2, echk, const, khi, c1, cm1
ttp	movzx	rax, BYTE PTR [rdi]		;; Load big vs. little flags
	vmovapd	ymm0, [rsi]			;; Load value1
	vmovapd	ymm1, [rsi+32]			;; Load value2
	yfmaddpd ymm0, ymm0, [rbp], ymm3	;; Mul value1 by two-to-minus-phi + previous high FFT data
	vmulpd	ymm1, ymm1, [rbp]		;; Mul value2 by two-to-minus-phi

base2 no const ttp	ynorm_1d_zpad_base2	echk, khi, c1, cm1, rax*2
base2 no const no ttp	ynorm_1d_zpad_base2	echk, khi, c1, cm1, 0
base2 const ttp		ynorm_1d_zpad_base2	echk, khi, noexec, noexec, rax*2
base2 const no ttp	ynorm_1d_zpad_base2	echk, khi, noexec, noexec, 0
no base2 no const ttp	ynorm_1d_zpad_nobase2	echk, khi, c1, cm1, rax*2
no base2 no const no ttp ynorm_1d_zpad_nobase2	echk, khi, c1, cm1, 0
no base2 const ttp	ynorm_1d_zpad_nobase2	echk, khi, noexec, noexec, rax*2
no base2 const no ttp	ynorm_1d_zpad_nobase2	echk, khi, noexec, noexec, 0

ttp	vmulpd	ymm0, ymm0, [rbp+32]		;; new value1 = val * two-to-phi
	ystore	[rsi], ymm0			;; Save value1
	vxorpd	ymm1, ymm1, ymm1		;; new value2 = zero
	ystore	[rsi+32], ymm1			;; Zero value2
	ENDM
ENDIF

ynorm_1d_zpad_base2 MACRO echk, khi, c1, cm1, basereg
	vaddpd	ymm3, ymm0, ymm14			;; Round to multiple of big word			; 1-3
	vaddpd	ymm8, ymm1, ymm14			;; Round to multiple of big word			; 2-4
	vroundpd ymm4, ymm0, 0				;; Round to an integer					; 3-5
	vsubpd	ymm3, ymm3, ymm14										; 4-6
	vsubpd	ymm8, ymm8, ymm14										; 5-7
	vroundpd ymm9, ymm1, 0				;; Round input to an integer				; 6-8
	vsubpd	ymm10, ymm4, ymm3			;; Compute low bigword bits of low FFT word		; 7-9
echk	vsubpd	ymm5, ymm0, ymm4			;; This is the convolution error			;  8-10
	vsubpd	ymm4, ymm9, ymm8			;; Compute low bigword bits of high FFT word		; 9-11
echk	vsubpd	ymm9, ymm1, ymm9			;; This is the convolution error			;  10-12
	vmovapd	ymm1, YMM_LIMIT_INVERSE[basereg]	;; Load limit inverse
	vmulpd	ymm3, ymm3, ymm1			;; Saved shifted FFT hi data for next iteration		; 7-11

no c1 no cm1 vmulpd ymm8, ymm8, ymm11			;; Mul by YMM_MINUS_C or YMM_MINUS_C_TIMES_MULCONST	; 8-12

	vmulpd	ymm0, ymm10, ymm13			;; Mul by YMM_K_LO or YMM_K_TIMES_MULCONST_LO		; 10-14

echk	absval	ymm5					;; Compute absolute value				;  11
echk	vmaxpd	ymm6, ymm6, ymm5			;; Compute maximum error				;  12-14

khi	vmulpd	ymm5, ymm10, ymm12			;; Mul by YMM_K_HI or YMM_K_TIMES_MULCONST_HI		; 11-15

echk	absval	ymm9					;; Compute absolute value				;  13
echk	vmaxpd	ymm6, ymm6, ymm9			;; Compute maximum error				;  14-16

no c1 no cm1 vmulpd ymm4, ymm4, ymm11			;; Mul by YMM_MINUS_C or YMM_MINUS_C_TIMES_MULCONST	; 12-16

	vaddpd	ymm0, ymm0, ymm2			;; x1 = values + carry					; 15-17

c1 khi	vsubpd	ymm8, ymm5, ymm8			;; Add upper FFT word * MINUS_C to lower FFT word	; 16-18
no c1 khi vaddpd ymm8, ymm5, ymm8			;; Add upper FFT word to lower FFT word			; 16-18

c1	vsubpd	ymm0, ymm0, ymm4			;; Add upper FFT word * MINUS_C to lower FFT word	; 18-20
no c1	vaddpd	ymm0, ymm0, ymm4			;; Add upper FFT word to lower FFT word			; 18-20

	vmovapd	ymm5, YMM_LIMIT_BIGMAX[basereg]		;; Load rounding constant
	vaddpd	ymm4, ymm0, ymm5			;; y = top bits of x					; 21-23
c1 no khi vsubpd ymm2, ymm4, ymm8			;; next carry = y + upper mul-by-const bits * MINUS_C	; 24-26
c1 khi	vaddpd	ymm2, ymm4, ymm8			;; next carry = y + upper mul-by-const bits 		; 24-26
no c1	vaddpd	ymm2, ymm4, ymm8			;; next carry = y + upper mul-by-const bits 		; 24-26
	vsubpd	ymm4, ymm4, ymm5			;; z = y - (maximum*BIGVAL-BIGVAL)			; 25-27
	vmulpd	ymm2, ymm2, ymm1			;; shift next carry appropriately			; 27-31
	vsubpd	ymm0, ymm0, ymm4			;; rounded value = x - z				; 28-30
ENDM

IF (@INSTR(,%yarch,<FMA3>) NE 0)
ynorm_1d_zpad_base2 MACRO echk, khi, c1, cm1, basereg
	vaddpd	ymm3, ymm0, ymm14			;; Round to multiple of big word			; 1-3
	vroundpd ymm4, ymm0, 0				;; Round to an integer					; 2-4,5-7
	vaddpd	ymm8, ymm1, ymm14			;; Round to multiple of big word			; 3-5
	vroundpd ymm9, ymm1, 0				;; Round input to an integer				; 4-6,7-9
	vsubpd	ymm3, ymm3, ymm14										; 6-8
	vsubpd	ymm8, ymm8, ymm14										; 8-10

	vsubpd	ymm10, ymm4, ymm3			;; Compute low bigword bits of low FFT word		; 9-11
echk	vsubpd	ymm5, ymm0, ymm4			;; This is the convolution error			;  10-12
	vsubpd	ymm4, ymm9, ymm8			;; Compute low bigword bits of high FFT word		; 11-13
echk	vsubpd	ymm9, ymm1, ymm9			;; This is the convolution error			;  12-14

	vmovapd	ymm1, YMM_LIMIT_INVERSE[basereg]	;; Load limit inverse
	vmulpd	ymm3, ymm3, ymm1			;; Saved shifted FFT hi data for next iteration		; 9-13

no c1 no cm1 vmulpd ymm8, ymm8, ymm11			;; Mul by YMM_MINUS_C or YMM_MINUS_C_TIMES_MULCONST	; 11-15

no echk	vmulpd	ymm0, ymm10, ymm13			;; Mul by YMM_K_LO or YMM_K_TIMES_MULCONST_LO		; 12-16		FMA only helps when error checking
echk	yfmaddpd ymm0, ymm10, ymm13, ymm2		;; Mul by YMM_K_LO or YMM_K_TIMES_MULCONST_LO + carry	; 12-16

echk	absval	ymm5					;; Compute absolute value				;  13
echk	vmaxpd	ymm6, ymm6, ymm5			;; Compute maximum error				;  14-16

khi	vmulpd	ymm5, ymm10, ymm12			;; Mul by YMM_K_HI or YMM_K_TIMES_MULCONST_HI		; 13-17

no c1 no cm1 vmulpd ymm4, ymm4, ymm11			;; Mul by YMM_MINUS_C or YMM_MINUS_C_TIMES_MULCONST	; 14-18		FMA-able, but doesn't help

echk	absval	ymm9					;; Compute absolute value				;  15

c1 khi	vsubpd	ymm8, ymm5, ymm8			;; Add upper FFT word * MINUS_C to lower FFT word	; 16-18
no c1 khi vaddpd ymm8, ymm5, ymm8			;; Add upper FFT word to lower FFT word			; 16-18

no echk	vaddpd	ymm0, ymm0, ymm2			;; x1 = values + carry					; 17-19		FMA only helps when error checking

echk	vmaxpd	ymm6, ymm6, ymm9			;; Compute maximum error				;  17-19

c1	vsubpd	ymm0, ymm0, ymm4			;; Add upper FFT word * MINUS_C to lower FFT word	; 20-22
no c1	vaddpd	ymm0, ymm0, ymm4			;; Add upper FFT word to lower FFT word			; 20-22

	vmovapd	ymm5, YMM_LIMIT_BIGMAX[basereg]		;; Load rounding constant
	vaddpd	ymm4, ymm0, ymm5			;; y = top bits of x					; 23-25
c1 no khi vsubpd ymm2, ymm4, ymm8			;; next carry = y + upper mul-by-const bits * MINUS_C	; 26-28
c1 khi	vaddpd	ymm2, ymm4, ymm8			;; next carry = y + upper mul-by-const bits 		; 26-28
no c1	vaddpd	ymm2, ymm4, ymm8			;; next carry = y + upper mul-by-const bits 		; 26-28
	vsubpd	ymm4, ymm4, ymm5			;; z = y - (maximum*BIGVAL-BIGVAL)			; 27-29
	vmulpd	ymm2, ymm2, ymm1			;; shift next carry appropriately			; 29-33
	vsubpd	ymm0, ymm0, ymm4			;; rounded value = x - z				; 30-32
ENDM
ENDIF


ynorm_1d_zpad_nobase2 MACRO echk, khi, c1, cm1, basereg
	vmovapd	ymm14, YMM_LIMIT_INVERSE[basereg]	;; Load 1 / base

	vmulpd	ymm3, ymm0, ymm14			;; Compute FFTvalue / base				; 1-5
	vmulpd	ymm8, ymm1, ymm14			;; Compute FFTvalue / base				; 2-6
	vroundpd ymm4, ymm0, 0				;; Round input to an integer				; 1-3
	vroundpd ymm9, ymm1, 0				;; Round input to an integer				; 2-4
khi	vmulpd	ymm5, ymm12, ymm14			;; Non-base2 rounding will need shifted carry		;    3-7
echk	vsubpd	ymm0, ymm0, ymm4			;; This is the convolution error			;  4-6
echk	vsubpd	ymm1, ymm1, ymm9			;; This is the convolution error			;  5-7
	vroundpd ymm3, ymm3, 0				;; Round FFTvalue / base to integer			; 6-8
	vroundpd ymm8, ymm8, 0				;; Round FFTvalue / base to integer			; 7-9
khi	vroundpd ymm5, ymm5, 0				;; WASTEFUL.  The mul and round could be precomputed!	;    8-10
echk	absval	ymm0					;; Compute absolute value				;  7
echk	absval	ymm1					;; Compute absolute value				;  8
echk	vmaxpd	ymm6, ymm6, ymm0			;; Compute maximum error				;  8-10
echk	vmaxpd	ymm6, ymm6, ymm1			;; Compute maximum error				;  11-13

	vmovapd	ymm10, YMM_LIMIT_BIGMAX[basereg]	;; Load base
	vmulpd	ymm0, ymm3, ymm10			;; tmp = round (FFTvalue / base) * base			; 9-13
	vmulpd	ymm1, ymm8, ymm10			;; tmp = round (FFTvalue / base) * base			; 10-14

no c1 no cm1 vmulpd ymm8, ymm8, ymm11			;; Mul by YMM_MINUS_C or YMM_MINUS_C_TIMES_MULCONST	; 11-15

	vsubpd	ymm4, ymm4, ymm0			;; Compute FFTvalue % base  (FFTvalue - tmp)		; 14-16
	vsubpd	ymm9, ymm9, ymm1			;; Compute FFTvalue % base  (FFTvalue - tmp)		; 15-17

	vmulpd	ymm0, ymm4, ymm13			;; Mul by YMM_K_LO or YMM_K_TIMES_MULCONST_LO		; 17-21
khi	vmulpd	ymm5, ymm4, ymm5			;; Mul by YMM_K_HI or YMM_K_TIMES_MULCONST_HI		;    18-22

no c1 no cm1 vmulpd ymm9, ymm9, ymm11			;; Mul by YMM_MINUS_C or YMM_MINUS_C_TIMES_MULCONST	; 18-22

	vaddpd	ymm0, ymm0, ymm2			;; x = values + carry					; 22-24

c1 khi	vsubpd	ymm8, ymm5, ymm8			;; Add upper FFT word * MINUS_C to lower FFT word	;    23-25
no c1 khi vaddpd ymm8, ymm5, ymm8			;; Add upper FFT word to lower FFT word			;    23-25

c1	vsubpd	ymm0, ymm0, ymm9			;; Add upper FFT word * MINUS_C to lower FFT word	; 25-27
no c1	vaddpd	ymm0, ymm0, ymm9			;; Add upper FFT word to lower FFT word			; 25-27

	vmulpd	ymm4, ymm0, ymm14			;; val / base						; 28-32
	vroundpd ymm4, ymm4, 0				;; y = round (val / base)				; 33-35
c1 no khi vsubpd ymm2, ymm4, ymm8			;; next carry = y + upper mul-by-const bits * MINUS_C	; 36-38
c1 khi	vaddpd	ymm2, ymm4, ymm8			;; next carry = y + upper mul-by-const bits 		; 36-38
no c1	vaddpd	ymm2, ymm4, ymm8			;; next carry = y + upper mul-by-const bits 		; 36-38
	vmulpd	ymm4, ymm4, ymm10			;; z = round (val / base) * base			; 36-40
	vsubpd	ymm0, ymm0, ymm4			;; new value = val - z					; 41-43
ENDM

IF (@INSTR(,%yarch,<FMA3>) NE 0)
ynorm_1d_zpad_nobase2 MACRO echk, khi, c1, cm1, basereg
	vmovapd	ymm14, YMM_LIMIT_INVERSE[basereg]	;; Load 1 / base

	vmulpd	ymm3, ymm0, ymm14			;; Compute FFTvalue / base				; 1-5
	vmulpd	ymm8, ymm1, ymm14			;; Compute FFTvalue / base				; 1-5
	vroundpd ymm4, ymm0, 0				;; Round input to an integer				; 2-4,5-7
	vroundpd ymm9, ymm1, 0				;; Round input to an integer				; 3-5,6-8
khi	vmulpd	ymm5, ymm12, ymm14			;; Non-base2 rounding will need shifted carry		;    2-6
	vroundpd ymm3, ymm3, 0				;; Round FFTvalue / base to integer			; 6-8,9-11
	vroundpd ymm8, ymm8, 0				;; Round FFTvalue / base to integer			; 7-9,10-12
echk	vsubpd	ymm0, ymm0, ymm4			;; This is the convolution error			;  8-10
echk	vsubpd	ymm1, ymm1, ymm9			;; This is the convolution error			;  11-13
khi	vroundpd ymm5, ymm5, 0				;; WASTEFUL.  The mul and round could be precomputed!	;    12-14,15-17
echk	absval	ymm0					;; Compute absolute value				;  11
echk	vmaxpd	ymm6, ymm6, ymm0			;; Compute maximum error				;  12-14

	vmovapd	ymm10, YMM_LIMIT_BIGMAX[basereg]	;; Load base
	yfnmaddpd ymm4, ymm3, ymm10, ymm4		;; Compute FFTvalue % base				; 12-16
	yfnmaddpd ymm9, ymm8, ymm10, ymm9		;; Compute FFTvalue % base				; 13-17

no c1 no cm1 vmulpd ymm8, ymm8, ymm11			;; Mul by YMM_MINUS_C or YMM_MINUS_C_TIMES_MULCONST	; 13-17

echk	absval	ymm1					;; Compute absolute value				;  14
echk	vmaxpd	ymm6, ymm6, ymm1			;; Compute maximum error				;  15-17

	vmulpd	ymm0, ymm4, ymm13			;; Mul by YMM_K_LO or YMM_K_TIMES_MULCONST_LO		; 17-21		FMA-able, but timings are worse
khi	vmulpd	ymm5, ymm4, ymm5			;; Mul by YMM_K_HI or YMM_K_TIMES_MULCONST_HI		;    18-22

no c1 no cm1 vmulpd ymm9, ymm9, ymm11			;; Mul by YMM_MINUS_C or YMM_MINUS_C_TIMES_MULCONST	; 18-22

c1 khi	vsubpd	ymm8, ymm5, ymm8			;; Add upper FFT word * MINUS_C to lower FFT word	;    23-25
no c1 khi vaddpd ymm8, ymm5, ymm8			;; Add upper FFT word to lower FFT word			;    23-25

	vaddpd	ymm0, ymm0, ymm2			;; x = values + carry					; 22-24

c1	vsubpd	ymm0, ymm0, ymm9			;; Add upper FFT word * MINUS_C to lower FFT word	; 25-27
no c1	vaddpd	ymm0, ymm0, ymm9			;; Add upper FFT word to lower FFT word			; 25-27

	vmulpd	ymm4, ymm0, ymm14			;; val / base						; 28-32
	vroundpd ymm4, ymm4, 0				;; y = round (val / base)				; 33-35
c1 no khi vsubpd ymm2, ymm4, ymm8			;; next carry = y + upper mul-by-const bits * MINUS_C	; 36-38
c1 khi	vaddpd	ymm2, ymm4, ymm8			;; next carry = y + upper mul-by-const bits 		; 36-38
no c1	vaddpd	ymm2, ymm4, ymm8			;; next carry = y + upper mul-by-const bits 		; 36-38
	yfnmaddpd ymm0, ymm4, ymm10, ymm0		;; new value = val - round (val / base) * base		; 36-40
ENDM
ENDIF

ENDIF


; *************** 1D followup macros ******************
; This macro finishes the normalize process by adding the section
; carries back into the start of the section.  Three of the YMM section
; carries are added back in, one of the YMM section carries is applied
; to the next section.
; ymm2,ymm3 = carries
; rax, rdx, rsi, rbp, rdi = trash

ynorm_1d_mid_cleanup MACRO ttp, base2, srcptr, biglitptr, ttpptr
	LOCAL	section_start, section_loop, force_done, done

	rotate_carries_interleaved base2, ymm2, ymm4, ymm3, ymm5, ymm0, ymm1

	mov	DWORD PTR YMM_TMP3, 15		;; Propagate carry at most 15 times (should almost never happen)
section_start:
	mov	rsi, srcptr			;; Load section pointers
ttp	mov	rdi, biglitptr
ttp	mov	rbp, ttpptr
	mov	edx, count1			;; Count of cache lines in section or padded group
	ystore	YMM_TMP1, ymm4			;; Save carry for next section
	ystore	YMM_TMP2, ymm5			;; Save carry for next section

section_loop:
base2	vmovapd	ymm1, YMM_BIGVAL		;; Load comparison value
no base2 vxorpd	ymm1, ymm1, ymm1		;; Create comparison value
	vcmppd	ymm0, ymm2, ymm1, 0Ch		;; Are any carries non-zero
	vmovmskpd rax, ymm0			;; Extract 4 comparison bits
	vcmppd	ymm0, ymm3, ymm1, 0Ch		;; Are any carries non-zero
	vmovmskpd rcx, ymm0			;; Extract 4 comparison bits
	or	rax, rcx			;; Are any bits on?
	jz	done				;; No, we're all done

	sub	DWORD PTR YMM_TMP3, 1		;; There is a bizarre case where adding a zero carry causes a carry (example:
						;; 10^114+1, a data value of 500 becomes -500 with a carry of 1 -- add zero again and
						;; it becomes -500 with a carry of -1).  This can lead to an infinite loop because
						;; the 4 carries never become zero.  After many attempts at getting 4 zero carries,
						;; give up and just add in the carries without propagation.
	jz	force_done

ttp		movzx	rax, BYTE PTR [rdi]		;; Load big vs. little flags
ttp		movzx	rcx, BYTE PTR [rdi+1]
		vmovapd	ymm0, [rsi]			;; Load values1
ttp		vmulpd	ymm0, ymm0, [rbp]		;; Mul values1 by two-to-minus-phi
ttp		vmulpd	ymm0, ymm0, YMM_NORM012_FF	;; Mul by FFTLEN/2
		vmovapd	ymm1, [rsi+32]			;; Load values2
ttp		vmulpd	ymm1, ymm1, [rbp+64]		;; Mul values2 by two-to-minus-phi
ttp		vmulpd	ymm1, ymm1, YMM_NORM012_FF	;; Mul by FFTLEN/2
		vaddpd	ymm0, ymm0, ymm2		;; x1 = values + carry
		vaddpd	ymm1, ymm1, ymm3		;; x2 = values + carry
		rounding_interleaved ttp, base2, noexec, ymm0, ymm2, ymm4, rax*2, ymm1, ymm3, ymm5, rcx*2
ttp		vmulpd	ymm0, ymm0, [rbp+32]		;; new value1 = val * two-to-phi
ttp		vmulpd	ymm1, ymm1, [rbp+96]		;; new value2 = val * two-to-phi
		ystore	[rsi], ymm0			;; Save new value1
		ystore	[rsi+32], ymm1			;; Save new value2

ttp	bump	rdi, 2				;; Advance pointers
	bump	rsi, 64
ttp	bump	rbp, 128
	sub	rdx, 1				;; Test counter
	jnz	section_loop			;; More cache lines in section, add carry in

	;; Section ended.  Rotate carries again and add the new next section carry values
	;; into the previously calculated next section carry values

	rotate_carries_interleaved base2, ymm2, ymm4, ymm3, ymm5, ymm0, ymm1
base2	vsubpd	ymm4, ymm4, YMM_BIGVAL
base2	vsubpd	ymm5, ymm5, YMM_BIGVAL
	vaddpd	ymm4, ymm4, YMM_TMP1
	vaddpd	ymm5, ymm5, YMM_TMP2
	jmp	section_start

force_done:	
base2		vsubpd	ymm2, ymm2, YMM_BIGVAL		;; Subtract rounding constant from carry
base2		vsubpd	ymm3, ymm3, YMM_BIGVAL
ttp		vmulpd	ymm2, ymm2, [rbp+32]		;; carry1 *= two-to-phi
ttp		vmulpd	ymm3, ymm3, [rbp+96]		;; carry2 *= two-to-phi
		vaddpd	ymm2, ymm2, [rsi]		;; Add in values1
		vaddpd	ymm3, ymm3, [rsi+32]		;; Add in values2
		ystore	[rsi], ymm2			;; Save new value1
		ystore	[rsi+32], ymm3			;; Save new value2

done:	vmovapd	ymm2, YMM_TMP1			;; Restore carry for next section
	vmovapd	ymm3, YMM_TMP2			;; Restore carry for next section
	ENDM

; This macro is similar to the above, but is for the zero padding case.
; ymm2 = carry #1 (traditional carry)
; ymm3 = carry #2 (previous high FFT data - not yet mul'ed by K)
; ymm2,ymm3 = carries
; rax, rdx, rsi, rbp, rdi = trash

ynorm_1d_zpad_mid_cleanup MACRO ttp, base2, const, srcptr, biglitptr, ttpptr
	LOCAL	section_start, section_loop, force_done, done

	rotate_carries base2, ymm2, ymm4, ymm0, ymm1
	rotate_carries noexec, ymm3, ymm5, ymm0, ymm1

	mov	DWORD PTR YMM_TMP3, 15		;; Propagate carry at most 15 times (should almost never happen)
section_start:
	mov	rsi, srcptr			;; Load section pointers
ttp	mov	rdi, biglitptr
ttp	mov	rbp, ttpptr
	mov	edx, count1			;; Count of cache lines in section or padded group
	ystore	YMM_TMP1, ymm4			;; Save carry for next section
	ystore	YMM_TMP2, ymm5			;; Save carry for next section

section_loop:
base2	vmovapd	ymm1, YMM_BIGVAL		;; Load comparison value
no base2 vxorpd	ymm1, ymm1, ymm1		;; Create comparison value
	vcmppd	ymm0, ymm2, ymm1, 0Ch		;; Are any carries non-zero
	vmovmskpd rax, ymm0			;; Extract 4 comparison bits
base2	vxorpd	ymm1, ymm1, ymm1		;; High carry words are always compared to zero
	vcmppd	ymm0, ymm3, ymm1, 0Ch		;; Are any carries non-zero
	vmovmskpd rcx, ymm0			;; Extract 4 comparison bits
	or	rax, rcx			;; Are any bits on?
	jz	done				;; No, we're all done

	sub	DWORD PTR YMM_TMP3, 1		;; There is a bizarre case where adding a zero carry causes a carry (example:
						;; 10^114+1, a data value of 500 becomes -500 with a carry of 1 -- add zero again and
						;; it becomes -500 with a carry of -1).  This can lead to an infinite loop because
						;; the 4 carries never become zero.  After many attempts at getting 4 zero carries,
						;; give up and just add in the carries without propagation.
	jz	force_done

	movzx	rax, BYTE PTR [rdi]		;; Load big vs. little flags
	vmovapd	ymm0, [rsi]			;; Load values1
ttp	vmulpd	ymm0, ymm0, [rbp]		;; Mul values1 by two-to-minus-phi
ttp	vmulpd	ymm0, ymm0, YMM_NORM012_FF	;; Mul by FFTLEN/2
	vaddpd	ymm0, ymm0, ymm2		;; x1 = values1 + carry
	split_upper_carry_zpad_word ttp, base2, ymm3, ymm1, ymm2, rax*2
no const vmulpd	ymm2, ymm3, YMM_K_LO		;; low bits of high FFT carry * k_lo
const	vmulpd	ymm2, ymm3, YMM_K_TIMES_MULCONST_LO ;; low bits of high_FFT_carry * k_lo
	vaddpd	ymm0, ymm0, ymm2		;; x1 = x1 + low bits of high_FFT_carry * k_lo
no const vmulpd ymm3, ymm3, YMM_K_HI		;; low bits of high FFT carry * k_hi
const	vmulpd	ymm3, ymm3, YMM_K_TIMES_MULCONST_HI ;; low bits of high FFT carry * k_hi
ttp	vmulpd	ymm3, ymm3, YMM_LIMIT_INVERSE[rax*2] ;; shift low bits of high FFT carry * k_hi
no ttp	vmulpd	ymm3, ymm3, YMM_LIMIT_INVERSE[0] ;; shift low bits of high FFT carry * k_hi
	vroundpd ymm3, ymm3, 0			;; WASTEFUL.  Round (k_hi * limit_inverse) should be precomputed
	rounding ttp, base2, noexec, ymm0, ymm2, ymm4, rax*2
	vaddpd	ymm2, ymm2, ymm3		;; Carry += shifted low bits of high_FFT_carry * k_hi
ttp	vmulpd	ymm0, ymm0, [rbp+32]		;; new value1 = val * two-to-phi
	ystore	[rsi], ymm0			;; Save new value1
	vmovapd	ymm3, ymm1			;; Next high FFT carry = high bits of current high FFT carry

ttp	bump	rdi, 1				;; Advance pointers
	bump	rsi, 64
ttp	bump	rbp, 64
	sub	rdx, 1				;; Test counter
	jnz	section_loop			;; More cache lines in section, add carry in

	;; Section ended.  Rotate carries again and add the new next section carry values
	;; into the previously calculated next section carry values

	rotate_carries base2, ymm2, ymm4, ymm0, ymm1
	rotate_carries noexec, ymm3, ymm5, ymm0, ymm1
base2	vsubpd	ymm4, ymm4, YMM_BIGVAL
	vaddpd	ymm4, ymm4, YMM_TMP1
	vaddpd	ymm5, ymm5, YMM_TMP2
	jmp	section_start

force_done:
base2	vsubpd	ymm2, ymm2, YMM_BIGVAL		;; Subtract rounding constant from carry
ttp	vmulpd	ymm2, ymm2, [rbp+32]		;; carry1 *= two-to-phi
	vaddpd	ymm2, ymm2, [rsi]		;; Add in values1
	ystore	[rsi], ymm2			;; Save new value1

done:	vmovapd	ymm2, YMM_TMP1			;; Restore carry for next section
	vmovapd	ymm3, YMM_TMP2			;; Restore carry for next section
	ENDM


; We could take advantage of the fact that the first two-to-phi multiplier
; and the first two-to-minus-phi multiplier are one.  We also know
; the first data value is a big word
; ymm2,ymm3 = carries
; rsi = pointer to the FFT data values
; rbp = pointer two-to-phi multipliers
; rdi = big vs. little array pointer
; Input arguments are destroyed!

ynorm_1d_cleanup MACRO ttp, base2
		LOCAL	section_begin, section_restart, section_loop, new_section, done, force_done

ttp		mov	rdi, norm_biglit_array		;; Address of the big/little flags array
ttp		mov	rbp, norm_col_mults		;; Restart the column multipliers

base2		vsubpd	ymm3, ymm3, YMM_BIGVAL
		vmulpd	ymm3, ymm3, YMM_MINUS_C		;; Adjust wrap-around carry
base2		vaddpd	ymm3, ymm3, YMM_BIGVAL

		mov	DWORD PTR YMM_TMP6, 3		;; Propagate carry through at most 3 sections (should almost never happen)

section_begin:
		mov	rbx, 4				;; Loop through section at most 4 times
		mov	PPTR YMM_TMP1, rsi		;; Save section pointers
ttp		mov	PPTR YMM_TMP2, rdi
ttp		mov	PPTR YMM_TMP3, rbp

		mov	PPTR YMM_TMP7, rsi		;; Init next section pointers to this section (in case we don't complete this section)
ttp		mov	PPTR YMM_TMP8, rbp

base2		vmovapd	ymm5, YMM_BIGVAL		;; Clear carries for next section
no base2	vxorpd	ymm5, ymm5, ymm5
		vmovapd	ymm4, ymm5

section_restart:
		ystore	YMM_TMP5, ymm5			;; Save next section's initial carries
		ystore	YMM_TMP4, ymm4
		mov	edx, count1			;; Count of cache lines in section or padded group

section_loop:	inc	dh				;; Count propagations in the section
		cmp	dh, 10				;; Only propagate carry for 10 words in the section (only pathological cases should ever get this high)
		je	force_done			;; Force the final carry

base2		vmovapd	ymm1, YMM_BIGVAL		;; Load comparison value
no base2	vxorpd	ymm1, ymm1, ymm1		;; Create comparison value
		vcmppd	ymm0, ymm3, ymm1, 0Ch		;; Are any carries non-zero
		vmovmskpd rax, ymm0			;; Extract 4 comparison bits
		vcmppd	ymm0, ymm2, ymm1, 0Ch		;; Are any carries non-zero
		vmovmskpd rcx, ymm0			;; Extract 4 comparison bits
		or	rax, rcx			;; Are any bits on?
		jz	done				;; No, we're all done

ttp		movzx	rax, BYTE PTR [rdi]		;; Load big vs. little flags
ttp		movzx	rcx, BYTE PTR [rdi+1]
		vmovapd	ymm0, [rsi]			;; Load values1
ttp		vmulpd	ymm0, ymm0, [rbp]		;; Mul values1 by two-to-minus-phi
ttp		vmulpd	ymm0, ymm0, YMM_NORM012_FF	;; Mul by FFTLEN/2
		vmovapd	ymm1, [rsi+32]			;; Load values2
ttp		vmulpd	ymm1, ymm1, [rbp+64]		;; Mul values2 by two-to-minus-phi
ttp		vmulpd	ymm1, ymm1, YMM_NORM012_FF	;; Mul by FFTLEN/2
		vaddpd	ymm0, ymm0, ymm3		;; x1 = values + carry
		vaddpd	ymm1, ymm1, ymm2		;; x2 = values + carry
		rounding_interleaved ttp, base2, noexec, ymm0, ymm3, ymm4, rax*2, ymm1, ymm2, ymm5, rcx*2
ttp		vmulpd	ymm0, ymm0, [rbp+32]		;; new value1 = val * two-to-phi
ttp		vmulpd	ymm1, ymm1, [rbp+96]		;; new value2 = val * two-to-phi
		ystore	[rsi], ymm0			;; Save new value1
		ystore	[rsi+32], ymm1			;; Save new value2

ttp		bump	rdi, 2				;; Advance pointers
		bump	rsi, 64
ttp		bump	rbp, 128
		sub	dl, 1				;; Test cache line counter
		jnz	section_loop			;; More cache lines in section, go add carry in

	;; Section ended.  Rotate carries and stay in the current section or
	;; move to the next section.  I think only a length 32 FFT will ever
	;; move onto the next section.

		rotate_carries_interleaved base2, ymm3, ymm5, ymm2, ymm4, ymm0, ymm1

		mov	PPTR YMM_TMP7, rsi		;; Save next section's pointers
ttp		mov	PPTR YMM_TMP8, rbp

base2		vsubpd	ymm5, ymm5, YMM_BIGVAL		;; Add carry for next section
base2		vsubpd	ymm4, ymm4, YMM_BIGVAL
		vaddpd	ymm5, ymm5, YMM_TMP5
		vaddpd	ymm4, ymm4, YMM_TMP4

		sub	rbx, 1				;; See if we should move to the next section
		jz	short new_section		;; Yes, go start next section

		mov	rsi, PPTR YMM_TMP1		;; Restore section pointers
ttp		mov	rdi, PPTR YMM_TMP2
ttp		mov	rbp, PPTR YMM_TMP3
		jmp	section_restart			;; Do the section again

new_section:
	;; There is a bizarre case where adding a zero carry causes a carry (example: 10^114+1, a data
	;; value of 500 becomes -500 with a carry of 1 -- add zero again and it becomes -500 with a carry
	;; of -1).  This means that even though we have shifted the carries 4 times, the carries may not
	;; be zero.  Add the carries back to the section start.

		mov	rsi, PPTR YMM_TMP1		;; Restore section start pointers
ttp		mov	rbp, PPTR YMM_TMP3
base2		vsubpd	ymm3, ymm3, YMM_BIGVAL		;; Subtract rounding constant from carry
base2		vsubpd	ymm2, ymm2, YMM_BIGVAL
ttp		vmulpd	ymm3, ymm3, [rbp+32]		;; carry1 *= two-to-phi
ttp		vmulpd	ymm2, ymm2, [rbp+96]		;; carry2 *= two-to-phi
		vaddpd	ymm3, ymm3, [rsi]		;; Add in values1
		vaddpd	ymm2, ymm2, [rsi+32]		;; Add in values2
		ystore	[rsi], ymm3			;; Save new value1
		ystore	[rsi+32], ymm2			;; Save new value2

		mov	rsi, PPTR YMM_TMP7		;; Load next section pointers
ttp		mov	rbp, PPTR YMM_TMP8
		vmovapd	ymm3, ymm5			;; Copy next section carry words
		vmovapd	ymm2, ymm4

		sub	DWORD PTR YMM_TMP6, 1		;; Text maximum number of sections to process
		jnz	section_begin			;; Do next section
		jmp	short force_done		;; Forced end.  Add already loaded carries into next section.

		;; Ugh, the next section's carries may not be zero (because adding a zero carry
		;; can cause a carry into the next section).  Add any carries into next section.

done:		vmovapd	ymm3, YMM_TMP5			;; Reload any rotated carries
		vmovapd	ymm2, YMM_TMP4
		mov	rsi, PPTR YMM_TMP7		;; Restore next section pointers
ttp		mov	rbp, PPTR YMM_TMP8

force_done:
base2		vsubpd	ymm3, ymm3, YMM_BIGVAL		;; Subtract rounding constant from carry
base2		vsubpd	ymm2, ymm2, YMM_BIGVAL
ttp		vmulpd	ymm3, ymm3, [rbp+32]		;; carry1 *= two-to-phi
ttp		vmulpd	ymm2, ymm2, [rbp+96]		;; carry2 *= two-to-phi
		vaddpd	ymm3, ymm3, [rsi]		;; Add in values1
		vaddpd	ymm2, ymm2, [rsi+32]		;; Add in values2
		ystore	[rsi], ymm3			;; Save new value1
		ystore	[rsi+32], ymm2			;; Save new value2
	ENDM

; This macro is similar to the above, but is for the zero padding case.
; xmm2 = carry #1 (traditional carry)
; xmm3 = carry #2 (previous high FFT data - not yet mul'ed by K)
; rax,rsi,rbp,rdi = trashed
; NOTE: If RATIONAL_FFT we could eliminate 8 multiplies.

ynorm_1d_zpad_cleanup MACRO const, base2
	LOCAL	smallk, mediumk, div_k_done

	;; Strip BIGVAL from the traditional carry, we'll add the traditional
	;; carry in later when we are working on the ZPAD0 - ZPAD6 values.
base2	vsubsd	xmm2, xmm2, Q YMM_BIGVAL	;; Integerize traditional carry

	;; Rather than calculate high FFT carry times k and then later dividing
	;; by k, we multiply FFT high carry by const and we'll add it
	;; to the lower FFT data later (after multiplying by -c).
const	vmulsd	xmm3, xmm3, Q YMM_MULCONST

	;; Work on zero-pad addin value.
	vmovsd	xmm4, ADDIN_VALUE		;; Load the add in value
const	vmulsd	xmm4, xmm4, YMM_MULCONST	;; Multiply the add in value by the small mul const
	vaddsd	xmm4, xmm4, POSTADDIN_VALUE	;; Add the post-mul-by-const addin value
	;; when c = -1, 1 = b^n
	;; when c = 1, -1 = b^n, 1 = -b^n
	;; when c = -3, 3 = b^n, 1 = b^n - 2
	;; when c = 3, -3 = b^n, 3 = -b^n, 1 = -b^n - 2
	;; The "- 2" has been precomputed in ZPAD_LSW_ADJUST.  Add ADDIN_VALUE * mul-by-const * ZPAD_LSW_ADJUST into the least significant FFT word
	mov	rsi, DESTARG			;; Address of squared number
	vmulsd	xmm0, xmm4, ZPAD_LSW_ADJUST
	vaddsd	xmm0, xmm0, [rsi]
	vmovsd	[rsi], xmm0

	;; Multiply ZPAD0 through ZPAD6 by const * -c.  This, in essense,
	;; wraps this data from above the FFT data area to the halfway point.
	;; Later on we'll divide this by K to decide which data needs wrapping
	;; all the way down to the bottom of the FFT data.

	;; NOTE that ZPAD0's ttp multiplier is 1.0.  Also, ZPAD6 will not
	;; be bigger than a big word.  We must be careful to handle c's up
	;; to about 30 bits

	lea	rdi, YMM_FIRST_BIGLIT_VALUES	;; Address of first big/little flags
	mov	rbp, norm_col_mults		;; Address of the ttmp/ttp multipliers

	movzx	rax, BYTE PTR [rdi]		;; Load big vs. little flags
	vmovsd	xmm0, ZPAD0			;; Load value
	vsubsd	xmm5, xmm5, xmm5
	single_split_lower_zpad_word base2, xmm0, xmm5, xmm1, rax*2
no const vmulsd	xmm0, xmm0, Q YMM_MINUS_C
const	vmulsd	xmm0, xmm0, Q YMM_MINUS_C_TIMES_MULCONST
	vaddsd	xmm0, xmm0, xmm4		;; Apply rest of ADDIN_VALUE * mul-by-const here (the FFT half way point)
	round_zpad7_word base2, xmm0, xmm2, xmm1, rax*2
	vmovsd	ZPAD0, xmm0

	add	rbp, YMM_NORM_INCR1		;; Next ttp/ttmp pointer
	movzx	rax, BYTE PTR [rdi+1]		;; Load big vs. little flags
	vmovsd	xmm0, ZPAD1			;; Load value
	vmulsd	xmm0, xmm0, Q [rbp]		;; Mul value by two-to-minus-phi
	vmulsd	xmm0, xmm0, Q YMM_NORM012_FF	;; Mul by FFTLEN/2
	single_split_lower_zpad_word base2, xmm0, xmm5, xmm1, rax*2
no const vmulsd	xmm0, xmm0, Q YMM_MINUS_C
const	vmulsd	xmm0, xmm0, Q YMM_MINUS_C_TIMES_MULCONST
	round_zpad7_word base2, xmm0, xmm2, xmm1, rax*2
	vmovsd	ZPAD1, xmm0

	add	rbp, YMM_NORM_INCR2		;; Next ttp/ttmp pointer
	movzx	rax, BYTE PTR [rdi+2]		;; Load big vs. little flags
	vmovsd	xmm0, ZPAD2			;; Load value
	vmulsd	xmm0, xmm0, Q [rbp]		;; Mul value by two-to-minus-phi
	vmulsd	xmm0, xmm0, Q YMM_NORM012_FF	;; Mul by FFTLEN/2
	single_split_lower_zpad_word base2, xmm0, xmm5, xmm1, rax*2
no const vmulsd	xmm0, xmm0, Q YMM_MINUS_C
const	vmulsd	xmm0, xmm0, Q YMM_MINUS_C_TIMES_MULCONST
	round_zpad7_word base2, xmm0, xmm2, xmm1, rax*2
	vmovsd	ZPAD2, xmm0

	add	rbp, YMM_NORM_INCR3		;; Next ttp/ttmp pointer
	movzx	rax, BYTE PTR [rdi+3]		;; Load big vs. little flags
	vmovsd	xmm0, ZPAD3			;; Load value
	vmulsd	xmm0, xmm0, Q [rbp]		;; Mul value by two-to-minus-phi
	vmulsd	xmm0, xmm0, Q YMM_NORM012_FF	;; Mul by FFTLEN/2
	single_split_lower_zpad_word base2, xmm0, xmm5, xmm1, rax*2
no const vmulsd	xmm0, xmm0, Q YMM_MINUS_C
const	vmulsd	xmm0, xmm0, Q YMM_MINUS_C_TIMES_MULCONST
	round_zpad7_word base2, xmm0, xmm2, xmm1, rax*2
	vmovsd	ZPAD3, xmm0

	add	rbp, YMM_NORM_INCR4		;; Next ttp/ttmp pointer
	movzx	rax, BYTE PTR [rdi+4]		;; Load big vs. little flags
	vmovsd	xmm0, ZPAD4			;; Load value
	vmulsd	xmm0, xmm0, Q [rbp]		;; Mul value by two-to-minus-phi
	vmulsd	xmm0, xmm0, Q YMM_NORM012_FF	;; Mul by FFTLEN/2
	single_split_lower_zpad_word base2, xmm0, xmm5, xmm1, rax*2
no const vmulsd	xmm0, xmm0, Q YMM_MINUS_C
const	vmulsd	xmm0, xmm0, Q YMM_MINUS_C_TIMES_MULCONST
	round_zpad7_word base2, xmm0, xmm2, xmm1, rax*2
	vmovsd	ZPAD4, xmm0

	add	rbp, YMM_NORM_INCR5		;; Next ttp/ttmp pointer
	movzx	rax, BYTE PTR [rdi+5]		;; Load big vs. little flags
	vmovsd	xmm0, ZPAD5			;; Load value
	vmulsd	xmm0, xmm0, Q [rbp]		;; Mul value by two-to-minus-phi
	vmulsd	xmm0, xmm0, Q YMM_NORM012_FF	;; Mul by FFTLEN/2
	single_split_lower_zpad_word base2, xmm0, xmm5, xmm1, rax*2
no const vmulsd	xmm0, xmm0, Q YMM_MINUS_C
const	vmulsd	xmm0, xmm0, Q YMM_MINUS_C_TIMES_MULCONST
	round_zpad7_word base2, xmm0, xmm2, xmm1, rax*2
	vmovsd	ZPAD5, xmm0

	add	rbp, YMM_NORM_INCR6		;; Next ttp/ttmp pointer
	vmovsd	xmm0, ZPAD6			;; Load value
	vmulsd	xmm0, xmm0, Q [rbp]		;; Mul value by two-to-minus-phi
	vmulsd	xmm0, xmm0, Q YMM_NORM012_FF	;; Mul by FFTLEN/2
	vroundsd xmm0, xmm0, xmm0, 0		;; Round to an integer
	vaddsd	xmm0, xmm0, xmm5		;; Add in shifted high ZPAD data
no const vmulsd	xmm0, xmm0, Q YMM_MINUS_C
const	vmulsd	xmm0, xmm0, Q YMM_MINUS_C_TIMES_MULCONST
	vaddsd	xmm0, xmm0, xmm2		;; Add in high part of last calculation
	vmovsd	ZPAD6, xmm0

	;; Divide the zpad data by k.  Store the integer part in YMM_TMP
	;; and the remainder in ZPAD0.  Later we will wrap the integer part
	;; down to the bottom of the FFT data area (and multiply by -c).
	;; And we will store the remainder in the upper half of the FFT
	;; data area.

	;; Note there are three cases to handle.  K is smaller than a big word.
	;; K is between one and 2 big words in size.  And K is more than
	;; 2 big words in size.

	cmp	ZPAD_TYPE, 2			;; Are we dealing with case 1,2,or 3
	jl	smallk				;; One word case
	je	mediumk				;; Two word case

	;; This case does the divide by k where k is three words

	vmovsd	xmm0, ZPAD6			;; Load zpad word (high bits)
	vmovsd	xmm1, ZPAD5			;; Load zpad word (middle bits)
	vmovsd	xmm2, ZPAD4			;; Load zpad word (low bits)
	vmulsd	xmm4, xmm0, ZPAD_INVERSE_K6	;; Mul ZPAD by shifted 1/k
	vroundsd xmm4, xmm4, xmm4, 0		;; Round to integer
	vmulsd	xmm5, xmm4, ZPAD_K6_HI		;; Mul by high bits of k
	vsubsd	xmm0, xmm0, xmm5		;; Calculate high bits of remainder
	vmulsd	xmm5, xmm4, ZPAD_K6_MID		;; Mul by middle bits of k
	vsubsd	xmm1, xmm1, xmm5		;; Calculate middle bits of remainder
	vmulsd	xmm5, xmm4, ZPAD_K6_LO		;; Mul by low bits of k
	vsubsd	xmm2, xmm2, xmm5		;; Calculate low bits of remainder
	vmovsd	Q YMM_TMP5, xmm4		;; Save word of zpad / k

	vmulsd	xmm0, xmm0, ZPAD_SHIFT6		;; Shift previous zpad word
	vaddsd	xmm1, xmm1, xmm0		;; Add to create new high zpad bits
	vmovsd	xmm0, ZPAD3			;; Load zpad word (new low bits)
	vmulsd	xmm5, xmm1, ZPAD_SHIFT5		;; Combine high and medium bits
	vaddsd	xmm5, xmm5, xmm2
	vmulsd	xmm4, xmm5, ZPAD_INVERSE_K5	;; Mul ZPAD by shifted 1/k
	vroundsd xmm4, xmm4, xmm4, 0		;; Round to integer
	vmulsd	xmm5, xmm4, ZPAD_K5_HI		;; Mul by high bits of k
	vsubsd	xmm1, xmm1, xmm5		;; Calculate high bits of remainder
	vmulsd	xmm5, xmm4, ZPAD_K5_MID		;; Mul by middle bits of k
	vsubsd	xmm2, xmm2, xmm5		;; Calculate middle bits of remainder
	vmulsd	xmm5, xmm4, ZPAD_K5_LO		;; Mul by low bits of k
	vsubsd	xmm0, xmm0, xmm5		;; Calculate low bits of remainder
	vmovsd	Q YMM_TMP4, xmm4		;; Save word of zpad / k

	vmulsd	xmm1, xmm1, ZPAD_SHIFT5		;; Shift previous zpad word
	vaddsd	xmm2, xmm2, xmm1		;; Add to create new high zpad bits
	vmovsd	xmm1, ZPAD2			;; Load zpad word (new low bits)
	vmulsd	xmm5, xmm2, ZPAD_SHIFT4		;; Combine high and medium bits
	vaddsd	xmm5, xmm5, xmm0
	vmulsd	xmm4, xmm5, ZPAD_INVERSE_K4	;; Mul ZPAD by shifted 1/k
	vroundsd xmm4, xmm4, xmm4, 0		;; Round to integer
	vmulsd	xmm5, xmm4, ZPAD_K4_HI		;; Mul by high bits of k
	vsubsd	xmm2, xmm2, xmm5		;; Calculate high bits of remainder
	vmulsd	xmm5, xmm4, ZPAD_K4_MID		;; Mul by middle bits of k
	vsubsd	xmm0,xmm0,xmm5			;; Calculate middle bits of remainder
	vmulsd	xmm5, xmm4, ZPAD_K4_LO		;; Mul by low bits of k
	vsubsd	xmm1, xmm1, xmm5		;; Calculate low bits of remainder
	vmovsd	Q YMM_TMP3, xmm4		;; Save word of zpad / k

	vmulsd	xmm2, xmm2, ZPAD_SHIFT4		;; Shift previous zpad word
	vaddsd	xmm0, xmm0, xmm2		;; Add to create new high zpad bits
	vmovsd	xmm2, ZPAD1			;; Load zpad word (new low bits)
	vmulsd	xmm5, xmm0, ZPAD_SHIFT3		;; Combine high and medium bits
	vaddsd	xmm5, xmm5, xmm1
	vmulsd	xmm4, xmm5, ZPAD_INVERSE_K3	;; Mul ZPAD by shifted 1/k
	vroundsd xmm4, xmm4, xmm4, 0		;; Round to integer
	vmulsd	xmm5, xmm4, ZPAD_K3_HI		;; Mul by high bits of k
	vsubsd	xmm0, xmm0, xmm5		;; Calculate high bits of remainder
	vmulsd	xmm5, xmm4, ZPAD_K3_MID		;; Mul by middle bits of k
	vsubsd	xmm1, xmm1, xmm5		;; Calculate middle bits of remainder
	vmulsd	xmm5, xmm4, ZPAD_K3_LO		;; Mul by low bits of k
	vsubsd	xmm2, xmm2, xmm5		;; Calculate low bits of remainder
	vmovsd	Q YMM_TMP2, xmm4		;; Save word of zpad / k

	vmulsd	xmm0, xmm0, ZPAD_SHIFT3		;; Shift previous zpad word
	vaddsd	xmm1, xmm1, xmm0		;; Add to create new high zpad bits
	vmovsd	xmm0, ZPAD0			;; Load zpad word (new low bits)
	vmulsd	xmm5, xmm1, ZPAD_SHIFT2		;; Combine high and medium bits
	vaddsd	xmm5, xmm5, xmm2
	vmulsd	xmm4, xmm5, ZPAD_INVERSE_K2	;; Mul ZPAD by shifted 1/k
	vroundsd xmm4, xmm4, xmm4, 0		;; Round to integer
	vmulsd	xmm5, xmm4, ZPAD_K2_HI		;; Mul by high bits of k
	vsubsd	xmm1, xmm1, xmm5		;; Calculate high bits of remainder
	vmulsd	xmm5, xmm4, ZPAD_K2_MID		;; Load middle bits of k
	vsubsd	xmm2, xmm2, xmm5		;; Calculate middle bits of remainder
	vmulsd	xmm5, xmm4, ZPAD_K2_LO		;; Load low bits of k
	vsubsd	xmm0, xmm0, xmm5		;; Calculate low bits of remainder
	vmovsd	Q YMM_TMP1, xmm4		;; Save word of zpad / k

	vmulsd	xmm1, xmm1, ZPAD_SHIFT2		;; Shift previous zpad word
	vaddsd	xmm2, xmm2, xmm1		;; Add to create new high zpad bits
	vmulsd	xmm2, xmm2, ZPAD_SHIFT1		;; Shift previous zpad word
	vaddsd	xmm0, xmm0, xmm2		;; Add to create new high zpad bits
	vmovsd	ZPAD0, xmm0			;; Save remainder of zpad / k

	vxorpd	xmm1, xmm1, xmm1		;; Zero words that other cases set
	vmovsd	Q YMM_TMP6, xmm1
	
	jmp	div_k_done

	;; This case does the divide by k where k is two words
mediumk:
	vmovsd	xmm0, ZPAD6			;; Load zpad word (high bits)
	vmovsd	xmm1, ZPAD5			;; Load zpad word (low bits)
	vmulsd	xmm4, xmm0, ZPAD_INVERSE_K6	;; Mul ZPAD by shifted 1/k
	vroundsd xmm4, xmm4, xmm4, 0		;; Round to integer
	vmulsd	xmm5, xmm4, ZPAD_K6_HI		;; Mul by high bits of k
	vsubsd	xmm0, xmm0, xmm5		;; Calculate high bits of remainder
	vmulsd	xmm5, xmm4, ZPAD_K6_LO		;; Mul by low bits of k
	vsubsd	xmm1, xmm1, xmm5		;; Calculate low bits of remainder
	vmovsd	Q YMM_TMP6, xmm4		;; Save word of zpad / k

	vmulsd	xmm0, xmm0, ZPAD_SHIFT6		;; Shift previous zpad word
	vaddsd	xmm0, xmm0, xmm1		;; Add to create new high zpad bits
	vmovsd	xmm1, ZPAD4			;; Load zpad word (new low bits)
	vmulsd	xmm4, xmm0, ZPAD_INVERSE_K5	;; Mul ZPAD by shifted 1/k
	vroundsd xmm4, xmm4, xmm4, 0		;; Round to integer
	vmulsd	xmm5, xmm4, ZPAD_K5_HI		;; Mul by high bits of k
	vsubsd	xmm0, xmm0, xmm5		;; Calculate high bits of remainder
	vmulsd	xmm5, xmm4, ZPAD_K5_LO		;; Mul by low bits of k
	vsubsd	xmm1, xmm1, xmm5		;; Calculate low bits of remainder
	vmovsd	Q YMM_TMP5, xmm4		;; Save word of zpad / k

	vmulsd	xmm0, xmm0, ZPAD_SHIFT5		;; Shift previous zpad word
	vaddsd	xmm0, xmm0, xmm1		;; Add to create new high zpad bits
	vmovsd	xmm1, ZPAD3			;; Load zpad word (new low bits)
	vmulsd	xmm4, xmm0, ZPAD_INVERSE_K4	;; Mul ZPAD by shifted 1/k
	vroundsd xmm4, xmm4, xmm4, 0		;; Round to integer
	vmulsd	xmm5, xmm4, ZPAD_K4_HI		;; Mul by high bits of k
	vsubsd	xmm0, xmm0, xmm5		;; Calculate high bits of remainder
	vmulsd	xmm5, xmm4, ZPAD_K4_LO		;; Mul by low bits of k
	vsubsd	xmm1, xmm1, xmm5		;; Calculate low bits of remainder
	vmovsd	Q YMM_TMP4, xmm4		;; Save word of zpad / k

	vmulsd	xmm0, xmm0, ZPAD_SHIFT4		;; Shift previous zpad word
	vaddsd	xmm0, xmm0, xmm1		;; Add to create new high zpad bits
	vmovsd	xmm1, ZPAD2			;; Load zpad word (new low bits)
	vmulsd	xmm4, xmm0, ZPAD_INVERSE_K3	;; Mul ZPAD by shifted 1/k
	vroundsd xmm4, xmm4, xmm4, 0		;; Round to integer
	vmulsd	xmm5, xmm4, ZPAD_K3_HI		;; Mul by high bits of k
	vsubsd	xmm0, xmm0, xmm5		;; Calculate high bits of remainder
	vmulsd	xmm5, xmm4, ZPAD_K3_LO		;; Mul by low bits of k
	vsubsd	xmm1, xmm1, xmm5		;; Calculate low bits of remainder
	vmovsd	Q YMM_TMP3, xmm4		;; Save word of zpad / k

	vmulsd	xmm0, xmm0, ZPAD_SHIFT3		;; Shift previous zpad word
	vaddsd	xmm0, xmm0, xmm1		;; Add to create new high zpad bits
	vmovsd	xmm1, ZPAD1			;; Load zpad word (new low bits)
	vmulsd	xmm4, xmm0, ZPAD_INVERSE_K2	;; Mul ZPAD by shifted 1/k
	vroundsd xmm4, xmm4, xmm4, 0		;; Round to integer
	vmulsd	xmm5, xmm4, ZPAD_K2_HI		;; Mul by high bits of k
	vsubsd	xmm0, xmm0, xmm5		;; Calculate high bits of remainder
	vmulsd	xmm5, xmm4, ZPAD_K2_LO		;; Mul by low bits of k
	vsubsd	xmm1, xmm1, xmm5		;; Calculate low bits of remainder
	vmovsd	Q YMM_TMP2, xmm4		;; Save word of zpad / k

	vmulsd	xmm0, xmm0, ZPAD_SHIFT2		;; Shift previous zpad word
	vaddsd	xmm0, xmm0, xmm1		;; Add to create new high zpad bits
	vmovsd	xmm1, ZPAD0			;; Load zpad word (new low bits)
	vmulsd	xmm4, xmm0, ZPAD_INVERSE_K1	;; Mul ZPAD by shifted 1/k
	vroundsd xmm4, xmm4, xmm4, 0		;; Round to integer
	vmulsd	xmm5, xmm4, ZPAD_K1_HI		;; Mul by high bits of k
	vsubsd	xmm0, xmm0, xmm5		;; Calculate high bits of remainder
	vmulsd	xmm5, xmm4, ZPAD_K1_LO		;; Mul by low bits of k
	vsubsd	xmm1, xmm1, xmm5		;; Calculate low bits of remainder
	vmovsd	Q YMM_TMP1, xmm4		;; Save word of zpad / k

	vmulsd	xmm0, xmm0, ZPAD_SHIFT1		;; Shift previous zpad word
	vaddsd	xmm0, xmm0, xmm1		;; Add to create new high zpad bits
	vmovsd	ZPAD0, xmm0			;; Save remainder of zpad / k

	jmp	div_k_done

	;; This case does the divide by k where k is one word
	;; Assume ZPAD5 and ZPAD6 are zero.
smallk:	vmovsd	xmm0, ZPAD4			;; Load zpad data
	vmulsd	xmm4, xmm0, ZPAD_INVERSE_K1	;; Mul ZPAD data by 1/k
	vroundsd xmm4, xmm4, xmm4, 0		;; Round to integer
	vmovsd	Q YMM_TMP5, xmm4		;; Save integer part
	vmulsd	xmm4, xmm4, ZPAD_K1_LO		;; Compute remainder
	vsubsd	xmm0, xmm0, xmm4

	vmulsd	xmm0, xmm0, ZPAD_SHIFT4		;; Shift previous zpad word
	vaddsd	xmm0, xmm0, ZPAD3		;; Add in zpad data
	vmulsd	xmm4, xmm0, ZPAD_INVERSE_K1	;; Mul ZPAD data by 1/k
	vroundsd xmm4, xmm4, xmm4, 0		;; Round to integer
	vmovsd	Q YMM_TMP4, xmm4		;; Save integer part
	vmulsd	xmm4, xmm4, ZPAD_K1_LO		;; Compute remainder
	vsubsd	xmm0, xmm0, xmm4

	vmulsd	xmm0, xmm0, ZPAD_SHIFT3		;; Shift previous zpad word
	vaddsd	xmm0, xmm0, ZPAD2		;; Add in zpad data
	vmulsd	xmm4, xmm0, ZPAD_INVERSE_K1	;; Mul ZPAD data by 1/k
	vroundsd xmm4, xmm4, xmm4, 0		;; Round to integer
	vmovsd	Q YMM_TMP3, xmm4		;; Save integer part
	vmulsd	xmm4, xmm4, ZPAD_K1_LO		;; Compute remainder
	vsubsd	xmm0, xmm0, xmm4

	vmulsd	xmm0, xmm0, ZPAD_SHIFT2		;; Shift previous zpad word
	vaddsd	xmm0, xmm0, ZPAD1		;; Add in zpad data
	vmulsd	xmm4, xmm0, ZPAD_INVERSE_K1	;; Mul ZPAD data by 1/k
	vroundsd xmm4, xmm4, xmm4, 0		;; Round to integer
	vmovsd	Q YMM_TMP2, xmm4		;; Save integer part
	vmulsd	xmm4, xmm4, ZPAD_K1_LO		;; Compute remainder
	vsubsd	xmm0, xmm0, xmm4

	vmulsd	xmm0, xmm0, ZPAD_SHIFT1		;; Shift previous zpad word
	vaddsd	xmm0, xmm0, ZPAD0		;; Add in zpad data
	vmulsd	xmm4, xmm0, ZPAD_INVERSE_K1	;; Mul ZPAD data by 1/k
	vroundsd xmm4, xmm4, xmm4, 0		;; Round to integer
	vmovsd	Q YMM_TMP1, xmm4		;; Save integer part
	vmulsd	xmm4, xmm4, ZPAD_K1_LO		;; Compute remainder
	vsubsd	xmm0, xmm0, xmm4
	vmovsd	ZPAD0, xmm0			;; Save remainder

	vxorpd	xmm1, xmm1, xmm1		;; Zero words that other cases set
	vmovsd	Q YMM_TMP6, xmm1

div_k_done:

	;; Now normalize the data above the halfway point.  Remember that the
	;; two-to-phi multiplier for the first value will be 1.0.

	lea	rdi, YMM_FIRST_BIGLIT_VALUES	;; Address of first big/little flags
	mov	rsi, DESTARG			;; Address of squared number
	mov	rbp, norm_col_mults		;; Address of the ttmp/ttp multipliers

	movzx	rax, BYTE PTR [rdi]		;; First word
	vmovsd	xmm0, ZPAD0			;; Load remainder of divide by k
base2	vaddsd	xmm0, xmm0, Q YMM_BIGVAL
	single_rounding base2, xmm0, xmm2, xmm4, rax*2
	vmovsd	Q [rsi+32], xmm0		;; Save value1

	add	rbp, YMM_NORM_INCR1		;; Next ttp/ttmp pointer
	add	rsi, YMM_SRC_INCR1		;; Next source pointer
	movzx	rax, BYTE PTR [rdi+1]		;; Load big vs. little flags
	single_rounding base2, xmm2, xmm0, xmm4, rax*2
	vmulsd	xmm2, xmm2, Q [rbp+32]		;; new value2 = val * two-to-phi
	vmovsd	Q [rsi+32], xmm2		;; Save value2

	add	rbp, YMM_NORM_INCR2		;; Next ttp/ttmp pointer
	add	rsi, YMM_SRC_INCR2		;; Next source pointer
	movzx	rax, BYTE PTR [rdi+2]		;; Load big vs. little flags
	single_rounding base2, xmm0, xmm2, xmm4, rax*2
	vmulsd	xmm0, xmm0, Q [rbp+32]		;; new value3 = val * two-to-phi
	vmovsd	Q [rsi+32], xmm0		;; Save value3

	add	rbp, YMM_NORM_INCR3		;; Next ttp/ttmp pointer
	add	rsi, YMM_SRC_INCR3		;; Next source pointer
base2	vsubsd	xmm2, xmm2, Q YMM_BIGVAL	;; Remove integer rounding constant
	vmulsd	xmm2, xmm2, Q [rbp+32]		;; value4 = carry * two-to-phi
	vmovsd	Q [rsi+32], xmm2		;; Save new value4

	;; Mul the integer part of (ZPAD data divided by k) by -c in
	;; preparation for adding it into the lower FFT data area.
	;; Also add in the shifted high FFT carry at this time.

	;; Now add in and normalize the bottom FFT data.  Remember that the
	;; two-to-phi multiplier for the first value will be 1.0.  We 
	;; must go 6 words deep in case k is 48-50 bits and c is 32 bits.

	mov	rsi, DESTARG			;; Address of squared number
	mov	rbp, norm_col_mults		;; Address of ttmp/ttp multipliers

	movzx	rax, BYTE PTR [rdi]		;; First word
	vmovsd	xmm0, Q YMM_TMP1		;; Load integer part of divide by k
	vaddsd	xmm0, xmm0, xmm3		;; Add in shifted high FFT carry
	vmulsd	xmm0, xmm0, Q YMM_MINUS_C	;; Mul by -c
base2	vaddsd	xmm0, xmm0, Q YMM_BIGVAL
	vaddsd	xmm0, xmm0, Q [rsi]		;; Add in the FFT data
	single_rounding base2, xmm0, xmm2, xmm4, rax*2
	vmovsd	Q [rsi], xmm0			;; Save value1

	add	rsi, YMM_SRC_INCR1		;; Next source pointer
	add	rbp, YMM_NORM_INCR1		;; Next ttp/ttmp pointer
	movzx	rax, BYTE PTR [rdi+1]		;; Load big vs. little flags
	vmovsd	xmm0, Q YMM_TMP2		;; Load integer part of divide by k
	vmulsd	xmm0, xmm0, Q YMM_MINUS_C	;; Mul by -c
	vmovsd	xmm1, Q [rsi]			;; Load FFT data
	vmulsd	xmm1, xmm1, Q [rbp]		;; Mul values2 by two-to-minus-phi
	vmulsd	xmm1, xmm1, Q YMM_NORM012_FF	;; Mul by FFTLEN/2
	vaddsd	xmm0, xmm0, xmm2		;; x2 = value + carry
	vaddsd	xmm0, xmm0, xmm1		;; Add in the FFT data
	single_rounding base2, xmm0, xmm2, xmm4, rax*2
	vmulsd	xmm0, xmm0, Q [rbp+32]		;; new value2 = val * two-to-phi
	vmovsd	Q [rsi], xmm0			;; Save value2

	add	rsi, YMM_SRC_INCR2		;; Next source pointer
	add	rbp, YMM_NORM_INCR2		;; Next ttp/ttmp pointer
	movzx	rax, BYTE PTR [rdi+2]		;; Load big vs. little flags
	vmovsd	xmm0, Q YMM_TMP3		;; Load integer part of divide by k
	vmulsd	xmm0, xmm0, Q YMM_MINUS_C	;; Mul by -c
	vmovsd	xmm1, Q [rsi]			;; Load FFT data
	vmulsd	xmm1, xmm1, Q [rbp]		;; Mul values3 by two-to-minus-phi
	vmulsd	xmm1, xmm1, Q YMM_NORM012_FF	;; Mul by FFTLEN/2
	vaddsd	xmm0, xmm0, xmm2		;; x3 = value + carry
	vaddsd	xmm0, xmm0, xmm1		;; Add in the FFT data
	single_rounding base2, xmm0, xmm2, xmm4, rax*2
	vmulsd	xmm0, xmm0, Q [rbp+32]		;; new value3 = val * two-to-phi
	vmovsd	Q [rsi], xmm0			;; Save value3

	add	rsi, YMM_SRC_INCR3		;; Next source pointer
	add	rbp, YMM_NORM_INCR3		;; Next ttp/ttmp pointer
	movzx	rax, BYTE PTR [rdi+3]		;; Load big vs. little flags
	vmovsd	xmm0, Q YMM_TMP4		;; Load integer part of divide by k
	vmulsd	xmm0, xmm0, Q YMM_MINUS_C	;; Mul by -c
	vmovsd	xmm1, Q [rsi]			;; Load FFT data
	vmulsd	xmm1, xmm1, Q [rbp]		;; Mul values4 by two-to-minus-phi
	vmulsd	xmm1, xmm1, Q YMM_NORM012_FF	;; Mul by FFTLEN/2
	vaddsd	xmm0, xmm0, xmm2		;; x4 = value + carry
	vaddsd	xmm0, xmm0, xmm1		;; Add in the FFT data
	single_rounding base2, xmm0, xmm2, xmm4, rax*2
	vmulsd	xmm0, xmm0, Q [rbp+32]		;; new value4 = val * two-to-phi
	vmovsd	Q [rsi], xmm0			;; Save value4

	add	rsi, YMM_SRC_INCR4		;; Next source pointer
	add	rbp, YMM_NORM_INCR4		;; Next ttp/ttmp pointer
	movzx	rax, BYTE PTR [rdi+4]		;; Load big vs. little flags
	vmovsd	xmm0, Q YMM_TMP5		;; Load integer part of divide by k
	vmulsd	xmm0, xmm0, Q YMM_MINUS_C	;; Mul by -c
	vmovsd	xmm1, Q [rsi]			;; Load FFT data
	vmulsd	xmm1, xmm1, Q [rbp]		;; Mul values4 by two-to-minus-phi
	vmulsd	xmm1, xmm1, Q YMM_NORM012_FF	;; Mul by FFTLEN/2
	vaddsd	xmm0, xmm0, xmm2		;; x5 = value + carry
	vaddsd	xmm0, xmm0, xmm1		;; Add in the FFT data
	single_rounding base2, xmm0, xmm2, xmm4, rax*2
	vmulsd	xmm0, xmm0, Q [rbp+32]		;; new value5 = val * two-to-phi
	vmovsd	Q [rsi], xmm0			;; Save value5

	add	rsi, YMM_SRC_INCR5		;; Next source pointer
	add	rbp, YMM_NORM_INCR5		;; Next ttp/ttmp pointer
	movzx	rax, BYTE PTR [rdi+5]		;; Load big vs. little flags
	vmovsd	xmm0, Q YMM_TMP6		;; Load integer part of divide by k
	vmulsd	xmm0, xmm0, Q YMM_MINUS_C	;; Mul by -c
	vmovsd	xmm1, Q [rsi]			;; Load FFT data
	vmulsd	xmm1, xmm1, Q [rbp]		;; Mul values6 by two-to-minus-phi
	vmulsd	xmm1, xmm1, Q YMM_NORM012_FF	;; Mul by FFTLEN/2
	vaddsd	xmm0, xmm0, xmm2		;; x6 = value + carry
	vaddsd	xmm0, xmm0, xmm1		;; Add in the FFT data
	single_rounding base2, xmm0, xmm2, xmm4, rax*2
	vmulsd	xmm0, xmm0, Q [rbp+32]		;; new value6 = val * two-to-phi
	vmovsd	Q [rsi], xmm0			;; Save value6

	add	rsi, YMM_SRC_INCR6		;; Next source pointer
	add	rbp, YMM_NORM_INCR6		;; Next ttp/ttmp pointer
	movzx	rax, BYTE PTR [rdi+6]		;; Load big vs. little flags
	vmovsd	xmm0, Q [rsi]			;; Load FFT data
	vmulsd	xmm0, xmm0, Q [rbp]		;; Mul values7 by two-to-minus-phi
	vmulsd	xmm0, xmm0, Q YMM_NORM012_FF	;; Mul by FFTLEN/2
	vaddsd	xmm0, xmm0, xmm2		;; x7 = value + carry
	single_rounding base2, xmm0, xmm2, xmm4, rax*2
	vmulsd	xmm0, xmm0, Q [rbp+32]		;; new value7 = val * two-to-phi
	vmovsd	Q [rsi], xmm0			;; Save value7

	add	rsi, YMM_SRC_INCR7		;; Next source pointer
	add	rbp, YMM_NORM_INCR7		;; Next ttp/ttmp pointer
base2	vsubsd	xmm2, xmm2, Q YMM_BIGVAL	;; Remove rounding constant
	vmulsd	xmm2, xmm2, Q [rbp+32]		;; new value8 = val * two-to-phi
	vaddsd	xmm2, xmm2, Q [rsi]		;; Add in FFT data
	vmovsd	Q [rsi], xmm2			;; Save value8
	ENDM



; *************** Top carry adjust macro ******************
; This macro corrects the carry out of the topmost word when k is not 1.
; The problem is the top carry is from b^ceil(logb(k)+n) rather than at k*b^n.
; So we recompute the top carry by multiplying by b^ceil(logb(k)) and then
; dividing by k.  The integer part is the new carry and the remainder is
; added back to the top three words.

; The single-pass case, the top carry is in low word of ymm3
ynorm_top_carry_1d MACRO ttp, base2
	ynorm_top_carry_cmn base2, rsi, xmm3, 0
	ENDM

; The multi-pass case.  The top carry is loaded into xmm7 from the carries array.
ynorm_top_carry_wpn MACRO ttp, base2
	ynorm_top_carry_cmn base2, rsi, xmm7, 1
	ENDM

; The multi-pass add/sub/smallmul case.  The top carry from the carries array is in xmm1.  No scratch area.
ynorm_top_carry_op_wpn MACRO ttp, base2
	ynorm_top_carry_cmn base2, rsi, xmm2, 2		;; We'll copy the incoming carry in xmm1 to xmm2
	ENDM

ynorm_top_carry_cmn MACRO base2, srcreg, xreg, twopass
	LOCAL	kok
	cmp	TOP_CARRY_NEEDS_ADJUSTING, 1 ;; Does top carry need work?
	jne	kok			;; Skip this code if K is 1

	IF twopass EQ 0
	ystore	YMM_TMP6, ymm3		;; Save the full carry register
	ENDIF

	IF twopass EQ 1			;; Two pass case - load the carry
	mov	rdi, carries		;; Addr of the carries
	mov	eax, addcount1		;; Load count of carry rows
	shl	rax, 6			;; Compute addr of the high carries
	add	rdi, rax
	vmovsd	xreg, Q [rdi-8]		;; Load very last carry
	vmovsd	Q YMM_TMP6, xreg	;; Save the carry for later use
	ENDIF

	IF twopass EQ 2			;; Two pass add/sub/smallmul case
	mov	rsi, DESTARG		;; Load destination
	vmovsd	xmm3, xmm0, xmm0	;; Preserve xmm0
	vmovsd	xmm2, xmm1, xmm1	;; Move carry to xmm2
	vmovsd	Q YMM_TMP6, xreg	;; Save the carry for later use
	ENDIF

base2	vsubsd	xreg, xreg, Q YMM_BIGVAL ;; Convert top carry from int+BIGVAL state

	;; We want to calculate carry * b^ceil(logb(k)) / k and
	;; carry * b^ceil(logb(k)) % k.  This must be done very carefully as
	;; carry * b^ceil(logb(k)) may not fit in 53 bits.

	;; Here is a strategy that works for k values up to and including 34 bits.
	;; We do lots of modulo k operations along the way to insure all intermediate
	;; results are 51 bits or less.
	;; Calculate y = carry % k.  This will fit in 34 bits.
	;; Let z = b^ceil(logb(k)) % k.  Precalculate high_17_bits(z) and low_17_bits(z)
	;; Remainder is (high_17_bits(z) * y % k * 2^17 + low_17_bits(z) * y) % k

	vmulsd	xmm0, xreg, INVERSE_K		;; Mul top carry by 1/k
	vroundsd xmm0, xmm0, xmm0, 0		;; Integer part
	vmulsd	xmm0, xmm0, K
	vsubsd	xreg, xreg, xmm0		;; y = carry % k

	vmulsd	xmm0, xreg, CARRY_ADJUST1_HI	;; y * high_17_bits(z)
	vmulsd	xmm1, xmm0, INVERSE_K		;; Mul y * high_17_bits(z) by 1/k
	vroundsd xmm1, xmm1, xmm1, 0		;; Integer part
	vmulsd	xmm1, xmm1, K
	vsubsd	xmm0, xmm0, xmm1		;; y * high_17_bits(z) % k
	vmulsd	xmm0, xmm0, TWO_TO_17		;; y * high_17_bits(z) % k * 2^17
	vmulsd	xreg, xreg, CARRY_ADJUST1_LO	;; y * low_17_bits(z)
	vaddsd	xmm0, xmm0, xreg		;; y * high_17_bits(z) % k * 2^17 + y * low_17_bits(z)

	vmulsd	xmm1, xmm0, INVERSE_K		;; Mul by 1/k
	vroundsd xmm1, xmm1, xmm1, 0		;; Integer part
	vmulsd	xmm1, xmm1, K
	vsubsd	xmm0, xmm0, xmm1		;; Remainder!!!

	;; Finally calculate integer_part = (carry * b^ceil(logb(k)) - remainder) / k

	vmovsd	xreg, Q YMM_TMP6		;; Reload top carry
base2	vsubsd	xreg, xreg, Q YMM_BIGVAL	;; Convert top carry from int+BIGVAL state
	vmulsd	xreg, xreg, CARRY_ADJUST1	;; Mul by b^ceil(logb(k))
	vsubsd	xreg, xreg, xmm0		;; Subtract the remainder
	vmulsd	xreg, xreg, INVERSE_K		;; Mul by 1/k
	vroundsd xreg, xreg, xreg, 0		;; Integer part of top carry over k

	;; Now add the remainder to the top words

	vmulsd	xmm0, xmm0, CARRY_ADJUST2	;; Shift remainder
	vroundsd xmm1, xmm0, xmm0, 0		;; Integer part of shifted remainder
	vsubsd	xmm0, xmm0, xmm1		;; Fractional part of shifted remainder
	vmulsd	xmm1, xmm1, CARRY_ADJUST3	;; Weight integer part

	;; Single pass case
	IF twopass EQ 0
	mov	eax, HIGH_WORD1_OFFSET		;; Add integer part to top word
	vaddsd	xmm1, xmm1, Q [srcreg][rax]
	vmovsd	Q [srcreg][rax], xmm1
	vmulsd	xmm0, xmm0, CARRY_ADJUST4	;; Shift fractional part
	vroundsd xmm1, xmm0, xmm0, 0		;; Integer part of shifted fractional
	vsubsd	xmm0, xmm0, xmm1		;; Fractional part
	vmulsd	xmm1, xmm1, CARRY_ADJUST5	;; Weight integer part
	mov	eax, HIGH_WORD2_OFFSET		;; Add frac part to top-1 word
	vaddsd	xmm1, xmm1, Q [srcreg][rax]
	vmovsd	Q [srcreg][rax], xmm1
	vmulsd	xmm0, xmm0, CARRY_ADJUST6	;; Shift fractional part
	vroundsd xmm0, xmm0, xmm0, 0
	vmulsd	xmm0, xmm0, CARRY_ADJUST7	;; Weight fractional part
	mov	eax, HIGH_WORD3_OFFSET		;; Add frac part to top-2 word
	vaddsd	xmm0, xmm0, Q [srcreg][rax]
	vmovsd	Q [srcreg][rax], xmm0
	ENDIF

	;; Two pass scratch area case
	IF twopass EQ 1
	mov	eax, HIGH_SCRATCH1_OFFSET	;; Add integer part to top word
	vaddsd	xmm1, xmm1, Q [srcreg][rax]
	vmovsd	Q [srcreg][rax], xmm1
	vmulsd	xmm0, xmm0, CARRY_ADJUST4	;; Shift fractional part
	vroundsd xmm0, xmm0, xmm0, 0
	vmulsd	xmm0, xmm0, CARRY_ADJUST5	;; Weight fractional part
	mov	eax, HIGH_SCRATCH2_OFFSET	;; Add frac part to top-1 word
	vaddsd	xmm0, xmm0, Q [srcreg][rax]
	vmovsd	Q [srcreg][rax], xmm0
	ENDIF

	;; Two pass FFT data case
	IF twopass EQ 2
	mov	eax, HIGH_WORD1_OFFSET		;; Add integer part to top word
	vaddsd	xmm1, xmm1, Q [srcreg][rax]
	vmovsd	Q [srcreg][rax], xmm1
	vmulsd	xmm0, xmm0, CARRY_ADJUST4	;; Shift fractional part
	vroundsd xmm0, xmm0, xmm0, 0
	vmulsd	xmm0, xmm0, CARRY_ADJUST5	;; Weight fractional part
	mov	eax, HIGH_WORD2_OFFSET		;; Add frac part to top-1 word
	vaddsd	xmm0, xmm0, Q [srcreg][rax]
	vmovsd	Q [srcreg][rax], xmm0
	ENDIF

base2	vaddsd	xreg, xreg, Q YMM_BIGVAL	;; Restore carry to int+BIGVAL state

	IF twopass EQ 0
	vblendpd ymm3, ymm3, YMM_TMP6, 1110b	;; Blend in the new carry
	ENDIF

	IF twopass EQ 1
	vmovsd	Q [rdi-8], xreg			;; Save very last carry
	ENDIF

	IF twopass EQ 2
	vmovsd	xmm0, xmm3, xmm3 		;; Restore xmm0
	vmovsd	xmm1, xreg, xreg		;; Return carry in xmm1
	ENDIF

kok:
	ENDM


; For 32-bit WPN macros, these registers are set on input:
; ymm6 = maxerr
; rbp = pointer to carries
; rdi = pointer to big/little flags
; rsi = pointer to the FFT data
; rdx = pointer two-to-phi group multipliers
; ebx = big vs. little & fudge flags
; ecx = big vs. little word flag #2
; eax = big vs. little word flag #1
; ymm2,ymm3 = carries

ynorm_wpn_preload MACRO ttp, base2, echk, const
	ENDM

ynorm_wpn MACRO ttp, base2, echk, const
		L1prefetchw rsi+64, L1PREFETCH_ALL
ttp		movzx	ecx, bl				;; Fudge flags 1
ttp		and	rcx, 0e0h
ttp		movzx	eax, bh				;; Big/lit flags 1-2
ttp		and	rbx, 01ch			;; Fudge flags 2
		vmovapd	ymm0, [rsi+0*32]		;; Load values1
ttp		vmulpd	ymm0, ymm0, [rdx+0*YMM_GMD][rcx] ;; Mul by fudged grp two-to-minus-phi
		vmovapd	ymm1, [rsi+1*32]		;; Load values2
ttp		vmulpd	ymm1, ymm1, [rdx+1*YMM_GMD][rbx*8] ;; Mul by fudged grp two-to-minus-phi
const		mul_by_const_interleaved ttp, base2, echk, ymm0, ymm2, ymm4, rax*8, ymm1, ymm3, ymm5, rax*8+32, ymm6
no const echk	error_check_interleaved ymm0, ymm4, ymm1, ymm5, ymm6
no const	vaddpd	ymm0, ymm0, ymm2		;; x1 = values + carry
no const	vaddpd	ymm1, ymm1, ymm3		;; x2 = values + carry
		rounding_interleaved ttp, base2, const, ymm0, ymm2, ymm4, rax*8, ymm1, ymm3, ymm5, rax*8+32
ttp		vmulpd	ymm0, ymm0, [rdx+0*YMM_GMD+YMM_GMD/2][rcx] ;; value1 = rounded value * fudged grp two-to-phi
ttp		vmulpd	ymm1, ymm1, [rdx+1*YMM_GMD+YMM_GMD/2][rbx*8] ;; value2 = rounded value * fudged grp two-to-phi
ttp		movzx	rbx, WORD PTR [rdi+2]		;; Load next 4 big vs. little & fudge flags
		ystore	[rsi+0*32], ymm0		;; Save new value1
		ystore	[rsi+1*32], ymm1		;; Save new value2
	ENDM

; In 64-bit mode we normalize 16 values at a time using the extra available registers

IFDEF X86_64

ynorm_wpn_preload MACRO ttp, base2, echk, const
echk		vmovapd	ymm15, YMM_ABSVAL
base2 const	vmovapd	ymm14, YMM_BIGBIGVAL
IF (@INSTR(,%yarch,<FMA3>) NE 0)
no base2 no const vmovapd ymm14, YMM_BIGVAL
ENDIF
const		vmovapd	ymm13, YMM_MULCONST
	ENDM

; For 64-bit WPN macros, these registers are set on input:
; ymm6 = maxerr
; rbp = pointer to carries
; rdi = pointer to big/little flags
; rsi = pointer to the FFT data
; r12 = pointer two-to-phi group multipliers
; r9 = group multiplier prefetch pointer
; rbx = combined fudge flags, then fudge flags #2
; r8 = fudge flags #1
; rax = big vs. little word flag #1-2
; ymm2,ymm3 = carries
; r13 = source #2
; r14 = biglit ptr #2
; r15 = pointer two-to-phi group multipliers #2
; rcx = combined fudge flags, then fudge flags #4
; r10 = fudge flags #3
; rdx = big vs. little word flag #3-4
; ymm9,ymm10 = carries #2

ynorm_wpn MACRO ttp, base2, echk, const
IFDEF YIMPL_WPN1_FFTS
no const	ynorm_wpn_noconst ttp, base2, echk, r15
const		ynorm_wpn_const ttp, base2, echk, r15
ENDIF
IFDEF YIMPL_WPN4_FFTS
no const	ynorm_wpn_noconst ttp, base2, echk, r12
const		ynorm_wpn_const ttp, base2, echk, r12
ENDIF
	ENDM

ynorm_wpn_noconst MACRO ttp, base2, echk, grpreg2
		L1prefetchw rsi+64, L1PREFETCH_ALL
		L1prefetchw r13+64, L1PREFETCH_ALL

ttp		movzx	r8d, bl				;; Fudge flags 1
ttp		and	r8, 0e0h
ttp		movzx	eax, bh				;; Big/lit flags 1-2
ttp		and	rbx, 01ch			;; Fudge flags 2
		vmovapd	ymm0, [rsi+0*32]		;; Load values1
ttp		vmulpd	ymm0, ymm0, [r12+0*YMM_GMD][r8] ;; Mul by fudged grp two-to-minus-phi
		vmovapd	ymm1, [rsi+1*32]		;; Load values2
ttp		vmulpd	ymm1, ymm1, [r12+1*YMM_GMD][rbx*8] ;; Mul by fudged grp two-to-minus-phi

ttp		movzx	r10d, cl			;; Fudge flags 1
ttp		and	r10, 0e0h
ttp		movzx	edx, ch				;; Big/lit flags 1-2
ttp		and	rcx, 01ch			;; Fudge flags 2
		vmovapd	ymm7, [r13+0*32]		;; Load values1
ttp		vmulpd	ymm7, ymm7, [grpreg2+0*YMM_GMD][r10] ;; Mul by fudged grp two-to-minus-phi
		vmovapd	ymm8, [r13+1*32]		;; Load values2
ttp		vmulpd	ymm8, ymm8, [grpreg2+1*YMM_GMD][rcx*8] ;; Mul by fudged grp two-to-minus-phi

ttp		L1prefetch r9, L1PREFETCH_ALL
ttp		L1prefetch r9+64, L1PREFETCH_ALL
ttp		bump	r9, 128

base2 no echk ttp	ynorm_wpn_base2_noconst	rax*8, rax*8+32, rdx*8, rdx*8+32
base2 no echk no ttp	ynorm_wpn_base2_noconst	0, 0, 0, 0
base2 echk ttp		ynorm_wpn_base2_noconst_echk rax*8, rax*8+32, rdx*8, rdx*8+32
base2 echk no ttp	ynorm_wpn_base2_noconst_echk 0, 0, 0, 0
no base2 no echk ttp	ynorm_wpn_nobase2_noconst rax*8, rax*8+32, rdx*8, rdx*8+32
no base2 no echk no ttp	ynorm_wpn_nobase2_noconst 0, 0, 0, 0
no base2 echk ttp	ynorm_wpn_nobase2_noconst_echk rax*8, rax*8+32, rdx*8, rdx*8+32
no base2 echk no ttp	ynorm_wpn_nobase2_noconst_echk 0, 0, 0, 0

ttp		vmulpd	ymm4, ymm4, [r12+0*YMM_GMD+YMM_GMD/2][r8] ;; value1 = rounded value * fudged grp two-to-phi
ttp		vmulpd	ymm5, ymm5, [r12+1*YMM_GMD+YMM_GMD/2][rbx*8] ;; value2 = rounded value * fudged grp two-to-phi
ttp		movzx	rbx, WORD PTR [rdi+2]		;; Load next 4 big vs. little & fudge flags

ttp		vmulpd	ymm11, ymm11, [grpreg2+0*YMM_GMD+YMM_GMD/2][r10] ;; value1 = rounded value * fudged grp two-to-phi
ttp		vmulpd	ymm12, ymm12, [grpreg2+1*YMM_GMD+YMM_GMD/2][rcx*8] ;; value2 = rounded value * fudged grp two-to-phi
ttp		movzx	rcx, WORD PTR [r14+2]		;; Load next 4 big vs. little & fudge flags

		ystore	[rsi+0*32], ymm4		;; Save new value1
		ystore	[rsi+1*32], ymm5		;; Save new value2
		ystore	[r13+0*32], ymm11		;; Save new value1
		ystore	[r13+1*32], ymm12		;; Save new value2
	ENDM

IF (@INSTR(,%yarch,<FMA3>) NE 0)
ynorm_wpn_noconst MACRO ttp, base2, echk, grpreg2
		L1prefetchw rsi+64, L1PREFETCH_ALL
		L1prefetchw r13+64, L1PREFETCH_ALL

ttp		movzx	r8d, bl				;; Fudge flags 1
ttp		and	r8, 0e0h
ttp		movzx	eax, bh				;; Big/lit flags 1-2
ttp		and	rbx, 01ch			;; Fudge flags 2
no echk			vmovapd	ymm0, [rsi+0*32]		;; Load values1
no echk ttp		yfmaddpd ymm0, ymm0, [r12+0*YMM_GMD][r8], ymm2 ;; Mul by fudged grp two-to-minus-phi + carry
no echk no ttp		vaddpd	ymm0, ymm0, ymm2		;; Add carry
no echk			vmovapd	ymm1, [rsi+1*32]		;; Load values2
no echk ttp		yfmaddpd ymm1, ymm1, [r12+1*YMM_GMD][rbx*8], ymm3 ;; Mul by fudged grp two-to-minus-phi + carry
no echk no ttp		vaddpd	ymm1, ymm1, ymm3		;; Add carry
base2 echk ttp		vmovapd	ymm11, [r12+0*YMM_GMD][r8]	;; Load fudged grp two-to-minus-phi
base2 echk ttp		yfmaddpd ymm0, ymm11, [rsi+0*32], ymm2	;; Mul by values1
base2 echk ttp		vmovapd	ymm12, [r12+1*YMM_GMD][rbx*8]	;; Load fudged grp two-to-minus-phi
base2 echk ttp		yfmaddpd ymm1, ymm12, [rsi+1*32], ymm3	;; Mul by values2
base2 echk no ttp	vmovapd	ymm11, [rsi+0*32]		;; Load values1
base2 echk no ttp	vaddpd	ymm0, ymm11, ymm2		;; Add carry
base2 echk no ttp	vmovapd	ymm12, [rsi+1*32]		;; Load values2
base2 echk no ttp	vaddpd	ymm1, ymm12, ymm3		;; Add carry
no base2 echk		vmovapd	ymm0, [rsi+0*32]		;; Load values1
no base2 echk ttp	yfmaddpd ymm0, ymm0, [r12+0*YMM_GMD][r8], ymm2 ;; Mul by fudged grp two-to-minus-phi + carry
no base2 echk no ttp	vaddpd	ymm0, ymm0, ymm2		;; Add carry
no base2 echk		vmovapd	ymm1, [rsi+1*32]		;; Load values2
no base2 echk ttp	yfmaddpd ymm1, ymm1, [r12+1*YMM_GMD][rbx*8], ymm3 ;; Mul by fudged grp two-to-minus-phi + carry
no base2 echk no ttp	vaddpd	ymm1, ymm1, ymm3		;; Add carry

ttp		movzx	r10d, cl			;; Fudge flags 1
ttp		and	r10, 0e0h
ttp		movzx	edx, ch				;; Big/lit flags 1-2
ttp		and	rcx, 01ch			;; Fudge flags 2
no echk			vmovapd	ymm7, [r13+0*32]		;; Load values1
no echk ttp		yfmaddpd ymm7, ymm7, [grpreg2+0*YMM_GMD][r10], ymm9 ;; Mul by fudged grp two-to-minus-phi + carry
no echk no ttp		vaddpd	ymm7, ymm7, ymm9		;; Add carry
no echk			vmovapd	ymm8, [r13+1*32]		;; Load values2
no echk ttp		yfmaddpd ymm8, ymm8, [grpreg2+1*YMM_GMD][rcx*8], ymm10 ;; Mul by fudged grp two-to-minus-phi + carry
no echk no ttp		vaddpd	ymm8, ymm8, ymm10		;; Add carry
base2 echk ttp		vmovapd	ymm13, [grpreg2+0*YMM_GMD][r10]	;; Load fudged grp two-to-minus-phi
base2 echk ttp		yfmaddpd ymm7, ymm13, [r13+0*32], ymm9	;; Mul by values1
base2 echk ttp		vmovapd	ymm14, [grpreg2+1*YMM_GMD][rcx*8] ;; Load fudged grp two-to-minus-phi
base2 echk ttp		yfmaddpd ymm8, ymm14, [r13+1*32], ymm10 ;; Mul by values2
base2 echk no ttp	vmovapd	ymm13, [r13+0*32]		;; Load values1
base2 echk no ttp	vaddpd	ymm7, ymm13, ymm9		;; Add carry
base2 echk no ttp	vmovapd	ymm14, [r13+1*32]		;; Load values2
base2 echk no ttp	vaddpd	ymm8, ymm14, ymm10		;; Add carry
no base2 echk		vmovapd	ymm7, [r13+0*32]		;; Load values1
no base2 echk ttp	yfmaddpd ymm7, ymm7, [grpreg2+0*YMM_GMD][r10], ymm9 ;; Mul by fudged grp two-to-minus-phi + carry
no base2 echk no ttp	vaddpd	ymm7, ymm7, ymm9		;; Add carry
no base2 echk		vmovapd	ymm8, [r13+1*32]		;; Load values2
no base2 echk ttp	yfmaddpd ymm8, ymm8, [grpreg2+1*YMM_GMD][rcx*8], ymm10 ;; Mul by fudged grp two-to-minus-phi + carry
no base2 echk no ttp	vaddpd	ymm8, ymm8, ymm10		;; Add carry

ttp		L1prefetch r9, L1PREFETCH_ALL
ttp		L1prefetch r9+64, L1PREFETCH_ALL
ttp		bump	r9, 128

base2 no echk ttp	ynorm_wpn_base2_noconst rax*8, rax*8+32, rdx*8, rdx*8+32
base2 no echk no ttp	ynorm_wpn_base2_noconst 0, 0, 0, 0
base2 echk ttp		ynorm_wpn_base2_noconst_echk rax*8, rax*8+32, rdx*8, rdx*8+32
base2 echk no ttp	ynorm_wpn_base2_noconst_echk_nottp 0, 0, 0, 0
no base2 no echk ttp	ynorm_wpn_nobase2_noconst rax*8, rax*8+32, rdx*8, rdx*8+32
no base2 no echk no ttp	ynorm_wpn_nobase2_noconst 0, 0, 0, 0
no base2 echk ttp	ynorm_wpn_nobase2_noconst_echk rax*8, rax*8+32, rdx*8, rdx*8+32
no base2 echk no ttp	ynorm_wpn_nobase2_noconst_echk 0, 0, 0, 0

ttp		vmulpd	ymm4, ymm4, [r12+0*YMM_GMD+YMM_GMD/2][r8] ;; value1 = rounded value * fudged grp two-to-phi
ttp		vmulpd	ymm5, ymm5, [r12+1*YMM_GMD+YMM_GMD/2][rbx*8] ;; value2 = rounded value * fudged grp two-to-phi
ttp		movzx	rbx, WORD PTR [rdi+2]		;; Load next 4 big vs. little & fudge flags

ttp		vmulpd	ymm11, ymm11, [grpreg2+0*YMM_GMD+YMM_GMD/2][r10] ;; value1 = rounded value * fudged grp two-to-phi
ttp		vmulpd	ymm12, ymm12, [grpreg2+1*YMM_GMD+YMM_GMD/2][rcx*8] ;; value2 = rounded value * fudged grp two-to-phi
ttp		movzx	rcx, WORD PTR [r14+2]		;; Load next 4 big vs. little & fudge flags

		ystore	[rsi+0*32], ymm4		;; Save new value1
		ystore	[rsi+1*32], ymm5		;; Save new value2
		ystore	[r13+0*32], ymm11		;; Save new value1
		ystore	[r13+1*32], ymm12		;; Save new value2
	ENDM
ENDIF

ynorm_wpn_const MACRO ttp, base2, echk, grpreg2
		L1prefetchw rsi+64, L1PREFETCH_ALL
		L1prefetchw r13+64, L1PREFETCH_ALL

ttp		movzx	r8d, bl				;; Fudge flags 1
ttp		and	r8, 0e0h
ttp		movzx	eax, bh				;; Big/lit flags 1-2
ttp		and	rbx, 01ch			;; Fudge flags 2
		vmovapd	ymm7, [rsi+0*32]		;; Load values1
ttp		vmulpd	ymm7, ymm7, [r12+0*YMM_GMD][r8] ;; Mul by fudged grp two-to-minus-phi
		vmovapd	ymm8, [rsi+1*32]		;; Load values2
ttp		vmulpd	ymm8, ymm8, [r12+1*YMM_GMD][rbx*8] ;; Mul by fudged grp two-to-minus-phi

ttp		L1prefetch r9, L1PREFETCH_ALL
ttp		L1prefetch r9+64, L1PREFETCH_ALL
ttp		bump	r9, 128

base2 ttp	ynorm_wpn_base2_const	echk, rax*8, rax*8+32, ymm2, ymm3
base2 no ttp	ynorm_wpn_base2_const	echk, 0, 0, ymm2, ymm3
no base2 ttp	ynorm_wpn_nobase2_const	echk, rax*8, rax*8+32, ymm2, ymm3
no base2 no ttp	ynorm_wpn_nobase2_const	echk, 0, 0, ymm2, ymm3

ttp		movzx	r10d, cl			;; Fudge flags 1
ttp		and	r10, 0e0h
ttp		movzx	edx, ch				;; Big/lit flags 1-2
ttp		and	rcx, 01ch			;; Fudge flags 2
		vmovapd	ymm7, [r13+0*32]		;; Load values1
ttp		vmulpd	ymm7, ymm7, [grpreg2+0*YMM_GMD][r10] ;; Mul by fudged grp two-to-minus-phi
		vmovapd	ymm8, [r13+1*32]		;; Load values2
ttp		vmulpd	ymm8, ymm8, [grpreg2+1*YMM_GMD][rcx*8] ;; Mul by fudged grp two-to-minus-phi

ttp		vmulpd	ymm4, ymm4, [r12+0*YMM_GMD+YMM_GMD/2][r8] ;; value1 = rounded value * fudged grp two-to-phi
ttp		vmulpd	ymm5, ymm5, [r12+1*YMM_GMD+YMM_GMD/2][rbx*8] ;; value2 = rounded value * fudged grp two-to-phi
ttp		movzx	rbx, WORD PTR [rdi+2]		;; Load next 4 big vs. little & fudge flags
		ystore	[rsi+0*32], ymm4		;; Save new value1
		ystore	[rsi+1*32], ymm5		;; Save new value2

base2 ttp	ynorm_wpn_base2_const	echk, rdx*8, rdx*8+32, ymm9, ymm10
base2 no ttp	ynorm_wpn_base2_const	echk, 0, 0, ymm9, ymm10
no base2 ttp	ynorm_wpn_nobase2_const	echk, rdx*8, rdx*8+32, ymm9, ymm10
no base2 no ttp	ynorm_wpn_nobase2_const	echk, 0, 0, ymm9, ymm10

ttp		vmulpd	ymm4, ymm4, [grpreg2+0*YMM_GMD+YMM_GMD/2][r10] ;; value1 = rounded value * fudged grp two-to-phi
ttp		vmulpd	ymm5, ymm5, [grpreg2+1*YMM_GMD+YMM_GMD/2][rcx*8] ;; value2 = rounded value * fudged grp two-to-phi
ttp		movzx	rcx, WORD PTR [r14+2]		;; Load next 4 big vs. little & fudge flags
		ystore	[r13+0*32], ymm4		;; Save new value1
		ystore	[r13+1*32], ymm5		;; Save new value2
	ENDM


ynorm_wpn_base2_noconst MACRO basereg, basereg2, basereg3, basereg4
	vaddpd	ymm0, ymm0, ymm2			;; x = value1 + carry				; 6-8	First dependence on previous iteration
	vaddpd	ymm1, ymm1, ymm3			;; x = value2 + carry				; 7-9
	vaddpd	ymm7, ymm7, ymm9			;; x = value1 + carry				; 8-10
	vaddpd	ymm8, ymm8, ymm10			;; x = value2 + carry				; 9-11
	vmovapd	ymm4, YMM_LIMIT_BIGMAX[basereg]		;; Load maximum * BIGVAL - BIGVAL
	vaddpd	ymm2, ymm0, ymm4			;; y = top bits of val				; 10-12
	vmovapd	ymm5, YMM_LIMIT_BIGMAX[basereg2]	;; Load maximum * BIGVAL - BIGVAL
	vaddpd	ymm3, ymm1, ymm5			;; y = top bits of val				; 11-13
	vmovapd	ymm11, YMM_LIMIT_BIGMAX[basereg3]	;; Load maximum * BIGVAL - BIGVAL
	vaddpd	ymm9, ymm7, ymm11			;; y = top bits of val				; 12-14
	vmovapd	ymm12, YMM_LIMIT_BIGMAX[basereg4]	;; Load maximum * BIGVAL - BIGVAL
	vaddpd	ymm10, ymm8, ymm12			;; y = top bits of val				; 13-15
	vsubpd	ymm4, ymm2, ymm4			;; z = y - (maximum * BIGVAL - BIGVAL)		; 14-16
	vmulpd	ymm2, ymm2, YMM_LIMIT_INVERSE[basereg]	;; carry = shifted y				;  13-17
	vsubpd	ymm5, ymm3, ymm5			;; z = y - (maximum * BIGVAL - BIGVAL)		; 15-17
	vmulpd	ymm3, ymm3, YMM_LIMIT_INVERSE[basereg2]	;; carry = shifted y				;  14-18
	vsubpd	ymm11, ymm9, ymm11			;; z = y - (maximum * BIGVAL - BIGVAL)		; 16-18
	vmulpd	ymm9, ymm9, YMM_LIMIT_INVERSE[basereg3]	;; carry = shifted y				;  15-19
	vsubpd	ymm12, ymm10, ymm12			;; z = y - (maximum * BIGVAL - BIGVAL)		; 17-19
	vmulpd	ymm10, ymm10, YMM_LIMIT_INVERSE[basereg4] ;; carry = shifted y				;  16-20
	vsubpd	ymm4, ymm0, ymm4			;; rounded val = x - z				; 18-20
	vsubpd	ymm5, ymm1, ymm5			;; rounded val = x - z				; 19-21
	vsubpd	ymm11, ymm7, ymm11			;; rounded val = x - z				; 20-22
	vsubpd	ymm12, ymm8, ymm12			;; rounded val = x - z				; 21-23
ENDM

IF (@INSTR(,%yarch,<FMA3>) NE 0)
ynorm_wpn_base2_noconst MACRO basereg, basereg2, basereg3, basereg4
	vmovapd	ymm4, YMM_LIMIT_BIGMAX[basereg]		;; Load maximum * BIGVAL - BIGVAL
	vaddpd	ymm2, ymm0, ymm4			;; y = top bits of val				; 6-8
	vmovapd	ymm5, YMM_LIMIT_BIGMAX[basereg2]	;; Load maximum * BIGVAL - BIGVAL
	vaddpd	ymm3, ymm1, ymm5			;; y = top bits of val				; 7-9
	vmovapd	ymm11, YMM_LIMIT_BIGMAX[basereg3]	;; Load maximum * BIGVAL - BIGVAL
	vaddpd	ymm9, ymm7, ymm11			;; y = top bits of val				; 8-10
	vmovapd	ymm12, YMM_LIMIT_BIGMAX[basereg4]	;; Load maximum * BIGVAL - BIGVAL
	vaddpd	ymm10, ymm8, ymm12			;; y = top bits of val				; 9-11
	vsubpd	ymm4, ymm2, ymm4			;; z = y - (maximum * BIGVAL - BIGVAL)		; 10-12
	vmulpd	ymm2, ymm2, YMM_LIMIT_INVERSE[basereg]	;; carry = shifted y				;  9-13
	vsubpd	ymm5, ymm3, ymm5			;; z = y - (maximum * BIGVAL - BIGVAL)		; 11-13
	vmulpd	ymm3, ymm3, YMM_LIMIT_INVERSE[basereg2]	;; carry = shifted y				;  10-14
	vsubpd	ymm11, ymm9, ymm11			;; z = y - (maximum * BIGVAL - BIGVAL)		; 12-14
	vmulpd	ymm9, ymm9, YMM_LIMIT_INVERSE[basereg3]	;; carry = shifted y				;  11-15
	vsubpd	ymm12, ymm10, ymm12			;; z = y - (maximum * BIGVAL - BIGVAL)		; 13-15
	vmulpd	ymm10, ymm10, YMM_LIMIT_INVERSE[basereg4] ;; carry = shifted y				;  12-16
	vsubpd	ymm4, ymm0, ymm4			;; rounded val = x - z				; 14-16
	vsubpd	ymm5, ymm1, ymm5			;; rounded val = x - z				; 15-17
	vsubpd	ymm11, ymm7, ymm11			;; rounded val = x - z				; 16-18
	vsubpd	ymm12, ymm8, ymm12			;; rounded val = x - z				; 17-19
ENDM
ENDIF

ynorm_wpn_base2_noconst_echk MACRO basereg, basereg2, basereg3, basereg4
	vroundpd ymm11, ymm0, 0				;; Convert to an integer			; 6-8	(and 9-11 on Haswell)
	vroundpd ymm12, ymm1, 0				;; Convert to an integer			; 7-9	(and 10-12 on Haswell)
	vroundpd ymm13, ymm7, 0				;; Convert to an integer			; 8-10	(and 11-13 on Haswell)
	vroundpd ymm14, ymm8, 0				;; Convert to an integer			; 9-11	(12-14,15-17 on Haswell)
	vsubpd	ymm11, ymm11, ymm0			;; This is the convolution error		; 10-12	(13-15)
	vsubpd	ymm12, ymm12, ymm1			;; This is the convolution error		; 11-13	(14-16)
	vsubpd	ymm13, ymm13, ymm7			;; This is the convolution error		; 12-14	(16-18)
	vsubpd	ymm14, ymm14, ymm8			;; This is the convolution error		; 13-15	(18-20)
	vaddpd	ymm0, ymm0, ymm2			;; x = value1 + carry				; 14-16	First dependence on previous iteration
	vaddpd	ymm1, ymm1, ymm3			;; x = value2 + carry				; 15-17
	vaddpd	ymm7, ymm7, ymm9			;; x = value1 + carry				; 16-18
	vaddpd	ymm8, ymm8, ymm10			;; x = value2 + carry				; 17-19
	vmovapd	ymm4, YMM_LIMIT_BIGMAX[basereg]		;; Load maximum * BIGVAL - BIGVAL
	vaddpd	ymm2, ymm0, ymm4			;; y = top bits of val				; 18-20
	vmovapd	ymm5, YMM_LIMIT_BIGMAX[basereg2]	;; Load maximum * BIGVAL - BIGVAL
	vaddpd	ymm3, ymm1, ymm5			;; y = top bits of val				; 19-21
	absval	ymm11					;; Compute absolute value			;	13
	vmaxpd	ymm6, ymm6, ymm11			;; Compute maximum error			; 20-22
	vmovapd	ymm11, YMM_LIMIT_BIGMAX[basereg3]	;; Load maximum * BIGVAL - BIGVAL
	vaddpd	ymm9, ymm7, ymm11			;; y = top bits of val				; 21-23
	absval	ymm12					;; Compute absolute value			;	14
	vmaxpd	ymm6, ymm6, ymm12			;; Compute maximum error			; 23-25
	vmovapd	ymm12, YMM_LIMIT_BIGMAX[basereg4]	;; Load maximum * BIGVAL - BIGVAL
	vaddpd	ymm10, ymm8, ymm12			;; y = top bits of val				; 22-24
	vsubpd	ymm4, ymm2, ymm4			;; z = y - (maximum * BIGVAL - BIGVAL)		; 24-26
	vmulpd	ymm2, ymm2, YMM_LIMIT_INVERSE[basereg]	;; carry = shifted y				;  21-25
	vsubpd	ymm5, ymm3, ymm5			;; z = y - (maximum * BIGVAL - BIGVAL)		; 25-27
	vmulpd	ymm3, ymm3, YMM_LIMIT_INVERSE[basereg2]	;; carry = shifted y				;  22-26
	vsubpd	ymm11, ymm9, ymm11			;; z = y - (maximum * BIGVAL - BIGVAL)		; 26-28
	vmulpd	ymm9, ymm9, YMM_LIMIT_INVERSE[basereg3]	;; carry = shifted y				;  24-28
	vsubpd	ymm12, ymm10, ymm12			;; z = y - (maximum * BIGVAL - BIGVAL)		; 27-29
	vmulpd	ymm10, ymm10, YMM_LIMIT_INVERSE[basereg4] ;; carry = shifted y				;  25-29
	vsubpd	ymm4, ymm0, ymm4			;; rounded val = x - z				; 28-30
	absval	ymm13					;; Compute absolute value			;	15
	vmaxpd	ymm6, ymm6, ymm13			;; Compute maximum error			; 29-31
	vsubpd	ymm5, ymm1, ymm5			;; rounded val = x - z				; 30-32
	vsubpd	ymm11, ymm7, ymm11			;; rounded val = x - z				; 31-33
	vsubpd	ymm12, ymm8, ymm12			;; rounded val = x - z				; 32-34
	absval	ymm13					;; Compute absolute value			;	16
	vmaxpd	ymm6, ymm6, ymm13			;; Compute maximum error			; 33-35
ENDM

;; Avoid using the vroundpd instruction (2 uops, 6 clocks on Haswell) when error checking.
;; This is done by computing "convolution error = val - carry - fftval * ttp"

IF (@INSTR(,%yarch,<FMA3>) NE 0)
ynorm_wpn_base2_noconst_echk MACRO basereg, basereg2, basereg3, basereg4
	vsubpd	ymm2, ymm0, ymm2			;; Start error calc (val - carry)		; 6-8
	vsubpd	ymm3, ymm1, ymm3			;; Start error calc (val - carry)		; 7-9
	vsubpd	ymm9, ymm7, ymm9			;; Start error calc (val - carry)		; 8-10
	vsubpd	ymm10, ymm8, ymm10			;; Start error calc (val - carry)		; 9-11
	yfmsubpd ymm11, ymm11, [rsi+0*32], ymm2		;; Convolution error = (val-carry) - fftval*ttp	; 9-13
	vmovapd	ymm4, YMM_LIMIT_BIGMAX[basereg]		;; Load maximum * BIGVAL - BIGVAL
	vaddpd	ymm2, ymm0, ymm4			;; y = top bits of val				; 10-12
	yfmsubpd ymm12, ymm12, [rsi+1*32], ymm3		;; Convolution error = (val-carry) - fftval*ttp	; 10-14
	vmovapd	ymm5, YMM_LIMIT_BIGMAX[basereg2]	;; Load maximum * BIGVAL - BIGVAL
	vaddpd	ymm3, ymm1, ymm5			;; y = top bits of val				; 11-13
	yfmsubpd ymm13, ymm13, [r13+0*32], ymm9		;; Convolution error = (val-carry) - fftval*ttp	; 11-15
	yfmsubpd ymm14, ymm14, [r13+1*32], ymm10	;; Convolution error = (val-carry) - fftval*ttp	; 12-16
	absval	ymm11					;; Compute absolute value			; 14
	vmaxpd	ymm6, ymm6, ymm11			;; Compute maximum error			; 15-17
	vmovapd	ymm11, YMM_LIMIT_BIGMAX[basereg3]	;; Load maximum * BIGVAL - BIGVAL
	vaddpd	ymm9, ymm7, ymm11			;; y = top bits of val				; 12-14
	absval	ymm12					;; Compute absolute value			; 15
	vmaxpd	ymm6, ymm6, ymm12			;; Compute maximum error			; 18-20
	vmovapd	ymm12, YMM_LIMIT_BIGMAX[basereg4]	;; Load maximum * BIGVAL - BIGVAL
	vaddpd	ymm10, ymm8, ymm12			;; y = top bits of val				; 13-15
	vsubpd	ymm4, ymm2, ymm4			;; z = y - (maximum * BIGVAL - BIGVAL)		; 14-16
	vmulpd	ymm2, ymm2, YMM_LIMIT_INVERSE[basereg]	;; carry = shifted y				;  13-17
	vsubpd	ymm5, ymm3, ymm5			;; z = y - (maximum * BIGVAL - BIGVAL)		; 15-17
	vmulpd	ymm3, ymm3, YMM_LIMIT_INVERSE[basereg2]	;; carry = shifted y				;  14-18
	vsubpd	ymm11, ymm9, ymm11			;; z = y - (maximum * BIGVAL - BIGVAL)		; 16-18
	vmulpd	ymm9, ymm9, YMM_LIMIT_INVERSE[basereg3]	;; carry = shifted y				;  15-19
	vsubpd	ymm12, ymm10, ymm12			;; z = y - (maximum * BIGVAL - BIGVAL)		; 17-19
	vmulpd	ymm10, ymm10, YMM_LIMIT_INVERSE[basereg4] ;; carry = shifted y				;  16-20
	vsubpd	ymm4, ymm0, ymm4			;; rounded val = x - z				; 19-21
	vsubpd	ymm5, ymm1, ymm5			;; rounded val = x - z				; 20-22
	absval	ymm13					;; Compute absolute value			; 16
	vmaxpd	ymm6, ymm6, ymm13			;; Compute maximum error			; 21-23
	vsubpd	ymm11, ymm7, ymm11			;; rounded val = x - z				; 22-24
	vsubpd	ymm12, ymm8, ymm12			;; rounded val = x - z				; 23-25
	absval	ymm14					;; Compute absolute value			; 17
	vmaxpd	ymm6, ymm6, ymm14			;; Compute maximum error			; 24-26
ENDM
ENDIF

IF (@INSTR(,%yarch,<FMA3>) NE 0)
ynorm_wpn_base2_noconst_echk_nottp MACRO basereg, basereg2, basereg3, basereg4
	vsubpd	ymm2, ymm0, ymm2			;; Start error calc (val - carry)		; 6-8
	vsubpd	ymm3, ymm1, ymm3			;; Start error calc (val - carry)		; 7-9
	vsubpd	ymm9, ymm7, ymm9			;; Start error calc (val - carry)		; 8-10
	vsubpd	ymm10, ymm8, ymm10			;; Start error calc (val - carry)		; 9-11
	vsubpd	ymm11, ymm2, ymm11			;; Convolution error = (val-carry) - fftval	; 10-12
	vmovapd	ymm4, YMM_LIMIT_BIGMAX[basereg]		;; Load maximum * BIGVAL - BIGVAL
	vaddpd	ymm2, ymm0, ymm4			;; y = top bits of val				; 11-13
	vsubpd	ymm12, ymm3, ymm12			;; Convolution error = (val-carry) - fftval	; 12-14
	vmovapd	ymm5, YMM_LIMIT_BIGMAX[basereg2]	;; Load maximum * BIGVAL - BIGVAL
	vaddpd	ymm3, ymm1, ymm5			;; y = top bits of val				; 13-15
	vsubpd	ymm13, ymm9, ymm13			;; Convolution error = (val-carry) - fftval	; 14-16
	absval	ymm11					;; Compute absolute value			; 13
	vmaxpd	ymm6, ymm6, ymm11			;; Compute maximum error			; 15-17
	vmovapd	ymm11, YMM_LIMIT_BIGMAX[basereg3]	;; Load maximum * BIGVAL - BIGVAL
	vaddpd	ymm9, ymm7, ymm11			;; y = top bits of val				; 16-18
	vsubpd	ymm14, ymm10, ymm14			;; Convolution error = (val-carry) - fftval	; 17-19
	absval	ymm12					;; Compute absolute value			; 15
	vmaxpd	ymm6, ymm6, ymm12			;; Compute maximum error			; 18-20
	vmovapd	ymm12, YMM_LIMIT_BIGMAX[basereg4]	;; Load maximum * BIGVAL - BIGVAL
	vaddpd	ymm10, ymm8, ymm12			;; y = top bits of val				; 19-21
	vsubpd	ymm4, ymm2, ymm4			;; z = y - (maximum * BIGVAL - BIGVAL)		; 20-22
	vmulpd	ymm2, ymm2, YMM_LIMIT_INVERSE[basereg]	;; carry = shifted y				;  14-18
	vsubpd	ymm5, ymm3, ymm5			;; z = y - (maximum * BIGVAL - BIGVAL)		; 21-23
	vmulpd	ymm3, ymm3, YMM_LIMIT_INVERSE[basereg2]	;; carry = shifted y				;  16-20
	vsubpd	ymm11, ymm9, ymm11			;; z = y - (maximum * BIGVAL - BIGVAL)		; 22-24
	vmulpd	ymm9, ymm9, YMM_LIMIT_INVERSE[basereg3]	;; carry = shifted y				;  19-23
	vsubpd	ymm12, ymm10, ymm12			;; z = y - (maximum * BIGVAL - BIGVAL)		; 23-25
	vmulpd	ymm10, ymm10, YMM_LIMIT_INVERSE[basereg4] ;; carry = shifted y				;  22-26
	vsubpd	ymm4, ymm0, ymm4			;; rounded val = x - z				; 24-26
	vsubpd	ymm5, ymm1, ymm5			;; rounded val = x - z				; 25-27
	absval	ymm13					;; Compute absolute value			; 17
	vmaxpd	ymm6, ymm6, ymm13			;; Compute maximum error			; 26-28
	vsubpd	ymm11, ymm7, ymm11			;; rounded val = x - z				; 27-29
	vsubpd	ymm12, ymm8, ymm12			;; rounded val = x - z				; 28-30
	absval	ymm14					;; Compute absolute value			; 20
	vmaxpd	ymm6, ymm6, ymm14			;; Compute maximum error			; 29-31
ENDM
ENDIF

ynorm_wpn_base2_const MACRO echk, basereg, basereg2, carryreg, carryreg2
	vaddpd	ymm0, ymm7, ymm14			;; Round to nearest multiple of 2^25		; 6-8
	vaddpd	ymm1, ymm8, ymm14			;; Round to nearest multiple of 2^25		; 7-9
	vroundpd ymm11, ymm7, 0				;; Round FFT value to an integer		; 8-10
	vsubpd	ymm0, ymm0, ymm14			;; Compute high bits of FFT value		; 9-11
	vsubpd	ymm1, ymm1, ymm14			;; Compute high bits of FFT value		; 10-12
	vroundpd ymm12, ymm8, 0				;; Round FFT value to an integer		; 11-13
	vsubpd	ymm4, ymm11, ymm0			;; Compute low 25 bits of FFT value		; 12-14
echk	vsubpd	ymm7, ymm11, ymm7			;; This is the convolution error		;  13-15
	vsubpd	ymm5, ymm12, ymm1			;; Compute low 25 bits of FFT value		; 14-16
	vmulpd	ymm4, ymm4, ymm13			;; Mul low bits of FFT value by the constant	; 15-19
echk	vsubpd	ymm8, ymm12, ymm8			;; This is the convolution error		;  15-17
echk	absval	ymm7					;; Compute absolute value			;  16
	vmulpd	ymm5, ymm5, ymm13			;; Mul low bits of FFT value by the constant	; 17-21
echk	vmaxpd	ymm6, ymm6, ymm7			;; Compute maximum error			;  17-19
echk	absval	ymm8					;; Compute absolute value			;  18
	vmulpd	ymm0, ymm0, ymm13			;; next carry = high bits of value times const	; 12-16 (result not needed until clock 27)
echk	vmaxpd	ymm6, ymm6, ymm8			;; Compute maximum error			;  19-21
	vmulpd	ymm1, ymm1, ymm13			;; next carry = high bits of value times const	; 13-17 (result not needed until clock 28)
	vaddpd	ymm4, ymm4, carryreg			;; Add in the previous carry			; 20-22
	vaddpd	ymm5, ymm5, carryreg2			;; Add in the previous carry			; 22-24
	vmovapd	ymm7, YMM_LIMIT_BIGMAX[basereg]		;; Load maximum * BIGVAL - BIGVAL
	vaddpd	carryreg, ymm4, ymm7			;; y = top bits of val				; 23-25
	vmovapd	ymm8, YMM_LIMIT_BIGMAX[basereg2]	;; Load maximum * BIGVAL - BIGVAL
	vaddpd	carryreg2, ymm5, ymm8			;; y = top bits of val				; 25-27
	vsubpd	ymm7, carryreg, ymm7			;; z = y - (maximum * BIGVAL - BIGVAL)		; 26-28
	vaddpd	ymm0, ymm0, carryreg			;; next carry += y				; 27-29
	vsubpd	ymm8, carryreg2, ymm8			;; z = y - (maximum * BIGVAL - BIGVAL)		; 28-30
	vaddpd	ymm1, ymm1, carryreg2			;; next carry += y				; 29-31
	vsubpd	ymm4, ymm4, ymm7			;; rounded val = x - z				; 30-32
	vmulpd	carryreg, ymm0, YMM_LIMIT_INVERSE[basereg] ;; shifted carry				; 30-34
	vsubpd	ymm5, ymm5, ymm8			;; rounded val = x - z				; 31-33
	vmulpd	carryreg2, ymm1, YMM_LIMIT_INVERSE[basereg2] ;; shifted carry				; 32-36
ENDM

IF (@INSTR(,%yarch,<FMA3>) NE 0)
ynorm_wpn_base2_const MACRO echk, basereg, basereg2, carryreg, carryreg2
	vaddpd	ymm0, ymm7, ymm14			;; Round to nearest multiple of 2^25		; 6-8
	vaddpd	ymm1, ymm8, ymm14			;; Round to nearest multiple of 2^25		; 7-9
	vroundpd ymm11, ymm7, 0				;; Round FFT value to an integer		; 8-10
	vsubpd	ymm0, ymm0, ymm14			;; Compute high bits of FFT value		; 9-11
	vsubpd	ymm1, ymm1, ymm14			;; Compute high bits of FFT value		; 10-12
	vroundpd ymm12, ymm8, 0				;; Round FFT value to an integer		; 11-13
	vsubpd	ymm4, ymm11, ymm0			;; Compute low 25 bits of FFT value		; 12-14
echk	vsubpd	ymm11, ymm11, ymm7			;; This is the convolution error		;  13-15
	vsubpd	ymm5, ymm12, ymm1			;; Compute low 25 bits of FFT value		; 14-16
	yfmaddpd ymm4, ymm4, ymm13, carryreg		;; Low bits of value * constant + carry		; 15-19
echk	vsubpd	ymm12, ymm12, ymm8			;; This is the convolution error		;  15-17
echk	absval	ymm11					;; Compute absolute value			;  16
	yfmaddpd ymm5, ymm5, ymm13, carryreg2		;; Low bits of value * constant + carry		; 17-21
echk	vmaxpd	ymm6, ymm6, ymm11			;; Compute maximum error			;  17-19
echk	absval	ymm12					;; Compute absolute value			;  18
	vmulpd	ymm0, ymm0, ymm13			;; next carry = high bits of value times const	; 12-16
echk	vmaxpd	ymm6, ymm6, ymm12			;; Compute maximum error			;  19-21
	vmulpd	ymm1, ymm1, ymm13			;; next carry = high bits of value times const	; 13-17
	vmovapd	ymm7, YMM_LIMIT_BIGMAX[basereg]		;; Load maximum * BIGVAL - BIGVAL
	vaddpd	carryreg, ymm4, ymm7			;; y = top bits of val				; 20-22
	vmovapd	ymm8, YMM_LIMIT_BIGMAX[basereg2]	;; Load maximum * BIGVAL - BIGVAL
	vaddpd	carryreg2, ymm5, ymm8			;; y = top bits of val				; 22-24
	vsubpd	ymm7, carryreg, ymm7			;; z = y - (maximum * BIGVAL - BIGVAL)		; 23-25
	;;;yfmaddpd ymm0, ymm0, ymm13, carryreg		;; next carry = high bits of value * const + y	; 23-27		this fma would increase latency but ease add scheduling
	vaddpd	ymm0, ymm0, carryreg			;; next carry += y				; 24-26
	vsubpd	ymm8, carryreg2, ymm8			;; z = y - (maximum * BIGVAL - BIGVAL)		; 25-27
	;;;yfmaddpd ymm1, ymm1, ymm13, carryreg2	;; next carry = high bits of value * const + y	; 25-29		this fma would increase latency but ease add scheduling
	vaddpd	ymm1, ymm1, carryreg2			;; next carry += y				; 26-28
	vsubpd	ymm4, ymm4, ymm7			;; rounded val = x - z				; 27-29		could be done 26-28, but
	vmulpd	carryreg, ymm0, YMM_LIMIT_INVERSE[basereg] ;; shifted carry				; 27-31
	vsubpd	ymm5, ymm5, ymm8			;; rounded val = x - z				; 28-30
	vmulpd	carryreg2, ymm1, YMM_LIMIT_INVERSE[basereg2] ;; shifted carry				; 29-33		but this would be 30-34
ENDM
ENDIF


ynorm_wpn_nobase2_noconst MACRO basereg, basereg2, basereg3, basereg4
	vaddpd	ymm4, ymm0, ymm2			;; x = value + carry				; 6-8	First dependence on previous iteration
	vaddpd	ymm5, ymm1, ymm3			;; x = value + carry				; 7-9
	vaddpd	ymm11, ymm7, ymm9			;; x = value + carry				; 8-10
	vaddpd	ymm12, ymm8, ymm10			;; x = value + carry				; 9-11
	vmulpd	ymm2, ymm4, YMM_LIMIT_INVERSE[basereg]	;; val / base					; 9-13
	vroundpd ymm0, ymm4, 0				;; Round val					; 10-12
	vmulpd	ymm3, ymm5, YMM_LIMIT_INVERSE[basereg2]	;; val / base					; 10-14
	vroundpd ymm1, ymm5, 0				;; Round val					; 11-13
	vmulpd	ymm9, ymm11, YMM_LIMIT_INVERSE[basereg3] ;; val / base					; 11-15
	vroundpd ymm7, ymm11, 0				;; Round val					; 12-14
	vmulpd	ymm10, ymm12, YMM_LIMIT_INVERSE[basereg4] ;; val / base					; 12-16
	vroundpd ymm8, ymm12, 0				;; Round val					; 13-15
	vroundpd ymm2, ymm2, 0				;; next carry = round (val / base)		; 14-16
	vroundpd ymm3, ymm3, 0				;; next carry = round (val / base)		; 15-17
	vroundpd ymm9, ymm9, 0				;; next carry = round (val / base)		; 16-18
	vroundpd ymm10, ymm10, 0			;; next carry = round (val / base)		; 17-19
	vmulpd	ymm4, ymm2, YMM_LIMIT_BIGMAX[basereg]	;; z = round (val / base) * base		; 17-21
	vmulpd	ymm5, ymm3, YMM_LIMIT_BIGMAX[basereg2]	;; z = round (val / base) * base		; 18-22
	vmulpd	ymm11, ymm9, YMM_LIMIT_BIGMAX[basereg3]	;; z = round (val / base) * base		; 19-23
	vmulpd	ymm12, ymm10, YMM_LIMIT_BIGMAX[basereg4];; z = round (val / base) * base		; 20-24
	vsubpd	ymm4, ymm0, ymm4			;; new value = val - z				; 22-24
	vsubpd	ymm5, ymm1, ymm5			;; new value = val - z				; 23-25
	vsubpd	ymm11, ymm7, ymm11			;; new value = val - z				; 24-26
	vsubpd	ymm12, ymm8, ymm12			;; new value = val - z				; 25-27
ENDM

IF (@INSTR(,%yarch,<FMA3>) NE 0)
ynorm_wpn_nobase2_noconst MACRO basereg, basereg2, basereg3, basereg4
	vroundpd ymm4, ymm0, 0				;; Round val					; 6-8 and 9-11
	yfmaddpd ymm0, ymm0, YMM_LIMIT_INVERSE[basereg], ymm14 ;; val / base + rounding_const		; 6-10
	vroundpd ymm5, ymm1, 0				;; Round val					; 7-9 and 10-12
	yfmaddpd ymm1, ymm1, YMM_LIMIT_INVERSE[basereg2], ymm14	;; val / base + rounding_const		; 7-11
	vroundpd ymm11, ymm7, 0				;; Round val					; 8-10 and 11-13
	yfmaddpd ymm7, ymm7, YMM_LIMIT_INVERSE[basereg3], ymm14 ;; val / base + rounding_const		; 8-12
	vroundpd ymm12, ymm8, 0				;; Round val					; 12-14 and 15-17
	yfmaddpd ymm8, ymm8, YMM_LIMIT_INVERSE[basereg4], ymm14 ;; val / base + rounding_const		; 9-13
	vsubpd	ymm2, ymm0, ymm14			;; next carry = round (val / base)		; 13-15
	vsubpd	ymm3, ymm1, ymm14			;; next carry = round (val / base)		; 14-16
	vsubpd	ymm9, ymm7, ymm14			;; next carry = round (val / base)		; 16-18
	vsubpd	ymm10, ymm8, ymm14			;; next carry = round (val / base)		; 17-19
	yfnmaddpd ymm4, ymm2, YMM_LIMIT_BIGMAX[basereg], ymm4 ;; new value = val - round (val / base) * base ; 16-20
	yfnmaddpd ymm5, ymm3, YMM_LIMIT_BIGMAX[basereg2], ymm5 ;; new value = val - round (val / base) * base ; 17-21
	yfnmaddpd ymm11, ymm9, YMM_LIMIT_BIGMAX[basereg3], ymm11 ;; new value = val - round (val / base) * base ; 19-23
	yfnmaddpd ymm12, ymm10, YMM_LIMIT_BIGMAX[basereg4], ymm12 ;; new value = val - round (val / base) * base ; 20-24
ENDM
ENDIF

ynorm_wpn_nobase2_noconst_echk MACRO basereg, basereg2, basereg3, basereg4
	vaddpd	ymm0, ymm0, ymm2			;; x = value + carry				; 6-8	First dependence on previous iteration
	vaddpd	ymm1, ymm1, ymm3			;; x = value + carry				; 7-9
	vaddpd	ymm7, ymm7, ymm9			;; x = value + carry				; 8-10
	vaddpd	ymm8, ymm8, ymm10			;; x = value + carry				; 9-11
	vmulpd	ymm2, ymm0, YMM_LIMIT_INVERSE[basereg]	;; val / base					;  9-13
	vroundpd ymm4, ymm0, 0				;; Round val					; 10-12
	vmulpd	ymm3, ymm1, YMM_LIMIT_INVERSE[basereg2]	;; val / base					;  10-14
	vroundpd ymm5, ymm1, 0				;; Round val					; 11-13
	vmulpd	ymm9, ymm7, YMM_LIMIT_INVERSE[basereg3];; val / base					;  11-15
	vroundpd ymm11, ymm7, 0				;; Round val					; 12-14
	vmulpd	ymm10, ymm8, YMM_LIMIT_INVERSE[basereg4];; val / base					;  12-16
	vroundpd ymm12, ymm8, 0				;; Round val					; 13-15
	vroundpd ymm2, ymm2, 0				;; next carry = round (val / base)		; 14-16
	vroundpd ymm3, ymm3, 0				;; next carry = round (val / base)		; 15-17
	vroundpd ymm9, ymm9, 0				;; next carry = round (val / base)		; 16-18
	vroundpd ymm10, ymm10, 0			;; next carry = round (val / base)		; 17-19
	vmulpd	ymm13, ymm2, YMM_LIMIT_BIGMAX[basereg]	;; z = round (val / base) * base		;  17-21
	vmulpd	ymm14, ymm3, YMM_LIMIT_BIGMAX[basereg2]	;; z = round (val / base) * base		;  18-22
	vsubpd	ymm0, ymm0, ymm4			;; This is the convolution error		; 18-20
	vsubpd	ymm1, ymm1, ymm5			;; This is the convolution error		; 19-21
	vsubpd	ymm7, ymm7, ymm11			;; This is the convolution error		; 20-22
	vsubpd	ymm8, ymm8, ymm12			;; This is the convolution error		; 21-23
	vsubpd	ymm4, ymm4, ymm13			;; new value = val - z				; 22-24
	vmulpd	ymm13, ymm9, YMM_LIMIT_BIGMAX[basereg3]	;; z = round (val / base) * base		;  19-23
	absval	ymm0					;; Compute absolute value			;   21
	absval	ymm1					;; Compute absolute value			;   22
	vmaxpd	ymm0, ymm0, ymm1			;; Compute maximum error			; 23-25
	vmulpd	ymm1, ymm10, YMM_LIMIT_BIGMAX[basereg4]	;; z = round (val / base) * base		;  20-24
	absval	ymm7					;; Compute absolute value			;   23
	absval	ymm8					;; Compute absolute value			;   24
	vsubpd	ymm5, ymm5, ymm14			;; new value = val - z				; 24-26
	vmaxpd	ymm7, ymm7, ymm8			;; Compute maximum error			; 25-27
	vmaxpd	ymm6, ymm6, ymm0			;; Compute maximum error			; 26-28
	vsubpd	ymm11, ymm11, ymm13			;; new value = val - z				; 27-28
	vsubpd	ymm12, ymm12, ymm1			;; new value = val - z				; 28-30
	vmaxpd	ymm6, ymm6, ymm7			;; Compute maximum error			; 29-31
ENDM

IF (@INSTR(,%yarch,<FMA3>) NE 0)
ynorm_wpn_nobase2_noconst_echk MACRO basereg, basereg2, basereg3, basereg4
	yfmaddpd ymm2, ymm0, YMM_LIMIT_INVERSE[basereg], ymm14 ;; val / base + rounding_const		; 6-10
	vroundpd ymm4, ymm0, 0				;; Round val					; 6-8 and 9-11
	yfmaddpd ymm3, ymm1, YMM_LIMIT_INVERSE[basereg2], ymm14	;; val / base + rounding_const		; 7-11
	vroundpd ymm5, ymm1, 0				;; Round val					; 7-9 and 10-12
	yfmaddpd ymm9, ymm7, YMM_LIMIT_INVERSE[basereg3], ymm14 ;; val / base + rounding_const		; 8-12
	vroundpd ymm11, ymm7, 0				;; Round val					; 8-10 and 11-13
	yfmaddpd ymm10, ymm8, YMM_LIMIT_INVERSE[basereg4], ymm14 ;; val / base + rounding_const		; 9-13
	vroundpd ymm12, ymm8, 0				;; Round val					; 12-14 and 15-17
	vsubpd	ymm2, ymm2, ymm14			;; next carry = round (val / base)		; 13-15
	vsubpd	ymm3, ymm3, ymm14			;; next carry = round (val / base)		; 14-16
	vsubpd	ymm9, ymm9, ymm14			;; next carry = round (val / base)		; 16-18
	vsubpd	ymm10, ymm10, ymm14			;; next carry = round (val / base)		; 17-19
	vsubpd	ymm0, ymm0, ymm4			;; This is the convolution error		; 18-20
	vsubpd	ymm1, ymm1, ymm5			;; This is the convolution error		; 19-21
	vsubpd	ymm7, ymm7, ymm11			;; This is the convolution error		; 20-22
	vsubpd	ymm8, ymm8, ymm12			;; This is the convolution error		; 21-23
	yfnmaddpd ymm4, ymm2, YMM_LIMIT_BIGMAX[basereg], ymm4 ;; new value = val - round (val / base) * base	; 16-20
	yfnmaddpd ymm5, ymm3, YMM_LIMIT_BIGMAX[basereg2], ymm5 ;; new value = val - round (val / base) * base	; 17-21
	yfnmaddpd ymm11, ymm9, YMM_LIMIT_BIGMAX[basereg3], ymm11 ;; new value = val - round (val / base) * base	; 19-23
	yfnmaddpd ymm12, ymm10, YMM_LIMIT_BIGMAX[basereg4], ymm12 ;; new value = val - round (val / base) * base; 20-24
	absval	ymm0					;; Compute absolute value			;  21
	absval	ymm1					;; Compute absolute value			;  22
	absval	ymm7					;; Compute absolute value			;  23
	absval	ymm8					;; Compute absolute value			;  24
	vmaxpd	ymm6, ymm6, ymm0			;; Compute maximum error			; 22-24
	vmaxpd	ymm1, ymm1, ymm7			;; Compute maximum error			; 24-26
	vmaxpd	ymm6, ymm6, ymm8			;; Compute maximum error			; 25-27
	vmaxpd	ymm6, ymm6, ymm1			;; Compute maximum error			; 28-30
ENDM
ENDIF


ynorm_wpn_nobase2_const MACRO echk, basereg, basereg2, carryreg, carryreg2
	vmulpd	ymm0, ymm7, YMM_LIMIT_INVERSE[basereg]	;; HiFFTval = FFT value / base			; 6-10
	vroundpd ymm4, ymm7, 0				;; Round FFT value				; 6-8
	vmulpd	ymm1, ymm8, YMM_LIMIT_INVERSE[basereg2]	;; HiFFTval = FFT value / base			; 7-11
	vroundpd ymm5, ymm8, 0				;; Round FFT value				; 7-9
echk	vsubpd	ymm11, ymm7, ymm4			;; This is the convolution error		;  9-11
echk	vsubpd	ymm12, ymm8, ymm5			;; This is the convolution error		;  10-12
	vroundpd ymm0, ymm0, 0				;; Round (HiFFTval)				; 11-13
	vroundpd ymm1, ymm1, 0				;; Round (HiFFTval)				; 12-14
echk	absval	ymm11					;; Compute absolute value			;  12
echk	vmaxpd	ymm6, ymm6, ymm11			;; Compute maximum error			;  13-15
echk	absval	ymm12					;; Compute absolute value			;  13
	vmulpd	ymm7, ymm0, YMM_LIMIT_BIGMAX[basereg]	;; HiFFTval * base				; 14-18
echk	vmaxpd	ymm6, ymm6, ymm12			;; Compute maximum error			;  14-16
	vmulpd	ymm8, ymm1, YMM_LIMIT_BIGMAX[basereg2]	;; HiFFTval * base				; 15-19
	vmulpd	ymm0, ymm0, ymm13			;; carryB = HiFFTval * constant			;  16-20
	vmulpd	ymm1, ymm1, ymm13			;; carryB = HiFFTval * constant			;  17-21
	vsubpd	ymm4, ymm4, ymm7			;; LoFFTval = FFTval - HiFFTval * base		; 19-21
	vsubpd	ymm5, ymm5, ymm8			;; LoFFTval = FFTval - HiFFTval * base		; 20-22
	vmulpd	ymm4, ymm4, ymm13			;; val = LoFFTval * constant			; 22-26
	vmulpd	ymm5, ymm5, ymm13			;; val = LoFFTval * constant			; 23-27
	vaddpd	ymm4, ymm4, carryreg			;; val += previous carry			; 27-29	First dependence on previous iteration
	vaddpd	ymm5, ymm5, carryreg2			;; val += previous carry			; 28-30
	vmulpd	ymm7, ymm4, YMM_LIMIT_INVERSE[basereg]	;; val / base					; 30-34
	vmulpd	ymm8, ymm5, YMM_LIMIT_INVERSE[basereg2]	;; val / base					; 31-35
	vroundpd ymm7, ymm7, 0				;; carryA = round (val / base)			; 35-37
	vroundpd ymm8, ymm8, 0				;; carryA = round (val / base)			; 36-38
	vaddpd	carryreg, ymm0, ymm7			;; carry = carryA + carryB			; 38-40
	vmulpd	ymm11, ymm7, YMM_LIMIT_BIGMAX[basereg]	;; z = round (val / base) * base		; 38-42
	vaddpd	carryreg2, ymm1, ymm8			;; carry = carryA + carryB			; 39-41
	vmulpd	ymm12, ymm8, YMM_LIMIT_BIGMAX[basereg2]	;; z = round (val / base) * base		; 39-43
	vsubpd	ymm4, ymm4, ymm11			;; new value = val - z				; 43-45
	vsubpd	ymm5, ymm5, ymm12			;; new value = val - z				; 44-46
ENDM

IF (@INSTR(,%yarch,<FMA3>) NE 0)
ynorm_wpn_nobase2_const MACRO echk, basereg, basereg2, carryreg, carryreg2
	vmulpd	ymm0, ymm7, YMM_LIMIT_INVERSE[basereg]	;; HiFFTval = FFT value / base			; 6-10
	vroundpd ymm4, ymm7, 0				;; Round FFT value				; 6-8,9-11
	vmulpd	ymm1, ymm8, YMM_LIMIT_INVERSE[basereg2]	;; HiFFTval = FFT value / base			; 7-11
	vroundpd ymm5, ymm8, 0				;; Round FFT value				; 7-9,10-12
echk	vsubpd	ymm11, ymm7, ymm4			;; This is the convolution error		;  12-14*
echk	vsubpd	ymm12, ymm8, ymm5			;; This is the convolution error		;  13-15
	vroundpd ymm0, ymm0, 0				;; Round (HiFFTval)				; 11-13,14-16
	vroundpd ymm1, ymm1, 0				;; Round (HiFFTval)				; 12-14,15-17
echk	absval	ymm11					;; Compute absolute value			;  15
echk	vmaxpd	ymm6, ymm6, ymm11			;; Compute maximum error			;  16-18
echk	absval	ymm12					;; Compute absolute value			;  16
	yfnmaddpd ymm4, ymm0, YMM_LIMIT_BIGMAX[basereg], ymm4 ;; LoFFTval = FFTval - HiFFTval * base	; 17-21
echk	vmaxpd	ymm6, ymm6, ymm12			;; Compute maximum error			;  17-19*
	yfnmaddpd ymm5, ymm1, YMM_LIMIT_BIGMAX[basereg2], ymm5 ;; LoFFTval = FFTval - HiFFTval * base	; 18-22
	vmulpd	ymm0, ymm0, ymm13			;; carryB = HiFFTval * constant			; 17-21	  FMA-able but may make latency worse
	vmulpd	ymm1, ymm1, ymm13			;; carryB = HiFFTval * constant			; 18-22	  FMA-able but may make latency worse
	yfmaddpd ymm4, ymm4, ymm13, carryreg		;; val = LoFFTval * constant + previous carry	; 22-26	First dependence on previous iteration
	yfmaddpd ymm5, ymm5, ymm13, carryreg2		;; val = LoFFTval * constant + previous carry	; 23-27
	vmulpd	ymm7, ymm4, YMM_LIMIT_INVERSE[basereg]	;; val / base					; 27-31
	vmulpd	ymm8, ymm5, YMM_LIMIT_INVERSE[basereg2]	;; val / base					; 28-32
	vroundpd ymm7, ymm7, 0				;; carryA = round (val / base)			; 32-34,35-37
	vroundpd ymm8, ymm8, 0				;; carryA = round (val / base)			; 33-35,36-38
	vaddpd	carryreg, ymm0, ymm7			;; carry = carryA + carryB			; 38-40	  FMA-able but may make latency worse
	yfnmaddpd ymm4, ymm7, YMM_LIMIT_BIGMAX[basereg], ymm4 ;; newval = val - round (val/base) * base	; 38-42
	vaddpd	carryreg2, ymm1, ymm8			;; carry = carryA + carryB			; 39-41   FMA-able but may make latency worse
	yfnmaddpd ymm5, ymm8, YMM_LIMIT_BIGMAX[basereg2], ymm5 ;; newval = val - round (val/base) * base; 39-43
ENDM
ENDIF


;; Proposed new non-base-2 code which gens next carry in 8 clocks
;;
;; In this code, the carry includes the integer rounding constant (BIGVAL = 3*2^52).
;; The key is that the next carry can be computed in a single FMA instruction.
;; Given x = value + BIGVAL, the next carry will be (x - BIGVAL) / base + BIGVAL.
;; This simplifies to x / base + (BIGVAL - BIGVAL / base).  This works as long as we define
;; the BIGVAL constant as the first value above 3*2^52 that is divisible by the base.
;;
;; To calculate value, value = (x - BIGVAL) - (next carry - BIGVAL) * base
;; Rearranging:  value = x - next carry * base + BIGVAL * base - BIGVAL
;; And:  value = x - (next_carry - (BIGVAL - BIGVAL / base)) * base

;;; BUG - need 3rd basereg indexed set of constants.  This increases load pressure.
;;; normalization code is already under serious load pressure.

IF (@INSTR(,%yarch,<FMA3>) NE 0)
;; Timings indicate this takes ??? clocks vs. 22.0 clocks for the existing code
new_ynorm_wpn_nobase2_noconst MACRO basereg, basereg2, basereg3, basereg4
	vmovapd ymm4, YMM_LIMIT_BIGMAX[basereg]			;; Load constant for creating next carry
	yfmaddpd ymm2, ymm0, YMM_LIMIT_INVERSE[basereg], ymm4	;; next carry = x / base + BIGVAL	; 6-10
	vmovapd ymm5, YMM_LIMIT_BIGMAX[basereg2]		;; Load constant for creating next carry
	yfmaddpd ymm3, ymm1, YMM_LIMIT_INVERSE[basereg2], ymm5	;; next carry = x / base + BIGVAL	; 7-11
	vmovapd ymm11, YMM_LIMIT_BIGMAX[basereg3]		;; Load constant for creating next carry
	yfmaddpd ymm9, ymm7, YMM_LIMIT_INVERSE[basereg3], ymm11	;; next carry = x / base + BIGVAL	; 8-12
	vmovapd ymm12, YMM_LIMIT_BIGMAX[basereg4]		;; Load constant for creating next carry
	yfmaddpd ymm10, ymm8, YMM_LIMIT_INVERSE[basereg4], ymm12 ;; next carry = x / base + BIGVAL	; 9-13
	vsubpd ymm4, ymm2, ymm4					;; y = next carry - BIGVAL		; 11-13
	vsubpd ymm5, ymm3, ymm5					;; y = next carry - BIGVAL		; 12-14
	vsubpd ymm11, ymm9, ymm11				;; y = next carry - BIGVAL		; 13-15
	vsubpd ymm12, ymm10, ymm12				;; y = next carry - BIGVAL		; 14-16
;; BUG - s.b. BASE not BIGMAX
	yfnmaddpd ymm4, ymm4, YMM_LIMIT_BIGMAX[basereg], ymm0	;; new value = x - y * base		; 14-18
	yfnmaddpd ymm5, ymm5, YMM_LIMIT_BIGMAX[basereg2], ymm1	;; new value = x - y * base		; 15-19
	yfnmaddpd ymm11, ymm11, YMM_LIMIT_BIGMAX[basereg3], ymm7 ;; new value = x - y * base		; 16-20
	yfnmaddpd ymm12, ymm12, YMM_LIMIT_BIGMAX[basereg4], ymm8 ;; new value = x - y * base		; 17-21
ENDM
ENDIF

IF (@INSTR(,%yarch,<FMA3>) NE 0)
new_ynorm_wpn_nobase2_noconst_echk MACRO basereg, basereg2, basereg3, basereg4
BUG - add basereg3&4 code
	vaddpd	ymm0, ymm8, ymm2			;; x = value + carry				; 3-5	First dependence on previous iteration
	vaddpd	ymm1, ymm9, ymm3			;; x = value + carry				; 4-6
	vmovapd ymm4, YMM_LIMIT_BIGMAX[basereg]		;; Load constant for creating next carry
echk	vsubpd	ymm10, ymm0, ymm2			;; Start error calc (x - carry)			;  6-8
	yfmaddpd ymm2, ymm0, YMM_LIMIT_INVERSE[basereg], ymm4	;; next carry = x / base + BIGVAL	; 6-10
	vmovapd ymm5, YMM_LIMIT_BIGMAX[basereg2]	;; Load constant for creating next carry
echk	vsubpd	ymm11, ymm1, ymm3			;; Start error calc (x - carry)			;  7-9
	yfmaddpd ymm3, ymm1, YMM_LIMIT_INVERSE[basereg2], ymm5	;; next carry = x / base + BIGVAL	; 7-11
echk	vsubpd	ymm10, ymm9, ymm10			;; Convolution error = value - (x - carry)	;  9-11
echk	vsubpd	ymm11, ymm5, ymm11			;; Convolution error = value - (x - carry)	;  10-12
	vsubpd ymm4, ymm2, ymm4				;; y = next carry - BIGVAL			; 11-13		
	vsubpd ymm5, ymm3, ymm5				;; y = next carry - BIGVAL			; 12-14
echk	absval	ymm10					;; Compute absolute value			;  12
echk	absval	ymm11					;; Compute absolute value			;  13
echk	vmaxpd	ymm6, ymm6, ymm10			;; Compute maximum error			;  13-15
echk	vmaxpd	ymm6, ymm6, ymm11			;; Compute maximum error			;  14-16
;; BUG - s.b. BASE not BIGMAX
	yfnmaddpd ymm4, ymm4, YMM_LIMIT_BIGMAX[basereg], ymm0 ;; new value = x - y * base			; 14-18
	yfnmaddpd ymm5, ymm5, YMM_LIMIT_BIGMAX[basereg2], ymm1 ;; new value = x - y * base		; 15-19
ENDM
ENDIF

;; Proposed new non-base-2 code to handle const
;;
;; Like the non-const code, we do 3 operations on x = value + BIGVAL to generate 2 needed values:
;;	x / base + (BIGVAL - BIGVAL / base)	gives us value / base + BIGVAL.
;; Then, given y = value / base + BIGVAL,
;;	tmp = y - (BIGVAL - BIGVAL / base)
;;	value = y - tmp * base			gives us value % base

;;BUG  -- set registers to:
;ymm14 = YMM_BIGVAL
;ymm13 = YMM_BIGVAL * const
;ymm12 = const

IF (@INSTR(,%yarch,<FMA3>) NE 0)
new_ynorm_wpn_nobase2_const MACRO echk, basereg, basereg2, basereg3, basereg4
BUG - add basereg3&4 code

uh oh, we can't have the caller do the mul-by-ttmp

	vmovapd	ymm0, [rsi]				;; Load value1
	yfmaddpd ymm4, ymm0, [rbp], ymm14		;; Mul value1 by two-to-minus-phi + BIGVAL	; presumably free
	vmovapd	ymm1, [rsi+32]				;; Load value2
	yfmaddpd ymm5, ymm1, [rbp+64], ymm14		;; Mul value2 by two-to-minus-phi + BIGVAL	; presumably free

	vmovapd ymm8, YMM_LIMIT_BIGMAX[basereg]		;; Load constant for creating next carry
	vmovapd ymm9, YMM_LIMIT_BIGMAX[basereg2]	;; Load constant for creating next carry

echk	vsubpd	ymm10, ymm4, ymm14			;; Start error calc, subtract BIGVAL		;  6-8
echk	vsubpd	ymm11, ymm5, ymm14			;; Start error calc, subtract BIGVAL		;  7-9

echk	yfmsubpd ymm10, ymm0, [rbp], ymm10		;; Convolution error, subtract val * ttmp	;  9-13
echk	yfmsubpd ymm11, ymm1, [rbp+64], ymm11		;; Convolution error, subtract val * ttmp	;  10-14

	yfmaddpd ymm0, ymm4, YMM_LIMIT_INVERSE[basereg], ymm8 ;; HiFFTval = FFT value / base + BIGVAL	; 6-10
	yfmaddpd ymm1, ymm5, YMM_LIMIT_INVERSE[basereg2], ymm9 ;; HiFFTval = FFT value / base + BIGVAL	; 7-11

echk	absval	ymm10					;; Compute absolute value			;  14
echk	absval	ymm11					;; Compute absolute value			;  15
echk	vmaxpd	ymm6, ymm6, ymm10			;; Compute maximum error			;  15-17
echk	vmaxpd	ymm6, ymm6, ymm11			;; Compute maximum error			;  16-18

	vsubpd	ymm10, ymm0, ymm8			;; Rounded HiFFTval				; 11-13
	vsubpd	ymm11, ymm1, ymm9			;; Rounded HiFFTval				; 12-14

	yfmaddpd ymm0, ymm0, ymm12, ymm13		;; carryB = HiFFTval * constant			; 11-15
	yfmaddpd ymm1, ymm1, ymm12, ymm13		;; carryB = HiFFTval * constant			; 12-16

	;; s.b. BASE
	yfnmaddpd ymm10, ymm10, YMM_LIMIT_BIGMAX[basereg], ymm4 ;; LoFFTval = FFTval - HiFFTval * base	; 14-18
	yfnmaddpd ymm11, ymm11, YMM_LIMIT_BIGMAX[basereg2], ymm5 ;; LoFFTval = FFTval - HiFFTval * base	; 15-19

	yfmaddpd ymm4, ymm10, ymm12, ymm2		;; val = LoFFTval * constant + previous carry	; 19-23	First dependence on previous iteration
	yfmaddpd ymm5, ymm11, ymm12, ymm3		;; val = LoFFTval * constant + previous carry	; 20-24

	yfmaddpd ymm2, ymm4, YMM_LIMIT_INVERSE[basereg], ymm8 ;; val / base + BIGVAL			; 24-28
	yfmaddpd ymm3, ymm5, YMM_LIMIT_INVERSE[basereg2], ymm9	;; val / base + BIGVAL			; 25-29

	vsubpd	ymm10, ymm2, ymm8			;; Rounded val / base				; 29-31
	vsubpd	ymm11, ymm3, ymm9			;; Rounded val / base				; 30-32

	vaddpd	ymm2, ymm2, ymm0			;; next carry = carryA + carryB			; 31-33
	vaddpd	ymm3, ymm3, ymm1			;; next carry = carryA + carryB			; 32-34

	;; s.b. BASE
	yfnmaddpd ymm4, ymm10, YMM_LIMIT_BIGMAX[basereg], ymm4 ;; newval = val - round (val/base) * base; 32-36
	yfnmaddpd ymm5, ymm11, YMM_LIMIT_BIGMAX[basereg2], ymm5 ;; newval = val - round (val/base) * base; 33-37
ENDM
ENDIF

;; new style code! 12 FMAs, 4 muls, 4 adds = 20 ops.  Best case would be 10 clocks.
IF (@INSTR(,%yarch,<FMA3>) NE 0)
new_ynorm_wpn_base2_noconst MACRO echk, basereg, basereg2, basereg3, basereg4

	vmovapd ymm4, YMM_LIMIT_BIGMAX[basereg]		;; Load constant for creating next carry
	yfmaddpd ymm2, ymm0, YMM_LIMIT_INVERSE[basereg], ymm4	;; next carry = x / base + BIGVAL	; 4-8
	vmovapd ymm5, YMM_LIMIT_BIGMAX[basereg2]	;; Load constant for creating next carry
	yfmaddpd ymm3, ymm1, YMM_LIMIT_INVERSE[basereg2], ymm5	;; next carry = x / base + BIGVAL	; 5-9
	vmovapd ymm11, YMM_LIMIT_BIGMAX[basereg3]		;; Load constant for creating next carry
	yfmaddpd ymm9, ymm7, YMM_LIMIT_INVERSE[basereg3], ymm11	;; next carry = x / base + BIGVAL	; 6-10
	vmovapd ymm12, YMM_LIMIT_BIGMAX[basereg4]	;; Load constant for creating next carry
	yfmaddpd ymm10, ymm8, YMM_LIMIT_INVERSE[basereg4], ymm12 ;; next carry = x / base + BIGVAL	; 7-11

	vsubpd ymm4, ymm2, ymm4				;; y = next carry - BIGVAL			; 9-11		
	vsubpd ymm5, ymm3, ymm5				;; y = next carry - BIGVAL			; 10-12
	vsubpd ymm11, ymm9, ymm11				;; y = next carry - BIGVAL		; 11-13		
	vsubpd ymm12, ymm10, ymm12				;; y = next carry - BIGVAL		; 12-14

;; BUG - s.b. BASE not BIGMAX
	yfnmaddpd ymm4, ymm4, YMM_LIMIT_BIGMAX[basereg], ymm0 ;; new value = x - y * base		; 12-16
	yfnmaddpd ymm5, ymm5, YMM_LIMIT_BIGMAX[basereg2], ymm1 ;; new value = x - y * base		; 13-17
	yfnmaddpd ymm11, ymm11, YMM_LIMIT_BIGMAX[basereg3], ymm7 ;; new value = x - y * base		; 14-18
	yfnmaddpd ymm12, ymm12, YMM_LIMIT_BIGMAX[basereg4], ymm8 ;; new value = x - y * base		; 15-19
ENDM
ENDIF

ENDIF


; This is the normalization routine when we are computing modulo k*b^n+c
; with a zero-padded b^2n FFT.  We do this by multiplying the lower FFT
; word by k and adding in the upper word times -c.  Of course, this is made
; very tedious because we have to carefully avoid any loss of precision.
;
;; NOTE: In zero pad FFTs, big/lit and fudge factor flags 1 and 2 are identical

; For WPN zpad macros, these registers are set on input:
; ymm6 = maxerr
; rbp = pointer to carries
; rdi = pointer to big/little flags
; rsi = pointer to the FFT data
; rdx = pointer two-to-phi group multipliers
; ebx = big vs. little & fudge flags
; eax = big vs. little word flag #1
; ymm2,ymm3 = carries

ynorm_wpn_zpad_preload MACRO ttp, base2, echk, const, khi, c1, cm1
	ENDM

ynorm_wpn_zpad MACRO ttp, base2, echk, const, khi, c1, cm1
			L1prefetchw rsi+64, L1PREFETCH_ALL
ttp			movzx	eax, bh				;; Big/lit flags 1-2
ttp			and	rbx, 0e0h			;; Fudge flags 2
			vmovapd	ymm0, [rsi+0*32]		;; Load values1
ttp			vmulpd	ymm0, ymm0, [rdx+0*YMM_GMD][rbx] ;; Mul by fudged grp two-to-minus-phi
			vmovapd	ymm1, [rsi+1*32]		;; Load values2
ttp			vmulpd	ymm1, ymm1, [rdx+0*YMM_GMD][rbx] ;; Mul by fudged grp two-to-minus-phi

			split_lower_zpad_word ttp, base2, echk, ymm0, ymm3, ymm4, rax*8

no const		vmulpd	ymm0, ymm4, YMM_K_LO
const			vmulpd	ymm0, ymm4, YMM_K_TIMES_MULCONST_LO

khi base2 no const	vmulpd	ymm5, ymm4, YMM_K_HI
khi base2 const		vmulpd	ymm5, ymm4, YMM_K_TIMES_MULCONST_HI
khi no base2 no const	vmovapd	ymm5, YMM_K_HI
khi no base2 const	vmovapd	ymm5, YMM_K_TIMES_MULCONST_HI
khi no base2 ttp	vmulpd	ymm5, ymm5, YMM_LIMIT_INVERSE[rax*8] ;; Non-base2 rounding needs shifted carry
khi no base2 no ttp	vmulpd	ymm5, ymm5, YMM_LIMIT_INVERSE[0] ;; Non-base2 rounding needs shifted carry
khi no base2		vroundpd ymm5, ymm5, 0			;; THIS IS WASTEFUL.  The mul and round should be precomputed!
khi no base2		vmulpd	ymm5, ymm5, ymm4

			vaddpd	ymm0, ymm0, ymm2		;; x1 = values + carry

c1			vmulpd	ymm1, ymm1, YMM_MINUS_C		;; Do one mul before split rather than two after split

			split_upper_zpad_word ttp, base2, echk, ymm1, ymm2, ymm4, rax*8

no const no c1 no cm1	vmulpd	ymm4, ymm4, YMM_MINUS_C
no const no c1 no cm1	vmulpd	ymm2, ymm2, YMM_MINUS_C
const			vmulpd	ymm4, ymm4, YMM_MINUS_C_TIMES_MULCONST
const			vmulpd	ymm2, ymm2, YMM_MINUS_C_TIMES_MULCONST

			vaddpd	ymm0, ymm0, ymm4		;; Add upper FFT word to lower FFT word
khi			vaddpd	ymm2, ymm2, ymm5		;; Add upper FFT word to lower FFT word

			rounding ttp, base2, exec, ymm0, ymm2, ymm4, rax*8

ttp			vmulpd	ymm0, ymm0, [rdx+0*YMM_GMD+YMM_GMD/2][rbx] ;; new value1 *= fudged grp two-to-phi
			ystore	[rsi], ymm0			;; Save new value1
			vxorpd	ymm1, ymm1, ymm1		;; new value2 = zero
			ystore	[rsi+32], ymm1			;; Zero value2

ttp			movzx	rbx, WORD PTR [rdi+2]		;; Load next big vs. little & fudge flags
	ENDM

;; 64-bit version using extra registers

IFDEF X86_64

ynorm_wpn_zpad_preload MACRO ttp, base2, echk, const, khi, c1, cm1
echk			vmovapd	ymm15, YMM_ABSVAL
base2			vmovapd ymm14, YMM_BIGBIGVAL
no const no c1 no cm1	vmovapd	ymm11, YMM_MINUS_C
const			vmovapd	ymm11, YMM_MINUS_C_TIMES_MULCONST
IF (@INSTR(,%yarch,<FMA3>) NE 0)
			vmovapd	ymm12, YMM_BIGVAL	;; Load round-to-integer constant
ENDIF
	ENDM

ynorm_wpn_zpad MACRO ttp, base2, echk, const, khi, c1, cm1
IFDEF YIMPL_WPN1_FFTS
	ynorm_wpn_zpad1 ttp, base2, echk, const, khi, c1, cm1, r15
ENDIF
IFDEF YIMPL_WPN4_FFTS
	ynorm_wpn_zpad1 ttp, base2, echk, const, khi, c1, cm1, r12
ENDIF
ENDM

ynorm_wpn_zpad1 MACRO ttp, base2, echk, const, khi, c1, cm1, grpreg2
	L1prefetchw rsi+64, L1PREFETCH_ALL
	L1prefetchw r13+64, L1PREFETCH_ALL

ttp	movzx	eax, bh				;; Big/lit flags 1-2
ttp	and	rbx, 0e0h			;; Fudge flags 2
ttp	vmovapd	ymm1, [r12+0*YMM_GMD][rbx]	;; Load fudged grp two-to-minus-phi
ttp	vmulpd	ymm0, ymm1, [rsi+0*32]		;; value1 * fudged grp two-to-minus-phi
no ttp	vmovapd	ymm0, [rsi+0*32]		;; value1
ttp	vmulpd	ymm1, ymm1, [rsi+1*32]		;; value2 * fudged grp two-to-minus-phi
no ttp	vmovapd	ymm1, [rsi+1*32]		;; value2

	vaddpd	ymm0, ymm0, ymm3		;; Add in previous high FFT data

ttp	movzx	edx, ch				;; Big/lit flags 1-2
ttp	and	rcx, 0e0h			;; Fudge flags 2
ttp	vmovapd	ymm8, [grpreg2+0*YMM_GMD][rcx]	;; Load fudged grp two-to-minus-phi
ttp	vmulpd	ymm7, ymm8, [r13+0*32]		;; value1 * fudged grp two-to-minus-phi
no ttp	vmovapd	ymm7, [r13+0*32]		;; value1
ttp	vmulpd	ymm8, ymm8, [r13+1*32]		;; value2 * fudged grp two-to-minus-phi
no ttp	vmovapd	ymm8, [r13+1*32]		;; value2

	vaddpd	ymm7, ymm7, ymm10		;; Add in previous high FFT data

ttp	L1prefetch r9, L1PREFETCH_ALL
ttp	bump	r9, 64

base2 no echk no const ttp	ynorm_wpn_zpad_base2		khi, c1, cm1, YMM_K_LO, YMM_K_HI, rax*8, rdx*8
base2 no echk no const no ttp	ynorm_wpn_zpad_base2		khi, c1, cm1, YMM_K_LO, YMM_K_HI, 0, 0
base2 no echk const ttp		ynorm_wpn_zpad_base2		khi, noexec, noexec, YMM_K_TIMES_MULCONST_LO, YMM_K_TIMES_MULCONST_HI, rax*8, rdx*8
base2 no echk const no ttp	ynorm_wpn_zpad_base2		khi, noexec, noexec, YMM_K_TIMES_MULCONST_LO, YMM_K_TIMES_MULCONST_HI, 0, 0
base2 echk no const ttp		ynorm_wpn_zpad_base2_echk	khi, c1, cm1, YMM_MINUS_C, YMM_K_LO, YMM_K_HI, rax*8, rdx*8
base2 echk no const no ttp	ynorm_wpn_zpad_base2_echk	khi, c1, cm1, YMM_MINUS_C, YMM_K_LO, YMM_K_HI, 0, 0
base2 echk const ttp		ynorm_wpn_zpad_base2_echk	khi, noexec, noexec, YMM_MINUS_C_TIMES_MULCONST, YMM_K_TIMES_MULCONST_LO, YMM_K_TIMES_MULCONST_HI, rax*8, rdx*8
base2 echk const no ttp		ynorm_wpn_zpad_base2_echk	khi, noexec, noexec, YMM_MINUS_C_TIMES_MULCONST, YMM_K_TIMES_MULCONST_LO, YMM_K_TIMES_MULCONST_HI, 0, 0
no base2 no echk no const ttp	ynorm_wpn_zpad_nobase2		khi, c1, cm1, YMM_MINUS_C, YMM_K_LO, YMM_K_HI, rax*8, rdx*8
no base2 no echk no const no ttp ynorm_wpn_zpad_nobase2		khi, c1, cm1, YMM_MINUS_C, YMM_K_LO, YMM_K_HI, 0, 0
no base2 no echk const ttp	ynorm_wpn_zpad_nobase2		khi, noexec, noexec, YMM_MINUS_C_TIMES_MULCONST, YMM_K_TIMES_MULCONST_LO, YMM_K_TIMES_MULCONST_HI, rax*8, rdx*8
no base2 no echk const no ttp	ynorm_wpn_zpad_nobase2		khi, noexec, noexec, YMM_MINUS_C_TIMES_MULCONST, YMM_K_TIMES_MULCONST_LO, YMM_K_TIMES_MULCONST_HI, 0, 0
no base2 echk no const ttp	ynorm_wpn_zpad_nobase2_echk	khi, c1, cm1, YMM_MINUS_C, YMM_K_LO, YMM_K_HI, rax*8, rdx*8
no base2 echk no const no ttp	ynorm_wpn_zpad_nobase2_echk	khi, c1, cm1, YMM_MINUS_C, YMM_K_LO, YMM_K_HI, 0, 0
no base2 echk const ttp		ynorm_wpn_zpad_nobase2_echk	khi, noexec, noexec, YMM_MINUS_C_TIMES_MULCONST, YMM_K_TIMES_MULCONST_LO, YMM_K_TIMES_MULCONST_HI, rax*8, rdx*8
no base2 echk const no ttp	ynorm_wpn_zpad_nobase2_echk	khi, noexec, noexec, YMM_MINUS_C_TIMES_MULCONST, YMM_K_TIMES_MULCONST_LO, YMM_K_TIMES_MULCONST_HI, 0, 0

ttp	vmulpd	ymm0, ymm0, [r12+0*YMM_GMD+YMM_GMD/2][rbx] ;; new value1 *= fudged grp two-to-phi
ttp	vmulpd	ymm7, ymm7, [grpreg2+0*YMM_GMD+YMM_GMD/2][rcx] ;; new value1 *= fudged grp two-to-phi
ttp	movzx	rbx, WORD PTR [rdi+2]		;; Load next big vs. little & fudge flags
ttp	movzx	rcx, WORD PTR [r14+2]		;; Load next big vs. little & fudge flags
	vxorpd	ymm1, ymm1, ymm1		;; new value2 = zero
	ystore	[rsi], ymm0			;; Save new value1
	ystore	[r13], ymm7			;; Save new value1
	ystore	[rsi+32], ymm1			;; Zero value2
	ystore	[r13+32], ymm1			;; Zero value2
	ENDM

IF (@INSTR(,%yarch,<FMA3>) NE 0)
ynorm_wpn_zpad1 MACRO ttp, base2, echk, const, khi, c1, cm1, grpreg2
	L1prefetchw rsi+64, L1PREFETCH_ALL
	L1prefetchw r13+64, L1PREFETCH_ALL

ttp	movzx	eax, bh				;; Big/lit flags 1-2
ttp	and	rbx, 0e0h			;; Fudge flags 2
no echk ttp	vmovapd	ymm1, [r12+0*YMM_GMD][rbx]	;; Load fudged grp two-to-minus-phi
no echk ttp	yfmaddpd ymm0, ymm1, [rsi+0*32], ymm3	;; value1 * fudged grp two-to-minus-phi + previous high FFT data + rounding const
no echk no ttp	vaddpd	ymm0, ymm3, [rsi+0*32]		;; value1 + previous high FFT data + rounding const
no echk ttp	yfmaddpd ymm1, ymm1, [rsi+1*32], ymm12	;; value2 * fudged grp two-to-minus-phi + rounding const
no echk no ttp	vaddpd	ymm1, ymm12, [rsi+1*32]		;; value2 + rounding const
echk ttp	vmovapd	ymm4, [r12+0*YMM_GMD][rbx]	;; Load fudged grp two-to-minus-phi
echk ttp	yfmaddpd ymm0, ymm4, [rsi+0*32], ymm3	;; value1 * fudged grp two-to-minus-phi + previous high FFT data + rounding const
echk no ttp	vaddpd	ymm0, ymm3, [rsi+0*32]		;; value1 + previous high FFT data + rounding const
echk ttp	yfmaddpd ymm1, ymm4, [rsi+1*32], ymm12	;; value2 * fudged grp two-to-minus-phi + rounding const
echk no ttp	vaddpd	ymm1, ymm12, [rsi+1*32]		;; value2 + rounding const

ttp	movzx	edx, ch				;; Big/lit flags 1-2
ttp	and	rcx, 0e0h			;; Fudge flags 2
no echk	ttp	vmovapd	ymm8, [grpreg2+0*YMM_GMD][rcx]	;; Load fudged grp two-to-minus-phi
no echk ttp	yfmaddpd ymm7, ymm8, [r13+0*32], ymm10	;; value1 * fudged grp two-to-minus-phi + previous high FFT data + rounding const
no echk no ttp	vaddpd	ymm7, ymm10, [r13+0*32]		;; value1 + previous high FFT data + rounding const
no echk ttp	yfmaddpd ymm8, ymm8, [r13+1*32], ymm12	;; value2 * fudged grp two-to-minus-phi + rounding const
no echk no ttp	vaddpd	ymm8, ymm12, [r13+1*32]		;; value2 + rounding const
echk ttp	vmovapd	ymm13, [grpreg2+0*YMM_GMD][rcx]	;; Load fudged grp two-to-minus-phi
echk ttp	yfmaddpd ymm7, ymm13, [r13+0*32], ymm10	;; value1 * fudged grp two-to-minus-phi + previous high FFT data + rounding const
echk no ttp	vaddpd	ymm7, ymm10, [r13+0*32]		;; value1 + previous high FFT data + rounding const
echk ttp	yfmaddpd ymm8, ymm13, [r13+1*32], ymm12	;; value2 * fudged grp two-to-minus-phi + rounding const
echk no ttp	vaddpd	ymm8, ymm12, [r13+1*32]		;; value2 + rounding const

base2 no echk no const ttp	ynorm_wpn_zpad_base2		  khi, c1, cm1, YMM_K_LO, YMM_K_HI, rax*8, rdx*8
base2 no echk no const no ttp	ynorm_wpn_zpad_base2		  khi, c1, cm1, YMM_K_LO, YMM_K_HI, 0, 0
base2 no echk const ttp		ynorm_wpn_zpad_base2		  khi, noexec, noexec, YMM_K_TIMES_MULCONST_LO, YMM_K_TIMES_MULCONST_HI, rax*8, rdx*8
base2 no echk const no ttp	ynorm_wpn_zpad_base2		  khi, noexec, noexec, YMM_K_TIMES_MULCONST_LO, YMM_K_TIMES_MULCONST_HI, 0, 0
base2 echk no const ttp		ynorm_wpn_zpad_base2_echk	  khi, c1, cm1, YMM_MINUS_C, YMM_K_LO, YMM_K_HI, rax*8, rdx*8
base2 echk no const no ttp	ynorm_wpn_zpad_base2_echk_nottp	  khi, c1, cm1, YMM_MINUS_C, YMM_K_LO, YMM_K_HI, 0, 0
base2 echk const ttp		ynorm_wpn_zpad_base2_echk	  khi, noexec, noexec, YMM_MINUS_C_TIMES_MULCONST, YMM_K_TIMES_MULCONST_LO, YMM_K_TIMES_MULCONST_HI, rax*8, rdx*8
base2 echk const no ttp		ynorm_wpn_zpad_base2_echk_nottp	  khi, noexec, noexec, YMM_MINUS_C_TIMES_MULCONST, YMM_K_TIMES_MULCONST_LO, YMM_K_TIMES_MULCONST_HI, 0, 0
no base2 no echk no const ttp	ynorm_wpn_zpad_nobase2		  khi, c1, cm1, YMM_MINUS_C, YMM_K_LO, YMM_K_HI, rax*8, rdx*8
no base2 no echk no const no ttp ynorm_wpn_zpad_nobase2		  khi, c1, cm1, YMM_MINUS_C, YMM_K_LO, YMM_K_HI, 0, 0
no base2 no echk const ttp	ynorm_wpn_zpad_nobase2		  khi, noexec, noexec, YMM_MINUS_C_TIMES_MULCONST, YMM_K_TIMES_MULCONST_LO, YMM_K_TIMES_MULCONST_HI, rax*8, rdx*8
no base2 no echk const no ttp	ynorm_wpn_zpad_nobase2		  khi, noexec, noexec, YMM_MINUS_C_TIMES_MULCONST, YMM_K_TIMES_MULCONST_LO, YMM_K_TIMES_MULCONST_HI, 0, 0
no base2 echk no const ttp	ynorm_wpn_zpad_nobase2_echk	  khi, c1, cm1, YMM_MINUS_C, YMM_K_LO, YMM_K_HI, rax*8, rdx*8
no base2 echk no const no ttp	ynorm_wpn_zpad_nobase2_echk_nottp khi, c1, cm1, YMM_MINUS_C, YMM_K_LO, YMM_K_HI, 0, 0
no base2 echk const ttp		ynorm_wpn_zpad_nobase2_echk	  khi, noexec, noexec, YMM_MINUS_C_TIMES_MULCONST, YMM_K_TIMES_MULCONST_LO, YMM_K_TIMES_MULCONST_HI, rax*8, rdx*8
no base2 echk const no ttp	ynorm_wpn_zpad_nobase2_echk_nottp khi, noexec, noexec, YMM_MINUS_C_TIMES_MULCONST, YMM_K_TIMES_MULCONST_LO, YMM_K_TIMES_MULCONST_HI, 0, 0

ttp	vmulpd	ymm0, ymm0, [r12+0*YMM_GMD+YMM_GMD/2][rbx] ;; new value1 *= fudged grp two-to-phi
ttp	vmulpd	ymm7, ymm7, [grpreg2+0*YMM_GMD+YMM_GMD/2][rcx] ;; new value1 *= fudged grp two-to-phi
ttp	movzx	rbx, WORD PTR [rdi+2]		;; Load next big vs. little & fudge flags
ttp	movzx	rcx, WORD PTR [r14+2]		;; Load next big vs. little & fudge flags
	vxorpd	ymm1, ymm1, ymm1		;; new value2 = zero
	ystore	[rsi], ymm0			;; Save new value1
	ystore	[r13], ymm7			;; Save new value1
	ystore	[rsi+32], ymm1			;; Zero value2
	ystore	[r13+32], ymm1			;; Zero value2
	ENDM
ENDIF

ynorm_wpn_zpad_base2 MACRO khi, c1, cm1, klo_const, khi_const, basereg, basereg2
	vaddpd	ymm3, ymm0, ymm14			;; Round low input to multiple of big word		; 1-3
	vroundpd ymm0, ymm0, 0				;; Round low input to an integer			; 2-4
	vaddpd	ymm10, ymm7, ymm14			;; Round low input to multiple of big word		; 3-5
	vroundpd ymm7, ymm7, 0				;; Round low input to an integer			; 4-6
	vsubpd	ymm3, ymm3, ymm14										; 5-7
	vsubpd	ymm10, ymm10, ymm14										; 6-8
	vaddpd	ymm5, ymm1, ymm14			;; Round high input to multiple of big word		; 7-9		n 12
	vsubpd	ymm0, ymm0, ymm3			;; Compute low bigword bits of low FFT word		; 8-10		n 11
	vmovapd ymm6, YMM_LIMIT_INVERSE[basereg] 	;; Load limit inverse
	vmulpd	ymm3, ymm3, ymm6			;; Saved shifted FFT hi data for next iteration		;  8-12
	vsubpd	ymm7, ymm7, ymm10			;; Compute low bigword bits of low FFT word		; 9-11		n 13
	vmovapd ymm15, YMM_LIMIT_INVERSE[basereg2] 	;; Load limit inverse
	vmulpd	ymm10, ymm10, ymm15			;; Saved shifted FFT hi data for next iteration		;  9-13
	vaddpd	ymm13, ymm8, ymm14			;; Round high input to multiple of big word		; 10-12		n 14
	vroundpd ymm1, ymm1, 0				;; Round high input to an integer			; 11-13		n 15
	vmovapd	ymm4, klo_const
	vmulpd	ymm12, ymm0, ymm4			;; Mul by YMM_K_LO or YMM_K_TIMES_MULCONST_LO		;  11-15	n 16
	vsubpd	ymm5, ymm5, ymm14										; 12-14		n 15
	vmulpd	ymm4, ymm7, ymm4			;; Mul by YMM_K_LO or YMM_K_TIMES_MULCONST_LO		;  12-16	n 18
	vroundpd ymm8, ymm8, 0				;; Round high input to an integer			; 13-15		n 17
	vsubpd	ymm13, ymm13, ymm14										; 14-16		n 17
	vsubpd	ymm1, ymm1, ymm5			;; Compute low bigword bits of high FFT word		; 15-17
	vaddpd	ymm2, ymm12, ymm2			;; x1 = values + carry					; 16-18
	vsubpd	ymm8, ymm8, ymm13			;; Compute low bigword bits of high FFT word		; 17-19
	vaddpd	ymm9, ymm4, ymm9			;; x1 = values + carry					; 18-20
khi	vmovapd	ymm4, khi_const
khi	vmulpd	ymm12, ymm0, ymm4			;; Mul by YMM_K_HI or YMM_K_TIMES_MULCONST_HI		;	13-17
khi	vmulpd	ymm4, ymm7, ymm4			;; Mul by YMM_K_HI or YMM_K_TIMES_MULCONST_HI		;	14-18
no c1 no cm1 vmulpd ymm5, ymm5, ymm11			;; Mul by YMM_MINUS_C or YMM_MINUS_C_TIMES_MULCONST	;	15-19
no c1 no cm1 vmulpd ymm13, ymm13, ymm11			;; Mul by YMM_MINUS_C or YMM_MINUS_C_TIMES_MULCONST	;	17-21
no c1 no cm1 vmulpd ymm1, ymm1, ymm11			;; Mul by YMM_MINUS_C or YMM_MINUS_C_TIMES_MULCONST	;	18-22
no c1 no cm1 vmulpd ymm8, ymm8, ymm11			;; Mul by YMM_MINUS_C or YMM_MINUS_C_TIMES_MULCONST	;	20-24
khi c1	vsubpd	ymm5, ymm12, ymm5			;; Add upper FFT word * MINUS_C to lower FFT word	;	18-22
khi no c1 vaddpd ymm5, ymm12, ymm5			;; Add upper FFT word to lower FFT word			;	18-22 or 20-24
khi c1	vsubpd	ymm13, ymm4, ymm13			;; Add upper FFT word * MINUS_C to lower FFT word	;	19-23
khi no c1 vaddpd ymm13, ymm4, ymm13			;; Add upper FFT word to lower FFT word			;	19-23 or 22-26
c1	vsubpd	ymm0, ymm2, ymm1			;; Add upper FFT word * MINUS_C to lower FFT word	; 19-21
no c1	vaddpd	ymm0, ymm2, ymm1			;; Add upper FFT word to lower FFT word			; 19-21
c1	vsubpd	ymm7, ymm9, ymm8			;; Add upper FFT word * MINUS_C to lower FFT word	; 21-23
no c1	vaddpd	ymm7, ymm9, ymm8			;; Add upper FFT word to lower FFT word			; 21-23
	vmovapd	ymm1, YMM_LIMIT_BIGMAX[basereg]		;; Load rounding constant
	vaddpd	ymm2, ymm0, ymm1			;; y = top bits of x					; 22-24
	vmovapd	ymm8, YMM_LIMIT_BIGMAX[basereg2]	;; Load rounding constant
	vaddpd	ymm9, ymm7, ymm8			;; y = top bits of x					; 24-26
	vsubpd	ymm1, ymm2, ymm1			;; z = y - (maximum*BIGVAL-BIGVAL)			; 25-27
c1 no khi vsubpd ymm2, ymm2, ymm5			;; next carry = y + upper mul-by-const bits * MINUS_C	; 26-28
c1 khi	vaddpd	ymm2, ymm2, ymm5			;; next carry = y + upper mul-by-const bits 		; 26-28
no c1	vaddpd	ymm2, ymm2, ymm5			;; next carry = y + upper mul-by-const bits 		; 26-28
	vsubpd	ymm8, ymm9, ymm8			;; z = y - (maximum*BIGVAL-BIGVAL)			; 27-29
c1 no khi vsubpd ymm9, ymm9, ymm13			;; next carry = y + upper mul-by-const bits * MINUS_C	; 28-30
c1 khi	vaddpd	ymm9, ymm9, ymm13			;; next carry = y + upper mul-by-const bits 		; 28-30
no c1	vaddpd	ymm9, ymm9, ymm13			;; next carry = y + upper mul-by-const bits 		; 28-30
	vsubpd	ymm0, ymm0, ymm1			;; rounded value = x - z				; 29-31
	vsubpd	ymm7, ymm7, ymm8			;; rounded value = x - z				; 30-32
	vmulpd	ymm2, ymm2, ymm6			;; shift next carry appropriately			; 29-33
	vmulpd	ymm9, ymm9, ymm15			;; shift next carry appropriately			; 31-35
ENDM

ynorm_wpn_zpad_base2_echk MACRO khi, c1, cm1, negc_const, klo_const, khi_const, basereg, basereg2
	vaddpd	ymm3, ymm0, ymm14			;; Round low input to multiple of big word		; 1-3
	vroundpd ymm4, ymm0, 0				;; Round low input to an integer			; 2-4
	vaddpd	ymm10, ymm7, ymm14			;; Round low input to multiple of big word		; 3-5
	vroundpd ymm12, ymm7, 0				;; Round low input to an integer			; 4-6
	vsubpd	ymm0, ymm0, ymm4			;; This is the convolution error			; 5-7
	vsubpd	ymm3, ymm3, ymm14										; 6-8
	vsubpd	ymm10, ymm10, ymm14										; 7-9
	vsubpd	ymm7, ymm7, ymm12			;; This is the convolution error			; 8-10
	absval	ymm0					;; Compute absolute value				;	8
	vmaxpd	ymm6, ymm6, ymm0			;; Compute maximum error				; 9-11
	vsubpd	ymm0, ymm4, ymm3			;; Compute low bigword bits of low FFT word		; 10-12		n 16
	vmulpd	ymm3, ymm3, YMM_LIMIT_INVERSE[basereg]	;; Saved shifted FFT hi data for next iteration		;  9-13
	vaddpd	ymm5, ymm1, ymm14			;; Round high input to multiple of big word		; 11-13		n 17
	absval	ymm7					;; Compute absolute value				;	11
	vmaxpd	ymm6, ymm6, ymm7			;; Compute maximum error				; 12-14
	vsubpd	ymm7, ymm12, ymm10			;; Compute low bigword bits of low FFT word		; 13-15		n 17
	vmulpd	ymm10, ymm10, YMM_LIMIT_INVERSE[basereg2];; Saved shifted FFT hi data for next iteration	;  10-14
	vaddpd	ymm13, ymm8, ymm14			;; Round high input to multiple of big word		; 14-16		n 20
	vroundpd ymm4, ymm1, 0				;; Round high input to an integer			; 15-17		n 18
	vroundpd ymm12, ymm8, 0				;; Round high input to an integer			; 16-18		n 19
	vsubpd	ymm5, ymm5, ymm14										; 17-19		n 24
	vsubpd	ymm1, ymm1, ymm4			;; This is the convolution error			; 18-20		n 21
	vsubpd	ymm8, ymm8, ymm12			;; This is the convolution error			; 19-21		n 22
	vsubpd	ymm13, ymm13, ymm14										; 20-22		n 25
	absval	ymm1					;; Compute absolute value				;	21
	absval	ymm8					;; Compute absolute value				;	22
	vmaxpd	ymm11, ymm1, ymm8			;; Compute maximum error				; 23-25		n 26
	vmovapd	ymm1, klo_const
	vmulpd	ymm8, ymm0, ymm1			;; Mul by YMM_K_LO or YMM_K_TIMES_MULCONST_LO		;  13-17	n 21
	vmulpd	ymm1, ymm7, ymm1			;; Mul by YMM_K_LO or YMM_K_TIMES_MULCONST_LO		;  16-20	n 22
	vaddpd	ymm2, ymm8, ymm2			;; x1 = values + carry					; 21-23		n 27
	vaddpd	ymm9, ymm1, ymm9			;; x1 = values + carry					; 22-24		n 28
	vsubpd	ymm4, ymm4, ymm5			;; Compute low bigword bits of high FFT word		; 24-26		n 27
	vsubpd	ymm12, ymm12, ymm13			;; Compute low bigword bits of high FFT word		; 25-27		n 28
	vmaxpd	ymm6, ymm6, ymm11			;; Compute maximum error				; 26-28
khi	vmovapd	ymm1, khi_const
khi	vmulpd	ymm8, ymm0, ymm1			;; Mul by YMM_K_HI or YMM_K_TIMES_MULCONST_HI		;	14-18
khi	vmulpd	ymm1, ymm7, ymm1			;; Mul by YMM_K_HI or YMM_K_TIMES_MULCONST_HI		;	17-21
no c1 no cm1 vmovapd ymm11, negc_const			;; Load YMM_MINUS_C or YMM_MINUS_C_TIMES_MULCONST
no c1 no cm1 vmulpd ymm5, ymm5, ymm11			;; Mul by YMM_MINUS_C or YMM_MINUS_C_TIMES_MULCONST	;	20-24
no c1 no cm1 vmulpd ymm13, ymm13, ymm11			;; Mul by YMM_MINUS_C or YMM_MINUS_C_TIMES_MULCONST	;	23-27
no c1 no cm1 vmulpd ymm4, ymm4, ymm11			;; Mul by YMM_MINUS_C or YMM_MINUS_C_TIMES_MULCONST	;	27-31
no c1 no cm1 vmulpd ymm12, ymm12, ymm11			;; Mul by YMM_MINUS_C or YMM_MINUS_C_TIMES_MULCONST	;	28-32
khi c1	vsubpd	ymm5, ymm8, ymm5			;; Add upper FFT word * MINUS_C to lower FFT word	;	19-23
khi no c1 vaddpd ymm5, ymm8, ymm5			;; Add upper FFT word to lower FFT word			;	19-23 or 25-28
khi c1	vsubpd	ymm13, ymm1, ymm13			;; Add upper FFT word * MINUS_C to lower FFT word	;	22-26
khi no c1 vaddpd ymm13, ymm1, ymm13			;; Add upper FFT word to lower FFT word			;	22-26 or 28-32
c1	vsubpd	ymm0, ymm2, ymm4			;; Add upper FFT word * MINUS_C to lower FFT word	; 27-29
no c1	vaddpd	ymm0, ymm2, ymm4			;; Add upper FFT word to lower FFT word			; 27-29
c1	vsubpd	ymm7, ymm9, ymm12			;; Add upper FFT word * MINUS_C to lower FFT word	; 28-30
no c1	vaddpd	ymm7, ymm9, ymm12			;; Add upper FFT word to lower FFT word			; 28-30
	vmovapd	ymm1, YMM_LIMIT_BIGMAX[basereg]		;; Load rounding constant
	vaddpd	ymm2, ymm0, ymm1			;; y = top bits of x					; 30-32
	vmovapd	ymm8, YMM_LIMIT_BIGMAX[basereg2]	;; Load rounding constant
	vaddpd	ymm9, ymm7, ymm8			;; y = top bits of x					; 31-33
	vsubpd	ymm1, ymm2, ymm1			;; z = y - (maximum*BIGVAL-BIGVAL)			; 33-35
	vsubpd	ymm8, ymm9, ymm8			;; z = y - (maximum*BIGVAL-BIGVAL)			; 34-36
c1 no khi vsubpd ymm2, ymm2, ymm5			;; next carry = y + upper mul-by-const bits * MINUS_C	; 35-37
c1 khi	vaddpd	ymm2, ymm2, ymm5			;; next carry = y + upper mul-by-const bits 		; 35-37
no c1	vaddpd	ymm2, ymm2, ymm5			;; next carry = y + upper mul-by-const bits 		; 35-37
c1 no khi vsubpd ymm9, ymm9, ymm13			;; next carry = y + upper mul-by-const bits * MINUS_C	; 36-38
c1 khi	vaddpd	ymm9, ymm9, ymm13			;; next carry = y + upper mul-by-const bits 		; 36-38
no c1	vaddpd	ymm9, ymm9, ymm13			;; next carry = y + upper mul-by-const bits 		; 36-38
	vsubpd	ymm0, ymm0, ymm1			;; rounded value = x - z				; 37-39
	vsubpd	ymm7, ymm7, ymm8			;; rounded value = x - z				; 38-40
	vmulpd	ymm2, ymm2, YMM_LIMIT_INVERSE[basereg]	;; shift next carry appropriately			; 38-42
	vmulpd	ymm9, ymm9, YMM_LIMIT_INVERSE[basereg2]	;; shift next carry appropriately			; 39-43
ENDM

IF (@INSTR(,%yarch,<FMA3>) NE 0)
ynorm_wpn_zpad_base2 MACRO khi, c1, cm1, klo_const, khi_const, basereg, basereg2
	vsubpd	ymm0, ymm0, ymm12			;; Round low input to an integer			; 1-3
	vsubpd	ymm7, ymm7, ymm12			;; Round low input to an integer			; 2-4
	vsubpd	ymm1, ymm1, ymm12			;; Round high input to an integer			; 3-5
	vaddpd	ymm3, ymm0, ymm14			;; Round low input to multiple of big word		; 4-6
	vaddpd	ymm10, ymm7, ymm14			;; Round low input to multiple of big word		; 5-7
	vsubpd	ymm8, ymm8, ymm12			;; Round high input to an integer			; 6-8
	vsubpd	ymm3, ymm3, ymm14										; 7-9
	vsubpd	ymm10, ymm10, ymm14										; 8-10
	vaddpd	ymm5, ymm1, ymm14			;; Round high input to multiple of big word		; 9-11
	vaddpd	ymm13, ymm8, ymm14			;; Round high input to multiple of big word		; 10-12
	vsubpd	ymm0, ymm0, ymm3			;; Compute low bigword bits of low FFT word		; 11-13
	vmovapd ymm6, YMM_LIMIT_INVERSE[basereg] 	;; Load limit inverse
	yfmaddpd ymm3, ymm3, ymm6, ymm12		;; Saved shifted FFT hi data for next iteration		;  10-14
	vsubpd	ymm7, ymm7, ymm10			;; Compute low bigword bits of low FFT word		; 12-14
	vmovapd ymm15, YMM_LIMIT_INVERSE[basereg2] 	;; Load limit inverse
	yfmaddpd ymm10, ymm10, ymm15, ymm12		;; Saved shifted FFT hi data for next iteration		;  11-15
	vsubpd	ymm5, ymm5, ymm14										; 13-15
	vsubpd	ymm13, ymm13, ymm14										; 14-16
	vmovapd	ymm4, klo_const
	yfmaddpd ymm2, ymm0, ymm4, ymm2			;; Mul by YMM_K_LO or YMM_K_TIMES_MULCONST_LO + carry	;  14-18
	yfmaddpd ymm9, ymm7, ymm4, ymm9			;; Mul by YMM_K_LO or YMM_K_TIMES_MULCONST_LO + carry	;  15-19
	vsubpd	ymm1, ymm1, ymm5			;; Compute low bigword bits of high FFT word		; 16-18
	vsubpd	ymm8, ymm8, ymm13			;; Compute low bigword bits of high FFT word		; 17-19
no c1 no cm1 vmulpd ymm5, ymm5, ymm11			;; Mul by YMM_MINUS_C or YMM_MINUS_C_TIMES_MULCONST	;	16-20
no c1 no cm1 vmulpd ymm13, ymm13, ymm11			;; Mul by YMM_MINUS_C or YMM_MINUS_C_TIMES_MULCONST	;	17-21
no c1 no cm1 vmulpd ymm1, ymm1, ymm11			;; Mul by YMM_MINUS_C or YMM_MINUS_C_TIMES_MULCONST	;	19-23
no c1 no cm1 vmulpd ymm8, ymm8, ymm11			;; Mul by YMM_MINUS_C or YMM_MINUS_C_TIMES_MULCONST	;	20-24
khi	vmovapd	ymm4, khi_const
khi c1	yfmsubpd ymm5, ymm0, ymm4, ymm5			;; Add upper FFT word * MINUS_C to lower FFT word	;	21-25
khi no c1 yfmaddpd ymm5, ymm0, ymm4, ymm5		;; Add upper FFT word to lower FFT word			;	21-25
khi c1	yfmsubpd ymm13, ymm7, ymm4, ymm13		;; Add upper FFT word * MINUS_C to lower FFT word	;	22-26
khi no c1 yfmaddpd ymm13, ymm7, ymm4, ymm13		;; Add upper FFT word to lower FFT word			;	22-26
c1	vsubpd	ymm0, ymm2, ymm1			;; Add upper FFT word * MINUS_C to lower FFT word	; 19-21
no c1	vaddpd	ymm0, ymm2, ymm1			;; Add upper FFT word to lower FFT word			; 19-21
c1	vsubpd	ymm7, ymm9, ymm8			;; Add upper FFT word * MINUS_C to lower FFT word	; 20-22
no c1	vaddpd	ymm7, ymm9, ymm8			;; Add upper FFT word to lower FFT word			; 20-22
	vmovapd	ymm1, YMM_LIMIT_BIGMAX[basereg]		;; Load rounding constant
	vaddpd	ymm2, ymm0, ymm1			;; y = top bits of x					; 21-23
	vmovapd	ymm8, YMM_LIMIT_BIGMAX[basereg2]	;; Load rounding constant
	vaddpd	ymm9, ymm7, ymm8			;; y = top bits of x					; 22-24
	vsubpd	ymm1, ymm2, ymm1			;; z = y - (maximum*BIGVAL-BIGVAL)			; 24-26
	vsubpd	ymm8, ymm9, ymm8			;; z = y - (maximum*BIGVAL-BIGVAL)			; 25-27
c1 no khi vsubpd ymm2, ymm2, ymm5			;; next carry = y + upper mul-by-const bits * MINUS_C	; 26-28
c1 khi	vaddpd	ymm2, ymm2, ymm5			;; next carry = y + upper mul-by-const bits 		; 26-28
no c1	vaddpd	ymm2, ymm2, ymm5			;; next carry = y + upper mul-by-const bits 		; 26-28
c1 no khi vsubpd ymm9, ymm9, ymm13			;; next carry = y + upper mul-by-const bits * MINUS_C	; 27-29
c1 khi	vaddpd	ymm9, ymm9, ymm13			;; next carry = y + upper mul-by-const bits 		; 27-29
no c1	vaddpd	ymm9, ymm9, ymm13			;; next carry = y + upper mul-by-const bits 		; 27-29
	vsubpd	ymm0, ymm0, ymm1			;; rounded value = x - z				; 28-30
	vsubpd	ymm7, ymm7, ymm8			;; rounded value = x - z				; 29-31
	vmulpd	ymm2, ymm2, ymm6			;; shift next carry appropriately			; 29-33
	vmulpd	ymm9, ymm9, ymm15			;; shift next carry appropriately			; 30-34
ENDM
ENDIF

IF (@INSTR(,%yarch,<FMA3>) NE 0)
ynorm_wpn_zpad_base2_echk MACRO khi, c1, cm1, negc_const, klo_const, khi_const, basereg, basereg2
	vsubpd	ymm5, ymm0, ymm3			;; Begin 2-step calc of low input convolution error	; 1-3
	vsubpd	ymm0, ymm0, ymm12			;; Round low input to an integer			; 2-4
	vsubpd	ymm11, ymm7, ymm10			;; Begin 2-step calc of low input convolution error	; 3-5
	vsubpd	ymm7, ymm7, ymm12			;; Round low input to an integer			; 4-6
	yfmsubpd ymm5, ymm4, [rsi+0*32], ymm5		;; This is the convolution error			;  4-8
	vsubpd	ymm1, ymm1, ymm12			;; Round high input to an integer			; 5-7
	vaddpd	ymm3, ymm0, ymm14			;; Round low input to multiple of big word		; 6-8
	yfmsubpd ymm11, ymm13, [r13+0*32], ymm11	;; This is the convolution error			;  6-10
	vaddpd	ymm10, ymm7, ymm14			;; Round low input to multiple of big word		; 7-9
	vsubpd	ymm8, ymm8, ymm12			;; Round high input to an integer			; 8-10
	yfmsubpd ymm4, ymm4, [rsi+1*32], ymm1		;; This is the convolution error			;  8-12
	absval	ymm5					;; Compute absolute value				;	9
	vsubpd	ymm3, ymm3, ymm14										; 9-11
	vsubpd	ymm10, ymm10, ymm14										; 10-12
	absval	ymm11					;; Compute absolute value				;	11
	vmaxpd	ymm6, ymm6, ymm5			;; Compute maximum error				; 11-13
	yfmsubpd ymm13, ymm13, [r13+1*32], ymm8		;; This is the convolution error			;  11-15
	vaddpd	ymm5, ymm1, ymm14			;; Round high input to multiple of big word		; 12-14
	absval	ymm4					;; Compute absolute value				;	13
	absval	ymm13					;; Compute absolute value				;	16
	vmaxpd	ymm4, ymm4, ymm13			;; Compute maximum error				; 17-19
	vaddpd	ymm13, ymm8, ymm14			;; Round high input to multiple of big word		; 13-15
	vsubpd	ymm0, ymm0, ymm3			;; Compute low bigword bits of low FFT word		; 14-16
	yfmaddpd ymm3, ymm3, YMM_LIMIT_INVERSE[basereg], ymm12 ;; Saved shifted FFT hi data for next iteration	;  14-18
	vsubpd	ymm7, ymm7, ymm10			;; Compute low bigword bits of low FFT word		; 15-17
	yfmaddpd ymm10, ymm10, YMM_LIMIT_INVERSE[basereg2], ymm12 ;; Saved shifted FFT hi data for next iter	;  15-19
	vsubpd	ymm5, ymm5, ymm14										; 16-18
	vsubpd	ymm13, ymm13, ymm14										; 18-20
	vmaxpd	ymm6, ymm6, ymm11			;; Compute maximum error				; 19-21
	vmovapd	ymm11, klo_const
	yfmaddpd ymm2, ymm0, ymm11, ymm2		;; Mul by YMM_K_LO or YMM_K_TIMES_MULCONST_LO + carry	;  17-21
	yfmaddpd ymm9, ymm7, ymm11, ymm9		;; Mul by YMM_K_LO or YMM_K_TIMES_MULCONST_LO + carry	;  18-22
	vsubpd	ymm1, ymm1, ymm5			;; Compute low bigword bits of high FFT word		; 20-22
	vsubpd	ymm8, ymm8, ymm13			;; Compute low bigword bits of high FFT word		; 21-23
	vmaxpd	ymm6, ymm6, ymm4			;; Compute maximum error				; 22-24
no c1 no cm1 vmovapd ymm11, negc_const			;; Load YMM_MINUS_C or YMM_MINUS_C_TIMES_MULCONST
no c1 no cm1 vmulpd ymm5, ymm5, ymm11			;; Mul by YMM_MINUS_C or YMM_MINUS_C_TIMES_MULCONST	;	19-23
no c1 no cm1 vmulpd ymm13, ymm13, ymm11			;; Mul by YMM_MINUS_C or YMM_MINUS_C_TIMES_MULCONST	;	21-25
no c1 no cm1 vmulpd ymm1, ymm1, ymm11			;; Mul by YMM_MINUS_C or YMM_MINUS_C_TIMES_MULCONST	;	23-27
no c1 no cm1 vmulpd ymm8, ymm8, ymm11			;; Mul by YMM_MINUS_C or YMM_MINUS_C_TIMES_MULCONST	;	24-28
khi	vmovapd	ymm11, khi_const			;; Load YMM_K_HI or YMM_K_HI_TIMES_MULCONST
khi c1	yfmsubpd ymm5, ymm0, ymm11, ymm5		;; Add upper FFT word * MINUS_C to lower FFT word	;	25-29
khi no c1 yfmaddpd ymm5, ymm0, ymm11, ymm5		;; Add upper FFT word to lower FFT word			;	25-29
khi c1	yfmsubpd ymm13, ymm7, ymm11, ymm13		;; Add upper FFT word * MINUS_C to lower FFT word	;	26-30
khi no c1 yfmaddpd ymm13, ymm7, ymm11, ymm13		;; Add upper FFT word to lower FFT word			;	26-30
c1	vsubpd	ymm0, ymm2, ymm1			;; Add upper FFT word * MINUS_C to lower FFT word	; 23-25
no c1	vaddpd	ymm0, ymm2, ymm1			;; Add upper FFT word to lower FFT word			; 23-25
c1	vsubpd	ymm7, ymm9, ymm8			;; Add upper FFT word * MINUS_C to lower FFT word	; 24-26
no c1	vaddpd	ymm7, ymm9, ymm8			;; Add upper FFT word to lower FFT word			; 24-26
	vmovapd	ymm1, YMM_LIMIT_BIGMAX[basereg]		;; Load rounding constant
	vaddpd	ymm2, ymm0, ymm1			;; y = top bits of x					; 26-28
	vmovapd	ymm8, YMM_LIMIT_BIGMAX[basereg2]	;; Load rounding constant
	vaddpd	ymm9, ymm7, ymm8			;; y = top bits of x					; 27-29
	vsubpd	ymm1, ymm2, ymm1			;; z = y - (maximum*BIGVAL-BIGVAL)			; 29-31
	vsubpd	ymm8, ymm9, ymm8			;; z = y - (maximum*BIGVAL-BIGVAL)			; 30-32
c1 no khi vsubpd ymm2, ymm2, ymm5			;; next carry = y + upper mul-by-const bits * MINUS_C	; 31-33
c1 khi	vaddpd	ymm2, ymm2, ymm5			;; next carry = y + upper mul-by-const bits 		; 31-33
no c1	vaddpd	ymm2, ymm2, ymm5			;; next carry = y + upper mul-by-const bits 		; 31-33
c1 no khi vsubpd ymm9, ymm9, ymm13			;; next carry = y + upper mul-by-const bits * MINUS_C	; 32-34
c1 khi	vaddpd	ymm9, ymm9, ymm13			;; next carry = y + upper mul-by-const bits 		; 32-34
no c1	vaddpd	ymm9, ymm9, ymm13			;; next carry = y + upper mul-by-const bits 		; 32-34
	vsubpd	ymm0, ymm0, ymm1			;; rounded value = x - z				; 33-35
	vsubpd	ymm7, ymm7, ymm8			;; rounded value = x - z				; 34-36
	vmulpd	ymm2, ymm2, YMM_LIMIT_INVERSE[basereg]	;; shift next carry appropriately			;  34-38
	vmulpd	ymm9, ymm9, YMM_LIMIT_INVERSE[basereg2]	;; shift next carry appropriately			;  35-39
ENDM
ENDIF

IF (@INSTR(,%yarch,<FMA3>) NE 0)
ynorm_wpn_zpad_base2_echk_nottp MACRO khi, c1, cm1, negc_const, klo_const, khi_const, basereg, basereg2
	vsubpd	ymm5, ymm0, ymm3			;; Begin 2-step calc of low input convolution error	; 1-3
	vsubpd	ymm0, ymm0, ymm12			;; Round low input to an integer			; 2-4
	vsubpd	ymm11, ymm7, ymm10			;; Begin 2-step calc of low input convolution error	; 3-5
	vsubpd	ymm7, ymm7, ymm12			;; Round low input to an integer			; 4-6
	vsubpd	ymm5, ymm5, [rsi+0*32]			;; This is the convolution error			;
	vsubpd	ymm1, ymm1, ymm12			;; Round high input to an integer			; 5-7
	vaddpd	ymm3, ymm0, ymm14			;; Round low input to multiple of big word		; 6-8
	vsubpd	ymm11, ymm11, [r13+0*32]		;; This is the convolution error			;
	vaddpd	ymm10, ymm7, ymm14			;; Round low input to multiple of big word		; 7-9
	vsubpd	ymm8, ymm8, ymm12			;; Round high input to an integer			; 8-10
	vsubpd	ymm4, ymm1, [rsi+1*32]			;; This is the convolution error			;
	absval	ymm5					;; Compute absolute value				;	9
	vsubpd	ymm3, ymm3, ymm14										; 9-11
	vsubpd	ymm10, ymm10, ymm14										; 10-12
	absval	ymm11					;; Compute absolute value				;	11
	vmaxpd	ymm6, ymm6, ymm5			;; Compute maximum error				; 11-13
	vsubpd	ymm13, ymm8, [r13+1*32]			;; This is the convolution error			;
	vaddpd	ymm5, ymm1, ymm14			;; Round high input to multiple of big word		; 12-14
	absval	ymm4					;; Compute absolute value				;	13
	absval	ymm13					;; Compute absolute value				;	16
	vmaxpd	ymm4, ymm4, ymm13			;; Compute maximum error				; 17-19
	vaddpd	ymm13, ymm8, ymm14			;; Round high input to multiple of big word		; 13-15
	vsubpd	ymm0, ymm0, ymm3			;; Compute low bigword bits of low FFT word		; 14-16
	yfmaddpd ymm3, ymm3, YMM_LIMIT_INVERSE[basereg], ymm12 ;; Saved shifted FFT hi data for next iteration	;  14-18
	vsubpd	ymm7, ymm7, ymm10			;; Compute low bigword bits of low FFT word		; 15-17
	yfmaddpd ymm10, ymm10, YMM_LIMIT_INVERSE[basereg2], ymm12 ;; Saved shifted FFT hi data for next iter	;  15-19
	vsubpd	ymm5, ymm5, ymm14										; 16-18
	vsubpd	ymm13, ymm13, ymm14										; 18-20
	vmaxpd	ymm6, ymm6, ymm11			;; Compute maximum error				; 19-21
	vmovapd	ymm11, klo_const
	yfmaddpd ymm2, ymm0, ymm11, ymm2		;; Mul by YMM_K_LO or YMM_K_TIMES_MULCONST_LO + carry	;  17-21
	yfmaddpd ymm9, ymm7, ymm11, ymm9		;; Mul by YMM_K_LO or YMM_K_TIMES_MULCONST_LO + carry	;  18-22
	vsubpd	ymm1, ymm1, ymm5			;; Compute low bigword bits of high FFT word		; 20-22
	vsubpd	ymm8, ymm8, ymm13			;; Compute low bigword bits of high FFT word		; 21-23
	vmaxpd	ymm6, ymm6, ymm4			;; Compute maximum error				; 22-24
no c1 no cm1 vmovapd ymm11, negc_const			;; Load YMM_MINUS_C or YMM_MINUS_C_TIMES_MULCONST
no c1 no cm1 vmulpd ymm5, ymm5, ymm11			;; Mul by YMM_MINUS_C or YMM_MINUS_C_TIMES_MULCONST	;	19-23
no c1 no cm1 vmulpd ymm13, ymm13, ymm11			;; Mul by YMM_MINUS_C or YMM_MINUS_C_TIMES_MULCONST	;	21-25
no c1 no cm1 vmulpd ymm1, ymm1, ymm11			;; Mul by YMM_MINUS_C or YMM_MINUS_C_TIMES_MULCONST	;	23-27
no c1 no cm1 vmulpd ymm8, ymm8, ymm11			;; Mul by YMM_MINUS_C or YMM_MINUS_C_TIMES_MULCONST	;	24-28
khi	vmovapd	ymm11, khi_const			;; Load YMM_K_HI or YMM_K_HI_TIMES_MULCONST
khi c1	yfmsubpd ymm5, ymm0, ymm11, ymm5		;; Add upper FFT word * MINUS_C to lower FFT word	;	25-29
khi no c1 yfmaddpd ymm5, ymm0, ymm11, ymm5		;; Add upper FFT word to lower FFT word			;	25-29
khi c1	yfmsubpd ymm13, ymm7, ymm11, ymm13		;; Add upper FFT word * MINUS_C to lower FFT word	;	26-30
khi no c1 yfmaddpd ymm13, ymm7, ymm11, ymm13		;; Add upper FFT word to lower FFT word			;	26-30
c1	vsubpd	ymm0, ymm2, ymm1			;; Add upper FFT word * MINUS_C to lower FFT word	; 23-25
no c1	vaddpd	ymm0, ymm2, ymm1			;; Add upper FFT word to lower FFT word			; 23-25
c1	vsubpd	ymm7, ymm9, ymm8			;; Add upper FFT word * MINUS_C to lower FFT word	; 24-26
no c1	vaddpd	ymm7, ymm9, ymm8			;; Add upper FFT word to lower FFT word			; 24-26
	vmovapd	ymm1, YMM_LIMIT_BIGMAX[basereg]		;; Load rounding constant
	vaddpd	ymm2, ymm0, ymm1			;; y = top bits of x					; 26-28
	vmovapd	ymm8, YMM_LIMIT_BIGMAX[basereg2]	;; Load rounding constant
	vaddpd	ymm9, ymm7, ymm8			;; y = top bits of x					; 27-29
	vsubpd	ymm1, ymm2, ymm1			;; z = y - (maximum*BIGVAL-BIGVAL)			; 29-31
	vsubpd	ymm8, ymm9, ymm8			;; z = y - (maximum*BIGVAL-BIGVAL)			; 30-32
c1 no khi vsubpd ymm2, ymm2, ymm5			;; next carry = y + upper mul-by-const bits * MINUS_C	; 31-33
c1 khi	vaddpd	ymm2, ymm2, ymm5			;; next carry = y + upper mul-by-const bits 		; 31-33
no c1	vaddpd	ymm2, ymm2, ymm5			;; next carry = y + upper mul-by-const bits 		; 31-33
c1 no khi vsubpd ymm9, ymm9, ymm13			;; next carry = y + upper mul-by-const bits * MINUS_C	; 32-34
c1 khi	vaddpd	ymm9, ymm9, ymm13			;; next carry = y + upper mul-by-const bits 		; 32-34
no c1	vaddpd	ymm9, ymm9, ymm13			;; next carry = y + upper mul-by-const bits 		; 32-34
	vsubpd	ymm0, ymm0, ymm1			;; rounded value = x - z				; 33-35
	vsubpd	ymm7, ymm7, ymm8			;; rounded value = x - z				; 34-36
	vmulpd	ymm2, ymm2, YMM_LIMIT_INVERSE[basereg]	;; shift next carry appropriately			;  34-38
	vmulpd	ymm9, ymm9, YMM_LIMIT_INVERSE[basereg2]	;; shift next carry appropriately			;  35-39
ENDM
ENDIF


ynorm_wpn_zpad_nobase2 MACRO khi, c1, cm1, negc_const, klo_const, khi_const, basereg, basereg2
	vmovapd	ymm4, YMM_LIMIT_INVERSE[basereg]	;; Load 1 / base
	vmovapd	ymm12, YMM_LIMIT_INVERSE[basereg2]	;; Load 1 / base
	vmulpd	ymm3, ymm0, ymm4			;; Compute low FFTvalue / base				; 1-5
	vmulpd	ymm5, ymm1, ymm4			;; Compute high FFTvalue / base				; 2-6
	vmulpd	ymm10, ymm7, ymm12			;; Compute low FFTvalue / base				; 3-7
	vmulpd	ymm13, ymm8, ymm12			;; Compute high FFTvalue / base				; 4-8
	vroundpd ymm0, ymm0, 0				;; Round low input to an integer			; 1-3
	vroundpd ymm1, ymm1, 0				;; Round high input to an integer			; 2-4
	vroundpd ymm7, ymm7, 0				;; Round low input to an integer			; 3-5
	vroundpd ymm8, ymm8, 0				;; Round high input to an integer			; 4-6
	vroundpd ymm3, ymm3, 0				;; Next carry = round (low FFTvalue / base)		; 6-8
	vroundpd ymm5, ymm5, 0				;; Round high FFTvalue / base to integer		; 7-9
	vroundpd ymm10, ymm10, 0			;; Next carry = round (low FFTvalue / base)		; 8-10
	vroundpd ymm13, ymm13, 0			;; Round high FFTvalue / base to integer		; 9-11
	vmovapd	ymm6, YMM_LIMIT_BIGMAX[basereg]		;; Load base
	vmulpd	ymm14, ymm3, ymm6			;; tmp = round (FFTvalue / base) * base			; 9-13
	vsubpd	ymm0, ymm0, ymm14			;; Compute FFTvalue % base  (FFTvalue - tmp)		; 14-16
	vmulpd	ymm14, ymm5, ymm6			;; tmp = round (FFTvalue / base) * base			; 10-14
	vsubpd	ymm1, ymm1, ymm14			;; Compute FFTvalue % base  (FFTvalue - tmp)		; 15-17
	vmovapd	ymm15, YMM_LIMIT_BIGMAX[basereg2]	;; Load base
	vmulpd	ymm14, ymm10, ymm15			;; tmp = round (FFTvalue / base) * base			; 11-15
	vsubpd	ymm7, ymm7, ymm14			;; Compute FFTvalue % base  (FFTvalue - tmp)		; 16-18
	vmulpd	ymm14, ymm13, ymm15			;; tmp = round (FFTvalue / base) * base			; 12-16
	vsubpd	ymm8, ymm8, ymm14			;; Compute FFTvalue % base  (FFTvalue - tmp)		; 17-19
	vmovapd	ymm11, klo_const			;; Load YMM_K_LO or YMM_K_TIMES_MULCONST_LO
	vmulpd	ymm14, ymm0, ymm11			;; Mul by YMM_K_LO or YMM_K_TIMES_MULCONST_LO		; 17-21
	vaddpd	ymm2, ymm14, ymm2			;; val = values + carry					; 22-24
	vmulpd	ymm14, ymm7, ymm11			;; Mul by YMM_K_LO or YMM_K_TIMES_MULCONST_LO		; 19-23
	vaddpd	ymm9, ymm14, ymm9			;; val = values + carry					; 24-26
no c1 no cm1 vmovapd ymm11, negc_const			;; Load YMM_MINUS_C or YMM_MINUS_C_TIMES_MULCONST
no c1 no cm1 vmulpd ymm5, ymm5, ymm11			;; Mul by YMM_MINUS_C or YMM_MINUS_C_TIMES_MULCONST	;	13-17
no c1 no cm1 vmulpd ymm13, ymm13, ymm11			;; Mul by YMM_MINUS_C or YMM_MINUS_C_TIMES_MULCONST	;	14-18
no c1 no cm1 vmulpd ymm1, ymm1, ymm11			;; Mul by YMM_MINUS_C or YMM_MINUS_C_TIMES_MULCONST	;	18-22
no c1 no cm1 vmulpd ymm8, ymm8, ymm11			;; Mul by YMM_MINUS_C or YMM_MINUS_C_TIMES_MULCONST	;	20-24
khi	vmovapd	ymm11, khi_const			;; Load YMM_K_HI or YMM_K_TIMES_MULCONST_HI
khi	vmulpd	ymm14, ymm11, ymm4			;; Non-base2 rounding requires shifted carry		;	early
khi	vroundpd ymm14, ymm14, 0			;; WASTEFUL.  The mul and round could be precomputed.	;	early
khi	vmulpd	ymm14, ymm0, ymm14			;; Mul by YMM_K_HI or YMM_K_TIMES_MULCONST_HI		;	18-22
khi c1	vsubpd	ymm5, ymm14, ymm5			;; Add upper FFT word * MINUS_C to lower FFT word	;	23-25
khi no c1 vaddpd ymm5, ymm14, ymm5			;; Add upper FFT word to lower FFT word			;	23-25
khi	vmulpd	ymm14, ymm11, ymm12			;; Non-base2 rounding requires shifted carry		;	early
khi	vroundpd ymm14, ymm14, 0			;; WASTEFUL.  The mul and round could be precomputed.	;	early
khi	vmulpd	ymm14, ymm7, ymm14			;; Mul by YMM_K_HI or YMM_K_TIMES_MULCONST_HI		;	20-24
khi c1	vsubpd	ymm13, ymm14, ymm13			;; Add upper FFT word * MINUS_C to lower FFT word	;	25-27
khi no c1 vaddpd ymm13, ymm14, ymm13			;; Add upper FFT word to lower FFT word			;	25-27
c1	vsubpd	ymm0, ymm2, ymm1			;; Add upper FFT word * MINUS_C to lower FFT word	; 25-27
no c1	vaddpd	ymm0, ymm2, ymm1			;; Add upper FFT word to lower FFT word			; 25-27
c1	vsubpd	ymm7, ymm9, ymm8			;; Add upper FFT word * MINUS_C to lower FFT word	; 27-29
no c1	vaddpd	ymm7, ymm9, ymm8			;; Add upper FFT word to lower FFT word			; 27-29
	vmulpd	ymm2, ymm0, ymm4			;; val / base						; 28-32
	vmulpd	ymm9, ymm7, ymm12			;; val / base						; 30-34
	vroundpd ymm2, ymm2, 0				;; y = round (val / base)				; 33-35
	vmulpd	ymm1, ymm2, ymm6			;; z = round (val / base) * base			; 33-37
	vroundpd ymm9, ymm9, 0				;; y = round (val / base)				; 35-37
	vmulpd	ymm8, ymm9, ymm15			;; z = round (val / base) * base			; 35-39
c1 no khi vsubpd ymm2, ymm2, ymm5			;; next carry = y + upper mul-by-const bits * MINUS_C	; 36-38
c1 khi	vaddpd	ymm2, ymm2, ymm5			;; next carry = y + upper mul-by-const bits 		; 36-38
no c1	vaddpd	ymm2, ymm2, ymm5			;; next carry = y + upper mul-by-const bits 		; 36-38
	vsubpd	ymm0, ymm0, ymm1			;; new value = val - z					; 38-40
c1 no khi vsubpd ymm9, ymm9, ymm13			;; next carry = y + upper mul-by-const bits * MINUS_C	; 39-41
c1 khi	vaddpd	ymm9, ymm9, ymm13			;; next carry = y + upper mul-by-const bits 		; 39-41
no c1	vaddpd	ymm9, ymm9, ymm13			;; next carry = y + upper mul-by-const bits 		; 39-41
	vsubpd	ymm7, ymm7, ymm8			;; new value = val - z					; 40-42
ENDM

ynorm_wpn_zpad_nobase2_echk MACRO khi, c1, cm1, negc_const, klo_const, khi_const, basereg, basereg2
	vmovapd	ymm4, YMM_LIMIT_INVERSE[basereg]	;; Load 1 / base
	vmovapd	ymm12, YMM_LIMIT_INVERSE[basereg2]	;; Load 1 / base
	vmulpd	ymm3, ymm0, ymm4			;; Compute low FFTvalue / base				;  1-5
	vmulpd	ymm5, ymm1, ymm4			;; Compute high FFTvalue / base				;  2-6
	vmulpd	ymm10, ymm7, ymm12			;; Compute low FFTvalue / base				;  3-7
	vmulpd	ymm13, ymm8, ymm12			;; Compute high FFTvalue / base				;  4-8
	vroundpd ymm11, ymm0, 0				;; Round low input to an integer			; 1-3
	vroundpd ymm14, ymm1, 0				;; Round high input to an integer			; 2-4
	vsubpd	ymm0, ymm0, ymm11			;; This is the convolution error			; 4-6
	vsubpd	ymm1, ymm1, ymm14			;; This is the convolution error			; 5-7
	absval	ymm0					;; Compute absolute value				;	7
	absval	ymm1					;; Compute absolute value				;	8
	vmaxpd	ymm6, ymm6, ymm0			;; Compute maximum error				; 8-10
	vroundpd ymm0, ymm7, 0				;; Round low input to an integer			; 3-5
	vmaxpd	ymm6, ymm6, ymm1			;; Compute maximum error				; 11-13
	vroundpd ymm1, ymm8, 0				;; Round high input to an integer			; 6-8
	vroundpd ymm3, ymm3, 0				;; Next carry = round (low FFTvalue / base)		; 7-9
	vroundpd ymm5, ymm5, 0				;; Round high FFTvalue / base to integer		; 9-11
	vroundpd ymm10, ymm10, 0			;; Next carry = round (low FFTvalue / base)		; 10-12
	vroundpd ymm13, ymm13, 0			;; Round high FFTvalue / base to integer		; 12-14
	vsubpd	ymm7, ymm7, ymm0			;; This is the convolution error			; 13-15
	vsubpd	ymm8, ymm8, ymm1			;; This is the convolution error			; 14-16
	absval	ymm7					;; Compute absolute value				;	16
	absval	ymm8					;; Compute absolute value				;	17
	vmaxpd	ymm6, ymm6, ymm7			;; Compute maximum error				; 17-19
	vmaxpd	ymm6, ymm6, ymm8			;; Compute maximum error				; 20-22
	vmovapd	ymm7, YMM_LIMIT_BIGMAX[basereg]		;; Load base
	vmulpd	ymm8, ymm3, ymm7			;; tmp = round (FFTvalue / base) * base			;  10-14
	vsubpd	ymm11, ymm11, ymm8			;; Compute FFTvalue % base  (FFTvalue - tmp)		; 15-17
	vmulpd	ymm8, ymm5, ymm7			;; tmp = round (FFTvalue / base) * base			;  12-16
	vsubpd	ymm14, ymm14, ymm8			;; Compute FFTvalue % base  (FFTvalue - tmp)		; 18-20
	vmovapd	ymm8, YMM_LIMIT_BIGMAX[basereg2]	;; Load base
	vmulpd	ymm7, ymm10, ymm8			;; tmp = round (FFTvalue / base) * base			;  11-15
	vsubpd	ymm7, ymm0, ymm7			;; Compute FFTvalue % base  (FFTvalue - tmp)		; 16-18
	vmulpd	ymm8, ymm13, ymm8			;; tmp = round (FFTvalue / base) * base			; 15-19
	vsubpd	ymm8, ymm1, ymm8			;; Compute FFTvalue % base  (FFTvalue - tmp)		; 21-23
	vmovapd	ymm0, klo_const				;; Load YMM_K_LO or YMM_K_TIMES_MULCONST_LO
	vmulpd	ymm1, ymm11, ymm0			;; Mul by YMM_K_LO or YMM_K_TIMES_MULCONST_LO		;  18-22
	vaddpd	ymm2, ymm1, ymm2			;; val = values + carry					; 23-25
	vmulpd	ymm1, ymm7, ymm0			;; Mul by YMM_K_LO or YMM_K_TIMES_MULCONST_LO		;  19-23
	vaddpd	ymm9, ymm1, ymm9			;; val = values + carry					; 24-26
no c1 no cm1 vmovapd ymm0, negc_const			;; Load YMM_MINUS_C or YMM_MINUS_C_TIMES_MULCONST
no c1 no cm1 vmulpd ymm5, ymm5, ymm0			;; Mul by YMM_MINUS_C or YMM_MINUS_C_TIMES_MULCONST	;	13-17
no c1 no cm1 vmulpd ymm13, ymm13, ymm0			;; Mul by YMM_MINUS_C or YMM_MINUS_C_TIMES_MULCONST	;	15-19
no c1 no cm1 vmulpd ymm14, ymm14, ymm0			;; Mul by YMM_MINUS_C or YMM_MINUS_C_TIMES_MULCONST	;	21-25
no c1 no cm1 vmulpd ymm8, ymm8, ymm0			;; Mul by YMM_MINUS_C or YMM_MINUS_C_TIMES_MULCONST	;	24-28
khi	vmovapd	ymm0, khi_const				;; Load YMM_K_HI or YMM_K_TIMES_MULCONST_HI
khi	vmulpd	ymm1, ymm0, ymm4			;; Non-base2 rounding requires shifted carry		;	early
khi	vroundpd ymm1, ymm1, 0				;; WASTEFUL.  The mul and round could be precomputed.	;	early
khi	vmulpd	ymm1, ymm11, ymm1			;; Mul by YMM_K_HI or YMM_K_TIMES_MULCONST_HI		;	20-24
khi c1	vsubpd	ymm5, ymm1, ymm5			;; Add upper FFT word * MINUS_C to lower FFT word	;	25-27
khi no c1 vaddpd ymm5, ymm1, ymm5			;; Add upper FFT word to lower FFT word			;	25-27
khi	vmulpd	ymm1, ymm0, ymm12			;; Non-base2 rounding requires shifted carry		;	early
khi	vroundpd ymm1, ymm1, 0				;; WASTEFUL.  The mul and round could be precomputed.	;	early
khi	vmulpd	ymm1, ymm7, ymm1			;; Mul by YMM_K_HI or YMM_K_TIMES_MULCONST_HI		;	21-25
khi c1	vsubpd	ymm13, ymm1, ymm13			;; Add upper FFT word * MINUS_C to lower FFT word	;	26-28
khi no c1 vaddpd ymm13, ymm1, ymm13			;; Add upper FFT word to lower FFT word			;	26-28
c1	vsubpd	ymm0, ymm2, ymm14			;; Add upper FFT word * MINUS_C to lower FFT word	; 26-28
no c1	vaddpd	ymm0, ymm2, ymm14			;; Add upper FFT word to lower FFT word			; 26-28
c1	vsubpd	ymm7, ymm9, ymm8			;; Add upper FFT word * MINUS_C to lower FFT word	; 27-29
no c1	vaddpd	ymm7, ymm9, ymm8			;; Add upper FFT word to lower FFT word			; 27-29
	vmulpd	ymm2, ymm0, ymm4			;; val / base						;  29-33
	vmulpd	ymm9, ymm7, ymm12			;; val / base						;  30-34
	vroundpd ymm2, ymm2, 0				;; y = round (val / base)				; 34-36
	vmulpd	ymm1, ymm2, YMM_LIMIT_BIGMAX[basereg]	;; z = round (val / base) * base			;  34-38
	vroundpd ymm9, ymm9, 0				;; y = round (val / base)				; 35-37
	vmulpd	ymm8, ymm9, YMM_LIMIT_BIGMAX[basereg2]	;; z = round (val / base) * base			;  35-39
c1 no khi vsubpd ymm2, ymm2, ymm5			;; next carry = y + upper mul-by-const bits * MINUS_C	; 37-39
c1 khi	vaddpd	ymm2, ymm2, ymm5			;; next carry = y + upper mul-by-const bits 		; 37-39
no c1	vaddpd	ymm2, ymm2, ymm5			;; next carry = y + upper mul-by-const bits 		; 37-39
	vsubpd	ymm0, ymm0, ymm1			;; new value = val - z					; 38-40
c1 no khi vsubpd ymm9, ymm9, ymm13			;; next carry = y + upper mul-by-const bits * MINUS_C	; 39-41
c1 khi	vaddpd	ymm9, ymm9, ymm13			;; next carry = y + upper mul-by-const bits 		; 39-41
no c1	vaddpd	ymm9, ymm9, ymm13			;; next carry = y + upper mul-by-const bits 		; 39-41
	vsubpd	ymm7, ymm7, ymm8			;; new value = val - z					; 40-42
ENDM


IF (@INSTR(,%yarch,<FMA3>) NE 0)
ynorm_wpn_zpad_nobase2 MACRO khi, c1, cm1, negc_const, klo_const, khi_const, basereg, basereg2
	vsubpd	ymm0, ymm0, ymm12			;; Round low input to an integer			; 1-3
	vsubpd	ymm1, ymm1, ymm12			;; Round high input to an integer			; 2-4
	vsubpd	ymm7, ymm7, ymm12			;; Round low input to an integer			; 3-5
	vsubpd	ymm8, ymm8, ymm12			;; Round high input to an integer			; 4-6
	vmovapd	ymm4, YMM_LIMIT_INVERSE[basereg]	;; Load 1 / base
	vmovapd	ymm14, YMM_LIMIT_INVERSE[basereg2]	;; Load 1 / base
	yfmaddpd ymm3, ymm0, ymm4, ymm12		;; Compute low FFTvalue / base				; 1-5
	yfmaddpd ymm5, ymm1, ymm4, ymm12		;; Compute high FFTvalue / base				; 2-6
	yfmaddpd ymm10, ymm7, ymm14, ymm12		;; Compute low FFTvalue / base				; 3-7
	yfmaddpd ymm13, ymm8, ymm14, ymm12		;; Compute high FFTvalue / base				; 4-8
	vsubpd	ymm3, ymm3, ymm12			;; Next carry = round (low FFTvalue / base)		; 6-8
	vsubpd	ymm5, ymm5, ymm12			;; Round high FFTvalue / base to integer		; 7-9
	vsubpd	ymm10, ymm10, ymm12			;; Next carry = round (low FFTvalue / base)		; 8-10
	vsubpd	ymm13, ymm13, ymm12			;; Round high FFTvalue / base to integer		; 9-11
	vmovapd	ymm6, YMM_LIMIT_BIGMAX[basereg]		;; Load base
	yfnmaddpd ymm0, ymm3, ymm6, ymm0		;; Compute low FFTvalue % base				; 9-13
	yfnmaddpd ymm1, ymm5, ymm6, ymm1		;; Compute high FFTvalue % base				; 10-14
	vmovapd	ymm15, YMM_LIMIT_BIGMAX[basereg2]	;; Load base
	yfnmaddpd ymm7, ymm10, ymm15, ymm7		;; Compute low FFTvalue % base				; 11-15
	yfnmaddpd ymm8, ymm13, ymm15, ymm8		;; Compute high FFTvalue % base				; 12-16
	vaddpd	ymm3, ymm3, ymm12			;; Add rounding const to carry				; 10-12
	vaddpd	ymm10, ymm10, ymm12			;; Add rounding const to carry				; 11-13
	vmovapd	ymm11, klo_const			;; Load YMM_K_LO or YMM_K_TIMES_MULCONST_LO
	yfmaddpd ymm2, ymm0, ymm11, ymm2		;; Mul by YMM_K_LO or YMM_K_TIMES_MULCONST_LO + carry	; 14-18
	yfmaddpd ymm9, ymm7, ymm11, ymm9		;; Mul by YMM_K_LO or YMM_K_TIMES_MULCONST_LO + carry	; 16-20
no c1 no cm1 vmovapd ymm11, negc_const			;; Load YMM_MINUS_C or YMM_MINUS_C_TIMES_MULCONST
no c1 no cm1 vmulpd ymm5, ymm5, ymm11			;; Mul by YMM_MINUS_C or YMM_MINUS_C_TIMES_MULCONST	;	10-14
no c1 no cm1 vmulpd ymm13, ymm13, ymm11			;; Mul by YMM_MINUS_C or YMM_MINUS_C_TIMES_MULCONST	;	12-16
no c1 no cm1 vmulpd ymm1, ymm1, ymm11			;; Mul by YMM_MINUS_C or YMM_MINUS_C_TIMES_MULCONST	;	15-19
no c1 no cm1 vmulpd ymm8, ymm8, ymm11			;; Mul by YMM_MINUS_C or YMM_MINUS_C_TIMES_MULCONST	;	17-21
khi	yfmaddpd ymm11, ymm4, khi_const, ymm12		;; Non-base2 rounding requires shifted carry		;	early
khi	vsubpd	ymm11, ymm11, ymm12			;; WASTEFUL.  The mul and round could be precomputed.	;	12-14
khi c1	yfmsubpd ymm5, ymm0, ymm11, ymm5		;; Add upper FFT word * MINUS_C to lower FFT word	;	15-19
khi no c1 yfmaddpd ymm5, ymm0, ymm11, ymm5		;; Add upper FFT word to lower FFT word			;	15-19
khi	yfmaddpd ymm11, ymm14, khi_const, ymm12		;; Non-base2 rounding requires shifted carry		;	early
khi	vsubpd	ymm11, ymm11, ymm12			;; WASTEFUL.  The mul and round could be precomputed.	;	13-15
khi c1	yfmsubpd ymm13, ymm7, ymm11, ymm13		;; Add upper FFT word * MINUS_C to lower FFT word	;	17-21
khi no c1 yfmaddpd ymm13, ymm7, ymm11, ymm13		;; Add upper FFT word to lower FFT word			;	17-21
c1	vsubpd	ymm0, ymm2, ymm1			;; Add upper FFT word * MINUS_C to lower FFT word	; 19-21
no c1	vaddpd	ymm0, ymm2, ymm1			;; Add upper FFT word to lower FFT word			; 19-21
c1	vsubpd	ymm7, ymm9, ymm8			;; Add upper FFT word * MINUS_C to lower FFT word	; 21-23
no c1	vaddpd	ymm7, ymm9, ymm8			;; Add upper FFT word to lower FFT word			; 21-23
	yfmaddpd ymm2, ymm0, ymm4, ymm12		;; val / base						; 22-26
	yfmaddpd ymm9, ymm7, ymm14, ymm12		;; val / base						; 24-28
	vsubpd	ymm2, ymm2, ymm12			;; y = round (val / base)				; 27-29
	vsubpd	ymm9, ymm9, ymm12			;; y = round (val / base)				; 29-31
	yfnmaddpd ymm0, ymm2, ymm6, ymm0		;; new value = val - round (val / base) * base		; 30-34
c1 no khi vsubpd ymm2, ymm2, ymm5			;; next carry = y + upper mul-by-const bits * MINUS_C	; 30-32
c1 khi	vaddpd	ymm2, ymm2, ymm5			;; next carry = y + upper mul-by-const bits 		; 30-32
no c1	vaddpd	ymm2, ymm2, ymm5			;; next carry = y + upper mul-by-const bits 		; 30-32
	yfnmaddpd ymm7, ymm9, ymm15, ymm7		;; new value = val - round (val / base) * base		; 32-36
c1 no khi vsubpd ymm9, ymm9, ymm13			;; next carry = y + upper mul-by-const bits * MINUS_C	; 32-34
c1 khi	vaddpd	ymm9, ymm9, ymm13			;; next carry = y + upper mul-by-const bits 		; 32-34
no c1	vaddpd	ymm9, ymm9, ymm13			;; next carry = y + upper mul-by-const bits 		; 32-34
ENDM
ENDIF

IF (@INSTR(,%yarch,<FMA3>) NE 0)
ynorm_wpn_zpad_nobase2_echk MACRO khi, c1, cm1, negc_const, klo_const, khi_const, basereg, basereg2
	vsubpd	ymm5, ymm0, ymm3			;; Begin 2-step calc of low input convolution error	; 1-3
	vsubpd	ymm0, ymm0, ymm12			;; Round low input to an integer			; 2-4
	vsubpd	ymm1, ymm1, ymm12			;; Round high input to an integer			; 3-5
	vsubpd	ymm11, ymm7, ymm10			;; Begin 2-step calc of low input convolution error	; 4-6
	vsubpd	ymm7, ymm7, ymm12			;; Round low input to an integer			; 5-7
	vsubpd	ymm8, ymm8, ymm12			;; Round high input to an integer			; 6-8
	yfmsubpd ymm5, ymm4, [rsi+0*32], ymm5		;; This is the convolution error			;  4-8
	yfmsubpd ymm4, ymm4, [rsi+1*32], ymm1		;; This is the convolution error			;  6-10
	yfmsubpd ymm11, ymm13, [r13+0*32], ymm11	;; This is the convolution error			;  7-11
	yfmsubpd ymm13, ymm13, [r13+1*32], ymm8		;; This is the convolution error			;  9-13
	absval	ymm5					;; Compute absolute value				;	9
	absval	ymm4					;; Compute absolute value				;	11
	vmaxpd	ymm6, ymm6, ymm5			;; Compute maximum error				; 11-13
	vmaxpd	ymm6, ymm6, ymm4			;; Compute maximum error				; 14-16
	vmovapd	ymm4, YMM_LIMIT_INVERSE[basereg]	;; Load 1 / base
	yfmaddpd ymm3, ymm0, ymm4, ymm12		;; Compute low FFTvalue / base				;  5-9
	yfmaddpd ymm5, ymm1, ymm4, ymm12		;; Compute high FFTvalue / base				;  7-11
	vmovapd	ymm14, YMM_LIMIT_INVERSE[basereg2]	;; Load 1 / base
	yfmaddpd ymm10, ymm7, ymm14, ymm12		;; Compute low FFTvalue / base				;  8-12
	absval	ymm13					;; Compute absolute value				;	14
	vmaxpd	ymm6, ymm6, ymm13			;; Compute maximum error				; 17-19
	yfmaddpd ymm13, ymm8, ymm14, ymm12		;; Compute high FFTvalue / base				;  10-14
	vsubpd	ymm3, ymm3, ymm12			;; Next carry = round (low FFTvalue / base)		; 10-12
	vsubpd	ymm5, ymm5, ymm12			;; Round high FFTvalue / base to integer		; 12-14
	absval	ymm11					;; Compute absolute value				;	12
	vsubpd	ymm10, ymm10, ymm12			;; Next carry = round (low FFTvalue / base)		; 13-15
	vsubpd	ymm13, ymm13, ymm12			;; Round high FFTvalue / base to integer		; 15-17
	vmaxpd	ymm6, ymm6, ymm11			;; Compute maximum error				; 20-22
	vmovapd	ymm11, YMM_LIMIT_BIGMAX[basereg]	;; Load base
	yfnmaddpd ymm0, ymm3, ymm11, ymm0		;; Compute low FFTvalue % base				;  13-17
	yfnmaddpd ymm1, ymm5, ymm11, ymm1		;; Compute high FFTvalue % base				;  15-19
	vaddpd	ymm3, ymm3, ymm12			;; Add rounding const to carry				; 14-16
	vmovapd	ymm11, YMM_LIMIT_BIGMAX[basereg2]	;; Load base
	yfnmaddpd ymm7, ymm10, ymm11, ymm7		;; Compute low FFTvalue % base				;  16-20
	yfnmaddpd ymm8, ymm13, ymm11, ymm8		;; Compute high FFTvalue % base				;  18-22
	vaddpd	ymm10, ymm10, ymm12			;; Add rounding const to carry				; 16-18
	vmovapd	ymm11, klo_const			;; Load YMM_K_LO or YMM_K_TIMES_MULCONST_LO
	yfmaddpd ymm2, ymm0, ymm11, ymm2		;; Mul by YMM_K_LO or YMM_K_TIMES_MULCONST_LO + carry	;  19-23
	yfmaddpd ymm9, ymm7, ymm11, ymm9		;; Mul by YMM_K_LO or YMM_K_TIMES_MULCONST_LO + carry	;  21-25
no c1 no cm1 vmovapd ymm11, negc_const			;; Load YMM_MINUS_C or YMM_MINUS_C_TIMES_MULCONST
no c1 no cm1 vmulpd ymm5, ymm5, ymm11			;; Mul by YMM_MINUS_C or YMM_MINUS_C_TIMES_MULCONST	;	17-21
no c1 no cm1 vmulpd ymm13, ymm13, ymm11			;; Mul by YMM_MINUS_C or YMM_MINUS_C_TIMES_MULCONST	;	19-23
no c1 no cm1 vmulpd ymm1, ymm1, ymm11			;; Mul by YMM_MINUS_C or YMM_MINUS_C_TIMES_MULCONST	;	20-24
no c1 no cm1 vmulpd ymm8, ymm8, ymm11			;; Mul by YMM_MINUS_C or YMM_MINUS_C_TIMES_MULCONST	;	23-27
khi	yfmaddpd ymm11, ymm4, khi_const, ymm12		;; Non-base2 rounding requires shifted carry		;	early
khi	vsubpd	ymm11, ymm11, ymm12			;; WASTEFUL.  The mul and round could be precomputed.	;	early
khi c1	yfmsubpd ymm5, ymm0, ymm11, ymm5		;; Add upper FFT word * MINUS_C to lower FFT word	;	17-21
khi no c1 yfmaddpd ymm5, ymm0, ymm11, ymm5		;; Add upper FFT word to lower FFT word			;	17-21
khi	yfmaddpd ymm11, ymm14, khi_const, ymm12		;; Non-base2 rounding requires shifted carry		;	early
khi	vsubpd	ymm11, ymm11, ymm12			;; WASTEFUL.  The mul and round could be precomputed.	;	early
khi c1	yfmsubpd ymm13, ymm7, ymm11, ymm13		;; Add upper FFT word * MINUS_C to lower FFT word	;	19-23
khi no c1 yfmaddpd ymm13, ymm7, ymm11, ymm13		;; Add upper FFT word to lower FFT word			;	19-23
c1	vsubpd	ymm0, ymm2, ymm1			;; Add upper FFT word * MINUS_C to lower FFT word	; 24-26
no c1	vaddpd	ymm0, ymm2, ymm1			;; Add upper FFT word to lower FFT word			; 24-26
c1	vsubpd	ymm7, ymm9, ymm8			;; Add upper FFT word * MINUS_C to lower FFT word	; 26-28
no c1	vaddpd	ymm7, ymm9, ymm8			;; Add upper FFT word to lower FFT word			; 26-28
	yfmaddpd ymm2, ymm0, ymm4, ymm12		;; val / base						;  27-31
	yfmaddpd ymm9, ymm7, ymm14, ymm12		;; val / base						;  29-33
	vsubpd	ymm2, ymm2, ymm12			;; y = round (val / base)				; 32-34
	vsubpd	ymm9, ymm9, ymm12			;; y = round (val / base)				; 34-36
	yfnmaddpd ymm0, ymm2, YMM_LIMIT_BIGMAX[basereg], ymm0 ;; new value = val - round (val / base) * base	;  35-39
c1 no khi vsubpd ymm2, ymm2, ymm5			;; next carry = y + upper mul-by-const bits * MINUS_C	; 35-37
c1 khi	vaddpd	ymm2, ymm2, ymm5			;; next carry = y + upper mul-by-const bits 		; 35-37
no c1	vaddpd	ymm2, ymm2, ymm5			;; next carry = y + upper mul-by-const bits 		; 35-37
	yfnmaddpd ymm7, ymm9, YMM_LIMIT_BIGMAX[basereg2], ymm7 ;; new value = val - round (val / base) * base	;  37-41
c1 no khi vsubpd ymm9, ymm9, ymm13			;; next carry = y + upper mul-by-const bits * MINUS_C	; 37-39
c1 khi	vaddpd	ymm9, ymm9, ymm13			;; next carry = y + upper mul-by-const bits 		; 37-39
no c1	vaddpd	ymm9, ymm9, ymm13			;; next carry = y + upper mul-by-const bits 		; 37-39
ENDM
ENDIF

IF (@INSTR(,%yarch,<FMA3>) NE 0)
ynorm_wpn_zpad_nobase2_echk_nottp MACRO khi, c1, cm1, negc_const, klo_const, khi_const, basereg, basereg2
	vsubpd	ymm5, ymm0, ymm3			;; Begin 2-step calc of low input convolution error	; 1-3
	vsubpd	ymm0, ymm0, ymm12			;; Round low input to an integer			; 2-4
	vsubpd	ymm1, ymm1, ymm12			;; Round high input to an integer			; 3-5
	vsubpd	ymm11, ymm7, ymm10			;; Begin 2-step calc of low input convolution error	; 4-6
	vsubpd	ymm7, ymm7, ymm12			;; Round low input to an integer			; 5-7
	vsubpd	ymm8, ymm8, ymm12			;; Round high input to an integer			; 6-8
	vsubpd	ymm5, ymm5, [rsi+0*32]			;; This	is the convolution error			; ?
	vsubpd	ymm4, ymm1, [rsi+1*32]			;; This is the convolution error			; ?
	vsubpd	ymm11, ymm11, [r13+0*32]		;; This is the convolution error			; ?
	vsubpd	ymm13, ymm8, [r13+1*32]			;; This is the convolution error			; ?
	absval	ymm5					;; Compute absolute value				;	9
	absval	ymm4					;; Compute absolute value				;	11
	vmaxpd	ymm6, ymm6, ymm5			;; Compute maximum error				; 11-13
	vmaxpd	ymm6, ymm6, ymm4			;; Compute maximum error				; 14-16
	vmovapd	ymm4, YMM_LIMIT_INVERSE[basereg]	;; Load 1 / base
	yfmaddpd ymm3, ymm0, ymm4, ymm12		;; Compute low FFTvalue / base				;  5-9
	yfmaddpd ymm5, ymm1, ymm4, ymm12		;; Compute high FFTvalue / base				;  7-11
	vmovapd	ymm14, YMM_LIMIT_INVERSE[basereg2]	;; Load 1 / base
	yfmaddpd ymm10, ymm7, ymm14, ymm12		;; Compute low FFTvalue / base				;  8-12
	absval	ymm13					;; Compute absolute value				;	14
	vmaxpd	ymm6, ymm6, ymm13			;; Compute maximum error				; 17-19
	yfmaddpd ymm13, ymm8, ymm14, ymm12		;; Compute high FFTvalue / base				;  10-14
	vsubpd	ymm3, ymm3, ymm12			;; Next carry = round (low FFTvalue / base)		; 10-12
	vsubpd	ymm5, ymm5, ymm12			;; Round high FFTvalue / base to integer		; 12-14
	absval	ymm11					;; Compute absolute value				;	12
	vsubpd	ymm10, ymm10, ymm12			;; Next carry = round (low FFTvalue / base)		; 13-15
	vsubpd	ymm13, ymm13, ymm12			;; Round high FFTvalue / base to integer		; 15-17
	vmaxpd	ymm6, ymm6, ymm11			;; Compute maximum error				; 20-22
	vmovapd	ymm11, YMM_LIMIT_BIGMAX[basereg]	;; Load base
	yfnmaddpd ymm0, ymm3, ymm11, ymm0		;; Compute low FFTvalue % base				;  13-17
	yfnmaddpd ymm1, ymm5, ymm11, ymm1		;; Compute high FFTvalue % base				;  15-19
	vaddpd	ymm3, ymm3, ymm12			;; Add rounding const to carry				; 14-16
	vmovapd	ymm11, YMM_LIMIT_BIGMAX[basereg2]	;; Load base
	yfnmaddpd ymm7, ymm10, ymm11, ymm7		;; Compute low FFTvalue % base				;  16-20
	yfnmaddpd ymm8, ymm13, ymm11, ymm8		;; Compute high FFTvalue % base				;  18-22
	vaddpd	ymm10, ymm10, ymm12			;; Add rounding const to carry				; 16-18
	vmovapd	ymm11, klo_const			;; Load YMM_K_LO or YMM_K_TIMES_MULCONST_LO
	yfmaddpd ymm2, ymm0, ymm11, ymm2		;; Mul by YMM_K_LO or YMM_K_TIMES_MULCONST_LO + carry	;  19-23
	yfmaddpd ymm9, ymm7, ymm11, ymm9		;; Mul by YMM_K_LO or YMM_K_TIMES_MULCONST_LO + carry	;  21-25
no c1 no cm1 vmovapd ymm11, negc_const			;; Load YMM_MINUS_C or YMM_MINUS_C_TIMES_MULCONST
no c1 no cm1 vmulpd ymm5, ymm5, ymm11			;; Mul by YMM_MINUS_C or YMM_MINUS_C_TIMES_MULCONST	;	17-21
no c1 no cm1 vmulpd ymm13, ymm13, ymm11			;; Mul by YMM_MINUS_C or YMM_MINUS_C_TIMES_MULCONST	;	19-23
no c1 no cm1 vmulpd ymm1, ymm1, ymm11			;; Mul by YMM_MINUS_C or YMM_MINUS_C_TIMES_MULCONST	;	20-24
no c1 no cm1 vmulpd ymm8, ymm8, ymm11			;; Mul by YMM_MINUS_C or YMM_MINUS_C_TIMES_MULCONST	;	23-27
khi	yfmaddpd ymm11, ymm4, khi_const, ymm12		;; Non-base2 rounding requires shifted carry		;	early
khi	vsubpd	ymm11, ymm11, ymm12			;; WASTEFUL.  The mul and round could be precomputed.	;	early
khi c1	yfmsubpd ymm5, ymm0, ymm11, ymm5		;; Add upper FFT word * MINUS_C to lower FFT word	;	17-21
khi no c1 yfmaddpd ymm5, ymm0, ymm11, ymm5		;; Add upper FFT word to lower FFT word			;	17-21
khi	yfmaddpd ymm11, ymm14, khi_const, ymm12		;; Non-base2 rounding requires shifted carry		;	early
khi	vsubpd	ymm11, ymm11, ymm12			;; WASTEFUL.  The mul and round could be precomputed.	;	early
khi c1	yfmsubpd ymm13, ymm7, ymm11, ymm13		;; Add upper FFT word * MINUS_C to lower FFT word	;	19-23
khi no c1 yfmaddpd ymm13, ymm7, ymm11, ymm13		;; Add upper FFT word to lower FFT word			;	19-23
c1	vsubpd	ymm0, ymm2, ymm1			;; Add upper FFT word * MINUS_C to lower FFT word	; 24-26
no c1	vaddpd	ymm0, ymm2, ymm1			;; Add upper FFT word to lower FFT word			; 24-26
c1	vsubpd	ymm7, ymm9, ymm8			;; Add upper FFT word * MINUS_C to lower FFT word	; 26-28
no c1	vaddpd	ymm7, ymm9, ymm8			;; Add upper FFT word to lower FFT word			; 26-28
	yfmaddpd ymm2, ymm0, ymm4, ymm12		;; val / base						;  27-31
	yfmaddpd ymm9, ymm7, ymm14, ymm12		;; val / base						;  29-33
	vsubpd	ymm2, ymm2, ymm12			;; y = round (val / base)				; 32-34
	vsubpd	ymm9, ymm9, ymm12			;; y = round (val / base)				; 34-36
	yfnmaddpd ymm0, ymm2, YMM_LIMIT_BIGMAX[basereg], ymm0 ;; new value = val - round (val / base) * base	;  35-39
c1 no khi vsubpd ymm2, ymm2, ymm5			;; next carry = y + upper mul-by-const bits * MINUS_C	; 35-37
c1 khi	vaddpd	ymm2, ymm2, ymm5			;; next carry = y + upper mul-by-const bits 		; 35-37
no c1	vaddpd	ymm2, ymm2, ymm5			;; next carry = y + upper mul-by-const bits 		; 35-37
	yfnmaddpd ymm7, ymm9, YMM_LIMIT_BIGMAX[basereg2], ymm7 ;; new value = val - round (val / base) * base	;  37-41
c1 no khi vsubpd ymm9, ymm9, ymm13			;; next carry = y + upper mul-by-const bits * MINUS_C	; 37-39
c1 khi	vaddpd	ymm9, ymm9, ymm13			;; next carry = y + upper mul-by-const bits 		; 37-39
no c1	vaddpd	ymm9, ymm9, ymm13			;; next carry = y + upper mul-by-const bits 		; 37-39
ENDM
ENDIF

ENDIF


; *************** WPN followup macros ******************

; This macro does initial prep work by shuffling the carries from the last pass 1 block.
; rsi = start of the carries array

ynorm012_wpn_part1 MACRO ttp, base2
	LOCAL	shuflp, no_adjust, done
	cmp	THIS_BLOCK, 0		;; Are we carrying into the first data block?
	jne	done			;; If not, skip wrapping carries and negating the last carry
	mov	eax, addcount1		;; Load count of cache lines in the carries array
shuflp:	vmovsd	xmm4, Q [rsi+0*32]	;; Shift 3 carry words up
	vmovsd	Q [rsi+0*32], xmm0	;; Save prev cache line's high carry in low word
	vmovsd	xmm5, Q [rsi+0*32+8]
	vmovsd	Q [rsi+0*32+8], xmm4
	vmovsd	xmm6, Q [rsi+0*32+16]
	vmovsd	Q [rsi+0*32+16], xmm5
	vmovsd	xmm0, Q [rsi+0*32+24]
	vmovsd	Q [rsi+0*32+24], xmm6
	vmovsd	xmm4, Q [rsi+1*32]	;; Shift 3 carry words up
	vmovsd	Q [rsi+1*32], xmm1	;; Save prev cache line's high carry in low word
	vmovsd	xmm5, Q [rsi+1*32+8]
	vmovsd	Q [rsi+1*32+8], xmm4
	vmovsd	xmm6, Q [rsi+1*32+16]
	vmovsd	Q [rsi+1*32+16], xmm5
	vmovsd	xmm1, Q [rsi+1*32+24]
	vmovsd	Q [rsi+1*32+24], xmm6
	bump	rsi, 64			;; Next carry cache line
	dec	rax			;; Decrement count of cache lines
	jnz	short shuflp

	;; Because of POSTFFT, squaring and multiply operations adjust the top carry while the data is in the scratch area.
	;; For add/sub/addsub/smallmul operations, we adjust the top carry here.
ttp	cmp	add_sub_smallmul_op, 0		;; If this is an add/sub/smallmul operation, not a square/multiply, do top carry adjust
ttp	je	no_adjust			;; Jump if not an add/sub/smallmul operation
ttp	ynorm_top_carry_op_wpn ttp, base2	;; Adjust carry in xmm1 if k > 1
no_adjust:

base2	vsubsd	xmm1, xmm1, Q YMM_BIGVAL
	vmulsd	xmm1, xmm1, Q YMM_MINUS_C ;; Negate the very last carry
base2	vaddsd	xmm1, xmm1, Q YMM_BIGVAL

	mov	rsi, carries		;; Reload carries array pointer
	vmovsd	Q [rsi+0*32], xmm1	;; Move last cache line's high carries into first cache line
	vmovsd	Q [rsi+1*32], xmm0

done:
	ENDM

; This macro finishes the normalize process by adding back the carries
; from each pass 1 block.  Three of the YMM carries are shifted and added
; back in to the current block, one of the YMM section carries is applied
; to the next block.
; The num_postfft_blocks setting in gwdata controls how many words we can safely carry into.
; We access this information in asm_data's SPREAD_CARRY_OVER_EXTRA_WORDS variable.
; rsi = pointer to carries
; rbp = pointer to FFT data
; rdi = pointer to big/little flags
; rdx = pointer two-to-phi group multipliers
; rax,rbx,rcx = destroyed

ynorm012_wpn MACRO ttp, base2, srcptr, carryptr, biglitptr
	LOCAL	cloop, last_iter, done

	vmovapd	ymm2, [rsi]			;; Load low carry word
	vmovapd	ymm3, [rsi+32]			;; Load hi carry word

	mov	srcptr, rbp			;; Save pointers
	mov	carryptr, rsi
ttp	mov	biglitptr, rdi
	mov	al, SPREAD_CARRY_OVER_EXTRA_WORDS ;; True if spreading carry over 8 words

cloop:

ttp		movzx	rbx, WORD PTR [rdi]		;; Load big vs. little flags
ttp		movzx	ecx, bl				;; Fudge flags 1
ttp		and	rcx, 0e0h
ttp		movzx	esi, bh				;; Big/lit flags 1-2
ttp		and	rbx, 01ch			;; Fudge flags 2

		vmovapd	ymm0, [rbp]			;; Load values1
		vmovapd	ymm1, [rbp+32]			;; Load values2

		cmp	al, 3*(256/4)			;; Is this the last iteration
		je	last_iter			;; Yes, last iteration requires special handling

ttp		vmulpd	ymm0, ymm0, [rdx+0*YMM_GMD][rcx] ;; Mul values1 by fudged two-to-minus-phi
ttp		vmulpd	ymm1, ymm1, [rdx+1*YMM_GMD][rbx*8] ;; Mul values2 by fudged two-to-minus-phi
		vaddpd	ymm0, ymm0, ymm2		;; x1 = values1 + carry
		vaddpd	ymm1, ymm1, ymm3		;; x2 = values2 + carry
		rounding_interleaved ttp, base2, noexec, ymm0, ymm2, ymm4, rsi*8, ymm1, ymm3, ymm5, rsi*8+32
ttp		vmulpd	ymm0, ymm0, [rdx+0*YMM_GMD+YMM_GMD/2][rcx] ;; value1 = rounded value * fudged grp two-to-phi
ttp		vmulpd	ymm1, ymm1, [rdx+1*YMM_GMD+YMM_GMD/2][rbx*8] ;; value2 = rounded value * fudged grp two-to-phi
		ystore	[rbp], ymm0			;; Save new value1
		ystore	[rbp+32], ymm1			;; Save new value2

base2		vmovapd	ymm1, YMM_BIGVAL		;; Load comparison value
no base2	vxorpd	ymm1, ymm1, ymm1		;; Create comparison value
		vcmppd	ymm0, ymm2, ymm1, 0Ch		;; Test for non-zero carries
		vmovmskpd rbx, ymm0			;; Extract 4 comparison bits
		vcmppd	ymm0, ymm3, ymm1, 0Ch		;; Test for non-zero carries
		vmovmskpd rcx, ymm0			;; Extract 4 comparison bits
		or	rbx, rcx			;; Are any bits on?
		jz	short done			;; No, we are done

ttp	bump	rdi, 2				;; Advance pointers
	add	rbp, pass2blkdst		;; Next FFT data ptr
 	add	al, 256/4			;; Special increment needed every 4th iteration
	jnc	cloop				;; Loop if not 4th iteration
	sub	rbp, pass1blkdst		;; Each pass1 block consists of 4 pass 2 blocks
	bump	rbp, 64				;; Next cache line
	dec	al				;; Decrement count of sets of 4 carries we've processed
ttp	cmp	cache_line_multiplier, 4	;; We must increment rdi by a funky amount every 4*clm cache lines
ttp	jne	cloop				;; No, repeat carry propagation loop
ttp	add	rdi, normval2			;; Adjust ptr to little/big flags
	jmp	cloop				;; Repeat carry propagation loop

		;; Do last iteration without propagating carries out
last_iter:
base2		vsubpd	ymm2, ymm2, YMM_BIGVAL		;; Subtract rounding constant from carries
base2		vsubpd	ymm3, ymm3, YMM_BIGVAL
ttp		vmulpd	ymm2, ymm2, [rdx+0*YMM_GMD+YMM_GMD/2][rcx] ;; Carry *= fudged grp two-to-phi
ttp		vmulpd	ymm3, ymm3, [rdx+1*YMM_GMD+YMM_GMD/2][rbx*8]
		vaddpd	ymm0, ymm0, ymm2		;; Values1 += carry
		vaddpd	ymm1, ymm1, ymm3		;; Values2 += carry
		ystore	[rbp], ymm0			;; Save new value1
		ystore	[rbp+32], ymm1			;; Save new value2

done:	mov	rbp, srcptr			;; Restore pointers
	mov	rsi, carryptr
ttp	mov	rdi, biglitptr

base2	vmovapd	ymm4, YMM_BIGVAL		;; Clear conventional carry
no base2 vxorpd ymm4, ymm4, ymm4
	ystore	[rsi], ymm4
	ystore	[rsi+32], ymm4
	ENDM


;; Significantly different cleanup code for zero-padded FFTs.
;; Note: The group multiplier should be 1.0 for both the bottom FFT words and
;; the FFT words just above the half-way point.
;; rbx = pointer past the end of the carries array

ynorm012_wpn_zpad_part1 MACRO ttp, base2
	LOCAL	shuflp, noncon, zpaddn, done
	cmp	THIS_BLOCK, 0		;; Are we carrying into the first data block?
	jne	done			;; If not, skip wrapping carries and negating the last carry
	mov	eax, addcount1		;; Load count of cache lines in the carries array
shuflp:	vmovsd	xmm4, Q [rsi+0*32]	;; Shift 3 carry words up
	vmovsd	Q [rsi+0*32], xmm2	;; Save prev cache line's high carry in low word
	vmovsd	xmm5, Q [rsi+0*32+8]
	vmovsd	Q [rsi+0*32+8], xmm4
	vmovsd	xmm6, Q [rsi+0*32+16]
	vmovsd	Q [rsi+0*32+16], xmm5
	vmovsd	xmm2, Q [rsi+0*32+24]
	vmovsd	Q [rsi+0*32+24], xmm6
	vmovsd	xmm4, Q [rsi+1*32]	;; Shift 3 carry words up
	vmovsd	Q [rsi+1*32], xmm3	;; Save prev cache line's high carry in low word
	vmovsd	xmm5, Q [rsi+1*32+8]
	vmovsd	Q [rsi+1*32+8], xmm4
	vmovsd	xmm6, Q [rsi+1*32+16]
	vmovsd	Q [rsi+1*32+16], xmm5
	vmovsd	xmm3, Q [rsi+1*32+24]
	vmovsd	Q [rsi+1*32+24], xmm6
	bump	rsi, 64			;; Next carry cache line
	dec	rax			;; Decrement count of cache lines
	jnz	short shuflp
;;	mov	rsi, carries		;; Reload carries array pointer
;;	vmovsd	Q [rsi+0*32], xmm3	;; Move last cache line's high carries into first cache line
;;	vmovsd	Q [rsi+1*32], xmm2

	mov	rsi, DESTARG		;; Address of FFT data
ttp	lea	rdi, YMM_FIRST_BIGLIT_VALUES ;; Address of first big/little flags

	cmp	const_fft, 0		;; Are we also multiplying by a constant?
	je	noncon			;; Jump if not const
	ynorm012_wpn_zpad_part1_cmn ttp, base2, exec
	jmp	zpaddn
noncon:	ynorm012_wpn_zpad_part1_cmn ttp, base2, noexec

zpaddn:	mov	rsi, carries		;; Reload carries array pointer
base2	vmovsd	xmm6, Q YMM_BIGVAL	;; Clear two carries just processed by ynorm012_wpn_zpad_part1_cmn
no base2 vxorpd	xmm6, xmm6, xmm6
	vxorpd	xmm7, xmm7, xmm7
	vmovsd	Q [rsi], xmm6
	vmovsd	Q [rsi+32], xmm7
done:
	ENDM

;; On input, xmm2 and xmm3 contain high carries from the last carries array row
ynorm012_wpn_zpad_part1_cmn MACRO ttp, base2, const
	LOCAL	zsetdn, nomulc, smallk, mediumk, div_k_done

	;; Strip BIGVAL from the traditional carry, we'll add the traditional
	;; carry in later when we are working on the ZPAD0 - ZPAD6 values.
base2	vsubsd	xmm2, xmm2, Q YMM_BIGVAL ;; Integerize traditional carry

	;; Rather than calculate high FFT carry times k and then later dividing
	;; by k, we multiply FFT high carry by const and we'll add it
	;; to the lower FFT data later (after multiplying by -c).
const	vmulsd	xmm3, xmm3, Q YMM_MULCONST

	;; If this is an add/sub/smallmul operation, set ZPAD0-6 to the value to be divided by k (FFT words above halfway point)
	;; This lets add/sub/smallmul share carry code with square/multiply
no const cmp	add_sub_smallmul_op, 0		;; If this is an add/sub/smallmul operation, not a square/multiply, then copy data
no const je	zsetdn				;; Jump if not an add/sub/smallmul operation
no const vaddsd	xmm0, xmm2, [rsi+32] 		;; Copy value1 + low carry
no const vmovsd	ZPAD0, xmm0
no const add	rsi, YMM_SRC_INCR1		;; Next source pointer
no const vmovsd	xmm0, [rsi+32]	 		;; Copy value2
no const vmovsd	ZPAD1, xmm0
no const add	rsi, YMM_SRC_INCR2		;; Next source pointer
no const vmovsd	xmm0, [rsi+32]	 		;; Copy value3
no const vmovsd	ZPAD2, xmm0
no const add	rsi, YMM_SRC_INCR3		;; Next source pointer
no const vmovsd	xmm0, [rsi+32]	 		;; Copy value4
no const vmovsd	ZPAD3, xmm0
no const add	rsi, YMM_SRC_INCR4		;; Next source pointer
no const vmovsd	xmm0, [rsi+32]	 		;; Copy value5
no const vmovsd	ZPAD4, xmm0
no const add	rsi, YMM_SRC_INCR5		;; Next source pointer
no const vmovsd	xmm0, [rsi+32]	 		;; Copy value6
no const vmovsd	ZPAD5, xmm0
no const add	rsi, YMM_SRC_INCR6		;; Next source pointer
no const vmovsd	xmm0, [rsi+32]	 		;; Copy value7
no const vmovsd	ZPAD6, xmm0
no const mov	rsi, DESTARG			;; Restore source pointer
no const jmp	nomulc				;; Skip over the mul by -C
zsetdn:

	;; Work on zero-pad addin value.
	vmovsd	xmm4, ADDIN_VALUE		;; Load the add in value
const	vmulsd	xmm4, xmm4, YMM_MULCONST	;; Multiply the add in value by the small mul const
	vaddsd	xmm4, xmm4, POSTADDIN_VALUE	;; Add the post-mul-by-const addin value
	;; when c = -1, 1 = b^n
	;; when c = 1, -1 = b^n, 1 = -b^n
	;; when c = -3, 3 = b^n, 1 = b^n - 2
	;; when c = 3, -3 = b^n, 3 = -b^n, 1 = -b^n - 2
	;; The "- 2" has been precomputed in ZPAD_LSW_ADJUST.  Add ADDIN_VALUE * mul-by-const * ZPAD_LSW_ADJUST into the least significant FFT word
	mov	rsi, DESTARG			;; Address of squared number
	vmulsd	xmm0, xmm4, ZPAD_LSW_ADJUST
	vaddsd	xmm0, xmm0, [rsi]
	vmovsd	[rsi], xmm0

	;; Multiply ZPAD0 through ZPAD6 by const * -C.  This, in essense,
	;; wraps this data from above the FFT data area to the halfway point.
	;; Later on we'll divide this by K to decide which data needs wrapping
	;; all the way down to the bottom of the FFT data.

	;; NOTE: ZPAD0's grp multiplier is 1.0.  Also, ZPAD6 will not
	;; be bigger than a big word.  We must be careful to handle c's up
	;; to about 30 bits

ttp	movzx	rax, BYTE PTR [rdi]		;; Load big vs. little flags
	vmovsd	xmm0, ZPAD0			;; Load value
	vsubsd	xmm5, xmm5, xmm5
	new_single_split_lower_zpad_word ttp, base2, xmm0, xmm5, xmm1, rax*8
no const vmulsd	xmm0, xmm0, Q YMM_MINUS_C
const	vmulsd	xmm0, xmm0, Q YMM_MINUS_C_TIMES_MULCONST
	vaddsd	xmm0, xmm0, xmm4		;; Apply rest of ADDIN_VALUE * mul-by-const here (the FFT half way point)
	new_round_zpad7_word ttp, base2, xmm0, xmm2, xmm1, rax*8
	vmovsd	ZPAD0, xmm0

ttp	movzx	rax, BYTE PTR [rdi+1]		;; Load big vs. little flags
	vmovsd	xmm0, ZPAD1			;; Load value
	new_single_split_lower_zpad_word ttp, base2, xmm0, xmm5, xmm1, rax*8
no const vmulsd	xmm0, xmm0, Q YMM_MINUS_C
const	vmulsd	xmm0, xmm0, Q YMM_MINUS_C_TIMES_MULCONST
	new_round_zpad7_word ttp, base2, xmm0, xmm2, xmm1, rax*8
	vmovsd	ZPAD1, xmm0

ttp	movzx	rax, BYTE PTR [rdi+2]		;; Load big vs. little flags
	vmovsd	xmm0, ZPAD2			;; Load value
	new_single_split_lower_zpad_word ttp, base2, xmm0, xmm5, xmm1, rax*8
no const vmulsd	xmm0, xmm0, Q YMM_MINUS_C
const	vmulsd	xmm0, xmm0, Q YMM_MINUS_C_TIMES_MULCONST
	new_round_zpad7_word ttp, base2, xmm0, xmm2, xmm1, rax*8
	vmovsd	ZPAD2, xmm0

ttp	movzx	rax, BYTE PTR [rdi+3]		;; Load big vs. little flags
	vmovsd	xmm0, ZPAD3			;; Load value
	new_single_split_lower_zpad_word ttp, base2, xmm0, xmm5, xmm1, rax*8
no const vmulsd	xmm0, xmm0, Q YMM_MINUS_C
const	vmulsd	xmm0, xmm0, Q YMM_MINUS_C_TIMES_MULCONST
	new_round_zpad7_word ttp, base2, xmm0, xmm2, xmm1, rax*8
	vmovsd	ZPAD3, xmm0

ttp	movzx	rax, BYTE PTR [rdi+4]		;; Load big vs. little flags
	vmovsd	xmm0, ZPAD4			;; Load value
	new_single_split_lower_zpad_word ttp, base2, xmm0, xmm5, xmm1, rax*8
no const vmulsd	xmm0, xmm0, Q YMM_MINUS_C
const	vmulsd	xmm0, xmm0, Q YMM_MINUS_C_TIMES_MULCONST
	new_round_zpad7_word ttp, base2, xmm0, xmm2, xmm1, rax*8
	vmovsd	ZPAD4, xmm0

ttp	movzx	rax, BYTE PTR [rdi+5]		;; Load big vs. little flags
	vmovsd	xmm0, ZPAD5			;; Load value
	new_single_split_lower_zpad_word ttp, base2, xmm0, xmm5, xmm1, rax*8
no const vmulsd	xmm0, xmm0, Q YMM_MINUS_C
const	vmulsd	xmm0, xmm0, Q YMM_MINUS_C_TIMES_MULCONST
	new_round_zpad7_word ttp, base2, xmm0, xmm2, xmm1, rax*8
	vmovsd	ZPAD5, xmm0

	vmovsd	xmm0, ZPAD6			;; Load value
	vroundsd xmm0, xmm0, xmm0, 0		;; Round to an integer
	vaddsd	xmm0, xmm0, xmm5		;; Add in shifted high ZPAD data
no const vmulsd	xmm0, xmm0, Q YMM_MINUS_C
const	vmulsd	xmm0, xmm0, Q YMM_MINUS_C_TIMES_MULCONST
	vaddsd	xmm0, xmm0, xmm2		;; Add in high part of last calculation
	vmovsd	ZPAD6, xmm0

	;; Divide the zpad data by k.  Store the integer part in YMM_TMP
	;; and the remainder in ZPAD0.  Later we will wrap the integer part
	;; down to the bottom of the FFT data area (and multiply by -c).
	;; And we will store the remainder in the upper half of the FFT
	;; data area.

	;; Note there are three cases to handle.  K is smaller than a big word.
	;; K is between one and 2 big words in size.  And K is more than
	;; 2 big words in size.

nomulc:	cmp	ZPAD_TYPE, 2		;; Are we dealing with case 1,2,or 3
	jl	smallk			;; One word case
	je	mediumk			;; Two word case

	;; This case does the divide by k where k is three words

	vmovsd	xmm0, ZPAD6			;; Load zpad word (high bits)
	vmovsd	xmm1, ZPAD5			;; Load zpad word (middle bits)
	vmovsd	xmm2, ZPAD4			;; Load zpad word (low bits)
	vmulsd	xmm4, xmm0, ZPAD_INVERSE_K6	;; Mul ZPAD by shifted 1/k
	vroundsd xmm4, xmm4, xmm4, 0		;; Round to integer
	vmulsd	xmm5, xmm4, ZPAD_K6_HI		;; Mul by high bits of k
	vsubsd	xmm0, xmm0, xmm5		;; Calculate high bits of remainder
	vmulsd	xmm5, xmm4, ZPAD_K6_MID		;; Mul by middle bits of k
	vsubsd	xmm1, xmm1, xmm5		;; Calculate middle bits of remainder
	vmulsd	xmm5, xmm4, ZPAD_K6_LO		;; Mul by low bits of k
	vsubsd	xmm2, xmm2, xmm5		;; Calculate low bits of remainder
	vmovsd	Q YMM_TMP5, xmm4		;; Save word of zpad / k

	vmulsd	xmm0, xmm0, ZPAD_SHIFT6		;; Shift previous zpad word
	vaddsd	xmm1, xmm1, xmm0		;; Add to create new high zpad bits
	vmovsd	xmm0, ZPAD3			;; Load zpad word (new low bits)
	vmulsd	xmm5, xmm1, ZPAD_SHIFT5		;; Combine high and medium bits
	vaddsd	xmm5, xmm5, xmm2
	vmulsd	xmm4, xmm5, ZPAD_INVERSE_K5	;; Mul ZPAD by shifted 1/k
	vroundsd xmm4, xmm4, xmm4, 0		;; Round to integer
	vmulsd	xmm5, xmm4, ZPAD_K5_HI		;; Mul by high bits of k
	vsubsd	xmm1, xmm1, xmm5		;; Calculate high bits of remainder
	vmulsd	xmm5, xmm4, ZPAD_K5_MID		;; Mul by middle bits of k
	vsubsd	xmm2, xmm2, xmm5		;; Calculate middle bits of remainder
	vmulsd	xmm5, xmm4, ZPAD_K5_LO		;; Mul by low bits of k
	vsubsd	xmm0, xmm0, xmm5		;; Calculate low bits of remainder
	vmovsd	Q YMM_TMP4, xmm4		;; Save word of zpad / k

	vmulsd	xmm1, xmm1, ZPAD_SHIFT5		;; Shift previous zpad word
	vaddsd	xmm2, xmm2, xmm1		;; Add to create new high zpad bits
	vmovsd	xmm1, ZPAD2			;; Load zpad word (new low bits)
	vmulsd	xmm5, xmm2, ZPAD_SHIFT4		;; Combine high and medium bits
	vaddsd	xmm5, xmm5, xmm0
	vmulsd	xmm4, xmm5, ZPAD_INVERSE_K4	;; Mul ZPAD by shifted 1/k
	vroundsd xmm4, xmm4, xmm4, 0		;; Round to integer
	vmulsd	xmm5, xmm4, ZPAD_K4_HI		;; Mul by high bits of k
	vsubsd	xmm2, xmm2, xmm5		;; Calculate high bits of remainder
	vmulsd	xmm5, xmm4, ZPAD_K4_MID		;; Mul by middle bits of k
	vsubsd	xmm0,xmm0,xmm5			;; Calculate middle bits of remainder
	vmulsd	xmm5, xmm4, ZPAD_K4_LO		;; Mul by low bits of k
	vsubsd	xmm1, xmm1, xmm5		;; Calculate low bits of remainder
	vmovsd	Q YMM_TMP3, xmm4		;; Save word of zpad / k

	vmulsd	xmm2, xmm2, ZPAD_SHIFT4		;; Shift previous zpad word
	vaddsd	xmm0, xmm0, xmm2		;; Add to create new high zpad bits
	vmovsd	xmm2, ZPAD1			;; Load zpad word (new low bits)
	vmulsd	xmm5, xmm0, ZPAD_SHIFT3		;; Combine high and medium bits
	vaddsd	xmm5, xmm5, xmm1
	vmulsd	xmm4, xmm5, ZPAD_INVERSE_K3	;; Mul ZPAD by shifted 1/k
	vroundsd xmm4, xmm4, xmm4, 0		;; Round to integer
	vmulsd	xmm5, xmm4, ZPAD_K3_HI		;; Mul by high bits of k
	vsubsd	xmm0, xmm0, xmm5		;; Calculate high bits of remainder
	vmulsd	xmm5, xmm4, ZPAD_K3_MID		;; Mul by middle bits of k
	vsubsd	xmm1, xmm1, xmm5		;; Calculate middle bits of remainder
	vmulsd	xmm5, xmm4, ZPAD_K3_LO		;; Mul by low bits of k
	vsubsd	xmm2, xmm2, xmm5		;; Calculate low bits of remainder
	vmovsd	Q YMM_TMP2, xmm4		;; Save word of zpad / k

	vmulsd	xmm0, xmm0, ZPAD_SHIFT3		;; Shift previous zpad word
	vaddsd	xmm1, xmm1, xmm0		;; Add to create new high zpad bits
	vmovsd	xmm0, ZPAD0			;; Load zpad word (new low bits)
	vmulsd	xmm5, xmm1, ZPAD_SHIFT2		;; Combine high and medium bits
	vaddsd	xmm5, xmm5, xmm2
	vmulsd	xmm4, xmm5, ZPAD_INVERSE_K2	;; Mul ZPAD by shifted 1/k
	vroundsd xmm4, xmm4, xmm4, 0		;; Round to integer
	vmulsd	xmm5, xmm4, ZPAD_K2_HI		;; Mul by high bits of k
	vsubsd	xmm1, xmm1, xmm5		;; Calculate high bits of remainder
	vmulsd	xmm5, xmm4, ZPAD_K2_MID		;; Load middle bits of k
	vsubsd	xmm2, xmm2, xmm5		;; Calculate middle bits of remainder
	vmulsd	xmm5, xmm4, ZPAD_K2_LO		;; Load low bits of k
	vsubsd	xmm0, xmm0, xmm5		;; Calculate low bits of remainder
	vmovsd	Q YMM_TMP1, xmm4		;; Save word of zpad / k

	vmulsd	xmm1, xmm1, ZPAD_SHIFT2		;; Shift previous zpad word
	vaddsd	xmm2, xmm2, xmm1		;; Add to create new high zpad bits
	vmulsd	xmm2, xmm2, ZPAD_SHIFT1		;; Shift previous zpad word
	vaddsd	xmm0, xmm0, xmm2		;; Add to create new high zpad bits
	vmovsd	ZPAD0, xmm0			;; Save remainder of zpad / k

	vxorpd	xmm1, xmm1, xmm1		;; Zero words that other cases set
	vmovsd	Q YMM_TMP6, xmm1

	jmp	div_k_done

	;; This case does the divide by k where k is two words
mediumk:
	vmovsd	xmm0, ZPAD6			;; Load zpad word (high bits)
	vmovsd	xmm1, ZPAD5			;; Load zpad word (low bits)
	vmulsd	xmm4, xmm0, ZPAD_INVERSE_K6	;; Mul ZPAD by shifted 1/k
	vroundsd xmm4, xmm4, xmm4, 0		;; Round to integer
	vmulsd	xmm5, xmm4, ZPAD_K6_HI		;; Mul by high bits of k
	vsubsd	xmm0, xmm0, xmm5		;; Calculate high bits of remainder
	vmulsd	xmm5, xmm4, ZPAD_K6_LO		;; Mul by low bits of k
	vsubsd	xmm1, xmm1, xmm5		;; Calculate low bits of remainder
	vmovsd	Q YMM_TMP6, xmm4		;; Save word of zpad / k

	vmulsd	xmm0, xmm0, ZPAD_SHIFT6		;; Shift previous zpad word
	vaddsd	xmm0, xmm0, xmm1		;; Add to create new high zpad bits
	vmovsd	xmm1, ZPAD4			;; Load zpad word (new low bits)
	vmulsd	xmm4, xmm0, ZPAD_INVERSE_K5	;; Mul ZPAD by shifted 1/k
	vroundsd xmm4, xmm4, xmm4, 0		;; Round to integer
	vmulsd	xmm5, xmm4, ZPAD_K5_HI		;; Mul by high bits of k
	vsubsd	xmm0, xmm0, xmm5		;; Calculate high bits of remainder
	vmulsd	xmm5, xmm4, ZPAD_K5_LO		;; Mul by low bits of k
	vsubsd	xmm1, xmm1, xmm5		;; Calculate low bits of remainder
	vmovsd	Q YMM_TMP5, xmm4		;; Save word of zpad / k

	vmulsd	xmm0, xmm0, ZPAD_SHIFT5		;; Shift previous zpad word
	vaddsd	xmm0, xmm0, xmm1		;; Add to create new high zpad bits
	vmovsd	xmm1, ZPAD3			;; Load zpad word (new low bits)
	vmulsd	xmm4, xmm0, ZPAD_INVERSE_K4	;; Mul ZPAD by shifted 1/k
	vroundsd xmm4, xmm4, xmm4, 0		;; Round to integer
	vmulsd	xmm5, xmm4, ZPAD_K4_HI		;; Mul by high bits of k
	vsubsd	xmm0, xmm0, xmm5		;; Calculate high bits of remainder
	vmulsd	xmm5, xmm4, ZPAD_K4_LO		;; Mul by low bits of k
	vsubsd	xmm1, xmm1, xmm5		;; Calculate low bits of remainder
	vmovsd	Q YMM_TMP4, xmm4		;; Save word of zpad / k

	vmulsd	xmm0, xmm0, ZPAD_SHIFT4		;; Shift previous zpad word
	vaddsd	xmm0, xmm0, xmm1		;; Add to create new high zpad bits
	vmovsd	xmm1, ZPAD2			;; Load zpad word (new low bits)
	vmulsd	xmm4, xmm0, ZPAD_INVERSE_K3	;; Mul ZPAD by shifted 1/k
	vroundsd xmm4, xmm4, xmm4, 0		;; Round to integer
	vmulsd	xmm5, xmm4, ZPAD_K3_HI		;; Mul by high bits of k
	vsubsd	xmm0, xmm0, xmm5		;; Calculate high bits of remainder
	vmulsd	xmm5, xmm4, ZPAD_K3_LO		;; Mul by low bits of k
	vsubsd	xmm1, xmm1, xmm5		;; Calculate low bits of remainder
	vmovsd	Q YMM_TMP3, xmm4		;; Save word of zpad / k

	vmulsd	xmm0, xmm0, ZPAD_SHIFT3		;; Shift previous zpad word
	vaddsd	xmm0, xmm0, xmm1		;; Add to create new high zpad bits
	vmovsd	xmm1, ZPAD1			;; Load zpad word (new low bits)
	vmulsd	xmm4, xmm0, ZPAD_INVERSE_K2	;; Mul ZPAD by shifted 1/k
	vroundsd xmm4, xmm4, xmm4, 0		;; Round to integer
	vmulsd	xmm5, xmm4, ZPAD_K2_HI		;; Mul by high bits of k
	vsubsd	xmm0, xmm0, xmm5		;; Calculate high bits of remainder
	vmulsd	xmm5, xmm4, ZPAD_K2_LO		;; Mul by low bits of k
	vsubsd	xmm1, xmm1, xmm5		;; Calculate low bits of remainder
	vmovsd	Q YMM_TMP2, xmm4		;; Save word of zpad / k

	vmulsd	xmm0, xmm0, ZPAD_SHIFT2		;; Shift previous zpad word
	vaddsd	xmm0, xmm0, xmm1		;; Add to create new high zpad bits
	vmovsd	xmm1, ZPAD0			;; Load zpad word (new low bits)
	vmulsd	xmm4, xmm0, ZPAD_INVERSE_K1	;; Mul ZPAD by shifted 1/k
	vroundsd xmm4, xmm4, xmm4, 0		;; Round to integer
	vmulsd	xmm5, xmm4, ZPAD_K1_HI		;; Mul by high bits of k
	vsubsd	xmm0, xmm0, xmm5		;; Calculate high bits of remainder
	vmulsd	xmm5, xmm4, ZPAD_K1_LO		;; Mul by low bits of k
	vsubsd	xmm1, xmm1, xmm5		;; Calculate low bits of remainder
	vmovsd	Q YMM_TMP1, xmm4		;; Save word of zpad / k

	vmulsd	xmm0, xmm0, ZPAD_SHIFT1		;; Shift previous zpad word
	vaddsd	xmm0, xmm0, xmm1		;; Add to create new high zpad bits
	vmovsd	ZPAD0, xmm0			;; Save remainder of zpad / k

	jmp	div_k_done

	;; This case does the divide by k where k is one word
	;; Assume ZPAD5 and ZPAD6 are zero.
smallk:	vmovsd	xmm0, ZPAD4			;; Load zpad data
	vmulsd	xmm4, xmm0, ZPAD_INVERSE_K1	;; Mul ZPAD data by 1/k
	vroundsd xmm4, xmm4, xmm4, 0		;; Round to integer
	vmovsd	Q YMM_TMP5, xmm4		;; Save integer part
	vmulsd	xmm4, xmm4, ZPAD_K1_LO		;; Compute remainder
	vsubsd	xmm0, xmm0, xmm4

	vmulsd	xmm0, xmm0, ZPAD_SHIFT4		;; Shift previous zpad word
	vaddsd	xmm0, xmm0, ZPAD3		;; Add in zpad data
	vmulsd	xmm4, xmm0, ZPAD_INVERSE_K1	;; Mul ZPAD data by 1/k
	vroundsd xmm4, xmm4, xmm4, 0		;; Round to integer
	vmovsd	Q YMM_TMP4, xmm4		;; Save integer part
	vmulsd	xmm4, xmm4, ZPAD_K1_LO		;; Compute remainder
	vsubsd	xmm0, xmm0, xmm4

	vmulsd	xmm0, xmm0, ZPAD_SHIFT3		;; Shift previous zpad word
	vaddsd	xmm0, xmm0, ZPAD2		;; Add in zpad data
	vmulsd	xmm4, xmm0, ZPAD_INVERSE_K1	;; Mul ZPAD data by 1/k
	vroundsd xmm4, xmm4, xmm4, 0		;; Round to integer
	vmovsd	Q YMM_TMP3, xmm4		;; Save integer part
	vmulsd	xmm4, xmm4, ZPAD_K1_LO		;; Compute remainder
	vsubsd	xmm0, xmm0, xmm4

	vmulsd	xmm0, xmm0, ZPAD_SHIFT2		;; Shift previous zpad word
	vaddsd	xmm0, xmm0, ZPAD1		;; Add in zpad data
	vmulsd	xmm4, xmm0, ZPAD_INVERSE_K1	;; Mul ZPAD data by 1/k
	vroundsd xmm4, xmm4, xmm4, 0		;; Round to integer
	vmovsd	Q YMM_TMP2, xmm4		;; Save integer part
	vmulsd	xmm4, xmm4, ZPAD_K1_LO		;; Compute remainder
	vsubsd	xmm0, xmm0, xmm4

	vmulsd	xmm0, xmm0, ZPAD_SHIFT1		;; Shift previous zpad word
	vaddsd	xmm0, xmm0, ZPAD0		;; Add in zpad data
	vmulsd	xmm4, xmm0, ZPAD_INVERSE_K1	;; Mul ZPAD data by 1/k
	vroundsd xmm4, xmm4, xmm4, 0		;; Round to integer
	vmovsd	Q YMM_TMP1, xmm4		;; Save integer part
	vmulsd	xmm4, xmm4, ZPAD_K1_LO		;; Compute remainder
	vsubsd	xmm0, xmm0, xmm4
	vmovsd	ZPAD0, xmm0			;; Save remainder

	vxorpd	xmm1, xmm1, xmm1		;; Zero words that other cases set
	vmovsd	Q YMM_TMP6, xmm1

div_k_done:

	;; Now normalize the data above the halfway point.  Remember that the
	;; two-to-phi multipliers will be applied by the FFT.

ttp	movzx	rax, BYTE PTR [rdi]		;; First word
	vmovsd	xmm0, ZPAD0			;; Load remainder of divide by k
base2	vaddsd	xmm0, xmm0, Q YMM_BIGVAL
	new_single_rounding ttp, base2, xmm0, xmm2, xmm4, rax*8
	vmovsd	Q [rsi+32], xmm0		;; Save value1

	add	rsi, YMM_SRC_INCR1		;; Next source pointer
ttp	movzx	rax, BYTE PTR [rdi+1]		;; Load big vs. little flags
	new_single_rounding ttp, base2, xmm2, xmm0, xmm4, rax*8
	vmovsd	Q [rsi+32], xmm2		;; Save value2

	add	rsi, YMM_SRC_INCR2		;; Next source pointer
ttp	movzx	rax, BYTE PTR [rdi+2]		;; Load big vs. little flags
	new_single_rounding ttp, base2, xmm0, xmm2, xmm4, rax*8
	vmovsd	Q [rsi+32], xmm0		;; Save value3

	add	rsi, YMM_SRC_INCR3		;; Next source pointer
base2	vsubsd	xmm2, xmm2, Q YMM_BIGVAL	;; Remove integer rounding constant
	vmovsd	Q [rsi+32], xmm2		;; Save new value4

	;; Mul the integer part of (ZPAD data divided by k) by -c in
	;; preparation for adding it into the lower FFT data area.
	;; Also add in the shifted high FFT carry at this time.

	;; Now add in and normalize the bottom FFT data.  Remember that the
	;; column two-to-phi multiplier for the first value will be 1.0.  We 
	;; must go 6 words deep in case k is 48-50 bits and c is 32 bits.

	mov	rsi, DESTARG			;; Restore address of FFT data

ttp	movzx	rax, BYTE PTR [rdi]		;; First word
	vmovsd	xmm0, Q YMM_TMP1		;; Load integer part of divide by k
	vaddsd	xmm0, xmm0, xmm3		;; Add in shifted high FFT carry
	vmulsd	xmm0, xmm0, Q YMM_MINUS_C	;; Mul by -c
base2	vaddsd	xmm0, xmm0, Q YMM_BIGVAL
	vaddsd	xmm0, xmm0, Q [rsi]		;; Add in the FFT data
	new_single_rounding ttp, base2, xmm0, xmm2, xmm4, rax*8
	vmovsd	Q [rsi], xmm0			;; Save value1

	add	rsi, YMM_SRC_INCR1		;; Next source pointer
ttp	movzx	rax, BYTE PTR [rdi+1]		;; Load big vs. little flags
	vmovsd	xmm0, Q YMM_TMP2		;; Load integer part of divide by k
	vmulsd	xmm0, xmm0, Q YMM_MINUS_C	;; Mul by -c
	vaddsd	xmm0, xmm0, xmm2		;; x2 = value + carry
	vaddsd	xmm0, xmm0, Q [rsi]		;; Add in the FFT data
	new_single_rounding ttp, base2, xmm0, xmm2, xmm4, rax*8
	vmovsd	Q [rsi], xmm0			;; Save value2

	add	rsi, YMM_SRC_INCR2		;; Next source pointer
ttp	movzx	rax, BYTE PTR [rdi+2]		;; Load big vs. little flags
	vmovsd	xmm0, Q YMM_TMP3		;; Load integer part of divide by k
	vmulsd	xmm0, xmm0, Q YMM_MINUS_C	;; Mul by -c
	vaddsd	xmm0, xmm0, xmm2		;; x3 = value + carry
	vaddsd	xmm0, xmm0, Q [rsi]		;; Add in the FFT data
	new_single_rounding ttp, base2, xmm0, xmm2, xmm4, rax*8
	vmovsd	Q [rsi], xmm0			;; Save value3

	add	rsi, YMM_SRC_INCR3		;; Next source pointer
ttp	movzx	rax, BYTE PTR [rdi+3]		;; Load big vs. little flags
	vmovsd	xmm0, Q YMM_TMP4		;; Load integer part of divide by k
	vmulsd	xmm0, xmm0, Q YMM_MINUS_C	;; Mul by -c
	vaddsd	xmm0, xmm0, xmm2		;; x4 = value + carry
	vaddsd	xmm0, xmm0, Q [rsi]		;; Add in the FFT data
	new_single_rounding ttp, base2, xmm0, xmm2, xmm4, rax*8
	vmovsd	Q [rsi], xmm0			;; Save value4

	add	rsi, YMM_SRC_INCR4		;; Next source pointer
ttp	movzx	rax, BYTE PTR [rdi+4]		;; Load big vs. little flags
	vmovsd	xmm0, Q YMM_TMP5		;; Load integer part of divide by k
	vmulsd	xmm0, xmm0, Q YMM_MINUS_C	;; Mul by -c
	vaddsd	xmm0, xmm0, xmm2		;; x5 = value + carry
	vaddsd	xmm0, xmm0, Q [rsi]		;; Add in the FFT data
	new_single_rounding ttp, base2, xmm0, xmm2, xmm4, rax*8
	vmovsd	Q [rsi], xmm0			;; Save value5

	add	rsi, YMM_SRC_INCR5		;; Next source pointer
ttp	movzx	rax, BYTE PTR [rdi+5]		;; Load big vs. little flags
	vmovsd	xmm0, Q YMM_TMP6		;; Load integer part of divide by k
	vmulsd	xmm0, xmm0, Q YMM_MINUS_C	;; Mul by -c
	vaddsd	xmm0, xmm0, xmm2		;; x6 = value + carry
	vaddsd	xmm0, xmm0, Q [rsi]		;; Add in the FFT data
	new_single_rounding ttp, base2, xmm0, xmm2, xmm4, rax*8
	vmovsd	Q [rsi], xmm0			;; Save value6

	add	rsi, YMM_SRC_INCR6		;; Next source pointer
ttp	movzx	rax, BYTE PTR [rdi+6]		;; Load big vs. little flags
	vaddsd	xmm0, xmm2, Q [rsi]		;; x7 = value + carry
	new_single_rounding ttp, base2, xmm0, xmm2, xmm4, rax*8
	vmovsd	Q [rsi], xmm0			;; Save value7

	add	rsi, YMM_SRC_INCR7		;; Next source pointer
base2	vsubsd	xmm2, xmm2, Q YMM_BIGVAL	;; Remove rounding constant
	vaddsd	xmm2, xmm2, Q [rsi]		;; Add in FFT data
	vmovsd	Q [rsi], xmm2			;; Save value8
	ENDM


; Process one of the cache lines from the carries table for a two-pass zero-padded FFT.
; rsi = pointer to carries
; rbp = pointer to FFT data
; rdi = pointer to big/little flags
; rdx = pointer two-to-phi group multipliers
; rax,rbx,rcx = destroyed

ynorm012_wpn_zpad MACRO ttp, base2, srcptr, carryptr, biglitptr
	LOCAL	noncon, done
	cmp	const_fft, 0		;; Are we also multiplying by a constant?
	je	noncon			;; Jump if not const
	ynorm012_wpn_zpad_cmn ttp, base2, exec, srcptr, carryptr, biglitptr
	jmp	done
noncon:	ynorm012_wpn_zpad_cmn ttp, base2, noexec, srcptr, carryptr, biglitptr
done:
	ENDM
ynorm012_wpn_zpad_cmn MACRO ttp, base2, const, srcptr, carryptr, biglitptr
	LOCAL	cloop, last_iter, done

	vmovapd	ymm2, [rsi]			;; Load one carry word
	vmovapd	ymm3, [rsi+32]			;; Load other carry word

	mov	srcptr, rbp			;; Save pointers
	mov	carryptr, rsi
ttp	mov	biglitptr, rdi
	mov	al, 1				;; Spreading carry over 8 words

cloop:

ttp	movzx	rcx, WORD PTR [rdi]		;; Load big vs. little flags
ttp	movzx	esi, ch				;; Big/lit flags 1-2
ttp	and	rcx, 0e0h			;; Fudge flags 1 & 2

	vmovapd	ymm0, [rbp]			;; Load values1

	cmp	al, 3*(256/4)			;; Is this the last iteration
	je	last_iter			;; Yes, last iteration requires special handling

ttp	vmulpd	ymm0, ymm0, [rdx+0*YMM_GMD][rcx] ;; mul by fudged grp two-to-minus-phi
	vaddpd	ymm0, ymm0, ymm2		;; x1 = values1 + carry
	split_upper_carry_zpad_word ttp, base2, ymm3, ymm1, ymm2, rsi*8
no const vmulpd	ymm2, ymm3, YMM_K_LO		;; low bits of high FFT carry * k_lo
const	vmulpd	ymm2, ymm3, YMM_K_TIMES_MULCONST_LO ;; low bits of high_FFT_carry * k_lo
	vaddpd	ymm0, ymm0, ymm2		;; x1 = x1 + low bits of high_FFT_carry * k_lo
no const vmulpd ymm3, ymm3, YMM_K_HI		;; low bits of high FFT carry * k_hi
const	vmulpd	ymm3, ymm3, YMM_K_TIMES_MULCONST_HI ;; low bits of high FFT carry * k_hi
ttp	vmulpd	ymm3, ymm3, YMM_LIMIT_INVERSE[rsi*8] ;; shift low bits of high FFT carry * k_hi
no ttp	vmulpd	ymm3, ymm3, YMM_LIMIT_INVERSE[0] ;; shift low bits of high FFT carry * k_hi
	vroundpd ymm3, ymm3, 0			;; WASTEFUL.  round(k_hi * inverse) should be precomputed.
	rounding ttp, base2, noexec, ymm0, ymm2, ymm4, rsi*8
	vaddpd	ymm2, ymm2, ymm3		;; Carry += shifted low bits of high_FFT_carry * k_hi
ttp	vmulpd	ymm0, ymm0, [rdx+0*YMM_GMD+YMM_GMD/2][rcx] ;; mul by fudged grp two-to-phi
	ystore	[rbp], ymm0			;; Save FFT data
	vmovapd	ymm3, ymm1			;; Next high FFT carry = high bits of high FFT carry

base2	vmovapd	ymm1, YMM_BIGVAL		;; Load comparison value
no base2 vxorpd	ymm1, ymm1, ymm1		;; Create comparison value
	vcmppd	ymm0, ymm2, ymm1, 0Ch		;; Test for non-zero carries
	vmovmskpd rbx, ymm0			;; Extract 4 comparison bits
base2	vxorpd	ymm1, ymm1, ymm1		;; Create comparison value
	vcmppd	ymm0, ymm3, ymm1, 0Ch		;; Test for non-zero carries
	vmovmskpd rcx, ymm0			;; Extract 4 comparison bits
	or	rbx, rcx			;; Are any bits on?
	jz	short done			;; No, we are done

ttp	bump	rdi, 2				;; Advance pointers
	add	rbp, pass2blkdst		;; Next FFT data ptr
 	add	al, 256/4			;; Special increment needed every 4th iteration
	jnc	cloop				;; Loop if not 4th iteration
	sub	rbp, pass1blkdst		;; Each pass1 block consists of 4 pass 2 blocks
	bump	rbp, 64				;; Next cache line
	dec	al				;; Decrement count of sets of 4 carries we've processed
ttp	cmp	cache_line_multiplier, 4	;; We must increment rdi by a funky amount every 4*clm cache lines
ttp	jne	cloop				;; No, repeat carry propagation loop
ttp	add	rdi, normval2			;; Adjust ptr to little/big flags
	jmp	cloop				;; Repeat carry propagation loop

		;; Do last iteration without propagating carries out
last_iter:
base2	vsubpd	ymm2, ymm2, YMM_BIGVAL		;; Subtract rounding constant from carry
ttp	vmulpd	ymm2, ymm2, [rdx+0*YMM_GMD+YMM_GMD/2][rcx] ;; Carry *= fudged grp two-to-phi
	vaddpd	ymm0, ymm0, ymm2		;; Values1 += carry
	ystore	[rbp], ymm0			;; Save new values1

done:	mov	rbp, srcptr			;; Restore pointers
	mov	rsi, carryptr
ttp	mov	rdi, biglitptr

base2	vmovapd	ymm4, YMM_BIGVAL		;; Clear conventional carry
no base2 vxorpd ymm4, ymm4, ymm4
	ystore	[rsi], ymm4
base2	vxorpd ymm4, ymm4, ymm4			;; Clear high carry
	ystore	[rsi+32], ymm4
	ENDM


; *************** 1D normalized add/sub macro ******************
; This macro adds or subtracts, then "normalizes" eight FFT
; data values.  This involves multiplying the summed values by
; two-to-minus-phi.  Rounding the value to an integer.  Making sure
; the integer is smaller than the maximum allowable integer, generating
; a carry if necessary. Finally, the value is multiplied by two-to-phi
; and stored.
; ymm3 = carry #2
; ymm2 = carry #1
; rcx = pointer to the first number
; rdx = pointer to the second number
; rsi = pointer to destination
; rbp = pointer two-to-phi multipliers
; rdi = pointer to array of big vs. little flags
; rbx = big vs. little word flag #2
; eax = big vs. little word flag #1
; A pipelined version of this code:
;	movzx	rax, BYTE PTR [rdi]	;; Load big vs. little flags
;	xload	ymm0, [rdx+0*dist1]	;; Load second number
;	fop	ymm0, [rcx]		;; Add/sub first number
;	mulpd	ymm0, [rbp+0]		;; Mul values1 by two-to-minus-phi
;	addpd	ymm0, ymm4		;; x = values + carry
;	xload	ymm2, YMM_LIMIT_BIGMAX[rax*2];; Load maximum * BIGVAL - BIGVAL
;	addpd	ymm2, ymm0		;; y = top bits of x
;	xload	ymm6, YMM_LIMIT_BIGMAX_NEG[rax*2];; Load -(maximum*BIGVAL-BIGVAL)
;	addpd	ymm6, ymm2		;; z = y - (maximum * BIGVAL - BIGVAL)
;	subpd	ymm0, ymm6		;; rounded value = x - z
;	mulpd	ymm2, YMM_LIMIT_INVERSE[rax*2];; next carry = shifted y
;	mulpd	ymm0, [rbp+16]		;; new value = val * two-to-phi
;	xstore	[rsi+0*dist1], ymm0	;; Save new value

ynorm_op_1d MACRO fop, ttp, base2
ttp	movzx	rax, BYTE PTR [rdi]	;; Load big vs. little flags
ttp	movzx	rbx, BYTE PTR [rdi+1]
	vmovapd	ymm0, [rdx]		;; Load second number
	fop	ymm0, ymm0, [rcx]	;; Add/sub first number
ttp	vmovapd	ymm5, YMM_NORM012_FF	;; Load FFTLEN/2
ttp	vmulpd	ymm4, ymm5, [rbp]	;; Create fudged two-to-minus-phi
	vmovapd	ymm1, [rdx+32]		;; Load second number
	fop	ymm1, ymm1, [rcx+32]	;; Add/sub first number
ttp	vmulpd	ymm5, ymm5, [rbp+64]	;; Create fudged two-to-minus-phi
ttp	vmulpd	ymm0, ymm0, ymm4	;; Mul value1 by fudged two-to-minus-phi
ttp	vmulpd	ymm1, ymm1, ymm5	;; Mul value2 by fudged two-to-minus-phi
	vaddpd	ymm0, ymm0, ymm2	;; x1 = value1 + carry
	vaddpd	ymm1, ymm1, ymm3	;; x2 = value2 + carry
	rounding_interleaved ttp, base2, noexec, ymm0, ymm2, ymm4, rax*2, ymm1, ymm3, ymm5, rbx*2
ttp	vmulpd	ymm0, ymm0, [rbp+32]	;; new value1 = val * two-to-phi
ttp	vmulpd	ymm1, ymm1, [rbp+96]	;; new value2 = val * two-to-phi
	ystore	[rsi], ymm0		;; Save value1
	ystore	[rsi+32], ymm1		;; Save value2
ttp	bump	rdi, 2			;; Next flags ptr
	bump	rcx, 64			;; Next src ptr
	bump	rdx, 64			;; Next src ptr
ttp	bump	rbp, 128		;; Next two-to-phi ptr
	bump	rsi, 64			;; Next dest ptr
	ENDM

; The irrational zpad case which uses half the ttp/ttmp data
ynorm_op_1d_zpad MACRO fop, ttp, base2
ttp	movzx	rax, BYTE PTR [rdi]	;; Load big vs. little flags
	vmovapd	ymm0, [rdx]		;; Load second number
	fop	ymm0, ymm0, [rcx]	;; Add/sub first number
ttp	vmovapd	ymm5, YMM_NORM012_FF	;; Load FFTLEN/2
ttp	vmulpd	ymm5, ymm5, [rbp]	;; Create fudged two-to-minus-phi
	vmovapd	ymm1, [rdx+32]		;; Load second number
	fop	ymm1, ymm1, [rcx+32]	;; Add/sub first number
ttp	vmulpd	ymm0, ymm0, ymm5	;; Mul value1 by fudged two-to-minus-phi
ttp	vmulpd	ymm1, ymm1, ymm5	;; Mul value2 by fudged two-to-minus-phi
	vaddpd	ymm0, ymm0, ymm2	;; x1 = value1 + carry
	vaddpd	ymm1, ymm1, ymm3	;; x2 = value2 + carry
	rounding_interleaved ttp, base2, noexec, ymm0, ymm2, ymm4, rax*2, ymm1, ymm3, ymm5, rax*2
ttp	vmulpd	ymm0, ymm0, [rbp+32]	;; new value1 = val * two-to-phi
ttp	vmulpd	ymm1, ymm1, [rbp+32]	;; new value2 = val * two-to-phi
	ystore	[rsi], ymm0		;; Save value1
	ystore	[rsi+32], ymm1		;; Save value2
ttp	bump	rdi, 1			;; Next flags ptr
	bump	rcx, 64			;; Next src ptr
	bump	rdx, 64			;; Next src ptr
ttp	bump	rbp, 64			;; Next two-to-phi ptr
	bump	rsi, 64			;; Next dest ptr
	ENDM


; This macro finishes the normalize process by adding the final
; carry from the first pass back into the lower two data values.
; These carries (from an add or subtract operation) are always
; very small.  There is no need to do any further carry propagation.
; ymm2,ymm3 = carries
; rax = pointer to the FFT data values
; rbx = pointer two-to-phi multipliers

ynorm_op_1d_mid_cleanup MACRO ttp, base2
	rotate_carries_interleaved base2, ymm2, ymm4, ymm3, ymm5, ymm0, ymm1 ;; Rotate carries
	vmovapd	ymm0, [rax]				;; Load values1
	vmovapd	ymm1, [rax+32]				;; Load values2
base2	vsubpd	ymm2, ymm2, YMM_BIGVAL			;; Remove BIGVAL
base2	vsubpd	ymm3, ymm3, YMM_BIGVAL			;; Remove BIGVAL
ttp	vmulpd	ymm2, ymm2, [rbx+32]			;; carry1 *= two-to-phi
ttp	vmulpd	ymm3, ymm3, [rbx+96]			;; carry2 *= two-to-phi
	vaddpd	ymm0, ymm0, ymm2			;; x1 = values + carry
	vaddpd	ymm1, ymm1, ymm3			;; x2 = values + carry
	ystore	[rax], ymm0				;; Save new value1
	ystore	[rax+32], ymm1				;; Save new value2
	vmovapd	ymm2, ymm4				;; Install next section's carry
	vmovapd	ymm3, ymm5				;; Install next section's carry
	ENDM

; The zpad case where we have half the ttp/ttmp data
ynorm_op_1d_zpad_mid_cleanup MACRO ttp, base2
	rotate_carries_interleaved base2, ymm2, ymm4, ymm3, ymm5, ymm0, ymm1 ;; Rotate carries
	vmovapd	ymm0, [rax]				;; Load values1
	vmovapd	ymm1, [rax+32]				;; Load values2
base2	vsubpd	ymm2, ymm2, YMM_BIGVAL			;; Remove BIGVAL
base2	vsubpd	ymm3, ymm3, YMM_BIGVAL			;; Remove BIGVAL
ttp	vmulpd	ymm2, ymm2, [rbx+32]			;; carry1 *= two-to-phi
ttp	vmulpd	ymm3, ymm3, [rbx+32]			;; carry2 *= two-to-phi
	vaddpd	ymm0, ymm0, ymm2			;; x1 = values + carry
	vaddpd	ymm1, ymm1, ymm3			;; x2 = values + carry
	ystore	[rax], ymm0				;; Save new value1
	ystore	[rax+32], ymm1				;; Save new value2
	vmovapd	ymm2, ymm4				;; Install next section's carry
	vmovapd	ymm3, ymm5				;; Install next section's carry
	ENDM

; This macro finishes the add/sub/addsub normalize process by adding
; the final carry from the first pass back into the lower two data values.
; xmm2,xmm3 = carries (will be very, very small)
; rax,rsi,rbp,rdi = trash

ynorm_op_1d_cleanup MACRO ttp, base2, destptr
	mov	rsi, destptr			;; Address of FFT data
	ynorm_top_carry_1d ttp, base2		;; No, do a very standard carry

ttp	mov	rbp, norm_col_mults		;; Address of the ttp/ttmp multipliers
base2	vsubsd	xmm3, xmm3, Q YMM_BIGVAL
	vmulsd	xmm3, xmm3, Q YMM_MINUS_C	;; Adjust wrap-around carry
	vaddsd	xmm0, xmm3, Q [rsi]		;; wrap-around carry + values1
	vmovsd	xmm1, Q [rsi+32]		;; Load values2
ttp	vmulsd	xmm1, xmm1, Q [rbp+64]		;; Mul values2 by two-to-minus-phi
ttp	vmulsd	xmm1, xmm1, Q YMM_NORM012_FF	;; Mul by FFTLEN/2
base2	vsubsd	xmm2, xmm2, Q YMM_BIGVAL
	vaddsd	xmm1, xmm1, xmm2		;; x2 = values + carry
ttp	vmulsd	xmm1, xmm1, Q [rbp+96]		;; new value2 = val * two-to-phi
	vmovsd	Q [rsi], xmm0			;; Save new value1
	vmovsd	Q [rsi+32], xmm1		;; Save new value2
	ENDM

; This macro is similar to ynorm_1d_zpad_cleanup for handling zpad carries in an
; add/sub/addsub operation.
; xmm2 = carry into the top half (carry is tiny)
; rsi = pointer to the FFT data values
; rbp = pointer two-to-phi multipliers
; rdi = big vs. little array pointer

ynorm_op_1d_zpad_cleanup MACRO ttp, base2, destptr
	LOCAL	smallk, mediumk, div_k_done

base2	vsubsd	xmm2, xmm2, Q YMM_BIGVAL	;; Integerize carry

	;; Copy and integerize data from 7 words above halfway point to ZPAD0-ZPAD6
	;; Clear high words as we go
	;; Then we can make an almost exact copy of the ynorm_1d_zpad_cleanup code

	mov	rsi, destptr			;; Address of FFT data
ttp	mov	rbp, norm_col_mults		;; Address of the multipliers

	vaddsd	xmm0, xmm2, Q [rsi+32]		;; Carry + value1
	vmovsd	ZPAD0, xmm0

	add	rsi, YMM_SRC_INCR1		;; Next source pointer
ttp	add	rbp, YMM_NORM_INCR1		;; Next ttp/ttmp pointer
	vmovsd	xmm0, Q [rsi+32]		;; Value2
ttp	vmulsd	xmm0, xmm0, Q [rbp]		;; Mul values2 by two-to-minus-phi
ttp	vmulsd	xmm0, xmm0, Q YMM_NORM012_FF	;; Mul by FFTLEN/2
ttp	vroundsd xmm0, xmm0, xmm0, 0		;; Round to an integer
	vmovsd	ZPAD1, xmm0

	add	rsi, YMM_SRC_INCR2		;; Next source pointer
ttp	add	rbp, YMM_NORM_INCR2		;; Next ttp/ttmp pointer
	vmovsd	xmm0, Q [rsi+32]		;; Value3
ttp	vmulsd	xmm0, xmm0, Q [rbp]		;; Mul values3 by two-to-minus-phi
ttp	vmulsd	xmm0, xmm0, Q YMM_NORM012_FF	;; Mul by FFTLEN/2
ttp	vroundsd xmm0, xmm0, xmm0, 0		;; Round to an integer
	vmovsd	ZPAD2, xmm0

	add	rsi, YMM_SRC_INCR3		;; Next source pointer
ttp	add	rbp, YMM_NORM_INCR3		;; Next ttp/ttmp pointer
	vmovsd	xmm0, Q [rsi+32]		;; Value4
ttp	vmulsd	xmm0, xmm0, Q [rbp]		;; Mul values4 by two-to-minus-phi
ttp	vmulsd	xmm0, xmm0, Q YMM_NORM012_FF	;; Mul by FFTLEN/2
ttp	vroundsd xmm0, xmm0, xmm0, 0		;; Round to an integer
	vmovsd	ZPAD3, xmm0

	add	rsi, YMM_SRC_INCR4		;; Next source pointer
ttp	add	rbp, YMM_NORM_INCR4		;; Next ttp/ttmp pointer
	vmovsd	xmm0, Q [rsi+32]		;; Value5
ttp	vmulsd	xmm0, xmm0, Q [rbp]		;; Mul values5 by two-to-minus-phi
ttp	vmulsd	xmm0, xmm0, Q YMM_NORM012_FF	;; Mul by FFTLEN/2
ttp	vroundsd xmm0, xmm0, xmm0, 0		;; Round to an integer
	vmovsd	ZPAD4, xmm0
	vxorpd	xmm1, xmm1, xmm1		;; Clear highest words
	vmovsd	Q [rsi+32], xmm1		;; Clear value5

	vmovsd	ZPAD5, xmm1
	vmovsd	ZPAD6, xmm1

	;; Divide the zpad data by k.  Store the integer part in YMM_TMP
	;; and the remainder in ZPAD0.  Later we will wrap the integer part
	;; down to the bottom of the FFT data area (and multiply by -c).
	;; And we will store the remainder in the upper half of the FFT
	;; data area.

	;; Note there are three cases to handle.  K is smaller than a big word.
	;; K is between one and 2 big words in size.  And K is more than
	;; 2 big words in size.

	cmp	ZPAD_TYPE, 2			;; Are we dealing with case 1,2,or 3
	jl	smallk				;; One word case
	je	mediumk				;; Two word case

	;; This case does the divide by k where k is three words

	vmovsd	xmm0, ZPAD6			;; Load zpad word (high bits)
	vmovsd	xmm1, ZPAD5			;; Load zpad word (middle bits)
	vmovsd	xmm2, ZPAD4			;; Load zpad word (low bits)
	vmulsd	xmm4, xmm0, ZPAD_INVERSE_K6	;; Mul ZPAD by shifted 1/k
	vroundsd xmm4, xmm4, xmm4, 0		;; Round to integer
	vmulsd	xmm5, xmm4, ZPAD_K6_HI		;; Mul by high bits of k
	vsubsd	xmm0, xmm0, xmm5		;; Calculate high bits of remainder
	vmulsd	xmm5, xmm4, ZPAD_K6_MID		;; Mul by middle bits of k
	vsubsd	xmm1, xmm1, xmm5		;; Calculate middle bits of remainder
	vmulsd	xmm5, xmm4, ZPAD_K6_LO		;; Mul by low bits of k
	vsubsd	xmm2, xmm2, xmm5		;; Calculate low bits of remainder
	vmovsd	Q YMM_TMP5, xmm4		;; Save word of zpad / k

	vmulsd	xmm0, xmm0, ZPAD_SHIFT6		;; Shift previous zpad word
	vaddsd	xmm1, xmm1, xmm0		;; Add to create new high zpad bits
	vmovsd	xmm0, ZPAD3			;; Load zpad word (new low bits)
	vmulsd	xmm5, xmm1, ZPAD_SHIFT5		;; Combine high and medium bits
	vaddsd	xmm5, xmm5, xmm2
	vmulsd	xmm4, xmm5, ZPAD_INVERSE_K5	;; Mul ZPAD by shifted 1/k
	vroundsd xmm4, xmm4, xmm4, 0		;; Round to integer
	vmulsd	xmm5, xmm4, ZPAD_K5_HI		;; Mul by high bits of k
	vsubsd	xmm1, xmm1, xmm5		;; Calculate high bits of remainder
	vmulsd	xmm5, xmm4, ZPAD_K5_MID		;; Mul by middle bits of k
	vsubsd	xmm2, xmm2, xmm5		;; Calculate middle bits of remainder
	vmulsd	xmm5, xmm4, ZPAD_K5_LO		;; Mul by low bits of k
	vsubsd	xmm0, xmm0, xmm5		;; Calculate low bits of remainder
	vmovsd	Q YMM_TMP4, xmm4		;; Save word of zpad / k

	vmulsd	xmm1, xmm1, ZPAD_SHIFT5		;; Shift previous zpad word
	vaddsd	xmm2, xmm2, xmm1		;; Add to create new high zpad bits
	vmovsd	xmm1, ZPAD2			;; Load zpad word (new low bits)
	vmulsd	xmm5, xmm2, ZPAD_SHIFT4		;; Combine high and medium bits
	vaddsd	xmm5, xmm5, xmm0
	vmulsd	xmm4, xmm5, ZPAD_INVERSE_K4	;; Mul ZPAD by shifted 1/k
	vroundsd xmm4, xmm4, xmm4, 0		;; Round to integer
	vmulsd	xmm5, xmm4, ZPAD_K4_HI		;; Mul by high bits of k
	vsubsd	xmm2, xmm2, xmm5		;; Calculate high bits of remainder
	vmulsd	xmm5, xmm4, ZPAD_K4_MID		;; Mul by middle bits of k
	vsubsd	xmm0,xmm0,xmm5			;; Calculate middle bits of remainder
	vmulsd	xmm5, xmm4, ZPAD_K4_LO		;; Mul by low bits of k
	vsubsd	xmm1, xmm1, xmm5		;; Calculate low bits of remainder
	vmovsd	Q YMM_TMP3, xmm4		;; Save word of zpad / k

	vmulsd	xmm2, xmm2, ZPAD_SHIFT4		;; Shift previous zpad word
	vaddsd	xmm0, xmm0, xmm2		;; Add to create new high zpad bits
	vmovsd	xmm2, ZPAD1			;; Load zpad word (new low bits)
	vmulsd	xmm5, xmm0, ZPAD_SHIFT3		;; Combine high and medium bits
	vaddsd	xmm5, xmm5, xmm1
	vmulsd	xmm4, xmm5, ZPAD_INVERSE_K3	;; Mul ZPAD by shifted 1/k
	vroundsd xmm4, xmm4, xmm4, 0		;; Round to integer
	vmulsd	xmm5, xmm4, ZPAD_K3_HI		;; Mul by high bits of k
	vsubsd	xmm0, xmm0, xmm5		;; Calculate high bits of remainder
	vmulsd	xmm5, xmm4, ZPAD_K3_MID		;; Mul by middle bits of k
	vsubsd	xmm1, xmm1, xmm5		;; Calculate middle bits of remainder
	vmulsd	xmm5, xmm4, ZPAD_K3_LO		;; Mul by low bits of k
	vsubsd	xmm2, xmm2, xmm5		;; Calculate low bits of remainder
	vmovsd	Q YMM_TMP2, xmm4		;; Save word of zpad / k

	vmulsd	xmm0, xmm0, ZPAD_SHIFT3		;; Shift previous zpad word
	vaddsd	xmm1, xmm1, xmm0		;; Add to create new high zpad bits
	vmovsd	xmm0, ZPAD0			;; Load zpad word (new low bits)
	vmulsd	xmm5, xmm1, ZPAD_SHIFT2		;; Combine high and medium bits
	vaddsd	xmm5, xmm5, xmm2
	vmulsd	xmm4, xmm5, ZPAD_INVERSE_K2	;; Mul ZPAD by shifted 1/k
	vroundsd xmm4, xmm4, xmm4, 0		;; Round to integer
	vmulsd	xmm5, xmm4, ZPAD_K2_HI		;; Mul by high bits of k
	vsubsd	xmm1, xmm1, xmm5		;; Calculate high bits of remainder
	vmulsd	xmm5, xmm4, ZPAD_K2_MID		;; Load middle bits of k
	vsubsd	xmm2, xmm2, xmm5		;; Calculate middle bits of remainder
	vmulsd	xmm5, xmm4, ZPAD_K2_LO		;; Load low bits of k
	vsubsd	xmm0, xmm0, xmm5		;; Calculate low bits of remainder
	vmovsd	Q YMM_TMP1, xmm4		;; Save word of zpad / k

	vmulsd	xmm1, xmm1, ZPAD_SHIFT2		;; Shift previous zpad word
	vaddsd	xmm2, xmm2, xmm1		;; Add to create new high zpad bits
	vmulsd	xmm2, xmm2, ZPAD_SHIFT1		;; Shift previous zpad word
	vaddsd	xmm0, xmm0, xmm2		;; Add to create new high zpad bits
	vmovsd	ZPAD0, xmm0			;; Save remainder of zpad / k

	vxorpd	xmm1, xmm1, xmm1		;; Zero words that other cases set
	vmovsd	Q YMM_TMP6, xmm1
	
	jmp	div_k_done

	;; This case does the divide by k where k is two words
mediumk:
	vmovsd	xmm0, ZPAD6			;; Load zpad word (high bits)
	vmovsd	xmm1, ZPAD5			;; Load zpad word (low bits)
	vmulsd	xmm4, xmm0, ZPAD_INVERSE_K6	;; Mul ZPAD by shifted 1/k
	vroundsd xmm4, xmm4, xmm4, 0		;; Round to integer
	vmulsd	xmm5, xmm4, ZPAD_K6_HI		;; Mul by high bits of k
	vsubsd	xmm0, xmm0, xmm5		;; Calculate high bits of remainder
	vmulsd	xmm5, xmm4, ZPAD_K6_LO		;; Mul by low bits of k
	vsubsd	xmm1, xmm1, xmm5		;; Calculate low bits of remainder
	vmovsd	Q YMM_TMP6, xmm4		;; Save word of zpad / k

	vmulsd	xmm0, xmm0, ZPAD_SHIFT6		;; Shift previous zpad word
	vaddsd	xmm0, xmm0, xmm1		;; Add to create new high zpad bits
	vmovsd	xmm1, ZPAD4			;; Load zpad word (new low bits)
	vmulsd	xmm4, xmm0, ZPAD_INVERSE_K5	;; Mul ZPAD by shifted 1/k
	vroundsd xmm4, xmm4, xmm4, 0		;; Round to integer
	vmulsd	xmm5, xmm4, ZPAD_K5_HI		;; Mul by high bits of k
	vsubsd	xmm0, xmm0, xmm5		;; Calculate high bits of remainder
	vmulsd	xmm5, xmm4, ZPAD_K5_LO		;; Mul by low bits of k
	vsubsd	xmm1, xmm1, xmm5		;; Calculate low bits of remainder
	vmovsd	Q YMM_TMP5, xmm4		;; Save word of zpad / k

	vmulsd	xmm0, xmm0, ZPAD_SHIFT5		;; Shift previous zpad word
	vaddsd	xmm0, xmm0, xmm1		;; Add to create new high zpad bits
	vmovsd	xmm1, ZPAD3			;; Load zpad word (new low bits)
	vmulsd	xmm4, xmm0, ZPAD_INVERSE_K4	;; Mul ZPAD by shifted 1/k
	vroundsd xmm4, xmm4, xmm4, 0		;; Round to integer
	vmulsd	xmm5, xmm4, ZPAD_K4_HI		;; Mul by high bits of k
	vsubsd	xmm0, xmm0, xmm5		;; Calculate high bits of remainder
	vmulsd	xmm5, xmm4, ZPAD_K4_LO		;; Mul by low bits of k
	vsubsd	xmm1, xmm1, xmm5		;; Calculate low bits of remainder
	vmovsd	Q YMM_TMP4, xmm4		;; Save word of zpad / k

	vmulsd	xmm0, xmm0, ZPAD_SHIFT4		;; Shift previous zpad word
	vaddsd	xmm0, xmm0, xmm1		;; Add to create new high zpad bits
	vmovsd	xmm1, ZPAD2			;; Load zpad word (new low bits)
	vmulsd	xmm4, xmm0, ZPAD_INVERSE_K3	;; Mul ZPAD by shifted 1/k
	vroundsd xmm4, xmm4, xmm4, 0		;; Round to integer
	vmulsd	xmm5, xmm4, ZPAD_K3_HI		;; Mul by high bits of k
	vsubsd	xmm0, xmm0, xmm5		;; Calculate high bits of remainder
	vmulsd	xmm5, xmm4, ZPAD_K3_LO		;; Mul by low bits of k
	vsubsd	xmm1, xmm1, xmm5		;; Calculate low bits of remainder
	vmovsd	Q YMM_TMP3, xmm4		;; Save word of zpad / k

	vmulsd	xmm0, xmm0, ZPAD_SHIFT3		;; Shift previous zpad word
	vaddsd	xmm0, xmm0, xmm1		;; Add to create new high zpad bits
	vmovsd	xmm1, ZPAD1			;; Load zpad word (new low bits)
	vmulsd	xmm4, xmm0, ZPAD_INVERSE_K2	;; Mul ZPAD by shifted 1/k
	vroundsd xmm4, xmm4, xmm4, 0		;; Round to integer
	vmulsd	xmm5, xmm4, ZPAD_K2_HI		;; Mul by high bits of k
	vsubsd	xmm0, xmm0, xmm5		;; Calculate high bits of remainder
	vmulsd	xmm5, xmm4, ZPAD_K2_LO		;; Mul by low bits of k
	vsubsd	xmm1, xmm1, xmm5		;; Calculate low bits of remainder
	vmovsd	Q YMM_TMP2, xmm4		;; Save word of zpad / k

	vmulsd	xmm0, xmm0, ZPAD_SHIFT2		;; Shift previous zpad word
	vaddsd	xmm0, xmm0, xmm1		;; Add to create new high zpad bits
	vmovsd	xmm1, ZPAD0			;; Load zpad word (new low bits)
	vmulsd	xmm4, xmm0, ZPAD_INVERSE_K1	;; Mul ZPAD by shifted 1/k
	vroundsd xmm4, xmm4, xmm4, 0		;; Round to integer
	vmulsd	xmm5, xmm4, ZPAD_K1_HI		;; Mul by high bits of k
	vsubsd	xmm0, xmm0, xmm5		;; Calculate high bits of remainder
	vmulsd	xmm5, xmm4, ZPAD_K1_LO		;; Mul by low bits of k
	vsubsd	xmm1, xmm1, xmm5		;; Calculate low bits of remainder
	vmovsd	Q YMM_TMP1, xmm4		;; Save word of zpad / k

	vmulsd	xmm0, xmm0, ZPAD_SHIFT1		;; Shift previous zpad word
	vaddsd	xmm0, xmm0, xmm1		;; Add to create new high zpad bits
	vmovsd	ZPAD0, xmm0			;; Save remainder of zpad / k

	jmp	div_k_done

	;; This case does the divide by k where k is one word
	;; Assume ZPAD5 and ZPAD6 are zero.
smallk:	vmovsd	xmm0, ZPAD4			;; Load zpad data
	vmulsd	xmm4, xmm0, ZPAD_INVERSE_K1	;; Mul ZPAD data by 1/k
	vroundsd xmm4, xmm4, xmm4, 0		;; Round to integer
	vmovsd	Q YMM_TMP5, xmm4		;; Save integer part
	vmulsd	xmm4, xmm4, ZPAD_K1_LO		;; Compute remainder
	vsubsd	xmm0, xmm0, xmm4

	vmulsd	xmm0, xmm0, ZPAD_SHIFT4		;; Shift previous zpad word
	vaddsd	xmm0, xmm0, ZPAD3		;; Add in zpad data
	vmulsd	xmm4, xmm0, ZPAD_INVERSE_K1	;; Mul ZPAD data by 1/k
	vroundsd xmm4, xmm4, xmm4, 0		;; Round to integer
	vmovsd	Q YMM_TMP4, xmm4		;; Save integer part
	vmulsd	xmm4, xmm4, ZPAD_K1_LO		;; Compute remainder
	vsubsd	xmm0, xmm0, xmm4

	vmulsd	xmm0, xmm0, ZPAD_SHIFT3		;; Shift previous zpad word
	vaddsd	xmm0, xmm0, ZPAD2		;; Add in zpad data
	vmulsd	xmm4, xmm0, ZPAD_INVERSE_K1	;; Mul ZPAD data by 1/k
	vroundsd xmm4, xmm4, xmm4, 0		;; Round to integer
	vmovsd	Q YMM_TMP3, xmm4		;; Save integer part
	vmulsd	xmm4, xmm4, ZPAD_K1_LO		;; Compute remainder
	vsubsd	xmm0, xmm0, xmm4

	vmulsd	xmm0, xmm0, ZPAD_SHIFT2		;; Shift previous zpad word
	vaddsd	xmm0, xmm0, ZPAD1		;; Add in zpad data
	vmulsd	xmm4, xmm0, ZPAD_INVERSE_K1	;; Mul ZPAD data by 1/k
	vroundsd xmm4, xmm4, xmm4, 0		;; Round to integer
	vmovsd	Q YMM_TMP2, xmm4		;; Save integer part
	vmulsd	xmm4, xmm4, ZPAD_K1_LO		;; Compute remainder
	vsubsd	xmm0, xmm0, xmm4

	vmulsd	xmm0, xmm0, ZPAD_SHIFT1		;; Shift previous zpad word
	vaddsd	xmm0, xmm0, ZPAD0		;; Add in zpad data
	vmulsd	xmm4, xmm0, ZPAD_INVERSE_K1	;; Mul ZPAD data by 1/k
	vroundsd xmm4, xmm4, xmm4, 0		;; Round to integer
	vmovsd	Q YMM_TMP1, xmm4		;; Save integer part
	vmulsd	xmm4, xmm4, ZPAD_K1_LO		;; Compute remainder
	vsubsd	xmm0, xmm0, xmm4
	vmovsd	ZPAD0, xmm0			;; Save remainder

	vxorpd	xmm1, xmm1, xmm1		;; Zero words that other cases set
	vmovsd	Q YMM_TMP6, xmm1

div_k_done:

	;; Now normalize the data above the halfway point.  Remember that the
	;; two-to-phi multiplier for the first value will be 1.0.

	lea	rdi, YMM_FIRST_BIGLIT_VALUES	;; Address of first big/little flags
	mov	rsi, destptr			;; Address of squared number
ttp	mov	rbp, norm_col_mults		;; Address of the ttmp/ttp multipliers

	movzx	rax, BYTE PTR [rdi]		;; First word
	vmovsd	xmm0, ZPAD0			;; Load remainder of divide by k
base2	vaddsd	xmm0, xmm0, Q YMM_BIGVAL
	new_single_rounding ttp, base2, xmm0, xmm2, xmm4, rax*2
	vmovsd	Q [rsi+32], xmm0		;; Save value1

ttp	add	rbp, YMM_NORM_INCR1		;; Next ttp/ttmp pointer
	add	rsi, YMM_SRC_INCR1		;; Next source pointer
	movzx	rax, BYTE PTR [rdi+1]		;; Load big vs. little flags
	new_single_rounding ttp, base2, xmm2, xmm0, xmm4, rax*2
ttp	vmulsd	xmm2, xmm2, Q [rbp+32]		;; new value2 = val * two-to-phi
	vmovsd	Q [rsi+32], xmm2		;; Save value2

ttp	add	rbp, YMM_NORM_INCR2		;; Next ttp/ttmp pointer
	add	rsi, YMM_SRC_INCR2		;; Next source pointer
	movzx	rax, BYTE PTR [rdi+2]		;; Load big vs. little flags
	new_single_rounding ttp, base2, xmm0, xmm2, xmm4, rax*2
ttp	vmulsd	xmm0, xmm0, Q [rbp+32]		;; new value3 = val * two-to-phi
	vmovsd	Q [rsi+32], xmm0		;; Save value3

ttp	add	rbp, YMM_NORM_INCR3		;; Next ttp/ttmp pointer
	add	rsi, YMM_SRC_INCR3		;; Next source pointer
base2	vsubsd	xmm2, xmm2, Q YMM_BIGVAL	;; Remove integer rounding constant
ttp	vmulsd	xmm2, xmm2, Q [rbp+32]		;; value4 = carry * two-to-phi
	vmovsd	Q [rsi+32], xmm2		;; Save new value4

	;; Mul the integer part of (ZPAD data divided by k) by -c in
	;; preparation for adding it into the lower FFT data area.
	;; Also add in the shifted high FFT carry at this time.

	;; Now add in and normalize the bottom FFT data.  Remember that the
	;; two-to-phi multiplier for the first value will be 1.0.  We 
	;; must go 6 words deep in case k is 48-50 bits and c is 32 bits.

	mov	rsi, destptr			;; Address of squared number
ttp	mov	rbp, norm_col_mults		;; Address of ttmp/ttp multipliers

	movzx	rax, BYTE PTR [rdi]		;; First word
	vmovsd	xmm0, Q YMM_TMP1		;; Load integer part of divide by k
	vmulsd	xmm0, xmm0, Q YMM_MINUS_C	;; Mul by -c
base2	vaddsd	xmm0, xmm0, Q YMM_BIGVAL
	vaddsd	xmm0, xmm0, Q [rsi]		;; Add in the FFT data
	new_single_rounding ttp, base2, xmm0, xmm2, xmm4, rax*2
	vmovsd	Q [rsi], xmm0			;; Save value1

	add	rsi, YMM_SRC_INCR1		;; Next source pointer
ttp	add	rbp, YMM_NORM_INCR1		;; Next ttp/ttmp pointer
ttp	movzx	rax, BYTE PTR [rdi+1]		;; Load big vs. little flags
	vmovsd	xmm0, Q YMM_TMP2		;; Load integer part of divide by k
	vmulsd	xmm0, xmm0, Q YMM_MINUS_C	;; Mul by -c
	vmovsd	xmm1, Q [rsi]			;; Load FFT data
ttp	vmulsd	xmm1, xmm1, Q [rbp]		;; Mul values2 by two-to-minus-phi
ttp	vmulsd	xmm1, xmm1, Q YMM_NORM012_FF	;; Mul by FFTLEN/2
	vaddsd	xmm0, xmm0, xmm2		;; x2 = value + carry
	vaddsd	xmm0, xmm0, xmm1		;; Add in the FFT data
	new_single_rounding ttp, base2, xmm0, xmm2, xmm4, rax*2
ttp	vmulsd	xmm0, xmm0, Q [rbp+32]		;; new value2 = val * two-to-phi
	vmovsd	Q [rsi], xmm0			;; Save value2

	add	rsi, YMM_SRC_INCR2		;; Next source pointer
ttp	add	rbp, YMM_NORM_INCR2		;; Next ttp/ttmp pointer
ttp	movzx	rax, BYTE PTR [rdi+2]		;; Load big vs. little flags
	vmovsd	xmm0, Q YMM_TMP3		;; Load integer part of divide by k
	vmulsd	xmm0, xmm0, Q YMM_MINUS_C	;; Mul by -c
	vmovsd	xmm1, Q [rsi]			;; Load FFT data
ttp	vmulsd	xmm1, xmm1, Q [rbp]		;; Mul values3 by two-to-minus-phi
ttp	vmulsd	xmm1, xmm1, Q YMM_NORM012_FF	;; Mul by FFTLEN/2
	vaddsd	xmm0, xmm0, xmm2		;; x3 = value + carry
	vaddsd	xmm0, xmm0, xmm1		;; Add in the FFT data
	new_single_rounding ttp, base2, xmm0, xmm2, xmm4, rax*2
ttp	vmulsd	xmm0, xmm0, Q [rbp+32]		;; new value3 = val * two-to-phi
	vmovsd	Q [rsi], xmm0			;; Save value3

	add	rsi, YMM_SRC_INCR3		;; Next source pointer
ttp	add	rbp, YMM_NORM_INCR3		;; Next ttp/ttmp pointer
ttp	movzx	rax, BYTE PTR [rdi+3]		;; Load big vs. little flags
	vmovsd	xmm0, Q YMM_TMP4		;; Load integer part of divide by k
	vmulsd	xmm0, xmm0, Q YMM_MINUS_C	;; Mul by -c
	vmovsd	xmm1, Q [rsi]			;; Load FFT data
ttp	vmulsd	xmm1, xmm1, Q [rbp]		;; Mul values4 by two-to-minus-phi
ttp	vmulsd	xmm1, xmm1, Q YMM_NORM012_FF	;; Mul by FFTLEN/2
	vaddsd	xmm0, xmm0, xmm2		;; x4 = value + carry
	vaddsd	xmm0, xmm0, xmm1		;; Add in the FFT data
	new_single_rounding ttp, base2, xmm0, xmm2, xmm4, rax*2
ttp	vmulsd	xmm0, xmm0, Q [rbp+32]		;; new value4 = val * two-to-phi
	vmovsd	Q [rsi], xmm0			;; Save value4

	add	rsi, YMM_SRC_INCR4		;; Next source pointer
ttp	add	rbp, YMM_NORM_INCR4		;; Next ttp/ttmp pointer
ttp	movzx	rax, BYTE PTR [rdi+4]		;; Load big vs. little flags
	vmovsd	xmm0, Q YMM_TMP5		;; Load integer part of divide by k
	vmulsd	xmm0, xmm0, Q YMM_MINUS_C	;; Mul by -c
	vmovsd	xmm1, Q [rsi]			;; Load FFT data
ttp	vmulsd	xmm1, xmm1, Q [rbp]		;; Mul values4 by two-to-minus-phi
ttp	vmulsd	xmm1, xmm1, Q YMM_NORM012_FF	;; Mul by FFTLEN/2
	vaddsd	xmm0, xmm0, xmm2		;; x5 = value + carry
	vaddsd	xmm0, xmm0, xmm1		;; Add in the FFT data
	new_single_rounding ttp, base2, xmm0, xmm2, xmm4, rax*2
ttp	vmulsd	xmm0, xmm0, Q [rbp+32]		;; new value5 = val * two-to-phi
	vmovsd	Q [rsi], xmm0			;; Save value5

	add	rsi, YMM_SRC_INCR5		;; Next source pointer
ttp	add	rbp, YMM_NORM_INCR5		;; Next ttp/ttmp pointer
ttp	movzx	rax, BYTE PTR [rdi+5]		;; Load big vs. little flags
	vmovsd	xmm0, Q YMM_TMP6		;; Load integer part of divide by k
	vmulsd	xmm0, xmm0, Q YMM_MINUS_C	;; Mul by -c
	vmovsd	xmm1, Q [rsi]			;; Load FFT data
ttp	vmulsd	xmm1, xmm1, Q [rbp]		;; Mul values6 by two-to-minus-phi
ttp	vmulsd	xmm1, xmm1, Q YMM_NORM012_FF	;; Mul by FFTLEN/2
	vaddsd	xmm0, xmm0, xmm2		;; x6 = value + carry
	vaddsd	xmm0, xmm0, xmm1		;; Add in the FFT data
	new_single_rounding ttp, base2, xmm0, xmm2, xmm4, rax*2
ttp	vmulsd	xmm0, xmm0, Q [rbp+32]		;; new value6 = val * two-to-phi
	vmovsd	Q [rsi], xmm0			;; Save value6

	add	rsi, YMM_SRC_INCR6		;; Next source pointer
ttp	add	rbp, YMM_NORM_INCR6		;; Next ttp/ttmp pointer
ttp	movzx	rax, BYTE PTR [rdi+6]		;; Load big vs. little flags
	vmovsd	xmm0, Q [rsi]			;; Load FFT data
ttp	vmulsd	xmm0, xmm0, Q [rbp]		;; Mul values7 by two-to-minus-phi
ttp	vmulsd	xmm0, xmm0, Q YMM_NORM012_FF	;; Mul by FFTLEN/2
	vaddsd	xmm0, xmm0, xmm2		;; x7 = value + carry
	new_single_rounding ttp, base2, xmm0, xmm2, xmm4, rax*2
ttp	vmulsd	xmm0, xmm0, Q [rbp+32]		;; new value7 = val * two-to-phi
	vmovsd	Q [rsi], xmm0			;; Save value7

	add	rsi, YMM_SRC_INCR7		;; Next source pointer
ttp	add	rbp, YMM_NORM_INCR7		;; Next ttp/ttmp pointer
base2	vsubsd	xmm2, xmm2, Q YMM_BIGVAL	;; Remove rounding constant
ttp	vmulsd	xmm2, xmm2, Q [rbp+32]		;; new value8 = val * two-to-phi
	vaddsd	xmm2, xmm2, Q [rsi]		;; Add in FFT data
	vmovsd	Q [rsi], xmm2			;; Save value8
	ENDM


; *************** 1D normalized add/sub macro ******************
; This macro adds and subtracts, then "normalizes" eight FFT
; data values.  This involves multiplying the sum values by
; two-to-minus-phi.  Adding, subtracting and rounding the value to an
; integer.  Make sure the integer is smaller than the maximum allowable
; integer, generating carries if necessary.  Finally, the values are
; multiplied by two-to-phi and stored.
; ymm7 = sub carry #2
; ymm6 = sub carry #1
; ymm3 = add carry #2
; ymm2 = add carry #1
; rcx = pointer to the first number
; rdx = pointer to the second number
; rsi = pointer to destination #1
; rbp = pointer to destination #2
; rbx = pointer two-to-phi multipliers
; rdi = pointer to array of big vs. little flags
; eax = big vs. little word flag #1

ynorm_addsub_1d MACRO ttp, base2
ttp	movzx	rax, BYTE PTR [rdi]	;; Load big vs. little flags
	vmovapd	ymm0, [rcx]		;; Load first number
	vaddpd	ymm0, ymm0, [rdx]	;; Add second number
ttp	vmovapd	ymm5, [rbx]		;; Load fudged two-to-minus-phi
ttp	vmulpd	ymm5, ymm5, YMM_NORM012_FF ;; Mul by FFTLEN/2
ttp	vmulpd	ymm0, ymm0, ymm5	;; Mul values1 by two-to-minus-phi
	vmovapd	ymm1, [rcx]		;; Load first number
	vsubpd	ymm1, ymm1, [rdx]	;; Sub second number
ttp	vmulpd	ymm1, ymm1, ymm5	;; Mul values2 by two-to-minus-phi
	vaddpd	ymm0, ymm0, ymm2	;; x1 = values + carry
	vaddpd	ymm1, ymm1, ymm6	;; x2 = values + carry
	rounding_interleaved ttp, base2, noexec, ymm0, ymm2, ymm4, rax*2, ymm1, ymm6, ymm5, rax*2
ttp	vmulpd	ymm0, ymm0, [rbx+32]	;; new value1 = val * two-to-phi
ttp	vmulpd	ymm1, ymm1, [rbx+32]	;; new value2 = val * two-to-phi
	ystore	[rsi], ymm0		;; Save value1
	ystore	[rbp], ymm1		;; Save value2

ttp	movzx	rax, BYTE PTR [rdi+1]	;; Load big vs. little flags
	vmovapd	ymm0, [rcx+32]		;; Load first number
	vaddpd	ymm0, ymm0, [rdx+32]	;; Add second number
ttp	vmovapd	ymm5, [rbx+64]		;; Load fudged two-to-minus-phi
ttp	vmulpd	ymm5, ymm5, YMM_NORM012_FF ;; Mul by FFTLEN/2
ttp	vmulpd	ymm0, ymm0, ymm5	;; Mul values1 by two-to-minus-phi
	vmovapd	ymm1, [rcx+32]		;; Load first number
	vsubpd	ymm1, ymm1, [rdx+32]	;; Sub second number
ttp	vmulpd	ymm1, ymm1, ymm5	;; Mul values2 by two-to-minus-phi
	vaddpd	ymm0, ymm0, ymm3	;; x1 = values + carry
	vaddpd	ymm1, ymm1, ymm7	;; x2 = values + carry
	rounding_interleaved ttp, base2, noexec, ymm0, ymm3, ymm4, rax*2, ymm1, ymm7, ymm5, rax*2
ttp	vmulpd	ymm0, ymm0, [rbx+96]	;; new value1 = val * two-to-phi
ttp	vmulpd	ymm1, ymm1, [rbx+96]	;; new value2 = val * two-to-phi
	ystore	[rsi+32], ymm0		;; Save value1
	ystore	[rbp+32], ymm1		;; Save value2

ttp	bump	rdi, 2			;; Next flags ptr
	bump	rcx, 64			;; Next src ptr
	bump	rdx, 64			;; Next src ptr
ttp	bump	rbx, 128		;; Next two-to-phi ptr
	bump	rsi, 64			;; Next dest ptr
	bump	rbp, 64			;; Next dest ptr
	ENDM

; The zero pad case which has half the ttp/ttmp data
ynorm_addsub_1d_zpad MACRO ttp, base2
ttp	movzx	rax, BYTE PTR [rdi]	;; Load big vs. little flags
	vmovapd	ymm0, [rcx]		;; Load first number
	vaddpd	ymm0, ymm0, [rdx]	;; Add second number
ttp	vmovapd	ymm5, [rbx]		;; Load fudged two-to-minus-phi
ttp	vmulpd	ymm5, ymm5, YMM_NORM012_FF ;; Mul by FFTLEN/2
ttp	vmulpd	ymm0, ymm0, ymm5	;; Mul values1 by two-to-minus-phi
	vmovapd	ymm1, [rcx]		;; Load first number
	vsubpd	ymm1, ymm1, [rdx]	;; Sub second number
ttp	vmulpd	ymm1, ymm1, ymm5	;; Mul values2 by two-to-minus-phi
	vaddpd	ymm0, ymm0, ymm2	;; x1 = values + carry
	vaddpd	ymm1, ymm1, ymm6	;; x2 = values + carry
	rounding_interleaved ttp, base2, noexec, ymm0, ymm2, ymm4, rax*2, ymm1, ymm6, ymm5, rax*2
ttp	vmulpd	ymm0, ymm0, [rbx+32]	;; new value1 = val * two-to-phi
ttp	vmulpd	ymm1, ymm1, [rbx+32]	;; new value2 = val * two-to-phi
	ystore	[rsi], ymm0		;; Save value1
	ystore	[rbp], ymm1		;; Save value2

	vmovapd	ymm0, [rcx+32]		;; Load first number
	vaddpd	ymm0, ymm0, [rdx+32]	;; Add second number
ttp	vmovapd	ymm5, [rbx]		;; Load fudged two-to-minus-phi
ttp	vmulpd	ymm5, ymm5, YMM_NORM012_FF ;; Mul by FFTLEN/2
ttp	vmulpd	ymm0, ymm0, ymm5	;; Mul values1 by two-to-minus-phi
	vmovapd	ymm1, [rcx+32]		;; Load first number
	vsubpd	ymm1, ymm1, [rdx+32]	;; Sub second number
ttp	vmulpd	ymm1, ymm1, ymm5	;; Mul values2 by two-to-minus-phi
	vaddpd	ymm0, ymm0, ymm3	;; x1 = values + carry
	vaddpd	ymm1, ymm1, ymm7	;; x2 = values + carry
	rounding_interleaved ttp, base2, noexec, ymm0, ymm3, ymm4, rax*2, ymm1, ymm7, ymm5, rax*2
ttp	vmulpd	ymm0, ymm0, [rbx+32]	;; new value1 = val * two-to-phi
ttp	vmulpd	ymm1, ymm1, [rbx+32]	;; new value2 = val * two-to-phi
	ystore	[rsi+32], ymm0		;; Save value1
	ystore	[rbp+32], ymm1		;; Save value2

ttp	bump	rdi, 1			;; Next flags ptr
	bump	rcx, 64			;; Next src ptr
	bump	rdx, 64			;; Next src ptr
ttp	bump	rbx, 64			;; Next two-to-phi ptr
	bump	rsi, 64			;; Next dest ptr
	bump	rbp, 64			;; Next dest ptr
	ENDM

; This macro finishes the normalize process by adding the final
; carry from the first pass back into the lower two data values.
; ymm2,ymm3 = carries #1
; ymm6,ymm7 = carries #2
; rax = pointer to the FFT data values #1
; top of stack = pointer to the FFT destination #1 and #2
; rbx = pointer two-to-phi multipliers

ynorm_addsub_1d_mid_cleanup MACRO ttp, base2, dest1, dest2
	mov	rax, dest1				;; Restore dest #1 pointer
	rotate_carries_interleaved base2, ymm2, ymm4, ymm3, ymm5, ymm0, ymm1 ;; Rotate carries
	vmovapd	ymm0, [rax]				;; Load values1
	vmovapd	ymm1, [rax+32]				;; Load values2
base2	vsubpd	ymm2, ymm2, YMM_BIGVAL			;; Remove BIGVAL
base2	vsubpd	ymm3, ymm3, YMM_BIGVAL			;; Remove BIGVAL
ttp	vmulpd	ymm2, ymm2, [rbx+32]			;; carry1 *= two-to-phi
ttp	vmulpd	ymm3, ymm3, [rbx+96]			;; carry2 *= two-to-phi
	vaddpd	ymm0, ymm0, ymm2			;; x1 = values + carry
	vaddpd	ymm1, ymm1, ymm3			;; x2 = values + carry
	ystore	[rax], ymm0				;; Save new value1
	ystore	[rax+32], ymm1				;; Save new value2
	vmovapd	ymm2, ymm4				;; Install next section's carry
	vmovapd	ymm3, ymm5				;; Install next section's carry

	mov	rax, dest2				;; Get FFT data pointer #2
	rotate_carries_interleaved base2, ymm6, ymm4, ymm7, ymm5, ymm0, ymm1 ;; Rotate carries
	vmovapd	ymm0, [rax]				;; Load values1
	vmovapd	ymm1, [rax+32]				;; Load values2
base2	vsubpd	ymm6, ymm6, YMM_BIGVAL			;; Remove BIGVAL
base2	vsubpd	ymm7, ymm7, YMM_BIGVAL			;; Remove BIGVAL
ttp	vmulpd	ymm6, ymm6, [rbx+32]			;; carry1 *= two-to-phi
ttp	vmulpd	ymm7, ymm7, [rbx+96]			;; carry2 *= two-to-phi
	vaddpd	ymm0, ymm0, ymm6			;; x1 = values + carry
	vaddpd	ymm1, ymm1, ymm7			;; x2 = values + carry
	ystore	[rax], ymm0				;; Save new value1
	ystore	[rax+32], ymm1				;; Save new value2
	vmovapd	ymm6, ymm4				;; Install next section's carry
	vmovapd	ymm7, ymm5				;; Install next section's carry
	ENDM

; The zpad case where we have half the ttp/ttmp data
ynorm_addsub_1d_zpad_mid_cleanup MACRO ttp, base2, dest1, dest2
	mov	rax, dest1				;; Restore dest #1 pointer
	rotate_carries_interleaved base2, ymm2, ymm4, ymm3, ymm5, ymm0, ymm1 ;; Rotate carries
	vmovapd	ymm0, [rax]				;; Load values1
	vmovapd	ymm1, [rax+32]				;; Load values2
base2	vsubpd	ymm2, ymm2, YMM_BIGVAL			;; Remove BIGVAL
base2	vsubpd	ymm3, ymm3, YMM_BIGVAL			;; Remove BIGVAL
ttp	vmulpd	ymm2, ymm2, [rbx+32]			;; carry1 *= two-to-phi
ttp	vmulpd	ymm3, ymm3, [rbx+32]			;; carry2 *= two-to-phi
	vaddpd	ymm0, ymm0, ymm2			;; x1 = values + carry
	vaddpd	ymm1, ymm1, ymm3			;; x2 = values + carry
	ystore	[rax], ymm0				;; Save new value1
	ystore	[rax+32], ymm1				;; Save new value2
	vmovapd	ymm2, ymm4				;; Install next section's carry
	vmovapd	ymm3, ymm5				;; Install next section's carry

	mov	rax, dest2				;; Get FFT data pointer #2
	rotate_carries_interleaved base2, ymm6, ymm4, ymm7, ymm5, ymm0, ymm1 ;; Rotate carries
	vmovapd	ymm0, [rax]				;; Load values1
	vmovapd	ymm1, [rax+32]				;; Load values2
base2	vsubpd	ymm6, ymm6, YMM_BIGVAL			;; Remove BIGVAL
base2	vsubpd	ymm7, ymm7, YMM_BIGVAL			;; Remove BIGVAL
ttp	vmulpd	ymm6, ymm6, [rbx+32]			;; carry1 *= two-to-phi
ttp	vmulpd	ymm7, ymm7, [rbx+32]			;; carry2 *= two-to-phi
	vaddpd	ymm0, ymm0, ymm6			;; x1 = values + carry
	vaddpd	ymm1, ymm1, ymm7			;; x2 = values + carry
	ystore	[rax], ymm0				;; Save new value1
	ystore	[rax+32], ymm1				;; Save new value2
	vmovapd	ymm6, ymm4				;; Install next section's carry
	vmovapd	ymm7, ymm5				;; Install next section's carry
	ENDM

; *************** 1D normalized smallmul macro ******************
; This macro multiplies by a small constant, then "normalizes" eight FFT
; data values.
; ymm7 = small multiplier value
; ymm3 = carry #2
; ymm2 = carry #1
; rsi = pointer to destination
; rbp = pointer two-to-phi multipliers
; rdi = pointer to array of big vs. little flags
; rax = big vs. little word flag #1
; rcx = big vs. little word flag #2

ynorm_smallmul_1d MACRO ttp, base2
ttp	movzx	rax, BYTE PTR [rdi]		;; Load big vs. little flags
ttp	movzx	rcx, BYTE PTR [rdi+1]
	vmovapd	ymm0, [rsi]			;; Load values1
	vmulpd	ymm0, ymm0, ymm7		;; Mul by small value * FFTLEN/2
ttp	vmulpd	ymm0, ymm0, [rbp]		;; Mul values1 by two-to-minus-phi
	vmovapd	ymm1, [rsi+32]			;; Load values2
	vmulpd	ymm1, ymm1, ymm7		;; Mul by small value * FFTLEN/2
ttp	vmulpd	ymm1, ymm1, [rbp+64]		;; Mul values2 by two-to-minus-phi
	vaddpd	ymm0, ymm0, ymm2		;; x1 = values + carry
	vaddpd	ymm1, ymm1, ymm3		;; x2 = values + carry
	rounding_interleaved ttp, base2, noexec, ymm0, ymm2, ymm4, rax*2, ymm1, ymm3, ymm5, rcx*2
ttp	vmulpd	ymm0, ymm0, [rbp+32]		;; new value1 = val * two-to-phi
ttp	vmulpd	ymm1, ymm1, [rbp+96]		;; new value2 = val * two-to-phi
	ystore	[rsi], ymm0			;; Save value1
	ystore	[rsi+32], ymm1			;; Save value2
ttp	bump	rdi, 2				;; Next flags ptr
	bump	rsi, 64				;; Next dest ptr
ttp	bump	rbp, 128			;; Next two-to-phi ptr
	ENDM

; The zero pad case where we have half the ttp/ttmp data
ynorm_smallmul_1d_zpad MACRO ttp, base2
ttp	movzx	rax, BYTE PTR [rdi]		;; Load big vs. little flags
	vmovapd	ymm0, [rsi]			;; Load values1
	vmulpd	ymm0, ymm0, ymm7		;; Mul by small value * FFTLEN/2
ttp	vmulpd	ymm0, ymm0, [rbp]		;; Mul values1 by two-to-minus-phi
	vmovapd	ymm1, [rsi+32]			;; Load values2
	vmulpd	ymm1, ymm1, ymm7		;; Mul by small value * FFTLEN/2
ttp	vmulpd	ymm1, ymm1, [rbp]		;; Mul values2 by two-to-minus-phi
	vaddpd	ymm0, ymm0, ymm2		;; x1 = values + carry
	vaddpd	ymm1, ymm1, ymm3		;; x2 = values + carry
	rounding_interleaved ttp, base2, noexec, ymm0, ymm2, ymm4, rax*2, ymm1, ymm3, ymm5, rax*2
ttp	vmulpd	ymm0, ymm0, [rbp+32]		;; new value1 = val * two-to-phi
ttp	vmulpd	ymm1, ymm1, [rbp+32]		;; new value2 = val * two-to-phi
	ystore	[rsi], ymm0			;; Save value1
	ystore	[rsi+32], ymm1			;; Save value2
ttp	bump	rdi, 1				;; Next flags ptr
	bump	rsi, 64				;; Next dest ptr
ttp	bump	rbp, 64				;; Next two-to-phi ptr
	ENDM


; This macro performs the smallmul normalize process by adding the section
; carries back into the start of the section.  Three of the YMM section
; carries are added back in, one of the YMM section carries is applied
; to the next section.
; ymm2,ymm3 = carries
; rax, rdx, rsi, rbp, rdi = trash

ynorm_smallmul_1d_mid_cleanup MACRO ttp, base2, srcptr, biglitptr, ttpptr
	ynorm_1d_mid_cleanup ttp, base2, srcptr, biglitptr, ttpptr
	ENDM

; This macro performs the smallmul normalize process by adding the section
; carries back into the start of the section.  Three of the YMM section
; carries are added back in, one of the YMM section carries is applied
; to the next section.
; This macro is similar to ynorm_smallmul_1d_mid_cleanup except that
; we only half half as much ttp/ttmp data.
; ymm2,ymm3 = carries
; rax, rdx, rsi, rbp, rdi = trash

ynorm_smallmul_1d_zpad_mid_cleanup MACRO ttp, base2, srcptr, biglitptr, ttpptr
	LOCAL	section_start, section_loop, force_done, done

	rotate_carries_interleaved base2, ymm2, ymm4, ymm3, ymm5, ymm0, ymm1

	mov	DWORD PTR YMM_TMP3, 15		;; Propagate carry at most 15 times (should almost never happen)
section_start:
	mov	rsi, srcptr			;; Load section pointers
ttp	mov	rdi, biglitptr
ttp	mov	rbp, ttpptr
	mov	edx, count1			;; Count of cache lines in section padded group
	ystore	YMM_TMP1, ymm4			;; Save carry for next section
	ystore	YMM_TMP2, ymm5			;; Save carry for next section

section_loop:
base2	vmovapd	ymm1, YMM_BIGVAL		;; Load comparison value
no base2 vxorpd	ymm1, ymm1, ymm1		;; Create comparison value
	vcmppd	ymm0, ymm2, ymm1, 0Ch		;; Are any carries non-zero
	vmovmskpd rax, ymm0			;; Extract 4 comparison bits
	vcmppd	ymm0, ymm3, ymm1, 0Ch		;; Are any carries non-zero
	vmovmskpd rcx, ymm0			;; Extract 4 comparison bits
	or	rax, rcx			;; Are any bits on?
	jz	done				;; No, we're all done

	sub	DWORD PTR YMM_TMP3, 1		;; There is a bizarre case where adding a zero carry causes a carry (example:
						;; 10^114+1, a data value of 500 becomes -500 with a carry of 1 -- add zero again and
						;; it becomes -500 with a carry of -1).  This can lead to an infinite loop because
						;; the 4 carries never become zero.  After many attempts at getting 4 zero carries,
						;; give up and just add in the carries without propagation.
	jz	force_done

ttp		movzx	rax, BYTE PTR [rdi]		;; Load big vs. little flags
		vmovapd	ymm0, [rsi]			;; Load values1
ttp		vmulpd	ymm0, ymm0, [rbp]		;; Mul values1 by two-to-minus-phi
ttp		vmulpd	ymm0, ymm0, YMM_NORM012_FF	;; Mul by FFTLEN/2
		vmovapd	ymm1, [rsi+32]			;; Load values2
ttp		vmulpd	ymm1, ymm1, [rbp]		;; Mul values2 by two-to-minus-phi
ttp		vmulpd	ymm1, ymm1, YMM_NORM012_FF	;; Mul by FFTLEN/2
		vaddpd	ymm0, ymm0, ymm2		;; x1 = values + carry
		vaddpd	ymm1, ymm1, ymm3		;; x2 = values + carry
		rounding_interleaved ttp, base2, noexec, ymm0, ymm2, ymm4, rax*2, ymm1, ymm3, ymm5, rax*2
ttp		vmulpd	ymm0, ymm0, [rbp+32]		;; new value1 = val * two-to-phi
ttp		vmulpd	ymm1, ymm1, [rbp+32]		;; new value2 = val * two-to-phi
		ystore	[rsi], ymm0			;; Save new value1
		ystore	[rsi+32], ymm1			;; Save new value2

ttp	bump	rdi, 1				;; Advance pointers
	bump	rsi, 64
ttp	bump	rbp, 64

	sub	rdx, 1				;; Test counter
	jnz	section_loop			;; More cache lines in section, add carry in

	;; Section ended.  Rotate carries again and add the new next section carry values
	;; into the previously calculated next section carry values

	rotate_carries_interleaved base2, ymm2, ymm4, ymm3, ymm5, ymm0, ymm1
base2	vsubpd	ymm4, ymm4, YMM_BIGVAL
base2	vsubpd	ymm5, ymm5, YMM_BIGVAL
	vaddpd	ymm4, ymm4, YMM_TMP1
	vaddpd	ymm5, ymm5, YMM_TMP2
	jmp	section_start

force_done:	
base2	vsubpd	ymm2, ymm2, YMM_BIGVAL		;; Subtract rounding constant from carry
base2	vsubpd	ymm3, ymm3, YMM_BIGVAL
ttp	vmulpd	ymm2, ymm2, [rbp+32]		;; carry1 *= two-to-phi
ttp	vmulpd	ymm3, ymm3, [rbp+32]		;; carry2 *= two-to-phi
	vaddpd	ymm2, ymm2, [rsi]		;; Add in values1
	vaddpd	ymm3, ymm3, [rsi+32]		;; Add in values2
	ystore	[rsi], ymm2			;; Save new value1
	ystore	[rsi+32], ymm3			;; Save new value2

done:	vmovapd	ymm2, YMM_TMP1			;; Restore carry for next section
	vmovapd	ymm3, YMM_TMP2			;; Restore carry for next section
	ENDM


; This macro finishes the smallmul normalize process by adding
; the final carry from the first pass back into the lower two data values.
; ymm2,ymm3 = carries
; rax,rsi,rbp,rdi = trash

ynorm_smallmul_1d_cleanup MACRO ttp, base2, destptr
	mov	rsi, destptr			;; Address of FFT data
	mov	rbp, norm_col_mults		;; Address of the multipliers
	mov	rdi, norm_biglit_array		;; Addr of the big/little flags array
ttp	ynorm_top_carry_1d ttp, base2		;; No, do a very standard carry
	ynorm_1d_cleanup ttp, base2
	ENDM

; This macro is similar to ynorm_op_1d_zpad_cleanup for handling zpad carries in an
; smallmul operation.  The difference is the carry can be much larger requiring
; proper rounding.
; xmm2 = carry into the top half
; rsi = pointer to the FFT data values
; rbp = pointer two-to-phi multipliers
; rdi = big vs. little array pointer

ynorm_smallmul_1d_zpad_cleanup MACRO ttp, base2, destptr
	LOCAL	smallk, mediumk, div_k_done

base2	vsubsd	xmm2, xmm2, Q YMM_BIGVAL	;; Integerize carry

	;; Copy and integerize data from 7 words above halfway point to ZPAD0-ZPAD6
	;; Clear high words as we go
	;; Then we can make an almost exact copy of the ynorm_1d_zpad_cleanup code

	mov	rsi, destptr			;; Address of FFT data
ttp	mov	rbp, norm_col_mults		;; Address of the multipliers
ttp	lea	rdi, YMM_FIRST_BIGLIT_VALUES	;; Address of first big/little flags

ttp	movzx	rax, BYTE PTR [rdi]		;; Load big vs. little flags
	vmovsd	xmm0, Q [rsi+32]		;; Value1
	new_single_split_lower_zpad_word ttp, base2, xmm0, xmm2, xmm4, rax*2
	vmovsd	ZPAD0, xmm0

	add	rsi, YMM_SRC_INCR1		;; Next source pointer
ttp	add	rbp, YMM_NORM_INCR1		;; Next ttp/ttmp pointer
ttp	movzx	rax, BYTE PTR [rdi+1]		;; Load big vs. little flags
	vmovsd	xmm0, Q [rsi+32]		;; Value2
ttp	vmulsd	xmm0, xmm0, Q [rbp]		;; Mul values2 by two-to-minus-phi
ttp	vmulsd	xmm0, xmm0, Q YMM_NORM012_FF	;; Mul by FFTLEN/2
	new_single_split_lower_zpad_word ttp, base2, xmm0, xmm2, xmm4, rax*2
	vmovsd	ZPAD1, xmm0

	add	rsi, YMM_SRC_INCR2		;; Next source pointer
ttp	add	rbp, YMM_NORM_INCR2		;; Next ttp/ttmp pointer
ttp	movzx	rax, BYTE PTR [rdi+2]		;; Load big vs. little flags
	vmovsd	xmm0, Q [rsi+32]		;; Value3
ttp	vmulsd	xmm0, xmm0, Q [rbp]		;; Mul values3 by two-to-minus-phi
ttp	vmulsd	xmm0, xmm0, Q YMM_NORM012_FF	;; Mul by FFTLEN/2
	new_single_split_lower_zpad_word ttp, base2, xmm0, xmm2, xmm4, rax*2
	vmovsd	ZPAD2, xmm0

	add	rsi, YMM_SRC_INCR3		;; Next source pointer
ttp	add	rbp, YMM_NORM_INCR3		;; Next ttp/ttmp pointer
	vmovsd	xmm0, Q [rsi+32]		;; Value4
ttp	vmulsd	xmm0, xmm0, Q [rbp]		;; Mul values4 by two-to-minus-phi
ttp	vmulsd	xmm0, xmm0, Q YMM_NORM012_FF	;; Mul by FFTLEN/2
ttp	vroundsd xmm0, xmm0, xmm0, 0		;; Round to an integer
	vaddsd	xmm0, xmm0, xmm2		;; Value4 + carry
	vmovsd	ZPAD3, xmm0

	add	rsi, YMM_SRC_INCR4		;; Next source pointer
ttp	add	rbp, YMM_NORM_INCR4		;; Next ttp/ttmp pointer
	vmovsd	xmm0, Q [rsi+32]		;; Value5
ttp	vmulsd	xmm0, xmm0, Q [rbp]		;; Mul values5 by two-to-minus-phi
ttp	vmulsd	xmm0, xmm0, Q YMM_NORM012_FF	;; Mul by FFTLEN/2
ttp	vroundsd xmm0, xmm0, xmm0, 0		;; Round to an integer
	vmovsd	ZPAD4, xmm0
	vxorpd	xmm1, xmm1, xmm1		;; Clear highest words
	vmovsd	Q [rsi+32], xmm1		;; Clear value5

	add	rsi, YMM_SRC_INCR5		;; Next source pointer
ttp	add	rbp, YMM_NORM_INCR5		;; Next ttp/ttmp pointer
	vmovsd	xmm0, Q [rsi+32]		;; Value6
ttp	vmulsd	xmm0, xmm0, Q [rbp]		;; Mul values6 by two-to-minus-phi
ttp	vmulsd	xmm0, xmm0, Q YMM_NORM012_FF	;; Mul by FFTLEN/2
ttp	vroundsd xmm0, xmm0, xmm0, 0		;; Round to an integer
	vmovsd	ZPAD5, xmm0
	vmovsd	Q [rsi+32], xmm1		;; Clear value6

	add	rsi, YMM_SRC_INCR6		;; Next source pointer
ttp	add	rbp, YMM_NORM_INCR6		;; Next ttp/ttmp pointer
	vmovsd	xmm0, Q [rsi+32]		;; Value7
ttp	vmulsd	xmm0, xmm0, Q [rbp]		;; Mul values7 by two-to-minus-phi
ttp	vmulsd	xmm0, xmm0, Q YMM_NORM012_FF	;; Mul by FFTLEN/2
ttp	vroundsd xmm0, xmm0, xmm0, 0		;; Round to an integer
	vmovsd	ZPAD6, xmm0
	vmovsd	Q [rsi+32], xmm1		;; Clear value7

	;; Divide the zpad data by k.  Store the integer part in YMM_TMP
	;; and the remainder in ZPAD0.  Later we will wrap the integer part
	;; down to the bottom of the FFT data area (and multiply by -c).
	;; And we will store the remainder in the upper half of the FFT
	;; data area.

	;; Note there are three cases to handle.  K is smaller than a big word.
	;; K is between one and 2 big words in size.  And K is more than
	;; 2 big words in size.

	cmp	ZPAD_TYPE, 2			;; Are we dealing with case 1,2,or 3
	jl	smallk				;; One word case
	je	mediumk				;; Two word case

	;; This case does the divide by k where k is three words

	vmovsd	xmm0, ZPAD6			;; Load zpad word (high bits)
	vmovsd	xmm1, ZPAD5			;; Load zpad word (middle bits)
	vmovsd	xmm2, ZPAD4			;; Load zpad word (low bits)
	vmulsd	xmm4, xmm0, ZPAD_INVERSE_K6	;; Mul ZPAD by shifted 1/k
	vroundsd xmm4, xmm4, xmm4, 0		;; Round to integer
	vmulsd	xmm5, xmm4, ZPAD_K6_HI		;; Mul by high bits of k
	vsubsd	xmm0, xmm0, xmm5		;; Calculate high bits of remainder
	vmulsd	xmm5, xmm4, ZPAD_K6_MID		;; Mul by middle bits of k
	vsubsd	xmm1, xmm1, xmm5		;; Calculate middle bits of remainder
	vmulsd	xmm5, xmm4, ZPAD_K6_LO		;; Mul by low bits of k
	vsubsd	xmm2, xmm2, xmm5		;; Calculate low bits of remainder
	vmovsd	Q YMM_TMP5, xmm4		;; Save word of zpad / k

	vmulsd	xmm0, xmm0, ZPAD_SHIFT6		;; Shift previous zpad word
	vaddsd	xmm1, xmm1, xmm0		;; Add to create new high zpad bits
	vmovsd	xmm0, ZPAD3			;; Load zpad word (new low bits)
	vmulsd	xmm5, xmm1, ZPAD_SHIFT5		;; Combine high and medium bits
	vaddsd	xmm5, xmm5, xmm2
	vmulsd	xmm4, xmm5, ZPAD_INVERSE_K5	;; Mul ZPAD by shifted 1/k
	vroundsd xmm4, xmm4, xmm4, 0		;; Round to integer
	vmulsd	xmm5, xmm4, ZPAD_K5_HI		;; Mul by high bits of k
	vsubsd	xmm1, xmm1, xmm5		;; Calculate high bits of remainder
	vmulsd	xmm5, xmm4, ZPAD_K5_MID		;; Mul by middle bits of k
	vsubsd	xmm2, xmm2, xmm5		;; Calculate middle bits of remainder
	vmulsd	xmm5, xmm4, ZPAD_K5_LO		;; Mul by low bits of k
	vsubsd	xmm0, xmm0, xmm5		;; Calculate low bits of remainder
	vmovsd	Q YMM_TMP4, xmm4		;; Save word of zpad / k

	vmulsd	xmm1, xmm1, ZPAD_SHIFT5		;; Shift previous zpad word
	vaddsd	xmm2, xmm2, xmm1		;; Add to create new high zpad bits
	vmovsd	xmm1, ZPAD2			;; Load zpad word (new low bits)
	vmulsd	xmm5, xmm2, ZPAD_SHIFT4		;; Combine high and medium bits
	vaddsd	xmm5, xmm5, xmm0
	vmulsd	xmm4, xmm5, ZPAD_INVERSE_K4	;; Mul ZPAD by shifted 1/k
	vroundsd xmm4, xmm4, xmm4, 0		;; Round to integer
	vmulsd	xmm5, xmm4, ZPAD_K4_HI		;; Mul by high bits of k
	vsubsd	xmm2, xmm2, xmm5		;; Calculate high bits of remainder
	vmulsd	xmm5, xmm4, ZPAD_K4_MID		;; Mul by middle bits of k
	vsubsd	xmm0,xmm0,xmm5			;; Calculate middle bits of remainder
	vmulsd	xmm5, xmm4, ZPAD_K4_LO		;; Mul by low bits of k
	vsubsd	xmm1, xmm1, xmm5		;; Calculate low bits of remainder
	vmovsd	Q YMM_TMP3, xmm4		;; Save word of zpad / k

	vmulsd	xmm2, xmm2, ZPAD_SHIFT4		;; Shift previous zpad word
	vaddsd	xmm0, xmm0, xmm2		;; Add to create new high zpad bits
	vmovsd	xmm2, ZPAD1			;; Load zpad word (new low bits)
	vmulsd	xmm5, xmm0, ZPAD_SHIFT3		;; Combine high and medium bits
	vaddsd	xmm5, xmm5, xmm1
	vmulsd	xmm4, xmm5, ZPAD_INVERSE_K3	;; Mul ZPAD by shifted 1/k
	vroundsd xmm4, xmm4, xmm4, 0		;; Round to integer
	vmulsd	xmm5, xmm4, ZPAD_K3_HI		;; Mul by high bits of k
	vsubsd	xmm0, xmm0, xmm5		;; Calculate high bits of remainder
	vmulsd	xmm5, xmm4, ZPAD_K3_MID		;; Mul by middle bits of k
	vsubsd	xmm1, xmm1, xmm5		;; Calculate middle bits of remainder
	vmulsd	xmm5, xmm4, ZPAD_K3_LO		;; Mul by low bits of k
	vsubsd	xmm2, xmm2, xmm5		;; Calculate low bits of remainder
	vmovsd	Q YMM_TMP2, xmm4		;; Save word of zpad / k

	vmulsd	xmm0, xmm0, ZPAD_SHIFT3		;; Shift previous zpad word
	vaddsd	xmm1, xmm1, xmm0		;; Add to create new high zpad bits
	vmovsd	xmm0, ZPAD0			;; Load zpad word (new low bits)
	vmulsd	xmm5, xmm1, ZPAD_SHIFT2		;; Combine high and medium bits
	vaddsd	xmm5, xmm5, xmm2
	vmulsd	xmm4, xmm5, ZPAD_INVERSE_K2	;; Mul ZPAD by shifted 1/k
	vroundsd xmm4, xmm4, xmm4, 0		;; Round to integer
	vmulsd	xmm5, xmm4, ZPAD_K2_HI		;; Mul by high bits of k
	vsubsd	xmm1, xmm1, xmm5		;; Calculate high bits of remainder
	vmulsd	xmm5, xmm4, ZPAD_K2_MID		;; Load middle bits of k
	vsubsd	xmm2, xmm2, xmm5		;; Calculate middle bits of remainder
	vmulsd	xmm5, xmm4, ZPAD_K2_LO		;; Load low bits of k
	vsubsd	xmm0, xmm0, xmm5		;; Calculate low bits of remainder
	vmovsd	Q YMM_TMP1, xmm4		;; Save word of zpad / k

	vmulsd	xmm1, xmm1, ZPAD_SHIFT2		;; Shift previous zpad word
	vaddsd	xmm2, xmm2, xmm1		;; Add to create new high zpad bits
	vmulsd	xmm2, xmm2, ZPAD_SHIFT1		;; Shift previous zpad word
	vaddsd	xmm0, xmm0, xmm2		;; Add to create new high zpad bits
	vmovsd	ZPAD0, xmm0			;; Save remainder of zpad / k

	vxorpd	xmm1, xmm1, xmm1		;; Zero words that other cases set
	vmovsd	q YMM_TMP6, xmm1
	
	jmp	div_k_done

	;; This case does the divide by k where k is two words
mediumk:
	vmovsd	xmm0, ZPAD6			;; Load zpad word (high bits)
	vmovsd	xmm1, ZPAD5			;; Load zpad word (low bits)
	vmulsd	xmm4, xmm0, ZPAD_INVERSE_K6	;; Mul ZPAD by shifted 1/k
	vroundsd xmm4, xmm4, xmm4, 0		;; Round to integer
	vmulsd	xmm5, xmm4, ZPAD_K6_HI		;; Mul by high bits of k
	vsubsd	xmm0, xmm0, xmm5		;; Calculate high bits of remainder
	vmulsd	xmm5, xmm4, ZPAD_K6_LO		;; Mul by low bits of k
	vsubsd	xmm1, xmm1, xmm5		;; Calculate low bits of remainder
	vmovsd	Q YMM_TMP6, xmm4		;; Save word of zpad / k

	vmulsd	xmm0, xmm0, ZPAD_SHIFT6		;; Shift previous zpad word
	vaddsd	xmm0, xmm0, xmm1		;; Add to create new high zpad bits
	vmovsd	xmm1, ZPAD4			;; Load zpad word (new low bits)
	vmulsd	xmm4, xmm0, ZPAD_INVERSE_K5	;; Mul ZPAD by shifted 1/k
	vroundsd xmm4, xmm4, xmm4, 0		;; Round to integer
	vmulsd	xmm5, xmm4, ZPAD_K5_HI		;; Mul by high bits of k
	vsubsd	xmm0, xmm0, xmm5		;; Calculate high bits of remainder
	vmulsd	xmm5, xmm4, ZPAD_K5_LO		;; Mul by low bits of k
	vsubsd	xmm1, xmm1, xmm5		;; Calculate low bits of remainder
	vmovsd	Q YMM_TMP5, xmm4		;; Save word of zpad / k

	vmulsd	xmm0, xmm0, ZPAD_SHIFT5		;; Shift previous zpad word
	vaddsd	xmm0, xmm0, xmm1		;; Add to create new high zpad bits
	vmovsd	xmm1, ZPAD3			;; Load zpad word (new low bits)
	vmulsd	xmm4, xmm0, ZPAD_INVERSE_K4	;; Mul ZPAD by shifted 1/k
	vroundsd xmm4, xmm4, xmm4, 0		;; Round to integer
	vmulsd	xmm5, xmm4, ZPAD_K4_HI		;; Mul by high bits of k
	vsubsd	xmm0, xmm0, xmm5		;; Calculate high bits of remainder
	vmulsd	xmm5, xmm4, ZPAD_K4_LO		;; Mul by low bits of k
	vsubsd	xmm1, xmm1, xmm5		;; Calculate low bits of remainder
	vmovsd	Q YMM_TMP4, xmm4		;; Save word of zpad / k

	vmulsd	xmm0, xmm0, ZPAD_SHIFT4		;; Shift previous zpad word
	vaddsd	xmm0, xmm0, xmm1		;; Add to create new high zpad bits
	vmovsd	xmm1, ZPAD2			;; Load zpad word (new low bits)
	vmulsd	xmm4, xmm0, ZPAD_INVERSE_K3	;; Mul ZPAD by shifted 1/k
	vroundsd xmm4, xmm4, xmm4, 0		;; Round to integer
	vmulsd	xmm5, xmm4, ZPAD_K3_HI		;; Mul by high bits of k
	vsubsd	xmm0, xmm0, xmm5		;; Calculate high bits of remainder
	vmulsd	xmm5, xmm4, ZPAD_K3_LO		;; Mul by low bits of k
	vsubsd	xmm1, xmm1, xmm5		;; Calculate low bits of remainder
	vmovsd	Q YMM_TMP3, xmm4		;; Save word of zpad / k

	vmulsd	xmm0, xmm0, ZPAD_SHIFT3		;; Shift previous zpad word
	vaddsd	xmm0, xmm0, xmm1		;; Add to create new high zpad bits
	vmovsd	xmm1, ZPAD1			;; Load zpad word (new low bits)
	vmulsd	xmm4, xmm0, ZPAD_INVERSE_K2	;; Mul ZPAD by shifted 1/k
	vroundsd xmm4, xmm4, xmm4, 0		;; Round to integer
	vmulsd	xmm5, xmm4, ZPAD_K2_HI		;; Mul by high bits of k
	vsubsd	xmm0, xmm0, xmm5		;; Calculate high bits of remainder
	vmulsd	xmm5, xmm4, ZPAD_K2_LO		;; Mul by low bits of k
	vsubsd	xmm1, xmm1, xmm5		;; Calculate low bits of remainder
	vmovsd	Q YMM_TMP2, xmm4		;; Save word of zpad / k

	vmulsd	xmm0, xmm0, ZPAD_SHIFT2		;; Shift previous zpad word
	vaddsd	xmm0, xmm0, xmm1		;; Add to create new high zpad bits
	vmovsd	xmm1, ZPAD0			;; Load zpad word (new low bits)
	vmulsd	xmm4, xmm0, ZPAD_INVERSE_K1	;; Mul ZPAD by shifted 1/k
	vroundsd xmm4, xmm4, xmm4, 0		;; Round to integer
	vmulsd	xmm5, xmm4, ZPAD_K1_HI		;; Mul by high bits of k
	vsubsd	xmm0, xmm0, xmm5		;; Calculate high bits of remainder
	vmulsd	xmm5, xmm4, ZPAD_K1_LO		;; Mul by low bits of k
	vsubsd	xmm1, xmm1, xmm5		;; Calculate low bits of remainder
	vmovsd	Q YMM_TMP1, xmm4		;; Save word of zpad / k

	vmulsd	xmm0, xmm0, ZPAD_SHIFT1		;; Shift previous zpad word
	vaddsd	xmm0, xmm0, xmm1		;; Add to create new high zpad bits
	vmovsd	ZPAD0, xmm0			;; Save remainder of zpad / k

	jmp	div_k_done

	;; This case does the divide by k where k is one word
	;; Assume ZPAD5 and ZPAD6 are zero.
smallk:	vmovsd	xmm0, ZPAD4			;; Load zpad data
	vmulsd	xmm4, xmm0, ZPAD_INVERSE_K1	;; Mul ZPAD data by 1/k
	vroundsd xmm4, xmm4, xmm4, 0		;; Round to integer
	vmovsd	Q YMM_TMP5, xmm4		;; Save integer part
	vmulsd	xmm4, xmm4, ZPAD_K1_LO		;; Compute remainder
	vsubsd	xmm0, xmm0, xmm4

	vmulsd	xmm0, xmm0, ZPAD_SHIFT4		;; Shift previous zpad word
	vaddsd	xmm0, xmm0, ZPAD3		;; Add in zpad data
	vmulsd	xmm4, xmm0, ZPAD_INVERSE_K1	;; Mul ZPAD data by 1/k
	vroundsd xmm4, xmm4, xmm4, 0		;; Round to integer
	vmovsd	Q YMM_TMP4, xmm4		;; Save integer part
	vmulsd	xmm4, xmm4, ZPAD_K1_LO		;; Compute remainder
	vsubsd	xmm0, xmm0, xmm4

	vmulsd	xmm0, xmm0, ZPAD_SHIFT3		;; Shift previous zpad word
	vaddsd	xmm0, xmm0, ZPAD2		;; Add in zpad data
	vmulsd	xmm4, xmm0, ZPAD_INVERSE_K1	;; Mul ZPAD data by 1/k
	vroundsd xmm4, xmm4, xmm4, 0		;; Round to integer
	vmovsd	Q YMM_TMP3, xmm4		;; Save integer part
	vmulsd	xmm4, xmm4, ZPAD_K1_LO		;; Compute remainder
	vsubsd	xmm0, xmm0, xmm4

	vmulsd	xmm0, xmm0, ZPAD_SHIFT2		;; Shift previous zpad word
	vaddsd	xmm0, xmm0, ZPAD1		;; Add in zpad data
	vmulsd	xmm4, xmm0, ZPAD_INVERSE_K1	;; Mul ZPAD data by 1/k
	vroundsd xmm4, xmm4, xmm4, 0		;; Round to integer
	vmovsd	Q YMM_TMP2, xmm4		;; Save integer part
	vmulsd	xmm4, xmm4, ZPAD_K1_LO		;; Compute remainder
	vsubsd	xmm0, xmm0, xmm4

	vmulsd	xmm0, xmm0, ZPAD_SHIFT1		;; Shift previous zpad word
	vaddsd	xmm0, xmm0, ZPAD0		;; Add in zpad data
	vmulsd	xmm4, xmm0, ZPAD_INVERSE_K1	;; Mul ZPAD data by 1/k
	vroundsd xmm4, xmm4, xmm4, 0		;; Round to integer
	vmovsd	Q YMM_TMP1, xmm4		;; Save integer part
	vmulsd	xmm4, xmm4, ZPAD_K1_LO		;; Compute remainder
	vsubsd	xmm0, xmm0, xmm4
	vmovsd	ZPAD0, xmm0			;; Save remainder

	vxorpd	xmm1, xmm1, xmm1		;; Zero words that other cases set
	vmovsd	Q YMM_TMP6, xmm1

div_k_done:

	;; Now normalize the data above the halfway point.  Remember that the
	;; two-to-phi multiplier for the first value will be 1.0.

ttp	lea	rdi, YMM_FIRST_BIGLIT_VALUES	;; Address of first big/little flags
	mov	rsi, destptr			;; Address of squared number
ttp	mov	rbp, norm_col_mults		;; Address of the ttmp/ttp multipliers

ttp	movzx	rax, BYTE PTR [rdi]		;; First word
	vmovsd	xmm0, ZPAD0			;; Load remainder of divide by k
base2	vaddsd	xmm0, xmm0, Q YMM_BIGVAL
	new_single_rounding ttp, base2, xmm0, xmm2, xmm4, rax*2
	vmovsd	Q [rsi+32], xmm0		;; Save value1

ttp	add	rbp, YMM_NORM_INCR1		;; Next ttp/ttmp pointer
	add	rsi, YMM_SRC_INCR1		;; Next source pointer
ttp	movzx	rax, BYTE PTR [rdi+1]		;; Load big vs. little flags
	new_single_rounding ttp, base2, xmm2, xmm0, xmm4, rax*2
ttp	vmulsd	xmm2, xmm2, Q [rbp+32]		;; new value2 = val * two-to-phi
	vmovsd	Q [rsi+32], xmm2		;; Save value2

ttp	add	rbp, YMM_NORM_INCR2		;; Next ttp/ttmp pointer
	add	rsi, YMM_SRC_INCR2		;; Next source pointer
ttp	movzx	rax, BYTE PTR [rdi+2]		;; Load big vs. little flags
	new_single_rounding ttp, base2, xmm0, xmm2, xmm4, rax*2
ttp	vmulsd	xmm0, xmm0, Q [rbp+32]		;; new value3 = val * two-to-phi
	vmovsd	Q [rsi+32], xmm0		;; Save value3

ttp	add	rbp, YMM_NORM_INCR3		;; Next ttp/ttmp pointer
	add	rsi, YMM_SRC_INCR3		;; Next source pointer
base2	vsubsd	xmm2, xmm2, Q YMM_BIGVAL	;; Remove integer rounding constant
ttp	vmulsd	xmm2, xmm2, Q [rbp+32]		;; value4 = carry * two-to-phi
	vmovsd	Q [rsi+32], xmm2		;; Save new value4

	;; Mul the integer part of (ZPAD data divided by k) by -c in
	;; preparation for adding it into the lower FFT data area.
	;; Also add in the shifted high FFT carry at this time.

	;; Now add in and normalize the bottom FFT data.  Remember that the
	;; two-to-phi multiplier for the first value will be 1.0.  We 
	;; must go 6 words deep in case k is 48-50 bits and c is 32 bits.

	mov	rsi, destptr			;; Address of squared number
ttp	mov	rbp, norm_col_mults		;; Address of ttmp/ttp multipliers

ttp	movzx	rax, BYTE PTR [rdi]		;; First word
	vmovsd	xmm0, Q YMM_TMP1		;; Load integer part of divide by k
	vmulsd	xmm0, xmm0, Q YMM_MINUS_C	;; Mul by -c
base2	vaddsd	xmm0, xmm0, Q YMM_BIGVAL
	vaddsd	xmm0, xmm0, Q [rsi]		;; Add in the FFT data
	new_single_rounding ttp, base2, xmm0, xmm2, xmm4, rax*2
	vmovsd	Q [rsi], xmm0			;; Save value1

	add	rsi, YMM_SRC_INCR1		;; Next source pointer
ttp	add	rbp, YMM_NORM_INCR1		;; Next ttp/ttmp pointer
ttp	movzx	rax, BYTE PTR [rdi+1]		;; Load big vs. little flags
	vmovsd	xmm0, Q YMM_TMP2		;; Load integer part of divide by k
	vmulsd	xmm0, xmm0, Q YMM_MINUS_C	;; Mul by -c
	vmovsd	xmm1, Q [rsi]			;; Load FFT data
ttp	vmulsd	xmm1, xmm1, Q [rbp]		;; Mul values2 by two-to-minus-phi
ttp	vmulsd	xmm1, xmm1, Q YMM_NORM012_FF	;; Mul by FFTLEN/2
	vaddsd	xmm0, xmm0, xmm2		;; x2 = value + carry
	vaddsd	xmm0, xmm0, xmm1		;; Add in the FFT data
	new_single_rounding ttp, base2, xmm0, xmm2, xmm4, rax*2
ttp	vmulsd	xmm0, xmm0, Q [rbp+32]		;; new value2 = val * two-to-phi
	vmovsd	Q [rsi], xmm0			;; Save value2

	add	rsi, YMM_SRC_INCR2		;; Next source pointer
ttp	add	rbp, YMM_NORM_INCR2		;; Next ttp/ttmp pointer
ttp	movzx	rax, BYTE PTR [rdi+2]		;; Load big vs. little flags
	vmovsd	xmm0, Q YMM_TMP3		;; Load integer part of divide by k
	vmulsd	xmm0, xmm0, Q YMM_MINUS_C	;; Mul by -c
	vmovsd	xmm1, Q [rsi]			;; Load FFT data
ttp	vmulsd	xmm1, xmm1, Q [rbp]		;; Mul values3 by two-to-minus-phi
ttp	vmulsd	xmm1, xmm1, Q YMM_NORM012_FF	;; Mul by FFTLEN/2
	vaddsd	xmm0, xmm0, xmm2		;; x3 = value + carry
	vaddsd	xmm0, xmm0, xmm1		;; Add in the FFT data
	new_single_rounding ttp, base2, xmm0, xmm2, xmm4, rax*2
ttp	vmulsd	xmm0, xmm0, Q [rbp+32]		;; new value3 = val * two-to-phi
	vmovsd	Q [rsi], xmm0			;; Save value3

	add	rsi, YMM_SRC_INCR3		;; Next source pointer
ttp	add	rbp, YMM_NORM_INCR3		;; Next ttp/ttmp pointer
ttp	movzx	rax, BYTE PTR [rdi+3]		;; Load big vs. little flags
	vmovsd	xmm0, Q YMM_TMP4		;; Load integer part of divide by k
	vmulsd	xmm0, xmm0, Q YMM_MINUS_C	;; Mul by -c
	vmovsd	xmm1, Q [rsi]			;; Load FFT data
ttp	vmulsd	xmm1, xmm1, Q [rbp]		;; Mul values4 by two-to-minus-phi
ttp	vmulsd	xmm1, xmm1, Q YMM_NORM012_FF	;; Mul by FFTLEN/2
	vaddsd	xmm0, xmm0, xmm2		;; x4 = value + carry
	vaddsd	xmm0, xmm0, xmm1		;; Add in the FFT data
	new_single_rounding ttp, base2, xmm0, xmm2, xmm4, rax*2
ttp	vmulsd	xmm0, xmm0, Q [rbp+32]		;; new value4 = val * two-to-phi
	vmovsd	Q [rsi], xmm0			;; Save value4

	add	rsi, YMM_SRC_INCR4		;; Next source pointer
ttp	add	rbp, YMM_NORM_INCR4		;; Next ttp/ttmp pointer
ttp	movzx	rax, BYTE PTR [rdi+4]		;; Load big vs. little flags
	vmovsd	xmm0, Q YMM_TMP5		;; Load integer part of divide by k
	vmulsd	xmm0, xmm0, Q YMM_MINUS_C	;; Mul by -c
	vmovsd	xmm1, Q [rsi]			;; Load FFT data
ttp	vmulsd	xmm1, xmm1, Q [rbp]		;; Mul values4 by two-to-minus-phi
ttp	vmulsd	xmm1, xmm1, Q YMM_NORM012_FF	;; Mul by FFTLEN/2
	vaddsd	xmm0, xmm0, xmm2		;; x5 = value + carry
	vaddsd	xmm0, xmm0, xmm1		;; Add in the FFT data
	new_single_rounding ttp, base2, xmm0, xmm2, xmm4, rax*2
ttp	vmulsd	xmm0, xmm0, Q [rbp+32]		;; new value5 = val * two-to-phi
	vmovsd	Q [rsi], xmm0			;; Save value5

	add	rsi, YMM_SRC_INCR5		;; Next source pointer
ttp	add	rbp, YMM_NORM_INCR5		;; Next ttp/ttmp pointer
ttp	movzx	rax, BYTE PTR [rdi+5]		;; Load big vs. little flags
	vmovsd	xmm0, Q YMM_TMP6		;; Load integer part of divide by k
	vmulsd	xmm0, xmm0, Q YMM_MINUS_C	;; Mul by -c
	vmovsd	xmm1, Q [rsi]			;; Load FFT data
ttp	vmulsd	xmm1, xmm1, Q [rbp]		;; Mul values6 by two-to-minus-phi
ttp	vmulsd	xmm1, xmm1, Q YMM_NORM012_FF	;; Mul by FFTLEN/2
	vaddsd	xmm0, xmm0, xmm2		;; x6 = value + carry
	vaddsd	xmm0, xmm0, xmm1		;; Add in the FFT data
	new_single_rounding ttp, base2, xmm0, xmm2, xmm4, rax*2
ttp	vmulsd	xmm0, xmm0, Q [rbp+32]		;; new value6 = val * two-to-phi
	vmovsd	Q [rsi], xmm0			;; Save value6

	add	rsi, YMM_SRC_INCR6		;; Next source pointer
ttp	add	rbp, YMM_NORM_INCR6		;; Next ttp/ttmp pointer
ttp	movzx	rax, BYTE PTR [rdi+6]		;; Load big vs. little flags
	vmovsd	xmm0, Q [rsi]			;; Load FFT data
ttp	vmulsd	xmm0, xmm0, Q [rbp]		;; Mul values7 by two-to-minus-phi
ttp	vmulsd	xmm0, xmm0, Q YMM_NORM012_FF	;; Mul by FFTLEN/2
	vaddsd	xmm0, xmm0, xmm2		;; x7 = value + carry
	new_single_rounding ttp, base2, xmm0, xmm2, xmm4, rax*2
ttp	vmulsd	xmm0, xmm0, Q [rbp+32]		;; new value7 = val * two-to-phi
	vmovsd	Q [rsi], xmm0			;; Save value7

	add	rsi, YMM_SRC_INCR7		;; Next source pointer
ttp	add	rbp, YMM_NORM_INCR7		;; Next ttp/ttmp pointer
base2	vsubsd	xmm2, xmm2, Q YMM_BIGVAL	;; Remove rounding constant
ttp	vmulsd	xmm2, xmm2, Q [rbp+32]		;; new value8 = val * two-to-phi
	vaddsd	xmm2, xmm2, Q [rsi]		;; Add in FFT data
	vmovsd	Q [rsi], xmm2			;; Save value8
	ENDM


; *************** WPN normalized add/sub macro ******************
; This macro adds or subtracts, then "normalizes" eight FFT
; data values.  This involves multiplying the summed values by
; two-to-minus-phi.  Rounding the value to an integer.  Making sure
; the integer is smaller than the maximum allowable integer, generating
; a carry if necessary. Finally, the value is multiplied by two-to-phi
; and stored.
; rsi = pointer to the first number
; rdx = pointer to the second number
; rdi = pointer to array of big vs. little flags
; rbp = pointer two-to-phi group multipliers
; ymm2,ymm3 = carries
; dist_to_dest = distance from first number to destination
; rax,rbx,rcx destroyed if ttp
; NOTE: caller can assume xmm6,xmm7 are left untouched (for better prolog/epilog handling)
; A pipelined version of this code:
;	xload	ymm0, [rdx]		;; Load second number
;	fop	ymm0, [rsi]		;; Add/sub first number
;	movzx	rax, BYTE PTR [rdi]	;; Load big vs. little flags
;	mulpd	ymm0, [rbp+0*YMM_GMD][rax*2] ;; Mul by fudged grp two-to-minus-phi
;	addpd	ymm0, YMM_TMP1		;; x1 = values + carry
;	xload	ymm2, YMM_LIMIT_BIGMAX[rax*2];; Load maximum * BIGVAL - BIGVAL
;	addpd	ymm2, ymm0		;; y1 = top bits of x
;	xload	ymm6, YMM_LIMIT_BIGMAX_NEG[rax*2];; Load -(maximum*BIGVAL-BIGVAL)
;	addpd	ymm6, ymm2		;; z1 = y1-(maximum * BIGVAL - BIGVAL)
;	subpd	ymm0, ymm6		;; rounded value = x1 - z1
;	mulpd	ymm2, YMM_LIMIT_INVERSE[rax*2];; next carry = shifted y1
;	mulpd	ymm0, [rbp+0*YMM_GMD+YMM_GMD/2][rax*2] ;; new value1 = val * fudged grp two-to-phi
;	xstore	[rsi+dist_to_dest+0*dist1], ymm0 ;; Save new value1
;	xstore	YMM_TMP1, ymm2		;; Save carry

ynorm_op_wpn MACRO fop, ttp, base2, dist_to_dest
		vmovapd	ymm0, [rdx]			;; Load second number
		fop	ymm0, ymm0, [rsi]		;; Add/sub first number
		vmovapd	ymm1, [rdx+32]			;; Load second number
		fop	ymm1, ymm1, [rsi+32]		;; Add/sub first number

ttp		movzx	rbx, WORD PTR [rdi]		;; Load big vs. little & fudge flags
ttp		movzx	ecx, bl				;; Fudge flags 1
ttp		and	rcx, 0e0h
ttp		movzx	eax, bh				;; Big/lit flags 1-2
ttp		and	rbx, 01ch			;; Fudge flags 2
		add	rsi, dist_to_dest		;; Change to a destination pointer
ttp		vmulpd	ymm0, ymm0, [rbp+0*YMM_GMD][rcx] ;; Mul by fudged grp two-to-minus-phi
ttp		vmulpd	ymm1, ymm1, [rbp+1*YMM_GMD][rbx*8] ;; Mul by fudged grp two-to-minus-phi
		vaddpd	ymm0, ymm0, ymm2		;; x1 = values + carry
		vaddpd	ymm1, ymm1, ymm3		;; x2 = values + carry
		rounding_interleaved ttp, base2, noexec, ymm0, ymm2, ymm4, rax*8, ymm1, ymm3, ymm5, rax*8+32
ttp		vmulpd	ymm0, ymm0, [rbp+0*YMM_GMD+YMM_GMD/2][rcx] ;; value1 *= fudged grp two-to-phi
ttp		vmulpd	ymm1, ymm1, [rbp+1*YMM_GMD+YMM_GMD/2][rbx*8] ;; value2 *= fudged grp two-to-phi
		ystore	[rsi+0*32], ymm0		;; Save new value1
		ystore	[rsi+1*32], ymm1		;; Save new value2
		sub	rsi, dist_to_dest		;; Change back to a pointer into the first input number
	ENDM

; Zero-padded version of ynorm_op_wpn
; rsi = pointer to the first number
; rdx = pointer to the second number
; rsi+rbx = pointer to the destination
; rdi = pointer to array of big vs. little flags
; rbp = pointer two-to-phi group multipliers
; ymm2,ymm3 = carries
; rax,rcx destroyed if ttp
; NOTE: caller can assume xmm6,xmm7 are left untouched (for better prolog/epilog handling)
ynorm_op_wpn_zpad MACRO fop, ttp, base2
		vmovapd	ymm0, [rdx]			;; Load second number
		fop	ymm0, ymm0, [rsi]		;; Add/sub first number
		vmovapd	ymm1, [rdx+32]			;; Load second number
		fop	ymm1, ymm1, [rsi+32]		;; Add/sub first number

ttp		movzx	rcx, WORD PTR [rdi]		;; Load big vs. little & fudge flags
ttp		movzx	eax, ch				;; Big/lit flags 1-2
ttp		and	rcx, 0e0h			;; Fudge flags 1 & 2
ttp		vmulpd	ymm0, ymm0, [rbp+0*YMM_GMD][rcx] ;; Mul by fudged grp two-to-minus-phi
ttp		vmulpd	ymm1, ymm1, [rbp+0*YMM_GMD][rcx] ;; Mul by fudged grp two-to-minus-phi
		vaddpd	ymm0, ymm0, ymm2		;; x1 = values + carry
		vaddpd	ymm1, ymm1, ymm3		;; x2 = values + carry
		rounding_interleaved ttp, base2, noexec, ymm0, ymm2, ymm4, rax*8, ymm1, ymm3, ymm5, rax*8
ttp		vmulpd	ymm0, ymm0, [rbp+0*YMM_GMD+YMM_GMD/2][rcx] ;; value1 *= fudged grp two-to-phi
ttp		vmulpd	ymm1, ymm1, [rbp+0*YMM_GMD+YMM_GMD/2][rcx] ;; value2 *= fudged grp two-to-phi
		ystore	[rsi+rbx+0*32], ymm0		;; Save new value1
		ystore	[rsi+rbx+1*32], ymm1		;; Save new value2
	ENDM

;; Final step in the ynorm_op process.  Write carries to the carries array (to be processed later)

ynorm_op_wpn_save_carries MACRO
	mov	rbp, DATA_ADDR			; Address of carries array
	ystore	[rbp], ymm2			; Save carry out of low word
	ystore	[rbp+32], ymm3			; Save carry out of high word
	ENDM

ynorm_op_wpn_save_carries2 MACRO
	mov	rbp, DATA_ADDR			; Address of carries #1 array
	ystore	[rbp], ymm2			; Save carry out of low word
	ystore	[rbp+32], ymm3			; Save carry out of high word
	mov	rbp, PREMULT_ADDR		; Address of carries #2 array
	ystore	[rbp], ymm6			; Save carry out of low word
	ystore	[rbp+32], ymm7			; Save carry out of high word
	ENDM

ynorm_op_wpn_save_carries_zpad MACRO
	mov	rbp, DATA_ADDR			; Address of carries array
	ystore	[rbp], ymm2			; Save carry out of low word
	vxorpd	ymm0, ymm0, ymm0		; High carry should be zero (BIGVAL)
	ystore	[rbp+32], ymm0			; Store high carry not in BIGVAL format
	ENDM

ynorm_op_wpn_save_carries2_zpad MACRO
	mov	rbp, DATA_ADDR			; Address of carries #1 array
	ystore	[rbp], ymm2			; Save carry out of low word
	vxorpd	ymm0, ymm0, ymm0		; High carry should be zero (BIGVAL)
	ystore	[rbp+32], ymm0			; Store high carry not in BIGVAL format
	mov	rbp, PREMULT_ADDR		; Address of carries #2 array
	ystore	[rbp], ymm6			; Save carry out of low word
	ystore	[rbp+32], ymm0			; Save carry out of high word
	ENDM

; *************** WPN followup macros ******************
; This macro finishes the normalize add/sub process by adding three carry values
; from the two carry registers from the end of a pass 1 block back to the start
; of the pass 1 block.  The remaining two carry values are rotated for starting the next block.
; rsi = pointer to the FFT data values
; rbp = pointer two-to-phi multipliers
; rdi = pointer to biglit flags
; ymm2,ymm3 or ymm6,ymm7 = carries

ynorm_op_wpn_blk MACRO ttp, base2, carry1, carry2
	rotate_carries_interleaved base2, carry1, ymm4, carry2, ymm5, ymm0, ymm1 ;; Rotate carries
base2	vsubpd	carry1, carry1, YMM_BIGVAL		;; Remove BIGVAL
base2	vsubpd	carry2, carry2, YMM_BIGVAL		;; Remove BIGVAL
ttp	movzx	rbx, WORD PTR [rdi]			;; Load big vs. little & fudge flags
ttp	movzx	ecx, bl					;; Fudge flags 1
ttp	and	rcx, 0e0h
ttp	and	rbx, 01ch				;; Fudge flags 2
ttp	vmulpd	carry1, carry1, [rbp+0*YMM_GMD+YMM_GMD/2][rcx] ;; carry1 *= fudged grp two-to-phi
ttp	vmulpd	carry2, carry2, [rbp+1*YMM_GMD+YMM_GMD/2][rbx*8] ;; carry2 *= fudged grp two-to-phi
	vaddpd	ymm0, carry1, [rsi]			;; x1 = carry1 + values1
	vaddpd	ymm1, carry2, [rsi+32]			;; x2 = carry2 + values2
	ystore	[rsi], ymm0				;; Save new value1
	ystore	[rsi+32], ymm1				;; Save new value2
	vmovapd	carry1, ymm4				;; Set next block's carry
	vmovapd	carry2, ymm5				;; Set next block's carry
	ENDM

; Zero-padded FFT version.  There cannot be any carry out of the top word.
ynorm_op_wpn_zpad_blk MACRO ttp, base2, carry1, carry2
	rotate_carries base2, carry1, ymm4, ymm0, ymm1	;; Rotate carries
base2	vsubpd	carry1, carry1, YMM_BIGVAL		;; Remove BIGVAL
ttp	movzx	rcx, WORD PTR [rdi]			;; Load big vs. little & fudge flags
ttp	and	rcx, 0e0h				;; Fudge flags 1 & 2
ttp	vmulpd	carry1, carry1, [rbp+0*YMM_GMD+YMM_GMD/2][rcx] ;; carry1 *= fudged grp two-to-phi
	vaddpd	ymm0, carry1, [rsi]			;; x1 = carry1 + values1
	ystore	[rsi], ymm0				;; Save new value1
	vmovapd	carry1, ymm4				;; Set next block's carry
	ENDM

; This macro finishes the normalize add/sub process by adding the last two carries
; from the end back to the start of the result.
; rsi = pointer to FFT data
; rbp = pointer two-to-phi group multipliers
; rdi = biglit flags ptr
; NOTE: Assumes carry can be propagated over 4 FFT words.  Should not be a problem
; as an add/sub carry will be just 2 or 3 bits at most and multiplying by -c should
; keep the carry at about 25 bits max.
; Also note: The group multiplier will be 1.0 for the first 4 FFT words.

ynorm_op_wpn_final MACRO ttp, base2, carry1, carry2
	LOCAL	cloop, last_iter, done

xxx	ynorm_top_carry_cmn base2, rsi, carry2, 2

base2	vsubsd	carry2, carry2, Q YMM_BIGVAL	;; Remove BIGVAL
	vmulsd	carry2, carry2, Q YMM_MINUS_C	;; mul wrap around carry by -c
base2	vaddsd	carry2, carry2, Q YMM_BIGVAL	;; Restore BIGVAL

	sub	dl, dl				;; Clear loop counter

cloop:
ttp	movzx	rbx, WORD PTR [rdi]		;; Load big vs. little & fudge flags
ttp	movzx	eax, bh				;; Big/lit flags 1-2
ttp	and	rbx, 01ch			;; Fudge flags 2

	cmp	dl, 3				;; Is this the 4th iteration
	je	last_iter			;; Yes it's our last.  Last iteration requires special handling

	vaddsd	xmm0, carry2, Q [rsi]		;; x1 = values1 + carry2
no ttp	vaddsd	xmm1, carry1, Q [rsi+32]	;; x2 = values2 + carry1
ttp	vmovsd	xmm1, Q [rsi+32]		;; values2
ttp	vmulsd	xmm1, xmm1, Q [rbp+1*YMM_GMD][rbx*8] ;; values2 *= fudged grp two-to-minus-phi
ttp	vaddsd	xmm1, carry1, xmm1		;; x2 = values2 + carry1
	new_single_rounding_interleaved ttp, base2, xmm0, carry2, xmm4, rax*8, xmm1, carry1, xmm5, rax*8+32
ttp	vmulsd	xmm1, xmm1, Q [rbp+1*YMM_GMD+YMM_GMD/2][rbx*8] ;; value2 *= fudged grp two-to-phi
	vmovsd	Q [rsi], xmm0			;; Save new value1
	vmovsd	Q [rsi+32], xmm1		;; Save new value2

base2	vmovsd	xmm1, Q YMM_BIGVAL		;; Load comparison value
no base2 vxorpd	xmm1, xmm1, xmm1		;; Create comparison value
	vcmpsd	xmm0, carry2, xmm1, 0Ch		;; Test for non-zero carry
	vmovmskpd rax, xmm0			;; Extract comparison bit
	vcmpsd	xmm0, carry1, xmm1, 0Ch		;; Test for non-zero carry
	vmovmskpd rcx, xmm0			;; Extract comparison bit
	or	rax, rcx			;; Is either bit on?
	jz	short done			;; No, we are done

ttp	bump	rdi, 2				;; Advance pointers
	add	rsi, pass2blkdst		;; Next FFT data ptr
 	inc	dl				;; Inrement loop counter
	jmp	cloop				;; No, repeat carry propagation loop

		;; Do last iteration without propagating carries out
last_iter:
base2	vsubsd	carry2, carry2, Q YMM_BIGVAL	;; Subtract rounding constant from carries
base2	vsubsd	carry1, carry1, Q YMM_BIGVAL
ttp	vmulsd	carry1, carry1, Q [rbp+1*YMM_GMD+YMM_GMD/2][rbx*8]
	vaddsd	xmm0, carry2, Q [rsi]		;; x1 = values1 + carry2
	vaddsd	xmm1, carry1, Q [rsi+32]	;; x2 = values2 + carry1
	vmovsd	Q [rsi], xmm0			;; Save new value1
	vmovsd	Q [rsi+32], xmm1		;; Save new value2

done:	
	ENDM

; The zero-padded FFT version.  There cannot be a wrap-around carry.
; The last carry from an add or subtract will be 2-3 bits at most, so the
; carry can be safely added into one FFT data word.  Also the group
; multiplier for the first cache line is 1.0.
ynorm_op_wpn_zpad_final MACRO ttp, base2, carry1, carry2
base2	vsubsd	carry1, carry1, Q YMM_BIGVAL		;; Remove BIGVAL
	vaddsd	xmm0, carry1, Q [rsi+32]		;; x1 = carry1 + values1
	vmovsd	Q [rsi+32], xmm0			;; Save new value1
	ENDM


; *************** WPN normalized add & sub macro ******************
; This macro adds and subtracts, then "normalizes" eight FFT data values.
; This involves multiplying the summed values by two-to-minus-phi, rounding
; the value to an integer while making sure the integer is smaller than the
; maximum allowable integer, generating a carry if necessary.
; Finally, the value is multiplied by two-to-phi.
; and stored.
; rsi = pointer to the first number
; rdx = pointer to the second number
; rdi = pointer to array of big vs. little flags
; rbp = pointer two-to-phi group multipliers
; ymm2,ymm3 = add carries
; ymm6,ymm7 = sub carries
; dist_to_dest1 = distance from first number to destination1
; dist_to_dest2 = distance from first number to destination2
; rax,rbx,rcx destroyed if ttp

ynorm_addsub_wpn MACRO ttp, base2, dist_to_dest1, dist_to_dest2
ttp		movzx	rbx, WORD PTR [rdi]		;; Load big vs. little & fudge flags
ttp		movzx	ecx, bl				;; Fudge flags 1
ttp		and	rcx, 0e0h
ttp		movzx	eax, bh				;; Big/lit flags 1-2
ttp		and	rbx, 01ch			;; Fudge flags 2

		vmovapd	ymm0, [rsi+0*32]		;; Load first number
		vaddpd	ymm0, ymm0, [rdx+0*32]		;; first + second number
		vmovapd	ymm1, [rsi+1*32]		;; Load first number
		vaddpd	ymm1, ymm1, [rdx+1*32]		;; first + second number
ttp		vmulpd	ymm0, ymm0, [rbp+0*YMM_GMD][rcx] ;; Mul by fudged grp two-to-minus-phi
ttp		vmulpd	ymm1, ymm1, [rbp+1*YMM_GMD][rbx*8] ;; Mul by fudged grp two-to-minus-phi
		vaddpd	ymm0, ymm0, ymm2		;; x1 = values + carry
		vaddpd	ymm1, ymm1, ymm3		;; x2 = values + carry
		rounding_interleaved ttp, base2, noexec, ymm0, ymm2, ymm4, rax*8, ymm1, ymm3, ymm5, rax*8+32
ttp		vmulpd	ymm0, ymm0, [rbp+0*YMM_GMD+YMM_GMD/2][rcx] ;; value1 *= fudged grp two-to-phi
ttp		vmulpd	ymm1, ymm1, [rbp+1*YMM_GMD+YMM_GMD/2][rbx*8] ;; value2 *= fudged grp two-to-phi

		vmovapd	ymm4, [rsi+0*32]		;; Load first number
		vsubpd	ymm4, ymm4, [rdx+0*32]		;; first - second number
		vmovapd	ymm5, [rsi+1*32]		;; Load first number
		vsubpd	ymm5, ymm5, [rdx+1*32]		;; first - second number

		add	rsi, dist_to_dest1		;; Change to a destination pointer
		ystore	[rsi+0*32], ymm0		;; Save new value1
		ystore	[rsi+1*32], ymm1		;; Save new value1
		sub	rsi, dist_to_dest1		;; Change back to a pointer into the first input number

ttp		vmulpd	ymm4, ymm4, [rbp+0*YMM_GMD][rcx] ;; Mul by fudged grp two-to-minus-phi
ttp		vmulpd	ymm5, ymm5, [rbp+1*YMM_GMD][rbx*8] ;; Mul by fudged grp two-to-minus-phi
		vaddpd	ymm4, ymm4, ymm6		;; x1 = values + carry
		vaddpd	ymm5, ymm5, ymm7		;; x2 = values + carry
		rounding_interleaved ttp, base2, noexec, ymm4, ymm6, ymm0, rax*8, ymm5, ymm7, ymm1, rax*8+32
ttp		vmulpd	ymm4, ymm4, [rbp+0*YMM_GMD+YMM_GMD/2][rcx] ;; value1 *= fudged grp two-to-phi
ttp		vmulpd	ymm5, ymm5, [rbp+1*YMM_GMD+YMM_GMD/2][rbx*8] ;; value2 *= fudged grp two-to-phi
		add	rsi, dist_to_dest2		;; Change to a destination pointer
		ystore	[rsi+0*32], ymm4		;; Save new value1
		ystore	[rsi+1*32], ymm5		;; Save new value1
		sub	rsi, dist_to_dest2		;; Change back to a pointer into the first input number
	ENDM

; The zero-padded FFT version
; rsi = pointer to the first number
; rdx = pointer to the second number
; rsi+rbx = pointer to destination1
; rdi = pointer to array of big vs. little flags
; rbp = pointer two-to-phi group multipliers
; ymm2,ymm3 = add carries
; ymm6,ymm7 = sub carries
; dist_to_dest2 = distance from first number to destination2
; rax,rcx destroyed if ttp

ynorm_addsub_wpn_zpad MACRO ttp, base2, dist_to_dest2
ttp		movzx	rcx, WORD PTR [rdi]		;; Load big vs. little & fudge flags
ttp		movzx	eax, ch				;; Big/lit flags 1-2
ttp		and	rcx, 0e0h			;; Fudge flags 1 & 2

		vmovapd	ymm0, [rsi+0*32]		;; Load first number
		vaddpd	ymm0, ymm0, [rdx+0*32]		;; first + second number
		vmovapd	ymm1, [rsi+1*32]		;; Load first number
		vaddpd	ymm1, ymm1, [rdx+1*32]		;; first + second number
ttp		vmulpd	ymm0, ymm0, [rbp+0*YMM_GMD][rcx] ;; Mul by fudged grp two-to-minus-phi
ttp		vmulpd	ymm1, ymm1, [rbp+0*YMM_GMD][rcx] ;; Mul by fudged grp two-to-minus-phi
		vaddpd	ymm0, ymm0, ymm2		;; x1 = values + carry
		vaddpd	ymm1, ymm1, ymm3		;; x2 = values + carry
		rounding_interleaved ttp, base2, noexec, ymm0, ymm2, ymm4, rax*8, ymm1, ymm3, ymm5, rax*8
ttp		vmulpd	ymm0, ymm0, [rbp+0*YMM_GMD+YMM_GMD/2][rcx] ;; value1 *= fudged grp two-to-phi
ttp		vmulpd	ymm1, ymm1, [rbp+0*YMM_GMD+YMM_GMD/2][rcx] ;; value2 *= fudged grp two-to-phi

		vmovapd	ymm4, [rsi+0*32]		;; Load first number
		vsubpd	ymm4, ymm4, [rdx+0*32]		;; first - second number
		vmovapd	ymm5, [rsi+1*32]		;; Load first number
		vsubpd	ymm5, ymm5, [rdx+1*32]		;; first - second number

		ystore	[rsi+rbx+0*32], ymm0		;; Save new value1
		ystore	[rsi+rbx+1*32], ymm1		;; Save new value1

ttp		vmulpd	ymm4, ymm4, [rbp+0*YMM_GMD][rcx] ;; Mul by fudged grp two-to-minus-phi
ttp		vmulpd	ymm5, ymm5, [rbp+0*YMM_GMD][rcx] ;; Mul by fudged grp two-to-minus-phi
		vaddpd	ymm4, ymm4, ymm6		;; x1 = values + carry
		vaddpd	ymm5, ymm5, ymm7		;; x2 = values + carry
		rounding_interleaved ttp, base2, noexec, ymm4, ymm6, ymm0, rax*8, ymm5, ymm7, ymm1, rax*8
ttp		vmulpd	ymm4, ymm4, [rbp+0*YMM_GMD+YMM_GMD/2][rcx] ;; value1 *= fudged grp two-to-phi
ttp		vmulpd	ymm5, ymm5, [rbp+0*YMM_GMD+YMM_GMD/2][rcx] ;; value2 *= fudged grp two-to-phi
		add	rsi, dist_to_dest2		;; Change to a destination pointer
		ystore	[rsi+0*32], ymm4		;; Save new value1
		ystore	[rsi+1*32], ymm5		;; Save new value1
		sub	rsi, dist_to_dest2		;; Change back to a pointer into the first input number
	ENDM

; *************** WPN normalized small mul macro ******************
; This macro multiplies by a small value, then "normalizes" eight FFT data values. 
; rsi = pointer to destination
; rdi = pointer to array of big vs. little flags
; rbp = pointer two-to-phi group multipliers
; ymm2,ymm3 = carries
; ymm6 = small multiplier value
; A pipelined version of this code:
;	xload	ymm0, [rsi]		;; Load second number
;	mulpd	ymm0, ymm6		;; Mul by small value
;	movzx	rcx, BYTE PTR [rdi]	;; Load big vs. little flags
;	mulpd	ymm0, [rax]		;; Mul by fudged grp two-to-minus-phi
;	addpd	ymm0, [rbp+0*32]	;; x1 = values + carry
;	xload	ymm2, YMM_LIMIT_BIGMAX[rcx*2];; Load maximum * BIGVAL - BIGVAL
;	addpd	ymm2, ymm0		;; y1 = top bits of x
;	xload	ymm6, YMM_LIMIT_BIGMAX_NEG[rcx*2];; Load -(maximum*BIGVAL-BIGVAL)
;	addpd	ymm6, ymm2		;; z1 = y1-(maximum * BIGVAL - BIGVAL)
;	subpd	ymm0, ymm6		;; rounded value = x1 - z1
;	mulpd	ymm2, YMM_LIMIT_INVERSE[rcx*2];; next carry = shifted y1
;	xload	ymm4, YMM_TTP_FUDGE[rcx*2];; fudge two-to-phi
;	mulpd	ymm0, [rax+0*32+16]	;; new value1 = val * fudged grp two-to-phi
;	xstore	[rsi], ymm0		;; Save new value1

ynorm_smallmul_wpn MACRO ttp, base2
	vmulpd	ymm0, ymm6, [rsi]		;; Mul small multiplier by value1
	vmulpd	ymm1, ymm6, [rsi+32]		;; Mul small multiplier by value2

ttp	movzx	rbx, WORD PTR [rdi]		;; Load 4 big vs. little & fudge flags
ttp	movzx	ecx, bl				;; Fudge flags 1
ttp	and	rcx, 0e0h
ttp	movzx	eax, bh				;; Big/lit flags 1-2
ttp	and	rbx, 01ch			;; Fudge flags 2

ttp	vmulpd	ymm0, ymm0, [rbp+0*YMM_GMD][rcx] ;; Mul by fudged grp two-to-minus-phi
ttp	vmulpd	ymm1, ymm1, [rbp+1*YMM_GMD][rbx*8] ;; Mul by fudged grp two-to-minus-phi
	vaddpd	ymm0, ymm0, ymm2		;; x1 = values + carry
	vaddpd	ymm1, ymm1, ymm3		;; x2 = values + carry
	rounding_interleaved ttp, base2, noexec, ymm0, ymm2, ymm4, rax*8, ymm1, ymm3, ymm5, rax*8+32
ttp	vmulpd	ymm0, ymm0, [rbp+0*YMM_GMD+YMM_GMD/2][rcx] ;; value1 *= fudged grp two-to-phi
ttp	vmulpd	ymm1, ymm1, [rbp+1*YMM_GMD+YMM_GMD/2][rbx*8] ;; value2 *= fudged grp two-to-phi
	ystore	[rsi+0*32], ymm0		;; Save new value1
	ystore	[rsi+1*32], ymm1		;; Save new value2
	ENDM

; Zero-padded FFT version
ynorm_smallmul_wpn_zpad MACRO ttp, base2
	vmulpd	ymm0, ymm6, [rsi]		;; Mul small multiplier by value1
	vmulpd	ymm1, ymm6, [rsi+32]		;; Mul small multiplier by value2

ttp	movzx	rcx, WORD PTR [rdi]		;; Load 4 big vs. little & fudge flags
ttp	movzx	eax, ch				;; Big/lit flags 1-2
ttp	and	rcx, 0e0h			;; Fudge flags 1 & 2

ttp	vmulpd	ymm0, ymm0, [rbp+0*YMM_GMD][rcx] ;; Mul by fudged grp two-to-minus-phi
ttp	vmulpd	ymm1, ymm1, [rbp+0*YMM_GMD][rcx] ;; Mul by fudged grp two-to-minus-phi
	vaddpd	ymm0, ymm0, ymm2		;; x1 = values + carry
	vaddpd	ymm1, ymm1, ymm3		;; x2 = values + carry
	rounding_interleaved ttp, base2, noexec, ymm0, ymm2, ymm4, rax*8, ymm1, ymm3, ymm5, rax*8
ttp	vmulpd	ymm0, ymm0, [rbp+0*YMM_GMD+YMM_GMD/2][rcx] ;; value1 *= fudged grp two-to-phi
ttp	vmulpd	ymm1, ymm1, [rbp+0*YMM_GMD+YMM_GMD/2][rcx] ;; value2 *= fudged grp two-to-phi
	ystore	[rsi+0*32], ymm0		;; Save new value1
	ystore	[rsi+1*32], ymm1		;; Save new value2
	ENDM

; This macro finishes the smallmul process by adding three carry values from the
; two carry registers from the end of a pass 1 block back to the start of the
; pass 1 block.  The remaining two carry values are rotated for starting the next block.
; rsi = pointer to the FFT data values
; rbp = pointer two-to-phi multipliers
; rdi = pointer to biglit flags
; ymm2,ymm3 = carries

ynorm_smallmul_wpn_blk MACRO ttp, base2
	LOCAL	clmloop, cloop, done

	rotate_carries_interleaved base2, ymm2, ymm4, ymm3, ymm5, ymm0, ymm1 ;; Rotate carries
	ystore	YMM_TMP1, ymm4			;; Save next block's carries
	ystore	YMM_TMP2, ymm5

clmloop:
ttp	mov	edx, cache_line_multiplier	;; We must increment rdi by a funky amount every 4*clm cache lines
no ttp	sub	rdx, rdx			;; Init loop counter

cloop:
ttp	movzx	rbx, WORD PTR [rdi]		;; Load big vs. little flags
ttp	movzx	ecx, bl				;; Fudge flags 1
ttp	and	rcx, 0e0h
ttp	movzx	eax, bh				;; Big/lit flags 1-2
ttp	and	rbx, 01ch			;; Fudge flags 2

	vmovapd	ymm0, [rsi]			;; Load values1
ttp	vmulpd	ymm0, ymm0, [rbp+0*YMM_GMD][rcx] ;; Mul values1 by fudged two-to-minus-phi
	vmovapd	ymm1, [rsi+32]			;; Load values2
ttp	vmulpd	ymm1, ymm1, [rbp+1*YMM_GMD][rbx*8] ;; Mul values2 by fudged two-to-minus-phi
	vaddpd	ymm0, ymm0, ymm2		;; x1 = values1 + carry
	vaddpd	ymm1, ymm1, ymm3		;; x2 = values2 + carry
	rounding_interleaved ttp, base2, noexec, ymm0, ymm2, ymm4, rax*8, ymm1, ymm3, ymm5, rax*8+32
ttp	vmulpd	ymm0, ymm0, [rbp+0*YMM_GMD+YMM_GMD/2][rcx] ;; value1 = rounded value * fudged grp two-to-phi
ttp	vmulpd	ymm1, ymm1, [rbp+1*YMM_GMD+YMM_GMD/2][rbx*8] ;; value2 = rounded value * fudged grp two-to-phi
	ystore	[rsi], ymm0			;; Save new value1
	ystore	[rsi+32], ymm1			;; Save new value2

base2	vmovapd	ymm1, YMM_BIGVAL		;; Load comparison value
no base2 vxorpd	ymm1, ymm1, ymm1		;; Create comparison value
	vcmppd	ymm0, ymm2, ymm1, 0Ch		;; Test for non-zero carries
	vmovmskpd rax, ymm0			;; Extract 4 comparison bits
	vcmppd	ymm0, ymm3, ymm1, 0Ch		;; Test for non-zero carries
	vmovmskpd rcx, ymm0			;; Extract 4 comparison bits
	or	rax, rcx			;; Are any bits on?
	jz	short done			;; No, we are done

ttp	bump	rdi, 2				;; Advance pointers
	add	rsi, pass2blkdst		;; Next FFT data ptr
 	add	dl, 256/4			;; Special increment needed every 4th iteration
	jnc	cloop				;; Loop if not 4th iteration
	sub	rsi, pass1blkdst		;; Each pass1 block consists of 4 pass 2 blocks
	bump	rsi, 64				;; Next cache line
ttp	sub	dl, 4				;; Do we need to bump rdi yet?
ttp	jnz	cloop				;; No, repeat carry propagation loop
ttp	add	rdi, normval2			;; Adjust ptr to little/big flags
ttp	jmp	clmloop				;; Yes, reload 4*clm counter and propagate
no ttp	jmp	cloop				;; No, repeat carry propagation loop

done:	vmovapd	ymm2, YMM_TMP1			;; Set next block's carry
	vmovapd	ymm3, YMM_TMP2			;; Set next block's carry
	ENDM

; The zero-padded FFT version.  There can't be a carry into the high word.
ynorm_smallmul_wpn_zpad_blk MACRO ttp, base2
	LOCAL	clmloop, cloop, done

	rotate_carries_interleaved base2, ymm2, ymm4, ymm3, ymm5, ymm0, ymm1 ;; Rotate carries
	ystore	YMM_TMP1, ymm4			;; Save next block's carries
	ystore	YMM_TMP2, ymm5

clmloop:
ttp	mov	edx, cache_line_multiplier	;; We must increment rdi by a funky amount every 4*clm cache lines
no ttp	sub	rdx, rdx			;; Init loop counter

cloop:
ttp	movzx	rcx, WORD PTR [rdi]		;; Load big vs. little flags
ttp	movzx	eax, ch				;; Big/lit flags 1-2
ttp	and	rcx, 0e0h			;; Fudge flags 1 & 2

	vmovapd	ymm0, [rsi]			;; Load values1
ttp	vmulpd	ymm0, ymm0, [rbp+0*YMM_GMD][rcx] ;; Mul values1 by fudged two-to-minus-phi
	vaddpd	ymm0, ymm0, ymm2		;; x1 = values1 + carry
	rounding ttp, base2, noexec, ymm0, ymm2, ymm4, rax*8
ttp	vmulpd	ymm0, ymm0, [rbp+0*YMM_GMD+YMM_GMD/2][rcx] ;; value1 = rounded value * fudged grp two-to-phi
	ystore	[rsi], ymm0			;; Save new value1

base2	vmovapd	ymm1, YMM_BIGVAL		;; Load comparison value
no base2 vxorpd	ymm1, ymm1, ymm1		;; Create comparison value
	vcmppd	ymm0, ymm2, ymm1, 0Ch		;; Test for non-zero carries
	vmovmskpd rax, ymm0			;; Extract 4 comparison bits
	test	rax, rax			;; Are any bits on?
	jz	short done			;; No, we are done

ttp	bump	rdi, 2				;; Advance pointers
	add	rsi, pass2blkdst		;; Next FFT data ptr
 	add	dl, 256/4			;; Special increment needed every 4th iteration
	jnc	cloop				;; Loop if not 4th iteration
	sub	rsi, pass1blkdst		;; Each pass1 block consists of 4 pass 2 blocks
	bump	rsi, 64				;; Next cache line
ttp	sub	dl, 4				;; Do we need to bump rdi yet?
ttp	jnz	cloop				;; No, repeat carry propagation loop
ttp	add	rdi, normval2			;; Adjust ptr to little/big flags
ttp	jmp	clmloop				;; Yes, reload 4*clm counter and propagate
no ttp	jmp	cloop				;; No, repeat carry propagation loop

done:	vmovapd	ymm2, YMM_TMP1			;; Set next block's carry
	vmovapd	ymm3, YMM_TMP2			;; Set next block's carry
	ENDM

; This macro finishes the smallmul process by adding the last two carries
; from the end back to the start of the result.
; rsi = pointer to FFT data
; rbp = pointer two-to-phi group multipliers
; rdi = biglit flags ptr
; ymm2,ymm3 = carries

ynorm_smallmul_wpn_final MACRO ttp, base2
	LOCAL	clmloop, cloop, done

xxx	ynorm_top_carry_cmn base2, rsi, xmm3, 2

base2	vsubsd	xmm3, xmm3, Q YMM_BIGVAL	;; Remove BIGVAL
	vmulsd	xmm3, xmm3, Q YMM_MINUS_C	;; mul wrap around carry by -c
base2	vaddsd	xmm3, xmm3, Q YMM_BIGVAL	;; Restore BIGVAL

clmloop:
ttp	mov	edx, cache_line_multiplier	;; We must increment rdi by a funky amount every 4*clm cache lines
no ttp	sub	rdx, rdx			;; Init loop counter

cloop:
ttp	movzx	rbx, WORD PTR [rdi]		;; Load big vs. little flags
ttp	movzx	ecx, bl				;; Fudge flags 1
ttp	and	rcx, 0e0h
ttp	movzx	eax, bh				;; Big/lit flags 1-2
ttp	and	rbx, 01ch			;; Fudge flags 2

	vmovsd	xmm0, Q [rsi]			;; Load value1
ttp	vmulsd	xmm0, xmm0, Q [rbp+0*YMM_GMD][rcx] ;; Mul value1 by fudged two-to-minus-phi
	vmovsd	xmm1, Q [rsi+32]		;; Load value2
ttp	vmulsd	xmm1, xmm1, Q [rbp+1*YMM_GMD][rbx*8] ;; Mul value2 by fudged two-to-minus-phi
	vaddsd	xmm0, xmm0, xmm3		;; x1 = values1 + carry2
	vaddsd	xmm1, xmm1, xmm2		;; x2 = values2 + carry1
	new_single_rounding_interleaved ttp, base2, xmm0, xmm3, xmm4, rax*8, xmm1, xmm2, xmm5, rax*8+32
ttp	vmulsd	xmm0, xmm0, Q [rbp+0*YMM_GMD+YMM_GMD/2][rcx] ;; value1 = rounded value * fudged grp two-to-phi
ttp	vmulsd	xmm1, xmm1, Q [rbp+1*YMM_GMD+YMM_GMD/2][rbx*8] ;; value2 = rounded value * fudged grp two-to-phi
	vmovsd	Q [rsi], xmm0			;; Save new value1
	vmovsd	Q [rsi+32], xmm1		;; Save new value2

base2	vmovsd	xmm1, Q YMM_BIGVAL		;; Load comparison value
no base2 vxorpd	xmm1, xmm1, xmm1		;; Create comparison value
	vcmpsd	xmm0, xmm3, xmm1, 0Ch		;; Test for non-zero carry
	vmovmskpd rax, xmm0			;; Extract comparison bit
	vcmpsd	xmm0, xmm2, xmm1, 0Ch		;; Test for non-zero carry
	vmovmskpd rcx, xmm0			;; Extract comparison bit
	or	rax, rcx			;; Is either bit on?
	jz	short done			;; No, we are done

ttp	bump	rdi, 2				;; Advance pointers
	add	rsi, pass2blkdst		;; Next FFT data ptr
 	add	dl, 256/4			;; Special increment needed every 4th iteration
	jnc	cloop				;; Loop if not 4th iteration
	sub	rsi, pass1blkdst		;; Each pass1 block consists of 4 pass 2 blocks
	bump	rsi, 64				;; Next cache line
ttp	sub	dl, 4				;; Do we need to bump rdi yet?
ttp	jnz	cloop				;; No, repeat carry propagation loop
ttp	add	rdi, normval2			;; Adjust ptr to little/big flags
ttp	jmp	clmloop				;; Yes, reload 4*clm counter and propagate
no ttp	jmp	cloop				;; No, repeat carry propagation loop

done:	
	ENDM


; This macro finishes the smallmul normalize process by adding the final
; two carries back into the appropriate FFT values at the start of the fft.
; rsi = pointer to FFT data
; rbp = pointer two-to-phi group multipliers
; rdi = biglit flags ptr
; ymm2,ymm3 = carries (and the ymm3 carry must be zero)

ynorm_smallmul_wpn_zpad_final MACRO ttp, base2, destptr
	LOCAL	smallk, mediumk, div_k_done

base2	vsubsd	xmm2, xmm2, Q YMM_BIGVAL	;; Integerize carry

	;; Copy and integerize data from 7 words above halfway point to ZPAD0-ZPAD6
	;; Clear high words as we go
	;; Then we can make an almost exact copy of the ynorm_1d_zpad_cleanup code

	mov	rsi, destptr			;; Address of FFT data
ttp	lea	rdi, YMM_FIRST_BIGLIT_VALUES	;; Address of first big/little flags

ttp	movzx	rax, BYTE PTR [rdi]		;; Load big vs. little flags
	vmovsd	xmm0, Q [rsi+32]		;; Value1
	new_single_split_lower_zpad_word ttp, base2, xmm0, xmm2, xmm4, rax*8
	vmovsd	ZPAD0, xmm0

	add	rsi, YMM_SRC_INCR1		;; Next source pointer
ttp	movzx	rax, BYTE PTR [rdi+1]		;; Load big vs. little flags
	vmovsd	xmm0, Q [rsi+32]		;; Value2
	new_single_split_lower_zpad_word ttp, base2, xmm0, xmm2, xmm4, rax*8
	vmovsd	ZPAD1, xmm0

	add	rsi, YMM_SRC_INCR2		;; Next source pointer
ttp	movzx	rax, BYTE PTR [rdi+2]		;; Load big vs. little flags
	vmovsd	xmm0, Q [rsi+32]		;; Value3
	new_single_split_lower_zpad_word ttp, base2, xmm0, xmm2, xmm4, rax*8
	vmovsd	ZPAD2, xmm0

	add	rsi, YMM_SRC_INCR3		;; Next source pointer
	vmovsd	xmm0, Q [rsi+32]		;; Value4
ttp	vroundsd xmm0, xmm0, xmm0, 0		;; Round to an integer
	vaddsd	xmm0, xmm0, xmm2		;; Value4 + carry
	vmovsd	ZPAD3, xmm0

	add	rsi, YMM_SRC_INCR4		;; Next source pointer
	vmovsd	xmm0, Q [rsi+32]		;; Value5
ttp	vroundsd xmm0, xmm0, xmm0, 0		;; Round to an integer
	vmovsd	ZPAD4, xmm0
	vxorpd	xmm1, xmm1, xmm1		;; Clear highest words
	vmovsd	Q [rsi+32], xmm1		;; Clear value5

	add	rsi, YMM_SRC_INCR5		;; Next source pointer
	vmovsd	xmm0, Q [rsi+32]		;; Value6
ttp	vroundsd xmm0, xmm0, xmm0, 0		;; Round to an integer
	vmovsd	ZPAD5, xmm0
	vmovsd	Q [rsi+32], xmm1		;; Clear value6

	add	rsi, YMM_SRC_INCR6		;; Next source pointer
	vmovsd	xmm0, Q [rsi+32]		;; Value7
ttp	vroundsd xmm0, xmm0, xmm0, 0		;; Round to an integer
	vmovsd	ZPAD6, xmm0
	vmovsd	Q [rsi+32], xmm1		;; Clear value7

	;; Divide the zpad data by k.  Store the integer part in YMM_TMP
	;; and the remainder in ZPAD0.  Later we will wrap the integer part
	;; down to the bottom of the FFT data area (and multiply by -c).
	;; And we will store the remainder in the upper half of the FFT
	;; data area.

	;; Note there are three cases to handle.  K is smaller than a big word.
	;; K is between one and 2 big words in size.  And K is more than
	;; 2 big words in size.

	cmp	ZPAD_TYPE, 2			;; Are we dealing with case 1,2,or 3
	jl	smallk				;; One word case
	je	mediumk				;; Two word case

	;; This case does the divide by k where k is three words

	vmovsd	xmm0, ZPAD6			;; Load zpad word (high bits)
	vmovsd	xmm1, ZPAD5			;; Load zpad word (middle bits)
	vmovsd	xmm2, ZPAD4			;; Load zpad word (low bits)
	vmulsd	xmm4, xmm0, ZPAD_INVERSE_K6	;; Mul ZPAD by shifted 1/k
	vroundsd xmm4, xmm4, xmm4, 0		;; Round to integer
	vmulsd	xmm5, xmm4, ZPAD_K6_HI		;; Mul by high bits of k
	vsubsd	xmm0, xmm0, xmm5		;; Calculate high bits of remainder
	vmulsd	xmm5, xmm4, ZPAD_K6_MID		;; Mul by middle bits of k
	vsubsd	xmm1, xmm1, xmm5		;; Calculate middle bits of remainder
	vmulsd	xmm5, xmm4, ZPAD_K6_LO		;; Mul by low bits of k
	vsubsd	xmm2, xmm2, xmm5		;; Calculate low bits of remainder
	vmovsd	Q YMM_TMP5, xmm4		;; Save word of zpad / k

	vmulsd	xmm0, xmm0, ZPAD_SHIFT6		;; Shift previous zpad word
	vaddsd	xmm1, xmm1, xmm0		;; Add to create new high zpad bits
	vmovsd	xmm0, ZPAD3			;; Load zpad word (new low bits)
	vmulsd	xmm5, xmm1, ZPAD_SHIFT5		;; Combine high and medium bits
	vaddsd	xmm5, xmm5, xmm2
	vmulsd	xmm4, xmm5, ZPAD_INVERSE_K5	;; Mul ZPAD by shifted 1/k
	vroundsd xmm4, xmm4, xmm4, 0		;; Round to integer
	vmulsd	xmm5, xmm4, ZPAD_K5_HI		;; Mul by high bits of k
	vsubsd	xmm1, xmm1, xmm5		;; Calculate high bits of remainder
	vmulsd	xmm5, xmm4, ZPAD_K5_MID		;; Mul by middle bits of k
	vsubsd	xmm2, xmm2, xmm5		;; Calculate middle bits of remainder
	vmulsd	xmm5, xmm4, ZPAD_K5_LO		;; Mul by low bits of k
	vsubsd	xmm0, xmm0, xmm5		;; Calculate low bits of remainder
	vmovsd	Q YMM_TMP4, xmm4		;; Save word of zpad / k

	vmulsd	xmm1, xmm1, ZPAD_SHIFT5		;; Shift previous zpad word
	vaddsd	xmm2, xmm2, xmm1		;; Add to create new high zpad bits
	vmovsd	xmm1, ZPAD2			;; Load zpad word (new low bits)
	vmulsd	xmm5, xmm2, ZPAD_SHIFT4		;; Combine high and medium bits
	vaddsd	xmm5, xmm5, xmm0
	vmulsd	xmm4, xmm5, ZPAD_INVERSE_K4	;; Mul ZPAD by shifted 1/k
	vroundsd xmm4, xmm4, xmm4, 0		;; Round to integer
	vmulsd	xmm5, xmm4, ZPAD_K4_HI		;; Mul by high bits of k
	vsubsd	xmm2, xmm2, xmm5		;; Calculate high bits of remainder
	vmulsd	xmm5, xmm4, ZPAD_K4_MID		;; Mul by middle bits of k
	vsubsd	xmm0,xmm0,xmm5			;; Calculate middle bits of remainder
	vmulsd	xmm5, xmm4, ZPAD_K4_LO		;; Mul by low bits of k
	vsubsd	xmm1, xmm1, xmm5		;; Calculate low bits of remainder
	vmovsd	Q YMM_TMP3, xmm4		;; Save word of zpad / k

	vmulsd	xmm2, xmm2, ZPAD_SHIFT4		;; Shift previous zpad word
	vaddsd	xmm0, xmm0, xmm2		;; Add to create new high zpad bits
	vmovsd	xmm2, ZPAD1			;; Load zpad word (new low bits)
	vmulsd	xmm5, xmm0, ZPAD_SHIFT3		;; Combine high and medium bits
	vaddsd	xmm5, xmm5, xmm1
	vmulsd	xmm4, xmm5, ZPAD_INVERSE_K3	;; Mul ZPAD by shifted 1/k
	vroundsd xmm4, xmm4, xmm4, 0		;; Round to integer
	vmulsd	xmm5, xmm4, ZPAD_K3_HI		;; Mul by high bits of k
	vsubsd	xmm0, xmm0, xmm5		;; Calculate high bits of remainder
	vmulsd	xmm5, xmm4, ZPAD_K3_MID		;; Mul by middle bits of k
	vsubsd	xmm1, xmm1, xmm5		;; Calculate middle bits of remainder
	vmulsd	xmm5, xmm4, ZPAD_K3_LO		;; Mul by low bits of k
	vsubsd	xmm2, xmm2, xmm5		;; Calculate low bits of remainder
	vmovsd	Q YMM_TMP2, xmm4		;; Save word of zpad / k

	vmulsd	xmm0, xmm0, ZPAD_SHIFT3		;; Shift previous zpad word
	vaddsd	xmm1, xmm1, xmm0		;; Add to create new high zpad bits
	vmovsd	xmm0, ZPAD0			;; Load zpad word (new low bits)
	vmulsd	xmm5, xmm1, ZPAD_SHIFT2		;; Combine high and medium bits
	vaddsd	xmm5, xmm5, xmm2
	vmulsd	xmm4, xmm5, ZPAD_INVERSE_K2	;; Mul ZPAD by shifted 1/k
	vroundsd xmm4, xmm4, xmm4, 0		;; Round to integer
	vmulsd	xmm5, xmm4, ZPAD_K2_HI		;; Mul by high bits of k
	vsubsd	xmm1, xmm1, xmm5		;; Calculate high bits of remainder
	vmulsd	xmm5, xmm4, ZPAD_K2_MID		;; Load middle bits of k
	vsubsd	xmm2, xmm2, xmm5		;; Calculate middle bits of remainder
	vmulsd	xmm5, xmm4, ZPAD_K2_LO		;; Load low bits of k
	vsubsd	xmm0, xmm0, xmm5		;; Calculate low bits of remainder
	vmovsd	Q YMM_TMP1, xmm4		;; Save word of zpad / k

	vmulsd	xmm1, xmm1, ZPAD_SHIFT2		;; Shift previous zpad word
	vaddsd	xmm2, xmm2, xmm1		;; Add to create new high zpad bits
	vmulsd	xmm2, xmm2, ZPAD_SHIFT1		;; Shift previous zpad word
	vaddsd	xmm0, xmm0, xmm2		;; Add to create new high zpad bits
	vmovsd	ZPAD0, xmm0			;; Save remainder of zpad / k

	vxorpd	xmm1, xmm1, xmm1		;; Zero words that other cases set
	vmovsd	Q YMM_TMP6, xmm1
	
	jmp	div_k_done

	;; This case does the divide by k where k is two words
mediumk:
	vmovsd	xmm0, ZPAD6			;; Load zpad word (high bits)
	vmovsd	xmm1, ZPAD5			;; Load zpad word (low bits)
	vmulsd	xmm4, xmm0, ZPAD_INVERSE_K6	;; Mul ZPAD by shifted 1/k
	vroundsd xmm4, xmm4, xmm4, 0		;; Round to integer
	vmulsd	xmm5, xmm4, ZPAD_K6_HI		;; Mul by high bits of k
	vsubsd	xmm0, xmm0, xmm5		;; Calculate high bits of remainder
	vmulsd	xmm5, xmm4, ZPAD_K6_LO		;; Mul by low bits of k
	vsubsd	xmm1, xmm1, xmm5		;; Calculate low bits of remainder
	vmovsd	Q YMM_TMP6, xmm4		;; Save word of zpad / k

	vmulsd	xmm0, xmm0, ZPAD_SHIFT6		;; Shift previous zpad word
	vaddsd	xmm0, xmm0, xmm1		;; Add to create new high zpad bits
	vmovsd	xmm1, ZPAD4			;; Load zpad word (new low bits)
	vmulsd	xmm4, xmm0, ZPAD_INVERSE_K5	;; Mul ZPAD by shifted 1/k
	vroundsd xmm4, xmm4, xmm4, 0		;; Round to integer
	vmulsd	xmm5, xmm4, ZPAD_K5_HI		;; Mul by high bits of k
	vsubsd	xmm0, xmm0, xmm5		;; Calculate high bits of remainder
	vmulsd	xmm5, xmm4, ZPAD_K5_LO		;; Mul by low bits of k
	vsubsd	xmm1, xmm1, xmm5		;; Calculate low bits of remainder
	vmovsd	Q YMM_TMP5, xmm4		;; Save word of zpad / k

	vmulsd	xmm0, xmm0, ZPAD_SHIFT5		;; Shift previous zpad word
	vaddsd	xmm0, xmm0, xmm1		;; Add to create new high zpad bits
	vmovsd	xmm1, ZPAD3			;; Load zpad word (new low bits)
	vmulsd	xmm4, xmm0, ZPAD_INVERSE_K4	;; Mul ZPAD by shifted 1/k
	vroundsd xmm4, xmm4, xmm4, 0		;; Round to integer
	vmulsd	xmm5, xmm4, ZPAD_K4_HI		;; Mul by high bits of k
	vsubsd	xmm0, xmm0, xmm5		;; Calculate high bits of remainder
	vmulsd	xmm5, xmm4, ZPAD_K4_LO		;; Mul by low bits of k
	vsubsd	xmm1, xmm1, xmm5		;; Calculate low bits of remainder
	vmovsd	Q YMM_TMP4, xmm4		;; Save word of zpad / k

	vmulsd	xmm0, xmm0, ZPAD_SHIFT4		;; Shift previous zpad word
	vaddsd	xmm0, xmm0, xmm1		;; Add to create new high zpad bits
	vmovsd	xmm1, ZPAD2			;; Load zpad word (new low bits)
	vmulsd	xmm4, xmm0, ZPAD_INVERSE_K3	;; Mul ZPAD by shifted 1/k
	vroundsd xmm4, xmm4, xmm4, 0		;; Round to integer
	vmulsd	xmm5, xmm4, ZPAD_K3_HI		;; Mul by high bits of k
	vsubsd	xmm0, xmm0, xmm5		;; Calculate high bits of remainder
	vmulsd	xmm5, xmm4, ZPAD_K3_LO		;; Mul by low bits of k
	vsubsd	xmm1, xmm1, xmm5		;; Calculate low bits of remainder
	vmovsd	Q YMM_TMP3, xmm4		;; Save word of zpad / k

	vmulsd	xmm0, xmm0, ZPAD_SHIFT3		;; Shift previous zpad word
	vaddsd	xmm0, xmm0, xmm1		;; Add to create new high zpad bits
	vmovsd	xmm1, ZPAD1			;; Load zpad word (new low bits)
	vmulsd	xmm4, xmm0, ZPAD_INVERSE_K2	;; Mul ZPAD by shifted 1/k
	vroundsd xmm4, xmm4, xmm4, 0		;; Round to integer
	vmulsd	xmm5, xmm4, ZPAD_K2_HI		;; Mul by high bits of k
	vsubsd	xmm0, xmm0, xmm5		;; Calculate high bits of remainder
	vmulsd	xmm5, xmm4, ZPAD_K2_LO		;; Mul by low bits of k
	vsubsd	xmm1, xmm1, xmm5		;; Calculate low bits of remainder
	vmovsd	Q YMM_TMP2, xmm4		;; Save word of zpad / k

	vmulsd	xmm0, xmm0, ZPAD_SHIFT2		;; Shift previous zpad word
	vaddsd	xmm0, xmm0, xmm1		;; Add to create new high zpad bits
	vmovsd	xmm1, ZPAD0			;; Load zpad word (new low bits)
	vmulsd	xmm4, xmm0, ZPAD_INVERSE_K1	;; Mul ZPAD by shifted 1/k
	vroundsd xmm4, xmm4, xmm4, 0		;; Round to integer
	vmulsd	xmm5, xmm4, ZPAD_K1_HI		;; Mul by high bits of k
	vsubsd	xmm0, xmm0, xmm5		;; Calculate high bits of remainder
	vmulsd	xmm5, xmm4, ZPAD_K1_LO		;; Mul by low bits of k
	vsubsd	xmm1, xmm1, xmm5		;; Calculate low bits of remainder
	vmovsd	Q YMM_TMP1, xmm4		;; Save word of zpad / k

	vmulsd	xmm0, xmm0, ZPAD_SHIFT1		;; Shift previous zpad word
	vaddsd	xmm0, xmm0, xmm1		;; Add to create new high zpad bits
	vmovsd	ZPAD0, xmm0			;; Save remainder of zpad / k

	jmp	div_k_done

	;; This case does the divide by k where k is one word
	;; Assume ZPAD5 and ZPAD6 are zero.
smallk:	vmovsd	xmm0, ZPAD4			;; Load zpad data
	vmulsd	xmm4, xmm0, ZPAD_INVERSE_K1	;; Mul ZPAD data by 1/k
	vroundsd xmm4, xmm4, xmm4, 0		;; Round to integer
	vmovsd	Q YMM_TMP5, xmm4		;; Save integer part
	vmulsd	xmm4, xmm4, ZPAD_K1_LO		;; Compute remainder
	vsubsd	xmm0, xmm0, xmm4

	vmulsd	xmm0, xmm0, ZPAD_SHIFT4		;; Shift previous zpad word
	vaddsd	xmm0, xmm0, ZPAD3		;; Add in zpad data
	vmulsd	xmm4, xmm0, ZPAD_INVERSE_K1	;; Mul ZPAD data by 1/k
	vroundsd xmm4, xmm4, xmm4, 0		;; Round to integer
	vmovsd	Q YMM_TMP4, xmm4		;; Save integer part
	vmulsd	xmm4, xmm4, ZPAD_K1_LO		;; Compute remainder
	vsubsd	xmm0, xmm0, xmm4

	vmulsd	xmm0, xmm0, ZPAD_SHIFT3		;; Shift previous zpad word
	vaddsd	xmm0, xmm0, ZPAD2		;; Add in zpad data
	vmulsd	xmm4, xmm0, ZPAD_INVERSE_K1	;; Mul ZPAD data by 1/k
	vroundsd xmm4, xmm4, xmm4, 0		;; Round to integer
	vmovsd	Q YMM_TMP3, xmm4		;; Save integer part
	vmulsd	xmm4, xmm4, ZPAD_K1_LO		;; Compute remainder
	vsubsd	xmm0, xmm0, xmm4

	vmulsd	xmm0, xmm0, ZPAD_SHIFT2		;; Shift previous zpad word
	vaddsd	xmm0, xmm0, ZPAD1		;; Add in zpad data
	vmulsd	xmm4, xmm0, ZPAD_INVERSE_K1	;; Mul ZPAD data by 1/k
	vroundsd xmm4, xmm4, xmm4, 0		;; Round to integer
	vmovsd	Q YMM_TMP2, xmm4		;; Save integer part
	vmulsd	xmm4, xmm4, ZPAD_K1_LO		;; Compute remainder
	vsubsd	xmm0, xmm0, xmm4

	vmulsd	xmm0, xmm0, ZPAD_SHIFT1		;; Shift previous zpad word
	vaddsd	xmm0, xmm0, ZPAD0		;; Add in zpad data
	vmulsd	xmm4, xmm0, ZPAD_INVERSE_K1	;; Mul ZPAD data by 1/k
	vroundsd xmm4, xmm4, xmm4, 0		;; Round to integer
	vmovsd	Q YMM_TMP1, xmm4		;; Save integer part
	vmulsd	xmm4, xmm4, ZPAD_K1_LO		;; Compute remainder
	vsubsd	xmm0, xmm0, xmm4
	vmovsd	ZPAD0, xmm0			;; Save remainder

	vxorpd	xmm1, xmm1, xmm1		;; Zero words that other cases set
	vmovsd	Q YMM_TMP6, xmm1

div_k_done:

	;; Now normalize the data above the halfway point.  Remember that the
	;; two-to-phi multiplier for the first value will be 1.0.

ttp	lea	rdi, YMM_FIRST_BIGLIT_VALUES	;; Address of first big/little flags
	mov	rsi, destptr			;; Address of squared number

ttp	movzx	rax, BYTE PTR [rdi]		;; First word
	vmovsd	xmm0, ZPAD0			;; Load remainder of divide by k
base2	vaddsd	xmm0, xmm0, Q YMM_BIGVAL
	new_single_rounding ttp, base2, xmm0, xmm2, xmm4, rax*8
	vmovsd	Q [rsi+32], xmm0		;; Save value1

	add	rsi, YMM_SRC_INCR1		;; Next source pointer
ttp	movzx	rax, BYTE PTR [rdi+1]		;; Load big vs. little flags
	new_single_rounding ttp, base2, xmm2, xmm0, xmm4, rax*8
	vmovsd	Q [rsi+32], xmm2		;; Save value2

	add	rsi, YMM_SRC_INCR2		;; Next source pointer
ttp	movzx	rax, BYTE PTR [rdi+2]		;; Load big vs. little flags
	new_single_rounding ttp, base2, xmm0, xmm2, xmm4, rax*8
	vmovsd	Q [rsi+32], xmm0		;; Save value3

	add	rsi, YMM_SRC_INCR3		;; Next source pointer
base2	vsubsd	xmm2, xmm2, Q YMM_BIGVAL	;; Remove integer rounding constant
	vmovsd	Q [rsi+32], xmm2		;; Save new value4

	;; Mul the integer part of (ZPAD data divided by k) by -c in
	;; preparation for adding it into the lower FFT data area.
	;; Also add in the shifted high FFT carry at this time.

	;; Now add in and normalize the bottom FFT data.  Remember that the
	;; two-to-phi multiplier for the first value will be 1.0.  We 
	;; must go 6 words deep in case k is 48-50 bits and c is 32 bits.

	mov	rsi, destptr			;; Address of squared number

ttp	movzx	rax, BYTE PTR [rdi]		;; First word
	vmovsd	xmm0, Q YMM_TMP1		;; Load integer part of divide by k
	vmulsd	xmm0, xmm0, Q YMM_MINUS_C	;; Mul by -c
base2	vaddsd	xmm0, xmm0, Q YMM_BIGVAL
	vaddsd	xmm0, xmm0, Q [rsi]		;; Add in the FFT data
	new_single_rounding ttp, base2, xmm0, xmm2, xmm4, rax*8
	vmovsd	Q [rsi], xmm0			;; Save value1

	add	rsi, YMM_SRC_INCR1		;; Next source pointer
ttp	movzx	rax, BYTE PTR [rdi+1]		;; Load big vs. little flags
	vmovsd	xmm0, Q YMM_TMP2		;; Load integer part of divide by k
	vmulsd	xmm0, xmm0, Q YMM_MINUS_C	;; Mul by -c
	vaddsd	xmm0, xmm0, xmm2		;; x2 = value + carry
	vaddsd	xmm0, xmm0, Q [rsi]		;; Add in the FFT data
	new_single_rounding ttp, base2, xmm0, xmm2, xmm4, rax*8
	vmovsd	Q [rsi], xmm0			;; Save value2

	add	rsi, YMM_SRC_INCR2		;; Next source pointer
ttp	movzx	rax, BYTE PTR [rdi+2]		;; Load big vs. little flags
	vmovsd	xmm0, Q YMM_TMP3		;; Load integer part of divide by k
	vmulsd	xmm0, xmm0, Q YMM_MINUS_C	;; Mul by -c
	vaddsd	xmm0, xmm0, xmm2		;; x3 = value + carry
	vaddsd	xmm0, xmm0, Q [rsi]		;; Add in the FFT data
	new_single_rounding ttp, base2, xmm0, xmm2, xmm4, rax*8
	vmovsd	Q [rsi], xmm0			;; Save value3

	add	rsi, YMM_SRC_INCR3		;; Next source pointer
ttp	movzx	rax, BYTE PTR [rdi+3]		;; Load big vs. little flags
	vmovsd	xmm0, Q YMM_TMP4		;; Load integer part of divide by k
	vmulsd	xmm0, xmm0, Q YMM_MINUS_C	;; Mul by -c
	vaddsd	xmm0, xmm0, xmm2		;; x4 = value + carry
	vaddsd	xmm0, xmm0, Q [rsi]		;; Add in the FFT data
	new_single_rounding ttp, base2, xmm0, xmm2, xmm4, rax*8
	vmovsd	Q [rsi], xmm0			;; Save value4

	add	rsi, YMM_SRC_INCR4		;; Next source pointer
ttp	movzx	rax, BYTE PTR [rdi+4]		;; Load big vs. little flags
	vmovsd	xmm0, Q YMM_TMP5		;; Load integer part of divide by k
	vmulsd	xmm0, xmm0, Q YMM_MINUS_C	;; Mul by -c
	vaddsd	xmm0, xmm0, xmm2		;; x5 = value + carry
	vaddsd	xmm0, xmm0, Q [rsi]		;; Add in the FFT data
	new_single_rounding ttp, base2, xmm0, xmm2, xmm4, rax*8
	vmovsd	Q [rsi], xmm0			;; Save value5

	add	rsi, YMM_SRC_INCR5		;; Next source pointer
ttp	movzx	rax, BYTE PTR [rdi+5]		;; Load big vs. little flags
	vmovsd	xmm0, Q YMM_TMP6		;; Load integer part of divide by k
	vmulsd	xmm0, xmm0, Q YMM_MINUS_C	;; Mul by -c
	vaddsd	xmm0, xmm0, xmm2		;; x6 = value + carry
	vaddsd	xmm0, xmm0, Q [rsi]		;; Add in the FFT data
	new_single_rounding ttp, base2, xmm0, xmm2, xmm4, rax*8
	vmovsd	Q [rsi], xmm0			;; Save value6

	add	rsi, YMM_SRC_INCR6		;; Next source pointer
ttp	movzx	rax, BYTE PTR [rdi+6]		;; Load big vs. little flags
	vaddsd	xmm0, xmm2, Q [rsi]		;; x7 = value + carry
	new_single_rounding ttp, base2, xmm0, xmm2, xmm4, rax*8
	vmovsd	Q [rsi], xmm0			;; Save value7

	add	rsi, YMM_SRC_INCR7		;; Next source pointer
base2	vsubsd	xmm2, xmm2, Q YMM_BIGVAL	;; Remove rounding constant
	vaddsd	xmm2, xmm2, Q [rsi]		;; Add in FFT data
	vmovsd	Q [rsi], xmm2			;; Save value8
	ENDM
