; Copyright 2000-2024 - Mersenne Research, Inc.  All rights reserved.
; Author:  George Woltman
; Email: woltman@alum.mit.edu
;
; These macros efficiently implement the normalization to integers
; and multiplication by two-to-phi powers using SSE2 instructions.
;


; These macros implement the variants of the normalization routines
; in a non-pipelined way.  It is simply too much work to hand optimize
; all normalization variants.

; Compute the convolution error and if greater than MAXERR, set MAXERR

error_check MACRO sse4, xmmreg, tmpreg, errreg
sse4	roundpd	tmpreg, xmmreg, 0	;; Convert to an integer
no sse4	xload	tmpreg, XMM_BIGVAL	;; Convert to an integer
no sse4	addpd	tmpreg, xmmreg
no sse4	subpd	tmpreg, XMM_BIGVAL
	subpd	tmpreg, xmmreg		;; This is the convolution error
	andpd	tmpreg, XMM_ABSVAL	;; Compute absolute value
	maxpd	errreg, tmpreg		;; Compute maximum error
	ENDM

error_check_interleaved MACRO sse4, xmmreg1, tmpreg1, xmmreg2, tmpreg2, errreg
sse4	roundpd	tmpreg1, xmmreg1, 0	;; Convert to an integer
sse4	roundpd	tmpreg2, xmmreg2, 0	;; Convert to an integer
no sse4	xload	tmpreg1, XMM_BIGVAL	;; Convert to an integer
no sse4	addpd	tmpreg1, xmmreg1
no sse4	xload	tmpreg2, XMM_BIGVAL	;; Convert to an integer
no sse4	addpd	tmpreg2, xmmreg2
no sse4	subpd	tmpreg1, XMM_BIGVAL
no sse4	subpd	tmpreg2, XMM_BIGVAL
	subpd	tmpreg1, xmmreg1	;; This is the convolution error
	subpd	tmpreg2, xmmreg2	;; This is the convolution error
	andpd	tmpreg1, XMM_ABSVAL	;; Compute absolute value
	andpd	tmpreg2, XMM_ABSVAL	;; Compute absolute value
	maxpd	errreg, tmpreg1		;; Compute maximum error
	maxpd	errreg, tmpreg2		;; Compute maximum error
	ENDM

IFDEF X86_64
;; Same as above but with xmm15 preloaded with a useful constant
error_check_interleaved_64 MACRO sse4, xmmreg1, tmpreg1, xmmreg2, tmpreg2, errreg
sse4	roundpd	tmpreg1, xmmreg1, 0	;; Convert to an integer
sse4	roundpd	tmpreg2, xmmreg2, 0	;; Convert to an integer
no sse4	xcopy	tmpreg1, xmm15		;; Convert to an integer
no sse4	addpd	tmpreg1, xmmreg1
no sse4	xcopy	tmpreg2, xmm15		;; Convert to an integer
no sse4	addpd	tmpreg2, xmmreg2
no sse4	subpd	tmpreg1, xmm15
no sse4	subpd	tmpreg2, xmm15
	subpd	tmpreg1, xmmreg1	;; This is the convolution error
	subpd	tmpreg2, xmmreg2	;; This is the convolution error
sse4	andpd	tmpreg1, xmm15		;; Compute absolute value
sse4	andpd	tmpreg2, xmm15		;; Compute absolute value
no sse4	andpd	tmpreg1, XMM_ABSVAL	;; Compute absolute value
no sse4	andpd	tmpreg2, XMM_ABSVAL	;; Compute absolute value
	maxpd	errreg, tmpreg1		;; Compute maximum error
	maxpd	errreg, tmpreg2		;; Compute maximum error
	ENDM
ENDIF

; In general, normalization routines calculate:
;		newFFTvalue = (FFTvalue * const + carry) % base
;		carry = (FFTvalue * const + carry) / base
; Since FFTvalue * const can exceed 51 bits, we instead split FFTvalue into:
;		hi = FFTvalue / base
;		lo = FFTvalue % base
; and then calculate:
;		newFFTvalue = (lo * const + carry) % base
;		carry = hi * const + (lo * const + carry) / base
;
; For the b = 2 case, we can split hi and lo using any power of 2 larger
; than the FFT base, this allows for some simpler code in this case.



; These routines split an FFT value and then multiplies it by the small
; constant.  If we are error-checking the input FFT value has already
; been rounded.

mul_by_const MACRO ttp, base2, echk, sse4, xmmreg, xmmreghi, xmmtmp, biglitreg, errreg
base2	 base2_mul_by_const echk, sse4, xmmreg, xmmreghi, xmmtmp, errreg
no base2 ttp nobase2_mul_by_const echk, sse4, xmmreg, xmmreghi, xmmtmp, biglitreg, errreg
no base2 no ttp nobase2_mul_by_const echk, sse4, xmmreg, xmmreghi, xmmtmp, 0, errreg
	ENDM

base2_mul_by_const MACRO echk, sse4, xmmreg, xmmreghi, xmmtmp, errreg
	xload	xmmreghi, XMM_BIGBIGVAL
	addpd	xmmreghi, xmmreg	;; Round to nearest multiple of 2^25
sse4	roundpd xmmtmp, xmmreg, 0	;; Round to an integer
no sse4	xload	xmmtmp, XMM_BIGVAL
no sse4	addpd	xmmtmp, xmmreg		;; Round to an integer
	subpd	xmmreghi, XMM_BIGBIGVAL
no sse4	subpd	xmmtmp, XMM_BIGVAL
echk	subpd	xmmreg, xmmtmp		;; This is the convolution error
echk	andpd	xmmreg, XMM_ABSVAL	;; Compute absolute value
echk	maxpd	errreg, xmmreg		;; Compute maximum error
	subpd	xmmtmp, xmmreghi	;; xmmtmp now contains low 25 bits
	xload	xmmreg, XMM_MULCONST
	mulpd	xmmreghi, xmmreg	;; Multiply by the small constant
	mulpd	xmmreg, xmmtmp
	ENDM

nobase2_mul_by_const MACRO echk, sse4, xmmreg, xmmreghi, xmmtmp, basereg, errreg
	xload	xmmreghi, XMM_LIMIT_INVERSE[basereg]
	mulpd	xmmreghi, xmmreg	;; Compute FFTvalue / base
sse4	roundpd	xmmtmp, xmmreg, 0	;; Round to an integer
sse4	roundpd	xmmreghi, xmmreghi, 0	;; Round to an integer
no sse4	xload	xmmtmp, XMM_BIGVAL
no sse4	addpd	xmmreghi, xmmtmp	;; Begin round to an integer
no sse4	subpd	xmmreghi, xmmtmp	;; End round to an integer
no sse4	addpd	xmmtmp, xmmreg		;; Begin round to an integer
no sse4	subpd	xmmtmp, XMM_BIGVAL	;; End round to an integer
echk	subpd	xmmreg, xmmtmp		;; This is the convolution error
echk	andpd	xmmreg, XMM_ABSVAL	;; Compute absolute value
echk	maxpd	errreg, xmmreg		;; Compute maximum error
	xload	xmmreg, XMM_LIMIT_BIGMAX[basereg]
	mulpd	xmmreg, xmmreghi
	subpd	xmmtmp, xmmreg		;; This is FFTvalue % base
	xload	xmmreg, XMM_MULCONST
	mulpd	xmmreghi, xmmreg	;; Multiply by the small constant
	mulpd	xmmreg, xmmtmp
	ENDM

mul_by_const_interleaved MACRO ttp, base2, echk, sse4, xmmreg, xmmreghi, xmmtmp, biglitreg, xmmreg2, xmmreg2hi, xmmtmp2, biglitreg2, errreg
base2	 base2_mul_by_const_interleaved echk, sse4, xmmreg, xmmreghi, xmmtmp, xmmreg2, xmmreg2hi, xmmtmp2, errreg
no base2 ttp nobase2_mul_by_const_interleaved echk, sse4, xmmreg, xmmreghi, xmmtmp, biglitreg, xmmreg2, xmmreg2hi, xmmtmp2, biglitreg2, errreg
no base2 no ttp nobase2_mul_by_const_interleaved echk, sse4, xmmreg, xmmreghi, xmmtmp, 0, xmmreg2, xmmreg2hi, xmmtmp2, 0, errreg
	ENDM

base2_mul_by_const_interleaved MACRO echk, sse4, xmmreg, xmmreghi, xmmtmp, xmmreg2, xmmreg2hi, xmmtmp2, errreg
	xload	xmmreghi, XMM_BIGBIGVAL
	addpd	xmmreghi, xmmreg	;; Round to nearest multiple of 2^25
	xload	xmmreg2hi, XMM_BIGBIGVAL
	addpd	xmmreg2hi, xmmreg2	;; Round to nearest multiple of 2^25
sse4	roundpd xmmtmp, xmmreg, 0	;; Round to an integer
sse4	roundpd xmmtmp2, xmmreg2, 0	;; Round to an integer
no sse4	xload	xmmtmp, XMM_BIGVAL
no sse4	addpd	xmmtmp, xmmreg		;; Round to an integer
no sse4	xload	xmmtmp2, XMM_BIGVAL
no sse4	addpd	xmmtmp2, xmmreg2	;; Round to an integer
	subpd	xmmreghi, XMM_BIGBIGVAL
	subpd	xmmreg2hi, XMM_BIGBIGVAL
no sse4	subpd	xmmtmp, XMM_BIGVAL
no sse4	subpd	xmmtmp2, XMM_BIGVAL
echk	subpd	xmmreg, xmmtmp		;; This is the convolution error
echk	subpd	xmmreg2, xmmtmp2	;; This is the convolution error
echk	andpd	xmmreg, XMM_ABSVAL	;; Compute absolute value
echk	andpd	xmmreg2, XMM_ABSVAL	;; Compute absolute value
echk	maxpd	errreg, xmmreg		;; Compute maximum error
echk	maxpd	errreg, xmmreg2		;; Compute maximum error
	subpd	xmmtmp, xmmreghi	;; xmmreg now contains low 25 bits
	subpd	xmmtmp2, xmmreg2hi	;; xmmreg now contains low 25 bits
	xload	xmmreg, XMM_MULCONST
	xload	xmmreg2, XMM_MULCONST
	mulpd	xmmreghi, xmmreg	;; Multiply by the small constant
	mulpd	xmmreg2hi, xmmreg2	;; Multiply by the small constant
	mulpd	xmmreg, xmmtmp
	mulpd	xmmreg2, xmmtmp2
	ENDM

nobase2_mul_by_const_interleaved MACRO echk, sse4, xmmreg, xmmreghi, xmmtmp, basereg, xmmreg2, xmmreg2hi, xmmtmp2, basereg2, errreg
	xload	xmmreghi, XMM_LIMIT_INVERSE[basereg]
	mulpd	xmmreghi, xmmreg	;; Compute FFTvalue / base
	xload	xmmreg2hi, XMM_LIMIT_INVERSE[basereg2]
	mulpd	xmmreg2hi, xmmreg2	;; Compute FFTvalue / base
sse4	roundpd	xmmtmp, xmmreg, 0	;; Round to an integer
sse4	roundpd	xmmtmp2, xmmreg2, 0	;; Round to an integer
sse4	roundpd	xmmreghi, xmmreghi, 0	;; Round to an integer
sse4	roundpd	xmmreg2hi, xmmreg2hi, 0	;; Round to an integer
no sse4	xload	xmmtmp, XMM_BIGVAL
no sse4	addpd	xmmreghi, xmmtmp	;; Begin round to an integer
no sse4	xload	xmmtmp2, XMM_BIGVAL
no sse4	addpd	xmmreg2hi, xmmtmp2	;; Begin round to an integer
no sse4	subpd	xmmreghi, xmmtmp	;; End round to an integer
no sse4	subpd	xmmreg2hi, xmmtmp2	;; End round to an integer
no sse4	addpd	xmmtmp, xmmreg		;; Begin round to an integer
no sse4	addpd	xmmtmp2, xmmreg2	;; Begin round to an integer
no sse4	subpd	xmmtmp, XMM_BIGVAL	;; End round to an integer
no sse4	subpd	xmmtmp2, XMM_BIGVAL	;; End round to an integer
echk	subpd	xmmreg, xmmtmp		;; This is the convolution error
echk	subpd	xmmreg2, xmmtmp2	;; This is the convolution error
echk	andpd	xmmreg, XMM_ABSVAL	;; Compute absolute value
echk	andpd	xmmreg2, XMM_ABSVAL	;; Compute absolute value
echk	maxpd	errreg, xmmreg		;; Compute maximum error
echk	maxpd	errreg, xmmreg2		;; Compute maximum error
	xload	xmmreg, XMM_LIMIT_BIGMAX[basereg]
	mulpd	xmmreg, xmmreghi
	xload	xmmreg2, XMM_LIMIT_BIGMAX[basereg2]
	mulpd	xmmreg2, xmmreg2hi
	subpd	xmmtmp, xmmreg		;; This is FFTvalue % base
	subpd	xmmtmp2, xmmreg2	;; This is FFTvalue % base
	xload	xmmreg, XMM_MULCONST
	mulpd	xmmreghi, xmmreg	;; Multiply by the small constant
	xload	xmmreg2, XMM_MULCONST
	mulpd	xmmreg2hi, xmmreg2	;; Multiply by the small constant
	mulpd	xmmreg, xmmtmp
	mulpd	xmmreg2, xmmtmp2
	ENDM

;
; These macros do the base2 and nobase2 roundings
; const - set to exec if the output of mul_by_const needs to be added in
; xmmval - input: number to round, output: value to store in the FFT
; xmmcarry - input: part of the next carry if mulbyconst set, output: the next carry
; xmmtmp - a temporary register
;

rounding MACRO ttp, base2, const, sse4, xmmval, xmmcarry, xmmtmp, basereg
base2 const ttp		base2_const_rounding sse4, xmmval, xmmcarry, xmmtmp, basereg
base2 const no ttp	base2_const_rounding sse4, xmmval, xmmcarry, xmmtmp, 0
no base2 const ttp	nobase2_const_rounding sse4, xmmval, xmmcarry, xmmtmp, basereg
no base2 const no ttp	nobase2_const_rounding sse4, xmmval, xmmcarry, xmmtmp, 0
base2 no const ttp	base2_noconst_rounding sse4, xmmval, xmmcarry, xmmtmp, basereg
base2 no const no ttp	base2_noconst_rounding sse4, xmmval, xmmcarry, xmmtmp, 0
no base2 no const ttp	nobase2_noconst_rounding sse4, xmmval, xmmcarry, xmmtmp, basereg
no base2 no const no ttp nobase2_noconst_rounding sse4, xmmval, xmmcarry, xmmtmp, 0
	ENDM

base2_noconst_rounding MACRO sse4, xmmval, xmmcarry, xmmtmp, basereg
	xload	xmmcarry, XMM_LIMIT_BIGMAX[basereg]	;; Load maximum * BIGVAL - BIGVAL
	addpd	xmmcarry, xmmval			;; y1 = top bits of val
	xload	xmmtmp, XMM_LIMIT_BIGMAX_NEG[basereg]	;; Load -(maximum*BIGVAL-BIGVAL)
	addpd	xmmtmp, xmmcarry			;; z1 = y1-(maximum * BIGVAL - BIGVAL)
	subpd	xmmval, xmmtmp				;; rounded val = x1 - z1
	mulpd	xmmcarry, XMM_LIMIT_INVERSE[basereg]	;; carry = shifted y1
	ENDM

base2_const_rounding MACRO sse4, xmmval, xmmcarry, xmmtmp, basereg
	xload	xmmtmp, XMM_LIMIT_BIGMAX[basereg]	;; Load maximum * BIGVAL - BIGVAL
	addpd	xmmtmp, xmmval				;; y1 = top bits of x
	addpd	xmmcarry, xmmtmp			;; Add in upper mul-by-const bits
	mulpd	xmmcarry, XMM_LIMIT_INVERSE[basereg]	;; next carry = shifted y1
	subpd	xmmtmp, XMM_LIMIT_BIGMAX[basereg]	;; z1 = y1 - (maximum*BIGVAL-BIGVAL)
	subpd	xmmval, xmmtmp				;; rounded value = x1 - z1
	ENDM

nobase2_noconst_rounding MACRO sse4, xmmval, xmmcarry, xmmtmp, basereg
	xload	xmmtmp, XMM_BIGVAL_NEG
	addpd	xmmval, xmmtmp				;; Remove rounding constant from val
	xload	xmmcarry, XMM_LIMIT_INVERSE[basereg]	;; Load base inverse
	mulpd	xmmcarry, xmmval			;; val / base
	subpd	xmmcarry, xmmtmp			;; next carry = BIGVAL + val / base
	addpd	xmmtmp, xmmcarry			;; round (val / base)
	mulpd	xmmtmp, XMM_LIMIT_BIGMAX[basereg]	;; z = round (val / base) * base
	subpd	xmmval, xmmtmp				;; new value = val - z
	ENDM

nobase2_const_rounding MACRO sse4, xmmval, xmmcarry, xmmtmp, basereg
	subpd	xmmval, XMM_BIGVAL			;; Remove rounding constant from val
	xload	xmmtmp, XMM_LIMIT_INVERSE[basereg]	;; Load maximum inverse
	mulpd	xmmtmp, xmmval				;; val / base
	addpd	xmmtmp, XMM_BIGVAL			;; next carry = BIGVAL + val / base
	addpd	xmmcarry, xmmtmp			;; next carry = BIGVAL + round (fftval * mulbyconst / base) + val / base
	subpd	xmmtmp, XMM_BIGVAL
	mulpd	xmmtmp, XMM_LIMIT_BIGMAX[basereg]	;; z = top bits of val
	subpd	xmmval, xmmtmp				;; new value = val - z
	ENDM

; Same as above except interleaved for better scheduling

rounding_interleaved MACRO ttp, base2, const, sse4, xmmval, xmmcarry, xmmtmp, basereg, xmmval2, xmmcarry2, xmmtmp2, basereg2
base2 const ttp		base2_const_rounding_interleaved sse4, xmmval, xmmcarry, xmmtmp, basereg, xmmval2, xmmcarry2, xmmtmp2, basereg2
base2 const no ttp	base2_const_rounding_interleaved sse4, xmmval, xmmcarry, xmmtmp, 0, xmmval2, xmmcarry2, xmmtmp2, 0
no base2 const ttp	nobase2_const_rounding_interleaved sse4, xmmval, xmmcarry, xmmtmp, basereg, xmmval2, xmmcarry2, xmmtmp2, basereg2
no base2 const no ttp	nobase2_const_rounding_interleaved sse4, xmmval, xmmcarry, xmmtmp, 0, xmmval2, xmmcarry2, xmmtmp2, 0
base2 no const ttp	base2_noconst_rounding_interleaved sse4, xmmval, xmmcarry, xmmtmp, basereg, xmmval2, xmmcarry2, xmmtmp2, basereg2
base2 no const no ttp	base2_noconst_rounding_interleaved sse4, xmmval, xmmcarry, xmmtmp, 0, xmmval2, xmmcarry2, xmmtmp2, 0
no base2 no const ttp	nobase2_noconst_rounding_interleaved sse4, xmmval, xmmcarry, xmmtmp, basereg, xmmval2, xmmcarry2, xmmtmp2, basereg2
no base2 no const no ttp nobase2_noconst_rounding_interleaved sse4, xmmval, xmmcarry, xmmtmp, 0, xmmval2, xmmcarry2, xmmtmp2, 0
	ENDM

base2_noconst_rounding_interleaved MACRO sse4, xmmval, xmmcarry, xmmtmp, basereg, xmmval2, xmmcarry2, xmmtmp2, basereg2
	xload	xmmcarry, XMM_LIMIT_BIGMAX[basereg]	;; Load maximum * BIGVAL - BIGVAL
	addpd	xmmcarry, xmmval			;; y1 = top bits of val
	xload	xmmcarry2, XMM_LIMIT_BIGMAX[basereg2]	;; Load maximum * BIGVAL - BIGVAL
	addpd	xmmcarry2, xmmval2			;; y1 = top bits of val
	xload	xmmtmp, XMM_LIMIT_BIGMAX_NEG[basereg]	;; Load -(maximum*BIGVAL-BIGVAL)
	addpd	xmmtmp, xmmcarry			;; z1 = y1-(maximum * BIGVAL - BIGVAL)
	xload	xmmtmp2, XMM_LIMIT_BIGMAX_NEG[basereg2]	;; Load -(maximum*BIGVAL-BIGVAL)
	addpd	xmmtmp2, xmmcarry2			;; z1 = y1-(maximum * BIGVAL - BIGVAL)
	subpd	xmmval, xmmtmp				;; rounded val = x1 - z1
	subpd	xmmval2, xmmtmp2			;; rounded val = x1 - z1
	mulpd	xmmcarry, XMM_LIMIT_INVERSE[basereg]	;; carry = shifted y1
	mulpd	xmmcarry2, XMM_LIMIT_INVERSE[basereg2]	;; carry = shifted y1
	ENDM

base2_const_rounding_interleaved MACRO sse4, xmmval, xmmcarry, xmmtmp, basereg, xmmval2, xmmcarry2, xmmtmp2, basereg2
	xload	xmmtmp, XMM_LIMIT_BIGMAX[basereg]	;; Load maximum * BIGVAL - BIGVAL
	addpd	xmmtmp, xmmval				;; y1 = top bits of x
	xload	xmmtmp2, XMM_LIMIT_BIGMAX[basereg2]	;; Load maximum * BIGVAL - BIGVAL
	addpd	xmmtmp2, xmmval2			;; y1 = top bits of x
	addpd	xmmcarry, xmmtmp			;; Add in upper mul-by-const bits
	addpd	xmmcarry2, xmmtmp2			;; Add in upper mul-by-const bits
	mulpd	xmmcarry, XMM_LIMIT_INVERSE[basereg]	;; next carry = shifted y1
	mulpd	xmmcarry2, XMM_LIMIT_INVERSE[basereg2]	;; next carry = shifted y1
	subpd	xmmtmp, XMM_LIMIT_BIGMAX[basereg]	;; z1 = y1 - (maximum*BIGVAL-BIGVAL)
	subpd	xmmtmp2, XMM_LIMIT_BIGMAX[basereg2]	;; z1 = y1 - (maximum*BIGVAL-BIGVAL)
	subpd	xmmval, xmmtmp				;; rounded value = x1 - z1
	subpd	xmmval2, xmmtmp2			;; rounded value = x1 - z1
	ENDM

nobase2_noconst_rounding_interleaved MACRO sse4, xmmval, xmmcarry, xmmtmp, basereg, xmmval2, xmmcarry2, xmmtmp2, basereg2
	xload	xmmtmp, XMM_BIGVAL_NEG
	addpd	xmmval, xmmtmp				;; Remove rounding constant from val
	xload	xmmtmp2, XMM_BIGVAL_NEG
	addpd	xmmval2, xmmtmp2			;; Remove rounding constant from val
	xload	xmmcarry, XMM_LIMIT_INVERSE[basereg]	;; Load base inverse
	mulpd	xmmcarry, xmmval			;; val / base
	xload	xmmcarry2, XMM_LIMIT_INVERSE[basereg2]	;; Load base inverse
	mulpd	xmmcarry2, xmmval2			;; val / base
	subpd	xmmcarry, xmmtmp			;; next carry = BIGVAL + val / base
	subpd	xmmcarry2, xmmtmp2			;; next carry = BIGVAL + val / base
	addpd	xmmtmp, xmmcarry			;; round (val / base)
	addpd	xmmtmp2, xmmcarry2			;; round (val / base)
	mulpd	xmmtmp, XMM_LIMIT_BIGMAX[basereg]	;; z = round (val / base) * base
	mulpd	xmmtmp2, XMM_LIMIT_BIGMAX[basereg2]	;; z = round (val / base) * base
	subpd	xmmval, xmmtmp				;; new value = val - z
	subpd	xmmval2, xmmtmp2			;; new value = val - z
	ENDM

nobase2_const_rounding_interleaved MACRO sse4, xmmval, xmmcarry, xmmtmp, basereg, xmmval2, xmmcarry2, xmmtmp2, basereg2
	subpd	xmmval, XMM_BIGVAL			;; Remove rounding constant from val
	subpd	xmmval2, XMM_BIGVAL			;; Remove rounding constant from val
	xload	xmmtmp, XMM_LIMIT_INVERSE[basereg]	;; Load maximum inverse
	mulpd	xmmtmp, xmmval				;; val / base
	xload	xmmtmp2, XMM_LIMIT_INVERSE[basereg2]	;; Load maximum inverse
	mulpd	xmmtmp2, xmmval2			;; val / base
	addpd	xmmtmp, XMM_BIGVAL			;; next carry = BIGVAL + val / base
	addpd	xmmtmp2, XMM_BIGVAL			;; next carry = BIGVAL + val / base
	addpd	xmmcarry, xmmtmp			;; next carry = BIGVAL + round (fftval * mulbyconst / base) + val / base
	addpd	xmmcarry2, xmmtmp2			;; next carry = BIGVAL + round (fftval * mulbyconst / base) + val / base
	subpd	xmmtmp, XMM_BIGVAL
	subpd	xmmtmp2, XMM_BIGVAL
	mulpd	xmmtmp, XMM_LIMIT_BIGMAX[basereg]	;; z = top bits of val
	mulpd	xmmtmp2, XMM_LIMIT_BIGMAX[basereg2]	;; z = top bits of val
	subpd	xmmval, xmmtmp				;; new value = val - z
	subpd	xmmval2, xmmtmp2			;; new value = val - z
	ENDM

;
; These macros round just one value in an XMM register.  This is done
; as part of the cleanup process where the final carry must be added
; back into the results.
;

single_rounding MACRO base2, xmmval, xmmcarry, xmmtmp, basereg
base2		base2_single_rounding xmmval, xmmcarry, xmmtmp, basereg
no base2	nobase2_single_rounding xmmval, xmmcarry, xmmtmp, basereg
	ENDM

base2_single_rounding MACRO xmmval, xmmcarry, xmmtmp, basereg
	movlpd	xmmcarry, Q XMM_LIMIT_BIGMAX[basereg]	;; Load maximum * BIGVAL - BIGVAL
	addsd	xmmcarry, xmmval			;; y1 = top bits of x
	movsd	xmmtmp, Q XMM_LIMIT_BIGMAX_NEG[basereg]	;; Load -(maximum*BIGVAL-BIGVAL)
	addsd	xmmtmp, xmmcarry			;; z1 = y1-(maximum * BIGVAL - BIGVAL)
	subsd	xmmval, xmmtmp				;; rounded value = x1 - z1
	mulsd	xmmcarry, Q XMM_LIMIT_INVERSE[basereg]	;; next carry = shifted y1
	ENDM

nobase2_single_rounding MACRO xmmval, xmmcarry, xmmtmp, basereg
	movsd	xmmtmp, Q XMM_BIGVAL_NEG
	addsd	xmmval, xmmtmp				;; Remove rounding constant from val
	movlpd	xmmcarry, Q XMM_LIMIT_INVERSE[basereg]	;; Load base inverse
	mulsd	xmmcarry, xmmval			;; val / base
	subsd	xmmcarry, xmmtmp			;; next carry = BIGVAL + val / base
	addsd	xmmtmp, xmmcarry			;; round (val / base)
	mulsd	xmmtmp, Q XMM_LIMIT_BIGMAX[basereg]	;; z = round (val / base) * base
	subsd	xmmval, xmmtmp				;; new value = val - z
	ENDM


single_rounding_interleaved MACRO base2, xmmval, xmmcarry, xmmtmp, basereg, xmmval2, xmmcarry2, xmmtmp2, basereg2
base2		base2_single_rounding_interleaved xmmval, xmmcarry, xmmtmp, basereg, xmmval2, xmmcarry2, xmmtmp2, basereg2
no base2	nobase2_single_rounding_interleaved xmmval, xmmcarry, xmmtmp, basereg, xmmval2, xmmcarry2, xmmtmp2, basereg2
	ENDM

base2_single_rounding_interleaved MACRO xmmval, xmmcarry, xmmtmp, basereg, xmmval2, xmmcarry2, xmmtmp2, basereg2
	movlpd	xmmcarry, Q XMM_LIMIT_BIGMAX[basereg]	;; Load maximum * BIGVAL - BIGVAL
	addsd	xmmcarry, xmmval			;; y1 = top bits of x
	movlpd	xmmcarry2, Q XMM_LIMIT_BIGMAX[basereg2]	;; Load maximum * BIGVAL - BIGVAL
	addsd	xmmcarry2, xmmval2			;; y2 = top bits of x
	movsd	xmmtmp, Q XMM_LIMIT_BIGMAX_NEG[basereg]	;; Load -(maximum*BIGVAL-BIGVAL)
	addsd	xmmtmp, xmmcarry			;; z1 = y1-(maximum * BIGVAL - BIGVAL)
	movsd	xmmtmp2, Q XMM_LIMIT_BIGMAX_NEG[basereg2];; Load -(maximum*BIGVAL-BIGVAL)
	addsd	xmmtmp2, xmmcarry2			;; z2 = y2-(maximum * BIGVAL - BIGVAL)
	subsd	xmmval, xmmtmp				;; rounded value = x1 - z1
	mulsd	xmmcarry, Q XMM_LIMIT_INVERSE[basereg]	;; next carry = shifted y1
	subsd	xmmval2, xmmtmp2			;; rounded value = x2 - z2
	mulsd	xmmcarry2, Q XMM_LIMIT_INVERSE[basereg2];; next carry = shifted y2
	ENDM

nobase2_single_rounding_interleaved MACRO xmmval, xmmcarry, xmmtmp, basereg, xmmval2, xmmcarry2, xmmtmp2, basereg2
	movsd	xmmtmp, Q XMM_BIGVAL_NEG
	addsd	xmmval, xmmtmp				;; Remove rounding constant from val
	movsd	xmmtmp2, Q XMM_BIGVAL_NEG
	addsd	xmmval2, xmmtmp2			;; Remove rounding constant from val
	movlpd	xmmcarry, Q XMM_LIMIT_INVERSE[basereg]	;; Load base inverse
	mulsd	xmmcarry, xmmval			;; val / base
	movlpd	xmmcarry2, Q XMM_LIMIT_INVERSE[basereg2];; Load base inverse
	mulsd	xmmcarry2, xmmval2			;; val / base
	subsd	xmmcarry, xmmtmp			;; next carry = BIGVAL + val / base
	subsd	xmmcarry2, xmmtmp2			;; next carry = BIGVAL + val / base
	addsd	xmmtmp, xmmcarry			;; round (val / base)
	addsd	xmmtmp2, xmmcarry2			;; round (val / base)
	mulsd	xmmtmp, Q XMM_LIMIT_BIGMAX[basereg]	;; z = round (val / base) * base
	mulsd	xmmtmp2, Q XMM_LIMIT_BIGMAX[basereg2]	;; z = round (val / base) * base
	subsd	xmmval, xmmtmp				;; new value = val - z
	subsd	xmmval2, xmmtmp2			;; new value = val - z
	ENDM

;
; These macros process zero-padded FFT result words.  These FFT results must
; be split into high and low parts with the high part used as a carry into
; the splitting the next FFT result word.
;

split_lower_zpad_word MACRO echk, base2, sse4, xmmvalin, xmmcarry, xmmvalout, basereg
base2		base2_split_lower_zpad_word echk, sse4, xmmvalin, xmmcarry, xmmvalout, basereg
no base2	nobase2_split_lower_zpad_word echk, sse4, xmmvalin, xmmcarry, xmmvalout, basereg
	ENDM

base2_split_lower_zpad_word MACRO echk, sse4, xmmvalin, xmmcarry, xmmvalout, basereg
	addpd	xmmvalin, xmmcarry	;; Add in previous high FFT data
	xload	xmmcarry, XMM_BIGBIGVAL	;; Big word rounding constant
	addpd	xmmcarry, xmmvalin	;; Round to multiple of big word
	subpd	xmmcarry, XMM_BIGBIGVAL
sse4	roundpd	xmmvalout, xmmvalin, 0	;; Round to an integer
no sse4	xload	xmmvalout, XMM_BIGVAL	;; Round to an integer
no sse4	addpd	xmmvalout, xmmvalin
no sse4	subpd	xmmvalout, XMM_BIGVAL
echk	subpd	xmmvalin, xmmvalout	;; This is the convolution error
echk	andpd	xmmvalin, XMM_ABSVAL	;; Compute absolute value
echk	maxpd	xmm6, xmmvalin		;; Compute maximum error
	subpd	xmmvalout, xmmcarry	;; xmmvalout now contains low bigword bits
	mulpd	xmmcarry, XMM_LIMIT_INVERSE[basereg];; Saved shifted FFT hi data
	ENDM

nobase2_split_lower_zpad_word MACRO echk, sse4, xmmvalin, xmmcarry, xmmvalout, basereg
	addpd	xmmvalin, xmmcarry	;; Add in previous high FFT data
	xload	xmmcarry, XMM_LIMIT_INVERSE[basereg] ;; Load base
	mulpd	xmmcarry, xmmvalin	;; Compute FFTvalue / base
sse4	roundpd	xmmcarry, xmmcarry, 0	;; Round to integer
no sse4	addpd	xmmcarry, XMM_BIGVAL	;; Round to integer
no sse4	subpd	xmmcarry, XMM_BIGVAL	;; Carry = shifted FFTvalue / base
sse4	roundpd	xmmvalout, xmmvalin, 0	;; Round input to an integer
no sse4	xload	xmmvalout, XMM_BIGVAL	;; Round input to an integer
no sse4	addpd	xmmvalout, xmmvalin
no sse4	subpd	xmmvalout, XMM_BIGVAL
echk	subpd	xmmvalin, xmmvalout	;; This is the convolution error
echk	andpd	xmmvalin, XMM_ABSVAL	;; Compute absolute value
echk	maxpd	xmm6, xmmvalin		;; Compute maximum error
	xload	xmmvalin, XMM_LIMIT_BIGMAX[basereg]
	mulpd	xmmvalin, xmmcarry
	subpd	xmmvalout, xmmvalin	;; xmmvalout now contains FFTvalue % base
	ENDM

;; Interleaved version used in 64-bit implementation of zpad normalization.
;; Relies on some registers being preloaded.

split_lower_zpad_word_interleaved MACRO echk, base2, sse4, xmmvalin, xmmcarry, xmmvalout, basereg, xmmvalin2, xmmcarry2, xmmvalout2, basereg2
base2		base2_split_lower_zpad_word_interleaved echk, sse4, xmmvalin, xmmcarry, xmmvalout, basereg, xmmvalin2, xmmcarry2, xmmvalout2, basereg2
no base2	nobase2_split_lower_zpad_word_interleaved echk, sse4, xmmvalin, xmmcarry, xmmvalout, basereg, xmmvalin2, xmmcarry2, xmmvalout2, basereg2
	ENDM

base2_split_lower_zpad_word_interleaved MACRO echk, sse4, xmmvalin, xmmcarry, xmmvalout, basereg, xmmvalin2, xmmcarry2, xmmvalout2, basereg2
	addpd	xmmvalin, xmmcarry	;; Add in previous high FFT data
	addpd	xmmvalin2, xmmcarry2	;; Add in previous high FFT data
	xcopy	xmmcarry, xmm13		;; XMM_BIGBIGVAL ;; Big word rounding constant
	addpd	xmmcarry, xmmvalin	;; Round to multiple of big word
	xcopy	xmmcarry2, xmm13	;; XMM_BIGBIGVAL ;; Big word rounding constant
	addpd	xmmcarry2, xmmvalin2	;; Round to multiple of big word
	subpd	xmmcarry, xmm13		;; XMM_BIGBIGVAL
	subpd	xmmcarry2, xmm13	;; XMM_BIGBIGVAL
sse4	roundpd	xmmvalout, xmmvalin, 0	;; Round to an integer
sse4	roundpd	xmmvalout2, xmmvalin2, 0 ;; Round to an integer

no sse4	echk	xload	xmmvalout, XMM_BIGVAL	;; Round to an integer
no sse4	echk	addpd	xmmvalout, xmmvalin
no sse4	echk	xload	xmmvalout2, XMM_BIGVAL	;; Round to an integer
no sse4	echk	addpd	xmmvalout2, xmmvalin2
no sse4	echk	subpd	xmmvalout, XMM_BIGVAL
no sse4	echk	subpd	xmmvalout2, XMM_BIGVAL

no sse4 no echk	xcopy	xmmvalout, xmm6		;; XMM_BIGVAL ;; Round to an integer
no sse4	no echk	addpd	xmmvalout, xmmvalin
no sse4	no echk	xcopy	xmmvalout2, xmm6	;; XMM_BIGVAL ;; Round to an integer
no sse4	no echk	addpd	xmmvalout2, xmmvalin2
no sse4	no echk	subpd	xmmvalout, xmm6		;; XMM_BIGVAL
no sse4	no echk	subpd	xmmvalout2, xmm6	;; XMM_BIGVAL

echk	subpd	xmmvalin, xmmvalout	;; This is the convolution error
echk	subpd	xmmvalin2, xmmvalout2	;; This is the convolution error
echk	andpd	xmmvalin, XMM_ABSVAL	;; Compute absolute value
echk	andpd	xmmvalin2, XMM_ABSVAL	;; Compute absolute value
echk	maxpd	xmm6, xmmvalin		;; Compute maximum error
echk	maxpd	xmm6, xmmvalin2		;; Compute maximum error
	subpd	xmmvalout, xmmcarry	;; xmmvalout now contains low bigword bits
	subpd	xmmvalout2, xmmcarry2	;; xmmvalout now contains low bigword bits
	mulpd	xmmcarry, XMM_LIMIT_INVERSE[basereg];; Saved shifted FFT hi data
	mulpd	xmmcarry2, XMM_LIMIT_INVERSE[basereg2];; Saved shifted FFT hi data
	ENDM

nobase2_split_lower_zpad_word_interleaved MACRO echk, sse4, xmmvalin, xmmcarry, xmmvalout, basereg, xmmvalin2, xmmcarry2, xmmvalout2, basereg2
	addpd	xmmvalin, xmmcarry	;; Add in previous high FFT data
	addpd	xmmvalin2, xmmcarry2	;; Add in previous high FFT data
	xload	xmmcarry, XMM_LIMIT_INVERSE[basereg] ;; Load base
	mulpd	xmmcarry, xmmvalin	;; Compute FFTvalue / base
	xload	xmmcarry2, XMM_LIMIT_INVERSE[basereg2] ;; Load base
	mulpd	xmmcarry2, xmmvalin2	;; Compute FFTvalue / base
sse4	roundpd	xmmcarry, xmmcarry, 0	;; Round to integer
sse4	roundpd	xmmcarry2, xmmcarry2, 0	;; Round to integer
no sse4	addpd	xmmcarry, xmm13		;; XMM_BIGVAL ;; Round to integer
no sse4	addpd	xmmcarry2, xmm13	;; XMM_BIGVAL ;; Round to integer
no sse4	subpd	xmmcarry, xmm13		;; XMM_BIGVAL ;; Carry = shifted FFTvalue / base
no sse4	subpd	xmmcarry2, xmm13	;; XMM_BIGVAL ;; Carry = shifted FFTvalue / base
sse4	roundpd	xmmvalout, xmmvalin, 0	;; Round input to an integer
sse4	roundpd	xmmvalout2, xmmvalin2, 0 ;; Round input to an integer
no sse4	xcopy	xmmvalout, xmm13	;; XMM_BIGVAL ;; Round input to an integer
no sse4	addpd	xmmvalout, xmmvalin
no sse4	xcopy	xmmvalout2, xmm13	;; XMM_BIGVAL ;; Round input to an integer
no sse4	addpd	xmmvalout2, xmmvalin2
no sse4	subpd	xmmvalout, xmm13	;; XMM_BIGVAL
no sse4	subpd	xmmvalout2, xmm13	;; XMM_BIGVAL
echk	subpd	xmmvalin, xmmvalout	;; This is the convolution error
echk	subpd	xmmvalin2, xmmvalout2	;; This is the convolution error
echk no sse4	andpd	xmmvalin, XMM_ABSVAL	;; Compute absolute value
echk no sse4	andpd	xmmvalin2, XMM_ABSVAL	;; Compute absolute value
echk sse4	andpd	xmmvalin, xmm13		;; XMM_ABSVAL ;; Compute absolute value
echk sse4	andpd	xmmvalin2, xmm13	;; XMM_ABSVAL ;; Compute absolute value
echk	maxpd	xmm6, xmmvalin		;; Compute maximum error
echk	maxpd	xmm6, xmmvalin2		;; Compute maximum error
	xload	xmmvalin, XMM_LIMIT_BIGMAX[basereg]
	mulpd	xmmvalin, xmmcarry
	xload	xmmvalin2, XMM_LIMIT_BIGMAX[basereg2]
	mulpd	xmmvalin2, xmmcarry2
	subpd	xmmvalout, xmmvalin	;; xmmvalout now contains FFTvalue % base
	subpd	xmmvalout2, xmmvalin2	;; xmmvalout now contains FFTvalue % base
	ENDM

; Split upper is like split lower except that previous carry is not added in and
; result carry is not shifted down.

split_upper_zpad_word MACRO echk, base2, sse4, xmmvalin, xmmcarry, xmmvalout, basereg
base2		base2_split_upper_zpad_word echk, sse4, xmmvalin, xmmcarry, xmmvalout, basereg
no base2	nobase2_split_upper_zpad_word echk, sse4, xmmvalin, xmmcarry, xmmvalout, basereg
	ENDM

base2_split_upper_zpad_word MACRO echk, sse4, xmmvalin, xmmcarry, xmmvalout, basereg
	xload	xmmcarry, XMM_BIGBIGVAL	;; Big word rounding constant
	addpd	xmmcarry, xmmvalin	;; Round to multiple of big word
	subpd	xmmcarry, XMM_BIGBIGVAL
sse4	roundpd	xmmvalout, xmmvalin, 0	;; Round input to an integer
no sse4	xload	xmmvalout, XMM_BIGVAL	;; Round input to an integer
no sse4	addpd	xmmvalout, xmmvalin
no sse4	subpd	xmmvalout, XMM_BIGVAL
echk	subpd	xmmvalin, xmmvalout	;; This is the convolution error
echk	andpd	xmmvalin, XMM_ABSVAL	;; Compute absolute value
echk	maxpd	xmm6, xmmvalin		;; Compute maximum error
	subpd	xmmvalout, xmmcarry	;; xmmvalout now contains low bigword bits
	ENDM

nobase2_split_upper_zpad_word MACRO echk, sse4, xmmvalin, xmmcarry, xmmvalout, basereg
	xload	xmmcarry, XMM_LIMIT_INVERSE[basereg] ;; Load base
	mulpd	xmmcarry, xmmvalin	;; Compute FFTvalue / base
sse4	roundpd	xmmcarry, xmmcarry, 0	;; Round to integer
no sse4	addpd	xmmcarry, XMM_BIGVAL	;; Round to integer
no sse4	subpd	xmmcarry, XMM_BIGVAL
sse4	roundpd	xmmvalout, xmmvalin, 0	;; Round input to an integer
no sse4	xload	xmmvalout, XMM_BIGVAL	;; Round input to an integer
no sse4	addpd	xmmvalout, xmmvalin
no sse4	subpd	xmmvalout, XMM_BIGVAL
echk	subpd	xmmvalin, xmmvalout	;; This is the convolution error
echk	andpd	xmmvalin, XMM_ABSVAL	;; Compute absolute value
echk	maxpd	xmm6, xmmvalin		;; Compute maximum error
	xload	xmmvalin, XMM_LIMIT_BIGMAX[basereg]
	mulpd	xmmvalin, xmmcarry
	subpd	xmmvalout, xmmvalin	;; xmmvalout now contains FFTvalue % base
	ENDM

;; Interleaved version used in 64-bit implementation of zpad normalization.
;; Relies on some registers being preloaded.

split_upper_zpad_word_interleaved MACRO echk, base2, sse4, xmmvalin, xmmcarry, xmmvalout, basereg, xmmvalin2, xmmcarry2, xmmvalout2, basereg2
base2		base2_split_upper_zpad_word_interleaved echk, sse4, xmmvalin, xmmcarry, xmmvalout, basereg, xmmvalin2, xmmcarry2, xmmvalout2, basereg2
no base2	nobase2_split_upper_zpad_word_interleaved echk, sse4, xmmvalin, xmmcarry, xmmvalout, basereg, xmmvalin2, xmmcarry2, xmmvalout2, basereg2
	ENDM

base2_split_upper_zpad_word_interleaved MACRO echk, sse4, xmmvalin, xmmcarry, xmmvalout, basereg, xmmvalin2, xmmcarry2, xmmvalout2, basereg2
	xcopy	xmmcarry, xmm13		;; XMM_BIGBIGVAL ;; Big word rounding constant
	addpd	xmmcarry, xmmvalin	;; Round to multiple of big word
	xcopy	xmmcarry2, xmm13	;; XMM_BIGBIGVAL ;; Big word rounding constant
	addpd	xmmcarry2, xmmvalin2	;; Round to multiple of big word
	subpd	xmmcarry, xmm13		;; XMM_BIGBIGVAL
	subpd	xmmcarry2, xmm13	;; XMM_BIGBIGVAL
sse4	roundpd	xmmvalout, xmmvalin, 0	;; Round input to an integer
sse4	roundpd	xmmvalout2, xmmvalin2, 0 ;; Round input to an integer

no sse4	echk	xload	xmmvalout, XMM_BIGVAL	;; Round input to an integer
no sse4	echk	addpd	xmmvalout, xmmvalin
no sse4	echk	xload	xmmvalout2, XMM_BIGVAL	;; Round input to an integer
no sse4	echk	addpd	xmmvalout2, xmmvalin2
no sse4	echk	subpd	xmmvalout, XMM_BIGVAL
no sse4	echk	subpd	xmmvalout2, XMM_BIGVAL

no sse4	no echk	xcopy	xmmvalout, xmm6		;; XMM_BIGVAL ;; Round input to an integer
no sse4	no echk	addpd	xmmvalout, xmmvalin
no sse4	no echk	xcopy	xmmvalout2, xmm6	;; XMM_BIGVAL ;; Round input to an integer
no sse4	no echk	addpd	xmmvalout2, xmmvalin2
no sse4	no echk	subpd	xmmvalout, xmm6		;; XMM_BIGVAL
no sse4	no echk	subpd	xmmvalout2, xmm6	;; XMM_BIGVAL

echk	subpd	xmmvalin, xmmvalout	;; This is the convolution error
echk	subpd	xmmvalin2, xmmvalout2	;; This is the convolution error
echk	andpd	xmmvalin, XMM_ABSVAL	;; Compute absolute value
echk	andpd	xmmvalin2, XMM_ABSVAL	;; Compute absolute value
echk	maxpd	xmm6, xmmvalin		;; Compute maximum error
echk	maxpd	xmm6, xmmvalin2		;; Compute maximum error
	subpd	xmmvalout, xmmcarry	;; xmmvalout now contains low bigword bits
	subpd	xmmvalout2, xmmcarry2	;; xmmvalout now contains low bigword bits
	ENDM

nobase2_split_upper_zpad_word_interleaved MACRO echk, sse4, xmmvalin, xmmcarry, xmmvalout, basereg, xmmvalin2, xmmcarry2, xmmvalout2, basereg2
	xload	xmmcarry, XMM_LIMIT_INVERSE[basereg] ;; Load base
	mulpd	xmmcarry, xmmvalin	;; Compute FFTvalue / base
	xload	xmmcarry2, XMM_LIMIT_INVERSE[basereg2] ;; Load base
	mulpd	xmmcarry2, xmmvalin2	;; Compute FFTvalue / base
sse4	roundpd	xmmcarry, xmmcarry, 0	;; Round to integer
sse4	roundpd	xmmcarry2, xmmcarry2, 0	;; Round to integer
no sse4	addpd	xmmcarry, xmm13		;; XMM_BIGVAL ;; Round to integer
no sse4	addpd	xmmcarry2, xmm13	;; XMM_BIGVAL ;; Round to integer
no sse4	subpd	xmmcarry, xmm13		;; XMM_BIGVAL
no sse4	subpd	xmmcarry2, xmm13	;; XMM_BIGVAL
sse4	roundpd	xmmvalout, xmmvalin, 0	;; Round input to an integer
sse4	roundpd	xmmvalout2, xmmvalin2, 0 ;; Round input to an integer
no sse4	xcopy	xmmvalout, xmm13	;; XMM_BIGVAL ;; Round input to an integer
no sse4	addpd	xmmvalout, xmmvalin
no sse4	xcopy	xmmvalout2, xmm13	;; XMM_BIGVAL ;; Round input to an integer
no sse4	addpd	xmmvalout2, xmmvalin2
no sse4	subpd	xmmvalout, xmm13	;; XMM_BIGVAL
no sse4	subpd	xmmvalout2, xmm13	;; XMM_BIGVAL
echk	subpd	xmmvalin, xmmvalout	;; This is the convolution error
echk	subpd	xmmvalin2, xmmvalout2	;; This is the convolution error
echk no sse4	andpd	xmmvalin, XMM_ABSVAL	;; Compute absolute value
echk no sse4	andpd	xmmvalin2, XMM_ABSVAL	;; Compute absolute value
echk sse4	andpd	xmmvalin, xmm13		;; XMM_ABSVAL ;; Compute absolute value
echk sse4	andpd	xmmvalin2, xmm13	;; XMM_ABSVAL ;; Compute absolute value
echk	maxpd	xmm6, xmmvalin		;; Compute maximum error
echk	maxpd	xmm6, xmmvalin2		;; Compute maximum error
	xload	xmmvalin, XMM_LIMIT_BIGMAX[basereg]
	mulpd	xmmvalin, xmmcarry
	xload	xmmvalin2, XMM_LIMIT_BIGMAX[basereg2]
	mulpd	xmmvalin2, xmmcarry2
	subpd	xmmvalout, xmmvalin	;; xmmvalout now contains FFTvalue % base
	subpd	xmmvalout2, xmmvalin2	;; xmmvalout now contains FFTvalue % base
	ENDM

; The single word version

single_split_lower_zpad_word MACRO base2, xmmval, xmmcarry, xmmtmp, basereg
base2		base2_single_split_lower_zpad_word xmmval, xmmcarry, xmmtmp, basereg
no base2	nobase2_single_split_lower_zpad_word xmmval, xmmcarry, xmmtmp, basereg
	ENDM

base2_single_split_lower_zpad_word MACRO xmmval, xmmcarry, xmmtmp, basereg
	addsd	xmmval, xmmcarry	;; Add in previous high FFT data
	movsd	xmmcarry, Q XMM_BIGBIGVAL ;; Big word rounding constant
	addsd	xmmcarry, xmmval	;; Round to multiple of big word
	subsd	xmmcarry, Q XMM_BIGBIGVAL
	addsd	xmmval, Q XMM_BIGVAL	;; Round to an integer
	subsd	xmmval, Q XMM_BIGVAL
	subsd	xmmval, xmmcarry	;; xmmval now contains low bigword bits
	mulsd	xmmcarry, Q XMM_LIMIT_INVERSE[basereg];; Next carry = shifted FFT hi data
	ENDM

nobase2_single_split_lower_zpad_word MACRO xmmval, xmmcarry, xmmtmp, basereg
	addsd	xmmval, xmmcarry	;; Add in previous high FFT data
	movsd	xmmcarry, Q XMM_LIMIT_INVERSE[basereg]
	mulsd	xmmcarry, xmmval	;; Compute FFTvalue / base
	addsd	xmmcarry, Q XMM_BIGVAL	;; Round to integer
	subsd	xmmcarry, Q XMM_BIGVAL	;; Next carry = round ( FFTvalue / base )
	movsd	xmmtmp, Q XMM_LIMIT_BIGMAX[basereg]
	mulsd	xmmtmp, xmmcarry
	addsd	xmmval, Q XMM_BIGVAL	;; Round input to an integer
	subsd	xmmval, Q XMM_BIGVAL
	subsd	xmmval, xmmtmp		;; xmmval now contains FFTvalue % base
	ENDM

; The version for splitting the high FFT carry.  The high carry input has already
; been rounded to an integer.

split_carry_zpad_word MACRO base2, xmmcarryin, xmmcarryout, xmmtmp, basereg
base2		base2_split_carry_zpad_word xmmcarryin, xmmcarryout, xmmtmp, basereg
no base2	nobase2_split_carry_zpad_word xmmcarryin, xmmcarryout, xmmtmp, basereg
	ENDM

base2_split_carry_zpad_word MACRO xmmcarryin, xmmcarryout, xmmtmp, basereg
	xload	xmmcarryout, XMM_BIGBIGVAL ;; Big word rounding constant
	addpd	xmmcarryout, xmmcarryin	;; Round to multiple of big word
	subpd	xmmcarryout, XMM_BIGBIGVAL
	subpd	xmmcarryin, xmmcarryout	;; xmmcarryin now contains low bigword bits
	mulpd	xmmcarryout, XMM_LIMIT_INVERSE[basereg];; Next carry = shifted carry
	ENDM

nobase2_split_carry_zpad_word MACRO xmmcarryin, xmmcarryout, xmmtmp, basereg
	xload	xmmcarryout, XMM_LIMIT_INVERSE[basereg] ;; Load 1 / base
	mulpd	xmmcarryout, xmmcarryin	;; Compute carry / base
	addpd	xmmcarryout, XMM_BIGVAL	;; Round to integer
	subpd	xmmcarryout, XMM_BIGVAL	;; Next carry = round(carry / base)
	xload	xmmtmp, XMM_LIMIT_BIGMAX[basereg]
	mulpd	xmmtmp, xmmcarryout
	subpd	xmmcarryin, xmmtmp	;; xmmcarryin now contains carry % base
	ENDM

; The single float version for splitting the high FFT carry.

single_split_carry_zpad_word MACRO base2, xmmcarryin, xmmcarryout, xmmtmp, basereg
base2		base2_single_split_carry_zpad_word xmmcarryin, xmmcarryout, xmmtmp, basereg
no base2	nobase2_single_split_carry_zpad_word xmmcarryin, xmmcarryout, xmmtmp, basereg
	ENDM

base2_single_split_carry_zpad_word MACRO xmmcarryin, xmmcarryout, xmmtmp, basereg
	movlpd	xmmcarryout, Q XMM_BIGBIGVAL	;; Big word rounding constant
	addsd	xmmcarryout, xmmcarryin		;; Round to multiple of big word
	subsd	xmmcarryout, Q XMM_BIGBIGVAL
	subsd	xmmcarryin, xmmcarryout		;; xmmcarryin now contains low bigword bits
	mulsd	xmmcarryout, Q XMM_LIMIT_INVERSE[basereg];; Next carry = shifted carry
	ENDM

nobase2_single_split_carry_zpad_word MACRO xmmcarryin, xmmcarryout, xmmtmp, basereg
	movlpd	xmmcarryout, Q XMM_LIMIT_INVERSE[basereg] ;; Load 1 / base
	mulsd	xmmcarryout, xmmcarryin		;; Compute carry / base
	addsd	xmmcarryout, Q XMM_BIGVAL	;; Round to integer
	subsd	xmmcarryout, Q XMM_BIGVAL	;; Next carry = round(carry / base)
	movlpd	xmmtmp, Q XMM_LIMIT_BIGMAX[basereg]
	mulsd	xmmtmp, xmmcarryout
	subsd	xmmcarryin, xmmtmp		;; xmmcarryin now contains carry % base
	ENDM

; Round the ZPAD0 - ZPAD6 values.  Simpler than other rounding macros
; in that we always round to a big word (and input value and output
; carry do not have XMM_BIGVAL added in).

round_zpad7_word MACRO base2, xmmvalin, xmmcarry, xmmvalout, basereg
base2		base2_round_zpad7_word xmmvalin, xmmcarry, xmmvalout, basereg
no base2	nobase2_round_zpad7_word xmmvalin, xmmcarry, xmmvalout, basereg
	ENDM

base2_round_zpad7_word MACRO xmmval, xmmcarry, xmmtmp, basereg
	addsd	xmmval, xmmcarry			;; Add in high part of last calculation
	movsd	xmmcarry, Q XMM_BIGBIGVAL		;; Big word rounding constant
	addsd	xmmcarry, xmmval			;; Round to multiple of big word
	subsd	xmmcarry, Q XMM_BIGBIGVAL
	subsd	xmmval, xmmcarry			;; xmmval now contains low bigword bits
	mulsd	xmmcarry, Q XMM_LIMIT_INVERSE[basereg]	;; Shift high ZPAD data
	ENDM

nobase2_round_zpad7_word MACRO xmmval, xmmcarry, xmmtmp, basereg
	addsd	xmmval, xmmcarry			;; Add in high part of last calculation
	movsd	xmmcarry, Q XMM_LIMIT_INVERSE[basereg]	;; Load base inverse
	mulsd	xmmcarry, xmmval			;; val / base
	addsd	xmmcarry, Q XMM_BIGVAL			;; next carry = round (val / base)
	subsd	xmmcarry, Q XMM_BIGVAL
	movsd	xmmtmp, Q XMM_LIMIT_BIGMAX[basereg]	;; Load base
	mulsd	xmmtmp, xmmcarry			;; round (val / base)
	subsd	xmmval, xmmtmp				;; new value = val - z
	ENDM


;
; Now for the actual normalization macros!
;


; For 1D macros, these registers are set on input:
; xmm6 = MAXERR
; xmm3 = carry #2
; xmm2 = carry #1
; rsi = pointer to the FFT data values
; rbp = pointer two-to-phi multipliers
; rdi = pointer to array of big vs. little flags
; ecx = big vs. little word flag #2
; eax = big vs. little word flag #1


; *************** 1D macro ******************
; A pipelined version of this code:
;	movzx	rax, BYTE PTR [rdi]	;; Load big vs. little flags
;	xload	xmm0, [rsi+0*dist1]	;; Load values
;	mulpd	xmm0, [rbp+0]		;; Mul values1 by two-to-minus-phi
;	addpd	xmm0, xmm4		;; x = values + carry
;	xload	xmm2, XMM_LIMIT_BIGMAX[rax];; Load maximum * BIGVAL - BIGVAL
;	addpd	xmm2, xmm0		;; y = top bits of x
;	xload	xmm6, XMM_LIMIT_BIGMAX_NEG[rax];; Load -(maximum*BIGVAL-BIGVAL)
;	addpd	xmm6, xmm2		;; z = y - (maximum * BIGVAL - BIGVAL)
;	subpd	xmm0, xmm6		;; rounded value = x - z
;	mulpd	xmm2, XMM_LIMIT_INVERSE[rax];; next carry = shifted y
;	mulpd	xmm0, [rbp+16]		;; new value = val * two-to-phi
;	xstore	[rsi+0*dist1], xmm0	;; Save new value

xnorm_1d MACRO ttp, echk, const, base2, sse4
ttp	movzx	rax, BYTE PTR [rdi]	;; Load big vs. little flags
ttp	movzx	rcx, BYTE PTR [rdi+2]
	xload	xmm0, [rsi]		;; Load values1
	unpcklo xmm0, [rsi+16]
	mulpd	xmm0, [rbp]		;; Mul values1 by two-to-minus-phi
	xload	xmm1, [rsi+32]		;; Load values2
	unpcklo xmm1, [rsi+48]
	mulpd	xmm1, [rbp+64]		;; Mul values2 by two-to-minus-phi
no const echk error_check_interleaved sse4, xmm0, xmm4, xmm1, xmm5, xmm6
const	mul_by_const ttp, base2, echk, sse4, xmm0, xmm4, xmm5, rax, xmm6
	addpd	xmm2, xmm0		;; x1 = values + carry
const	mul_by_const ttp, base2, echk, sse4, xmm1, xmm5, xmm0, rcx, xmm6
	addpd	xmm3, xmm1		;; x2 = values + carry
	rounding_interleaved ttp, base2, const, sse4, xmm2, xmm4, xmm0, rax, xmm3, xmm5, xmm1, rcx
ttp	mulpd	xmm2, [rbp+16]		;; new value1 = val * two-to-phi
ttp	mulpd	xmm3, [rbp+80]		;; new value2 = val * two-to-phi
ttp	movzx	rax, BYTE PTR [rdi+1]	;; Load big vs. little flags
ttp	movzx	rcx, BYTE PTR [rdi+3]
	xload	xmm0, [rsi]		;; Load values1
	unpckhi xmm0, [rsi+16]
	mulpd	xmm0, [rbp+32]		;; Mul values1 by two-to-minus-phi
	xstore	[rsi], xmm2		;; Save previous value1
	xload	xmm1, [rsi+32]		;; Load values2
	unpckhi xmm1, [rsi+48]
	mulpd	xmm1, [rbp+96]		;; Mul values2 by two-to-minus-phi
	xstore	[rsi+32], xmm3		;; Save previous value2
no const echk error_check_interleaved sse4, xmm0, xmm2, xmm1, xmm3, xmm6
const	mul_by_const ttp, base2, echk, sse4, xmm0, xmm2, xmm3, rax, xmm6
	addpd	xmm0, xmm4		;; x1 = values + carry
const	mul_by_const ttp, base2, echk, sse4, xmm1, xmm3, xmm4, rcx, xmm6
	addpd	xmm1, xmm5		;; x2 = values + carry
	rounding_interleaved ttp, base2, const, sse4, xmm0, xmm2, xmm4, rax, xmm1, xmm3, xmm5, rcx
ttp	mulpd	xmm0, [rbp+48]		;; new value1 = val * two-to-phi
ttp	mulpd	xmm1, [rbp+112]		;; new value2 = val * two-to-phi
	xstore	[rsi+16], xmm0		;; Save new value1
	xstore	[rsi+48], xmm1		;; Save new value2
	ENDM


; This is the normalization routine when we are computing modulo k*b^n+c
; with a zero-padded b^2n FFT.  We do this by multiplying the lower FFT
; word by k and adding in the upper word times -c.  Of course, this is made
; very tedious because we have to carefully avoid any loss of precision.
;
; xmm6 = MAXERR
; xmm3 = carry #2 (previous high FFT data - not yet mul'ed by K)
; xmm2 = carry #1 (traditional carry)
; rsi = pointer to the FFT data values
; rbp = pointer two-to-phi multipliers
; rdi = pointer to array of big vs. little flags
; eax = big vs. little word flag #1

xnorm_1d_zpad MACRO ttp, echk, const, base2, sse4, khi, c1, cm1
ttp	movzx	rax, BYTE PTR [rdi]	;; Load big vs. little flags
	xload	xmm0, [rsi]		;; Load values1
	unpcklo xmm0, [rsi+16]
	xload	xmm1, [rsi+32]		;; Load values2
	unpcklo xmm1, [rsi+48]
	mulpd	xmm0, [rbp]		;; Mul values1 by two-to-minus-phi
	mulpd	xmm1, [rbp]		;; Mul values2 by two-to-minus-phi

	split_lower_zpad_word echk, base2, sse4, xmm0, xmm3, xmm4, rax

no const	xload	xmm0, XMM_K_LO
const		xload	xmm0, XMM_K_TIMES_MULCONST_LO
		mulpd	xmm0, xmm4
khi no const	xload	xmm5, XMM_K_HI
khi const	xload	xmm5, XMM_K_TIMES_MULCONST_HI
khi no base2	mulpd	xmm5, XMM_LIMIT_INVERSE[rax] ;; Non-base2 rounding needs shifted carry
khi		mulpd	xmm5, xmm4

		addpd	xmm0, xmm2		;; x1 = values + carry

c1		mulpd   xmm1, XMM_MINUS_C	;; Do one mul before split rather than two after split

	split_upper_zpad_word echk, base2, sse4, xmm1, xmm4, xmm2, rax

no const no c1 no cm1	mulpd	xmm2, XMM_MINUS_C
no const no c1 no cm1	mulpd	xmm4, XMM_MINUS_C
const			mulpd	xmm2, XMM_MINUS_C_TIMES_MULCONST
const			mulpd	xmm4, XMM_MINUS_C_TIMES_MULCONST

	addpd	xmm0, xmm2		;; Add upper FFT word to lower FFT word
khi	addpd	xmm4, xmm5		;; Add upper FFT word to lower FFT word

 	rounding ttp, base2, exec, sse4, xmm0, xmm4, xmm2, rax

ttp	mulpd	xmm0, [rbp+16]		;; new value1 = val * two-to-phi

ttp	movzx	rax, BYTE PTR [rdi+1]	;; Load big vs. little flags
	xload	xmm5, [rsi]		;; Load values1
	unpckhi xmm5, [rsi+16]
	xstore	[rsi], xmm0		;; Save previous value1
	xload	xmm1, [rsi+32]		;; Load values2
	unpckhi xmm1, [rsi+48]
	mulpd	xmm5, [rbp+32]		;; Mul values1 by two-to-minus-phi
	mulpd	xmm1, [rbp+32]		;; Mul values2 by two-to-minus-phi

	split_lower_zpad_word echk, base2, sse4, xmm5, xmm3, xmm2, rax

no const	xload	xmm0, XMM_K_LO
const		xload	xmm0, XMM_K_TIMES_MULCONST_LO
		mulpd	xmm0, xmm2
khi no const	xload	xmm5, XMM_K_HI
khi const	xload	xmm5, XMM_K_TIMES_MULCONST_HI
khi no base2	mulpd	xmm5, XMM_LIMIT_INVERSE[rax] ; Non-base2 rounding needs shifted carry
khi		mulpd	xmm5, xmm2

		addpd	xmm0, xmm4		;; x1 = values + carry

c1		mulpd   xmm1, XMM_MINUS_C	;; Do one mul before split rather than two after split

	split_upper_zpad_word echk, base2, sse4, xmm1, xmm2, xmm4, rax

no const no c1 no cm1	mulpd	xmm4, XMM_MINUS_C
no const no c1 no cm1	mulpd	xmm2, XMM_MINUS_C
const			mulpd	xmm4, XMM_MINUS_C_TIMES_MULCONST
const			mulpd	xmm2, XMM_MINUS_C_TIMES_MULCONST

	addpd	xmm0, xmm4		;; Add upper FFT word to lower FFT word
khi	addpd	xmm2, xmm5		;; Add upper FFT word to lower FFT word

	rounding ttp, base2, exec, sse4, xmm0, xmm2, xmm4, rax

ttp	mulpd	xmm0, [rbp+48]		;; new value1 = val * two-to-phi

	xstore	[rsi+16], xmm0		;; Save new value1
	subpd	xmm1, xmm1		;; new value2 = zero
	xstore	[rsi+32], xmm1		;; Zero previous value2
	xstore	[rsi+48], xmm1		;; Zero current value2
	ENDM


; *************** 1D followup macros ******************
; This macro finishes the normalize process by adding the final
; carry from the first pass back into the lower two data values.
; xmm2,xmm3 = carries
; rsi = pointer to the FFT data values
; rbp = pointer two-to-phi multipliers
; rdi = big vs. little array pointer
; These registers are destroyed!

xnorm012_1d_mid MACRO ttp, base2
	LOCAL	not_80_or_112, only_do_4

	;; WARNING: Carry propagation into the 5th and 6th words does not work for
	;; FFT lengths 80 and 112.  gwnum.c makes sure that 5th carry will be
	;; zero for these FFT lengths.  The hack below adjusts the input pointers
	;; so that we only propagate 4 carry words
	test	FFTLEN, 64		;; Length 80 and 112 have different
	jz	not_80_or_112		;; memory addresses for the 5th and 6th words
	test	FFTLEN, 16
	jz	not_80_or_112

ttp	bump	rdi, -4			;; Modify input pointers so we
	bump	rsi, -64		;; propagate into only 4 words
ttp	bump	rbp, -128
	jmp	only_do_4

not_80_or_112:
ttp	movzx	rax, BYTE PTR [rdi]	;; Load big vs. little flags
ttp	movzx	rcx, BYTE PTR [rdi+2]
	movsd	xmm0, Q [rsi+8]		;; Load values1
ttp	mulsd	xmm0, Q [rbp+8]		;; Mul values1 by two-to-minus-phi
ttp	mulsd	xmm0, Q XMM_NORM012_FF	;; Mul by FFTLEN/2
	movsd	xmm1, Q [rsi+40]	;; Load values2
ttp	mulsd	xmm1, Q [rbp+72]	;; Mul values2 by two-to-minus-phi
ttp	mulsd	xmm1, Q XMM_NORM012_FF	;; Mul by FFTLEN/2
	addsd	xmm0, xmm2		;; x1 = values + carry
	addsd	xmm1, xmm3		;; x2 = values + carry
	single_rounding_interleaved base2, xmm0, xmm2, xmm4, rax+8, xmm1, xmm3, xmm5, rcx+8
ttp	mulsd	xmm0, Q [rbp+24]	;; new value1 = val * two-to-phi
ttp	mulsd	xmm1, Q [rbp+88]	;; new value2 = val * two-to-phi
	movsd	Q [rsi+8], xmm0		;; Save new value1
	movsd	Q [rsi+40], xmm1	;; Save new value2

ttp	movzx	rax, BYTE PTR [rdi+1]	;; Load big vs. little flags
ttp	movzx	rcx, BYTE PTR [rdi+3]
	movsd	xmm0, Q [rsi+24]	;; Load values1
ttp	mulsd	xmm0, Q [rbp+40]	;; Mul values1 by two-to-minus-phi
ttp	mulsd	xmm0, Q XMM_NORM012_FF	;; Mul by FFTLEN/2
	movsd	xmm1, Q [rsi+56]	;; Load values2
ttp	mulsd	xmm1, Q [rbp+104]	;; Mul values2 by two-to-minus-phi
ttp	mulsd	xmm1, Q XMM_NORM012_FF	;; Mul by FFTLEN/2
	addsd	xmm0, xmm2		;; x1 = values + carry
	addsd	xmm1, xmm3		;; x2 = values + carry
	single_rounding_interleaved base2, xmm0, xmm2, xmm4, rax+8, xmm1, xmm3, xmm5, rcx+8
ttp	mulsd	xmm0, Q [rbp+56]	;; new value1 = val * two-to-phi
ttp	mulsd	xmm1, Q [rbp+120]	;; new value2 = val * two-to-phi
	movsd	Q [rsi+24], xmm0	;; Save new value1
	movsd	Q [rsi+56], xmm1	;; Save new value2

only_do_4:
ttp	movzx	rax, BYTE PTR [rdi+4]	;; Load big vs. little flags
ttp	movzx	rcx, BYTE PTR [rdi+6]
	movsd	xmm0, Q [rsi+64+8]	;; Load values1
ttp	mulsd	xmm0, Q [rbp+128+8]	;; Mul values1 by two-to-minus-phi
ttp	mulsd	xmm0, Q XMM_NORM012_FF	;; Mul by FFTLEN/2
	movsd	xmm1, Q [rsi+64+40]	;; Load values2
ttp	mulsd	xmm1, Q [rbp+128+72]	;; Mul values2 by two-to-minus-phi
ttp	mulsd	xmm1, Q XMM_NORM012_FF	;; Mul by FFTLEN/2
	addsd	xmm0, xmm2		;; x1 = values + carry
	addsd	xmm1, xmm3		;; x2 = values + carry
	single_rounding_interleaved base2, xmm0, xmm2, xmm4, rax+8, xmm1, xmm3, xmm5, rcx+8
ttp	mulsd	xmm0, Q [rbp+128+24]	;; new value1 = val * two-to-phi
ttp	mulsd	xmm1, Q [rbp+128+88]	;; new value2 = val * two-to-phi
	movsd	Q [rsi+64+8], xmm0	;; Save new value1
	movsd	Q [rsi+64+40], xmm1	;; Save new value2

ttp	movzx	rax, BYTE PTR [rdi+5]	;; Load big vs. little flags
ttp	movzx	rcx, BYTE PTR [rdi+7]
	movsd	xmm0, Q [rsi+64+24]	;; Load values1
ttp	mulsd	xmm0, Q [rbp+128+40]	;; Mul values1 by two-to-minus-phi
ttp	mulsd	xmm0, Q XMM_NORM012_FF	;; Mul by FFTLEN/2
	movsd	xmm1, Q [rsi+64+56]	;; Load values2
ttp	mulsd	xmm1, Q [rbp+128+104]	;; Mul values2 by two-to-minus-phi
ttp	mulsd	xmm1, Q XMM_NORM012_FF	;; Mul by FFTLEN/2
	addsd	xmm0, xmm2		;; x1 = values + carry
	addsd	xmm1, xmm3		;; x2 = values + carry
	single_rounding_interleaved base2, xmm0, xmm2, xmm4, rax+8, xmm1, xmm3, xmm5, rcx+8
ttp	mulsd	xmm0, Q [rbp+128+56]	;; new value1 = val * two-to-phi
ttp	mulsd	xmm1, Q [rbp+128+120]	;; new value2 = val * two-to-phi
	movsd	Q [rsi+64+24], xmm0	;; Save new value1
	movsd	Q [rsi+64+56], xmm1	;; Save new value2

ttp	movzx	rax, BYTE PTR [rdi+8]	;; Load big vs. little flags
ttp	movzx	rcx, BYTE PTR [rdi+10]
	movsd	xmm0, Q [rsi+128+8]	;; Load values1
ttp	mulsd	xmm0, Q [rbp+256+8]	;; Mul values1 by two-to-minus-phi
ttp	mulsd	xmm0, Q XMM_NORM012_FF	;; Mul by FFTLEN/2
	movsd	xmm1, Q [rsi+128+40]	;; Load values2
ttp	mulsd	xmm1, Q [rbp+256+72]	;; Mul values2 by two-to-minus-phi
ttp	mulsd	xmm1, Q XMM_NORM012_FF	;; Mul by FFTLEN/2
	addsd	xmm0, xmm2		;; x1 = values + carry
	addsd	xmm1, xmm3		;; x2 = values + carry
	single_rounding_interleaved base2, xmm0, xmm2, xmm4, rax+8, xmm1, xmm3, xmm5, rcx+8
ttp	mulsd	xmm0, Q [rbp+256+24]	;; new value1 = val * two-to-phi
ttp	mulsd	xmm1, Q [rbp+256+88]	;; new value2 = val * two-to-phi
	movsd	Q [rsi+128+8], xmm0	;; Save new value1
	movsd	Q [rsi+128+40], xmm1	;; Save new value2

	subsd	xmm2, Q XMM_BIGVAL	;; Remove integer rounding constant
	subsd	xmm3, Q XMM_BIGVAL	;; Remove integer rounding constant
ttp	mulsd	xmm2, Q [rbp+256+56]	;; carry *= two-to-phi
ttp	mulsd	xmm3, Q [rbp+256+120]	;; carry *= two-to-phi
	addsd	xmm2, Q [rsi+128+24]	;; value1 = values + carry
	addsd	xmm3, Q [rsi+128+56]	;; value2 = values + carry
	movsd	Q [rsi+128+24], xmm2	;; Save new value1
	movsd	Q [rsi+128+56], xmm3	;; Save new value2

	shufpd	xmm2, xmm2, 1		;; Rotate carry
	movhpd	xmm2, Q XMM_BIGVAL
	shufpd	xmm3, xmm3, 1		;; Rotate carry
	movhpd	xmm3, Q XMM_BIGVAL
	ENDM

; This macro is similar to the above, but is for the zero padding case.
; xmm2 = carry #1 (traditional carry)
; xmm3 = carry #2 (previous high FFT data - not yet mul'ed by K)
; rbx = pointer to the FFT data values
; rdx = pointer two-to-phi multipliers
; rdi = big vs. little array pointer

xnorm012_1d_mid_zpad MACRO const, base2
	LOCAL	notfunny, funnyaddr1, funny1done

	movzx	rax, BYTE PTR [rdi]	;; Load big vs. little flags
	movsd	xmm0, Q [rbx+8]		;; Load values1
	mulsd	xmm0, Q [rdx+8]		;; Mul values1 by two-to-minus-phi
	mulsd	xmm0, Q XMM_NORM012_FF	;; Mul by FFTLEN/2
	addsd	xmm0, xmm2		;; x1 = values1 + carry
	single_split_carry_zpad_word base2, xmm3, xmm1, xmm2, rax+8
no const movlpd	xmm2, Q XMM_K_LO	;; Calc high FFT carry times k
const	movlpd	xmm2, Q XMM_K_TIMES_MULCONST_LO
	mulsd	xmm2, xmm3		;; high_FFT_carry * k_lo
	addsd	xmm0, xmm2		;; x1 = x1 + high_FFT_carry * k_lo
no const movsd	xmm4, Q XMM_K_HI
const	movsd	xmm4, Q XMM_K_TIMES_MULCONST_HI
	mulsd	xmm4, Q XMM_LIMIT_INVERSE[rax+8] ;; shift k_hi
	mulsd	xmm3, xmm4
	single_rounding base2, xmm0, xmm2, xmm4, rax+8
	addsd	xmm2, xmm3		;; Carry += high_FFT_carry * shifted k_hi
	mulsd	xmm0, Q [rdx+24]	;; new value1 = val * two-to-phi
	movsd	Q [rbx+8], xmm0		;; Save new value1

	movzx	rax, BYTE PTR [rdi+1]	;; Load big vs. little flags
	movsd	xmm0, Q [rbx+24]	;; Load values2
	mulsd	xmm0, Q [rdx+40]	;; Mul values2 by two-to-minus-phi
	mulsd	xmm0, Q XMM_NORM012_FF	;; Mul by FFTLEN/2
	addsd	xmm0, xmm2		;; x2 = values + carry
	single_split_carry_zpad_word base2, xmm1, xmm3, xmm2, rax+8
no const movlpd	xmm2, Q XMM_K_LO	;; Calc high FFT carry times k
const	movlpd	xmm2, Q XMM_K_TIMES_MULCONST_LO
	mulsd	xmm2, xmm1		;; high_FFT_carry * k_lo
	addsd	xmm0, xmm2		;; x2 = x2 + high_FFT_carry * k_lo
no const movsd	xmm4, Q XMM_K_HI
const	movsd	xmm4, Q XMM_K_TIMES_MULCONST_HI
	mulsd	xmm4, Q XMM_LIMIT_INVERSE[rax+8] ;; shift k_hi
	mulsd	xmm1, xmm4
	single_rounding base2, xmm0, xmm2, xmm4, rax+8
	addsd	xmm2, xmm1		;; Carry += high_FFT_carry * shifted k_hi
	mulsd	xmm0, Q [rdx+56]	;; new value2 = val * two-to-phi
	movsd	Q [rbx+24], xmm0	;; Save new value2

	movzx	rax, BYTE PTR [rdi+4]	;; Load big vs. little flags
	movsd	xmm0, Q [rbx+64+8]	;; Load values3
	mulsd	xmm0, Q [rdx+128+8]	;; Mul values3 by two-to-minus-phi
	mulsd	xmm0, Q XMM_NORM012_FF	;; Mul by FFTLEN/2
	addsd	xmm0, xmm2		;; x3 = values + carry
	single_split_carry_zpad_word base2, xmm3, xmm1, xmm2, rax+8
no const movlpd	xmm2, Q XMM_K_LO	;; Calc high FFT carry times k
const	movlpd	xmm2, Q XMM_K_TIMES_MULCONST_LO
	mulsd	xmm2, xmm3		;; high_FFT_carry * k_lo
	addsd	xmm0, xmm2		;; x3 = x3 + high_FFT_carry * k_lo
no const movsd	xmm4, Q XMM_K_HI
const	movsd	xmm4, Q XMM_K_TIMES_MULCONST_HI
	mulsd	xmm4, Q XMM_LIMIT_INVERSE[rax+8] ;; shift k_hi
	mulsd	xmm3, xmm4
	single_rounding base2, xmm0, xmm2, xmm4, rax+8
	addsd	xmm2, xmm3		;; Carry += high_FFT_carry * shifted k_hi
	mulsd	xmm0, Q [rdx+128+24]	;; new value3 = val * two-to-phi
	movsd	Q [rbx+64+8], xmm0	;; Save new value3

	movzx	rax, BYTE PTR [rdi+5]	;; Load big vs. little flags
	movsd	xmm0, Q [rbx+64+24]	;; Load values4
	mulsd	xmm0, Q [rdx+128+40]	;; Mul values4 by two-to-minus-phi
	mulsd	xmm0, Q XMM_NORM012_FF	;; Mul by FFTLEN/2
	addsd	xmm0, xmm2		;; x4 = values + carry
	single_split_carry_zpad_word base2, xmm1, xmm3, xmm2, rax+8
no const movlpd	xmm2, Q XMM_K_LO	;; Calc high FFT carry times k
const	movlpd	xmm2, Q XMM_K_TIMES_MULCONST_LO
	mulsd	xmm2, xmm1		;; high_FFT_carry * k_lo
	addsd	xmm0, xmm2		;; x4 = x4 + high_FFT_carry * k_lo
no const movsd	xmm4, Q XMM_K_HI
const	movsd	xmm4, Q XMM_K_TIMES_MULCONST_HI
	mulsd	xmm4, Q XMM_LIMIT_INVERSE[rax+8] ;; shift k_hi
	mulsd	xmm1, xmm4
	single_rounding base2, xmm0, xmm2, xmm4, rax+8
	addsd	xmm2, xmm1		;; Carry += high_FFT_carry * shifted k_hi
	mulsd	xmm0, Q [rdx+128+56]	;; new value4 = val * two-to-phi
	movsd	Q [rbx+64+24], xmm0	;; Save new value4

  	cmp	rbx, DESTARG		;; Only the first section is funny
	jne	short notfunny
	cmp	FFTLEN, 80		;; Length 80 and 112 have different
	je	funnyaddr1		;; memory addresses for the fourth
	cmp	FFTLEN, 112		;; and higher data elements
	je	funnyaddr1

notfunny:
	movzx	rax, BYTE PTR [rdi+8]	;; Load big vs. little flags
	movsd	xmm0, Q [rbx+128+8]	;; Load values5
	mulsd	xmm0, Q [rdx+256+8]	;; Mul values1 by two-to-minus-phi
	mulsd	xmm0, Q XMM_NORM012_FF	;; Mul by FFTLEN/2
	addsd	xmm0, xmm2		;; x5 = values + carry
no const movlpd	xmm2, Q XMM_K_LO	;; Calc high FFT carry times k
const	movlpd	xmm2, Q XMM_K_TIMES_MULCONST_LO
	mulsd	xmm2, xmm3		;; high_FFT_carry * k_lo
	addsd	xmm0, xmm2		;; x5 = x5 + high_FFT_carry * k_lo
no const movsd	xmm4, Q XMM_K_HI
const	movsd	xmm4, Q XMM_K_TIMES_MULCONST_HI
	mulsd	xmm4, Q XMM_LIMIT_INVERSE[rax+8] ;; shift k_hi
	mulsd	xmm3, xmm4
	single_rounding base2, xmm0, xmm2, xmm4, rax+8
	addsd	xmm2, xmm3		;; Carry += high_FFT_carry * shifted k_hi
	mulsd	xmm0, Q [rdx+256+24]	;; new value5 = val * two-to-phi
	movsd	Q [rbx+128+8], xmm0	;; Save new value5

	subsd	xmm2, Q XMM_BIGVAL	;; Remove integer rounding constant
	mulsd	xmm2, Q [rdx+256+56]	;; carry *= two-to-phi
	addsd	xmm2, Q [rbx+128+24]	;; value6 = values + carry
	movsd	Q [rbx+128+24], xmm2	;; Save new value6
	jmp	funny1done

funnyaddr1:				;; FFT length = 80 or 112
	movsd	xmm0, xmm2		;; Add the carry to the next section's carry
	subsd	xmm0, Q XMM_BIGVAL
	shufpd	xmm2, xmm2, 1
	addsd	xmm2, xmm0
	shufpd	xmm2, xmm2, 1
	movsd	xmm0, xmm3		;; Add the carry to the next section's carry
	shufpd	xmm3, xmm3, 1
	addsd	xmm3, xmm0
	shufpd	xmm3, xmm3, 1

funny1done:
	shufpd	xmm2, xmm2, 1		;; Rotate carry
	movhpd	xmm2, Q XMM_BIGVAL
	subsd	xmm3, xmm3
	shufpd	xmm3, xmm3, 1		;; Rotate carry
	ENDM


; We could take advantage of the fact that the first two-to-phi multiplier
; and the first two-to-minus-phi multiplier are one.  We also know
; the first data value is a big word (eax would be 48).
; xmm2,xmm3 = carries
; rsi = pointer to the FFT data values
; rbp = pointer two-to-phi multipliers
; rdi = big vs. little array pointer
; NOTE: If RATIONAL_FFT we could eliminate 8 multiplies.
; Input arguments are destroyed!

xnorm012_1d MACRO base2
	LOCAL	not_80_or_112, only_do_4

	subsd	xmm3, Q XMM_BIGVAL
	mulsd	xmm3, Q XMM_MINUS_C	;; Adjust wrap-around carry
	addsd	xmm3, Q XMM_BIGVAL

	;; WARNING: Carry propagation into the 5th and 6th words does not work for
	;; FFT lengths 80 and 112.  gwnum.c makes sure that 5th carry will be
	;; zero for these FFT lengths.  The hack below adjusts the input pointers
	;; so that we only propagate 4 carry words
	test	FFTLEN, 64		;; Length 80 and 112 have different
	jz	not_80_or_112		;; memory addresses for the 5th and 6th words
	test	FFTLEN, 16
	jz	not_80_or_112

	bump	rdi, -4			;; Modify input pointers so we
	bump	rsi, -64		;; propagate into only 4 words
	bump	rbp, -128
	jmp	only_do_4

not_80_or_112:
	movzx	rax, BYTE PTR [rdi]	;; Load big vs. little flags
	movzx	rcx, BYTE PTR [rdi+2]
	movsd	xmm0, Q [rsi]		;; Load values1
	mulsd	xmm0, Q [rbp]		;; Mul values1 by two-to-minus-phi
	mulsd	xmm0, Q XMM_NORM012_FF	;; Mul by FFTLEN/2
	movsd	xmm1, Q [rsi+32]	;; Load values2
	mulsd	xmm1, Q [rbp+64]	;; Mul values2 by two-to-minus-phi
	mulsd	xmm1, Q XMM_NORM012_FF	;; Mul by FFTLEN/2
	addsd	xmm0, xmm3		;; x1 = values + carry
	addsd	xmm1, xmm2		;; x2 = values + carry
	single_rounding_interleaved base2, xmm0, xmm3, xmm4, rax, xmm1, xmm2, xmm5, rcx
	mulsd	xmm0, Q [rbp+16]	;; new value1 = val * two-to-phi
	mulsd	xmm1, Q [rbp+80]	;; new value2 = val * two-to-phi
	movsd	Q [rsi], xmm0		;; Save value1
	movsd	Q [rsi+32], xmm1	;; Save value2

	movzx	rax, BYTE PTR [rdi+1]	;; Load big vs. little flags
	movzx	rcx, BYTE PTR [rdi+3]
	movsd	xmm0, Q [rsi+16]	;; Load values1
	mulsd	xmm0, Q [rbp+32]	;; Mul values1 by two-to-minus-phi
	mulsd	xmm0, Q XMM_NORM012_FF	;; Mul by FFTLEN/2
	movsd	xmm1, Q [rsi+48]	;; Load values2
	mulsd	xmm1, Q [rbp+96]	;; Mul values2 by two-to-minus-phi
	mulsd	xmm1, Q XMM_NORM012_FF	;; Mul by FFTLEN/2
	addsd	xmm0, xmm3		;; x1 = values + carry
	addsd	xmm1, xmm2		;; x2 = values + carry
	single_rounding_interleaved base2, xmm0, xmm3, xmm4, rax, xmm1, xmm2, xmm5, rcx
	mulsd	xmm0, Q [rbp+48]	;; new value1 = val * two-to-phi
	mulsd	xmm1, Q [rbp+112]	;; new value2 = val * two-to-phi
	movsd	Q [rsi+16], xmm0	;; Save value1
	movsd	Q [rsi+48], xmm1	;; Save value2

only_do_4:
	movzx	rax, BYTE PTR [rdi+4]	;; Load big vs. little flags
	movzx	rcx, BYTE PTR [rdi+6]
	movsd	xmm0, Q [rsi+64]	;; Load values1
	mulsd	xmm0, Q [rbp+128]	;; Mul values1 by two-to-minus-phi
	mulsd	xmm0, Q XMM_NORM012_FF	;; Mul by FFTLEN/2
	movsd	xmm1, Q [rsi+64+32]	;; Load values2
	mulsd	xmm1, Q [rbp+128+64]	;; Mul values2 by two-to-minus-phi
	mulsd	xmm1, Q XMM_NORM012_FF	;; Mul by FFTLEN/2
	addsd	xmm0, xmm3		;; x1 = values + carry
	addsd	xmm1, xmm2		;; x2 = values + carry
	single_rounding_interleaved base2, xmm0, xmm3, xmm4, rax, xmm1, xmm2, xmm5, rcx
	mulsd	xmm0, Q [rbp+128+16]	;; new value1 = val * two-to-phi
	mulsd	xmm1, Q [rbp+128+80]	;; new value2 = val * two-to-phi
	movsd	Q [rsi+64], xmm0	;; Save value1
	movsd	Q [rsi+64+32], xmm1	;; Save value2

	movzx	rax, BYTE PTR [rdi+5]	;; Load big vs. little flags
	movzx	rcx, BYTE PTR [rdi+7]
	movsd	xmm0, Q [rsi+64+16]	;; Load values1
	mulsd	xmm0, Q [rbp+128+32]	;; Mul values1 by two-to-minus-phi
	mulsd	xmm0, Q XMM_NORM012_FF	;; Mul by FFTLEN/2
	movsd	xmm1, Q [rsi+64+48]	;; Load values2
	mulsd	xmm1, Q [rbp+128+96]	;; Mul values2 by two-to-minus-phi
	mulsd	xmm1, Q XMM_NORM012_FF	;; Mul by FFTLEN/2
	addsd	xmm0, xmm3		;; x1 = values + carry
	addsd	xmm1, xmm2		;; x2 = values + carry
	single_rounding_interleaved base2, xmm0, xmm3, xmm4, rax, xmm1, xmm2, xmm5, rcx
	mulsd	xmm0, Q [rbp+128+48]	;; new value1 = val * two-to-phi
	mulsd	xmm1, Q [rbp+128+112]	;; new value2 = val * two-to-phi
	movsd	Q [rsi+64+16], xmm0	;; Save value1
	movsd	Q [rsi+64+48], xmm1	;; Save value2

	movzx	rax, BYTE PTR [rdi+8]	;; Load big vs. little flags
	movzx	rcx, BYTE PTR [rdi+10]
	movsd	xmm0, Q [rsi+128]	;; Load values1
	mulsd	xmm0, Q [rbp+256]	;; Mul values1 by two-to-minus-phi
	mulsd	xmm0, Q XMM_NORM012_FF	;; Mul by FFTLEN/2
	movsd	xmm1, Q [rsi+128+32]	;; Load values2
	mulsd	xmm1, Q [rbp+256+64]	;; Mul values2 by two-to-minus-phi
	mulsd	xmm1, Q XMM_NORM012_FF	;; Mul by FFTLEN/2
	addsd	xmm0, xmm3		;; x1 = values + carry
	addsd	xmm1, xmm2		;; x2 = values + carry
	single_rounding_interleaved base2, xmm0, xmm3, xmm4, rax, xmm1, xmm2, xmm5, rcx
	mulsd	xmm0, Q [rbp+256+16]	;; new value1 = val * two-to-phi
	mulsd	xmm1, Q [rbp+256+80]	;; new value2 = val * two-to-phi
	movsd	Q [rsi+128], xmm0	;; Save value1
	movsd	Q [rsi+128+32], xmm1	;; Save value2

	subsd	xmm3, Q XMM_BIGVAL	;; Remove integer rounding constant
	subsd	xmm2, Q XMM_BIGVAL	;; Remove integer rounding constant
	mulsd	xmm3, Q [rbp+256+48]	;; carry *= two-to-phi
	mulsd	xmm2, Q [rbp+256+112]	;; carry *= two-to-phi
	addsd	xmm3, Q [rsi+128+16]	;; value1 = values + carry
	addsd	xmm2, Q [rsi+128+48]	;; value2 = values + carry
	movsd	Q [rsi+128+16], xmm3	;; Save value1
	movsd	Q [rsi+128+48], xmm2	;; Save value2
	ENDM

; This macro is similar to the above, but is for the zero padding case.
; xmm2 = carry #1 (traditional carry)
; xmm3 = carry #2 (previous high FFT data - not yet mul'ed by K)
; rsi = pointer to the FFT data values
; rbp = pointer two-to-phi multipliers
; rdi = big vs. little array pointer
; NOTE: If RATIONAL_FFT we could eliminate 8 multiplies.

xnorm012_1d_zpad MACRO const, base2
	LOCAL	smallk, mediumk, div_k_done, funnyaddr1, funny1cmn
	LOCAL	funnyaddr2, funny2done

	;; Strip BIGVAL from the traditional carry, we'll add the traditional
	;; carry in later when we are working on the ZPAD0 - ZPAD6 values.
	subsd	xmm2, Q XMM_BIGVAL	;; Integerize traditional carry

	;; Rather than calculate high FFT carry times k and then later dividing
	;; by k, we multiply FFT high carry by const and we'll add it
	;; to the lower FFT data later (after multiplying by -c).
const	mulsd	xmm3, Q XMM_MULCONST

	;; Work on zero-pad addin value.
	movsd	xmm4, Q ADDIN_VALUE		;; Load the add in value
const	mulsd	xmm4, Q XMM_MULCONST		;; Multiply the add in value by the small mul const
	addsd	xmm4, POSTADDIN_VALUE		;; Add the post-mul-by-const addin value
	;; when c = -1, 1 = b^n
	;; when c = 1, -1 = b^n, 1 = -b^n
	;; when c = -3, 3 = b^n, 1 = b^n - 2
	;; when c = 3, -3 = b^n, 3 = -b^n, 1 = -b^n - 2
	;; The "- 2" has been precomputed in ZPAD_LSW_ADJUST.  Add ADDIN_VALUE * mul-by-const * ZPAD_LSW_ADJUST into the least significant FFT word
	mov	rsi, DESTARG			;; Address of squared number
	movsd	xmm0, Q ZPAD_LSW_ADJUST
	mulsd	xmm0, xmm4
	addsd	xmm0, [rsi]
	movsd	[rsi], xmm0

	;; Multiply ZPAD0 through ZPAD6 by const * -c.  This, in essense,
	;; wraps this data from above the FFT data area to the halfway point.
	;; Later on we'll divide this by K to decide which data needs wrapping
	;; all the way down to the bottom of the FFT data.

	;; NOTE that ZPAD0's column multiplier is 1.0.  Also, ZPAD6 will not
	;; be bigger than a big word.  We must be careful to handle c's up
	;; to about 30 bits

	movzx	rax, BYTE PTR [rdi]	;; Load big vs. little flags
	movsd	xmm0, ZPAD0		;; Load values1
	subsd	xmm5, xmm5		;; Create a zero high FFT data carry
	single_split_lower_zpad_word base2, xmm0, xmm5, xmm1, rax
no const mulsd	xmm0, Q XMM_MINUS_C
const	mulsd	xmm0, Q XMM_MINUS_C_TIMES_MULCONST
	addsd	xmm0, xmm4		;; Apply rest of ADDIN_VALUE * mul-by-const here (the FFT half way point)
	round_zpad7_word base2, xmm0, xmm2, xmm1, rax
	movsd	ZPAD0, xmm0

	movzx	rax, BYTE PTR [rdi+1]	;; Load big vs. little flags
	movsd	xmm0, ZPAD1		;; Load values1
	mulsd	xmm0, Q [rbp+32]	;; Mul values1 by two-to-minus-phi
	mulsd	xmm0, Q XMM_NORM012_FF	;; Mul by FFTLEN/2
	single_split_lower_zpad_word base2, xmm0, xmm5, xmm1, rax
no const mulsd	xmm0, Q XMM_MINUS_C
const	mulsd	xmm0, Q XMM_MINUS_C_TIMES_MULCONST
	round_zpad7_word base2, xmm0, xmm2, xmm1, rax
	movsd	ZPAD1, xmm0

	movzx	rax, BYTE PTR [rdi+4]	;; Load big vs. little flags
	movsd	xmm0, ZPAD2		;; Load values1
	mulsd	xmm0, Q [rbp+128]	;; Mul values1 by two-to-minus-phi
	mulsd	xmm0, Q XMM_NORM012_FF	;; Mul by FFTLEN/2
	single_split_lower_zpad_word base2, xmm0, xmm5, xmm1, rax
no const mulsd	xmm0, Q XMM_MINUS_C
const	mulsd	xmm0, Q XMM_MINUS_C_TIMES_MULCONST
	round_zpad7_word base2, xmm0, xmm2, xmm1, rax
	movsd	ZPAD2, xmm0

	movzx	rax, BYTE PTR [rdi+5]	;; Load big vs. little flags
	movsd	xmm0, ZPAD3		;; Load values1
	mulsd	xmm0, Q [rbp+128+32]	;; Mul values1 by two-to-minus-phi
	mulsd	xmm0, Q XMM_NORM012_FF	;; Mul by FFTLEN/2
	single_split_lower_zpad_word base2, xmm0, xmm5, xmm1, rax
no const mulsd	xmm0, Q XMM_MINUS_C
const	mulsd	xmm0, Q XMM_MINUS_C_TIMES_MULCONST
	round_zpad7_word base2, xmm0, xmm2, xmm1, rax
	movsd	ZPAD3, xmm0

	cmp	FFTLEN, 80		;; Length 80 and 112 have different
	je	funnyaddr1		;; memory addresses for the fourth
	cmp	FFTLEN, 112		;; and higher data elements
	je	funnyaddr1

	movzx	rax, BYTE PTR [rdi+8]	;; Load big vs. little flags
	movsd	xmm0, ZPAD4		;; Load values1
	mulsd	xmm0, Q [rbp+256]	;; Mul values1 by two-to-minus-phi
	mulsd	xmm0, Q XMM_NORM012_FF	;; Mul by FFTLEN/2
	single_split_lower_zpad_word base2, xmm0, xmm5, xmm1, rax
no const mulsd	xmm0, Q XMM_MINUS_C
const	mulsd	xmm0, Q XMM_MINUS_C_TIMES_MULCONST
	round_zpad7_word base2, xmm0, xmm2, xmm1, rax
	movsd	ZPAD4, xmm0

	movzx	rax, BYTE PTR [rdi+9]	;; Load big vs. little flags
	movsd	xmm0, ZPAD5		;; Load values1
	mulsd	xmm0, Q [rbp+256+32]	;; Mul values1 by two-to-minus-phi
	mulsd	xmm0, Q XMM_NORM012_FF	;; Mul by FFTLEN/2
	single_split_lower_zpad_word base2, xmm0, xmm5, xmm1, rax
no const mulsd	xmm0, Q XMM_MINUS_C
const	mulsd	xmm0, Q XMM_MINUS_C_TIMES_MULCONST
	round_zpad7_word base2, xmm0, xmm2, xmm1, rax
	movsd	ZPAD5, xmm0

	movsd	xmm0, Q [rbp+384]	;; Load two-to-minus-phi
	jmp	funny1cmn		;; Join common code

	;; Same as the above but with different addresses required by
	;; the length 80 and 112 FFT lengths

funnyaddr1:
	movzx	rax, BYTE PTR [rdi]	;; Load big vs. little flags
	movsd	xmm0, ZPAD4		;; Load values1
	mulsd	xmm0, Q [rbp+8]		;; Mul values1 by two-to-minus-phi
	mulsd	xmm0, Q XMM_NORM012_FF	;; Mul by FFTLEN/2
	single_split_lower_zpad_word base2, xmm0, xmm5, xmm1, rax+8
no const mulsd	xmm0, Q XMM_MINUS_C
const	mulsd	xmm0, Q XMM_MINUS_C_TIMES_MULCONST
	round_zpad7_word base2, xmm0, xmm2, xmm1, rax+8
	movsd	ZPAD4, xmm0

	movzx	rax, BYTE PTR [rdi+1]	;; Load big vs. little flags
	movsd	xmm0, ZPAD5		;; Load values1
	mulsd	xmm0, Q [rbp+32+8]	;; Mul values1 by two-to-minus-phi
	mulsd	xmm0, Q XMM_NORM012_FF	;; Mul by FFTLEN/2
	single_split_lower_zpad_word base2, xmm0, xmm5, xmm1, rax+8
no const mulsd	xmm0, Q XMM_MINUS_C
const	mulsd	xmm0, Q XMM_MINUS_C_TIMES_MULCONST
	round_zpad7_word base2, xmm0, xmm2, xmm1, rax+8
	movsd	ZPAD5, xmm0

	movsd	xmm0, Q [rbp+128+8]	;; Load two-to-minus-phi
funny1cmn:
	mulsd	xmm0, ZPAD6		;; Mul by values1
	mulsd	xmm0, Q XMM_NORM012_FF	;; Mul by FFTLEN/2
	addsd	xmm0, Q XMM_BIGVAL	;; Round to an integer
	subsd	xmm0, Q XMM_BIGVAL
	addsd	xmm0, xmm5		;; Add in shifted high ZPAD data
no const mulsd	xmm0, Q XMM_MINUS_C
const	mulsd	xmm0, Q XMM_MINUS_C_TIMES_MULCONST
	addsd	xmm0, xmm2		;; Add in high part of last calculation
	movsd	ZPAD6, xmm0

	;; Divide the zpad data by k.  Store the integer part in XMM_TMP
	;; and the remainder in ZPAD0.  Later we will wrap the integer part
	;; down to the bottom of the FFT data area (and multiply by -c).
	;; And we will store the remainder in the upper half of the FFT
	;; data area.

	;; Note there are three cases to handle.  K is smaller than a big word.
	;; K is between one and 2 big words in size.  And K is more than
	;; 2 big words in size.

	cmp	ZPAD_TYPE, 2		;; Are we dealing with case 1,2,or 3
	jl	smallk			;; One word case
	je	mediumk			;; Two word case

	;; This case does the divide by k where k is three words

	movsd	xmm0, ZPAD6		;; Load zpad word (high bits)
	movsd	xmm1, ZPAD5		;; Load zpad word (middle bits)
	movsd	xmm2, ZPAD4		;; Load zpad word (low bits)
	movsd	xmm4, ZPAD_INVERSE_K6	;; Load shifted 1/k
	mulsd	xmm4, xmm0		;; Mul ZPAD by shifted 1/k
	addsd	xmm4, Q XMM_BIGVAL	;; Round to integer	
	subsd	xmm4, Q XMM_BIGVAL
	movsd	xmm5, ZPAD_K6_HI	;; Load high bits of k
	mulsd	xmm5, xmm4
	subsd	xmm0, xmm5		;; Calculate high bits of remainder
	movsd	xmm5, ZPAD_K6_MID	;; Load middle bits of k
	mulsd	xmm5, xmm4
	subsd	xmm1, xmm5		;; Calculate middle bits of remainder
	movsd	xmm5, ZPAD_K6_LO	;; Load low bits of k
	mulsd	xmm5, xmm4
	subsd	xmm2, xmm5		;; Calculate low bits of remainder
	movsd	Q XMM_TMP5, xmm4	;; Save word of zpad / k

	mulsd	xmm0, ZPAD_SHIFT6	;; Shift previous zpad word
	addsd	xmm1, xmm0		;; Add to create new high zpad bits
	movsd	xmm0, ZPAD3		;; Load zpad word (new low bits)
	movsd	xmm5, ZPAD_SHIFT5	;; Combine high and medium bits
	mulsd	xmm5, xmm1
	addsd	xmm5, xmm2
	movsd	xmm4, ZPAD_INVERSE_K5	;; Load shifted 1/k
	mulsd	xmm4, xmm5		;; Mul ZPAD by shifted 1/k
	addsd	xmm4, Q XMM_BIGVAL	;; Round to integer
	subsd	xmm4, Q XMM_BIGVAL
	movsd	xmm5, ZPAD_K5_HI	;; Load high bits of k
	mulsd	xmm5, xmm4
	subsd	xmm1, xmm5		;; Calculate high bits of remainder
	movsd	xmm5, ZPAD_K5_MID	;; Load middle bits of k
	mulsd	xmm5, xmm4
	subsd	xmm2, xmm5		;; Calculate middle bits of remainder
	movsd	xmm5, ZPAD_K5_LO	;; Load low bits of k
	mulsd	xmm5, xmm4
	subsd	xmm0, xmm5		;; Calculate low bits of remainder
	movsd	Q XMM_TMP4, xmm4	;; Save word of zpad / k

	mulsd	xmm1, ZPAD_SHIFT5	;; Shift previous zpad word
	addsd	xmm2, xmm1		;; Add to create new high zpad bits
	movsd	xmm1, ZPAD2		;; Load zpad word (new low bits)
	movsd	xmm5, ZPAD_SHIFT4	;; Combine high and medium bits
	mulsd	xmm5, xmm2
	addsd	xmm5, xmm0
	movsd	xmm4, ZPAD_INVERSE_K4	;; Load shifted 1/k
	mulsd	xmm4, xmm5		;; Mul ZPAD by shifted 1/k
	addsd	xmm4, Q XMM_BIGVAL	;; Round to integer
	subsd	xmm4, Q XMM_BIGVAL
	movsd	xmm5, ZPAD_K4_HI	;; Load high bits of k
	mulsd	xmm5, xmm4
	subsd	xmm2, xmm5		;; Calculate high bits of remainder
	movsd	xmm5, ZPAD_K4_MID	;; Load middle bits of k
	mulsd	xmm5, xmm4
	subsd	xmm0, xmm5		;; Calculate middle bits of remainder
	movsd	xmm5, ZPAD_K4_LO	;; Load low bits of k
	mulsd	xmm5, xmm4
	subsd	xmm1, xmm5		;; Calculate low bits of remainder
	movsd	Q XMM_TMP3, xmm4	;; Save word of zpad / k

	mulsd	xmm2, ZPAD_SHIFT4	;; Shift previous zpad word
	addsd	xmm0, xmm2		;; Add to create new high zpad bits
	movsd	xmm2, ZPAD1		;; Load zpad word (new low bits)
	movsd	xmm5, ZPAD_SHIFT3	;; Combine high and medium bits
	mulsd	xmm5, xmm0
	addsd	xmm5, xmm1
	movsd	xmm4, ZPAD_INVERSE_K3	;; Load shifted 1/k
	mulsd	xmm4, xmm5		;; Mul ZPAD by shifted 1/k
	addsd	xmm4, Q XMM_BIGVAL	;; Round to integer
	subsd	xmm4, Q XMM_BIGVAL
	movsd	xmm5, ZPAD_K3_HI	;; Load high bits of k
	mulsd	xmm5, xmm4
	subsd	xmm0, xmm5		;; Calculate high bits of remainder
	movsd	xmm5, ZPAD_K3_MID	;; Load middle bits of k
	mulsd	xmm5, xmm4
	subsd	xmm1, xmm5		;; Calculate middle bits of remainder
	movsd	xmm5, ZPAD_K3_LO	;; Load low bits of k
	mulsd	xmm5, xmm4
	subsd	xmm2, xmm5		;; Calculate low bits of remainder
	movsd	Q XMM_TMP2, xmm4	;; Save word of zpad / k

	mulsd	xmm0, ZPAD_SHIFT3	;; Shift previous zpad word
	addsd	xmm1, xmm0		;; Add to create new high zpad bits
	movsd	xmm0, ZPAD0		;; Load zpad word (new low bits)
	movsd	xmm5, ZPAD_SHIFT2	;; Combine high and medium bits
	mulsd	xmm5, xmm1
	addsd	xmm5, xmm2
	movsd	xmm4, ZPAD_INVERSE_K2	;; Load shifted 1/k
	mulsd	xmm4, xmm5		;; Mul ZPAD by shifted 1/k
	addsd	xmm4, Q XMM_BIGVAL	;; Round to integer
	subsd	xmm4, Q XMM_BIGVAL
	movsd	xmm5, ZPAD_K2_HI	;; Load high bits of k
	mulsd	xmm5, xmm4
	subsd	xmm1, xmm5		;; Calculate high bits of remainder
	movsd	xmm5, ZPAD_K2_MID	;; Load middle bits of k
	mulsd	xmm5, xmm4
	subsd	xmm2, xmm5		;; Calculate middle bits of remainder
	movsd	xmm5, ZPAD_K2_LO	;; Load low bits of k
	mulsd	xmm5, xmm4
	subsd	xmm0, xmm5		;; Calculate low bits of remainder
	movsd	Q XMM_TMP1, xmm4	;; Save word of zpad / k

	mulsd	xmm1, ZPAD_SHIFT2	;; Shift previous zpad word
	addsd	xmm2, xmm1		;; Add to create new high zpad bits
	mulsd	xmm2, ZPAD_SHIFT1	;; Shift previous zpad word
	addsd	xmm0, xmm2		;; Add to create new high zpad bits
	movsd	ZPAD0, xmm0		;; Save remainder of zpad / k

	subsd	xmm1, xmm1		;; Zero words that other cases set
	movsd	Q XMM_TMP6, xmm1
	
	jmp	div_k_done

	;; This case does the divide by k where k is two words
mediumk:
	movsd	xmm0, ZPAD6		;; Load zpad word (high bits)
	movsd	xmm1, ZPAD5		;; Load zpad word (low bits)
	movsd	xmm4, ZPAD_INVERSE_K6	;; Load shifted 1/k
	mulsd	xmm4, xmm0		;; Mul ZPAD by shifted 1/k
	addsd	xmm4, Q XMM_BIGVAL	;; Round to integer	
	subsd	xmm4, Q XMM_BIGVAL
	movsd	xmm5, ZPAD_K6_HI	;; Load high bits of k
	mulsd	xmm5, xmm4
	subsd	xmm0, xmm5		;; Calculate high bits of remainder
	movsd	xmm5, ZPAD_K6_LO	;; Load low bits of k
	mulsd	xmm5, xmm4
	subsd	xmm1, xmm5		;; Calculate low bits of remainder
	movsd	Q XMM_TMP6, xmm4	;; Save word of zpad / k

	mulsd	xmm0, ZPAD_SHIFT6	;; Shift previous zpad word
	addsd	xmm0, xmm1		;; Add to create new high zpad bits
	movsd	xmm1, ZPAD4		;; Load zpad word (new low bits)
	movsd	xmm4, ZPAD_INVERSE_K5	;; Load shifted 1/k
	mulsd	xmm4, xmm0		;; Mul ZPAD by shifted 1/k
	addsd	xmm4, Q XMM_BIGVAL	;; Round to integer
	subsd	xmm4, Q XMM_BIGVAL
	movsd	xmm5, ZPAD_K5_HI	;; Load high bits of k
	mulsd	xmm5, xmm4
	subsd	xmm0, xmm5		;; Calculate high bits of remainder
	movsd	xmm5, ZPAD_K5_LO	;; Load low bits of k
	mulsd	xmm5, xmm4
	subsd	xmm1, xmm5		;; Calculate low bits of remainder
	movsd	Q XMM_TMP5, xmm4	;; Save word of zpad / k

	mulsd	xmm0, ZPAD_SHIFT5	;; Shift previous zpad word
	addsd	xmm0, xmm1		;; Add to create new high zpad bits
	movsd	xmm1, ZPAD3		;; Load zpad word (new low bits)
	movsd	xmm4, ZPAD_INVERSE_K4	;; Load shifted 1/k
	mulsd	xmm4, xmm0		;; Mul ZPAD by shifted 1/k
	addsd	xmm4, Q XMM_BIGVAL	;; Round to integer
	subsd	xmm4, Q XMM_BIGVAL
	movsd	xmm5, ZPAD_K4_HI	;; Load high bits of k
	mulsd	xmm5, xmm4
	subsd	xmm0, xmm5		;; Calculate high bits of remainder
	movsd	xmm5, ZPAD_K4_LO	;; Load low bits of k
	mulsd	xmm5, xmm4
	subsd	xmm1, xmm5		;; Calculate low bits of remainder
	movsd	Q XMM_TMP4, xmm4	;; Save word of zpad / k

	mulsd	xmm0, ZPAD_SHIFT4	;; Shift previous zpad word
	addsd	xmm0, xmm1		;; Add to create new high zpad bits
	movsd	xmm1, ZPAD2		;; Load zpad word (new low bits)
	movsd	xmm4, ZPAD_INVERSE_K3	;; Load shifted 1/k
	mulsd	xmm4, xmm0		;; Mul ZPAD by shifted 1/k
	addsd	xmm4, Q XMM_BIGVAL	;; Round to integer
	subsd	xmm4, Q XMM_BIGVAL
	movsd	xmm5, ZPAD_K3_HI	;; Load high bits of k
	mulsd	xmm5, xmm4
	subsd	xmm0, xmm5		;; Calculate high bits of remainder
	movsd	xmm5, ZPAD_K3_LO	;; Load low bits of k
	mulsd	xmm5, xmm4
	subsd	xmm1, xmm5		;; Calculate low bits of remainder
	movsd	Q XMM_TMP3, xmm4	;; Save word of zpad / k

	mulsd	xmm0, ZPAD_SHIFT3	;; Shift previous zpad word
	addsd	xmm0, xmm1		;; Add to create new high zpad bits
	movsd	xmm1, ZPAD1		;; Load zpad word (new low bits)
	movsd	xmm4, ZPAD_INVERSE_K2	;; Load shifted 1/k
	mulsd	xmm4, xmm0		;; Mul ZPAD by shifted 1/k
	addsd	xmm4, Q XMM_BIGVAL	;; Round to integer
	subsd	xmm4, Q XMM_BIGVAL
	movsd	xmm5, ZPAD_K2_HI	;; Load high bits of k
	mulsd	xmm5, xmm4
	subsd	xmm0, xmm5		;; Calculate high bits of remainder
	movsd	xmm5, ZPAD_K2_LO	;; Load low bits of k
	mulsd	xmm5, xmm4
	subsd	xmm1, xmm5		;; Calculate low bits of remainder
	movsd	Q XMM_TMP2, xmm4	;; Save word of zpad / k

	mulsd	xmm0, ZPAD_SHIFT2	;; Shift previous zpad word
	addsd	xmm0, xmm1		;; Add to create new high zpad bits
	movsd	xmm1, ZPAD0		;; Load zpad word (new low bits)
	movsd	xmm4, ZPAD_INVERSE_K1	;; Load shifted 1/k
	mulsd	xmm4, xmm0		;; Mul ZPAD by shifted 1/k
	addsd	xmm4, Q XMM_BIGVAL	;; Round to integer
	subsd	xmm4, Q XMM_BIGVAL
	movsd	xmm5, ZPAD_K1_HI	;; Load high bits of k
	mulsd	xmm5, xmm4
	subsd	xmm0, xmm5		;; Calculate high bits of remainder
	movsd	xmm5, ZPAD_K1_LO	;; Load low bits of k
	mulsd	xmm5, xmm4
	subsd	xmm1, xmm5		;; Calculate low bits of remainder
	movsd	Q XMM_TMP1, xmm4	;; Save word of zpad / k

	mulsd	xmm0, ZPAD_SHIFT1	;; Shift previous zpad word
	addsd	xmm0, xmm1		;; Add to create new high zpad bits
	movsd	ZPAD0, xmm0		;; Save remainder of zpad / k

	jmp	div_k_done

	;; This case does the divide by k where k is one word
	;; Assume ZPAD5 and ZPAD6 are zero.
smallk:	movsd	xmm0, ZPAD4		;; Load zpad data
	movsd	xmm4, ZPAD_INVERSE_K1	;; Load by 1/k
	mulsd	xmm4, xmm0		;; Mul ZPAD data by 1/k
	addsd	xmm4, Q XMM_BIGVAL	;; Round to integer	
	subsd	xmm4, Q XMM_BIGVAL
	movsd	Q XMM_TMP5, xmm4	;; Save integer part
	mulsd	xmm4, ZPAD_K1_LO	;; Compute remainder
	subsd	xmm0, xmm4

	mulsd	xmm0, ZPAD_SHIFT4	;; Shift previous zpad word
	addsd	xmm0, ZPAD3		;; Add in zpad data
	movsd	xmm4, ZPAD_INVERSE_K1	;; Load by 1/k
	mulsd	xmm4, xmm0		;; Mul ZPAD data by 1/k
	addsd	xmm4, Q XMM_BIGVAL	;; Round to integer	
	subsd	xmm4, Q XMM_BIGVAL
	movsd	Q XMM_TMP4, xmm4	;; Save integer part
	mulsd	xmm4, ZPAD_K1_LO	;; Compute remainder
	subsd	xmm0, xmm4

	mulsd	xmm0, ZPAD_SHIFT3	;; Shift previous zpad word
	addsd	xmm0, ZPAD2		;; Add in zpad data
	movsd	xmm4, ZPAD_INVERSE_K1	;; Load by 1/k
	mulsd	xmm4, xmm0		;; Mul ZPAD data by 1/k
	addsd	xmm4, Q XMM_BIGVAL	;; Round to integer	
	subsd	xmm4, Q XMM_BIGVAL
	movsd	Q XMM_TMP3, xmm4	;; Save integer part
	mulsd	xmm4, ZPAD_K1_LO	;; Compute remainder
	subsd	xmm0, xmm4

	mulsd	xmm0, ZPAD_SHIFT2	;; Shift previous zpad word
	addsd	xmm0, ZPAD1		;; Add in zpad data
	movsd	xmm4, ZPAD_INVERSE_K1	;; Load by 1/k
	mulsd	xmm4, xmm0		;; Mul ZPAD data by 1/k
	addsd	xmm4, Q XMM_BIGVAL	;; Round to integer	
	subsd	xmm4, Q XMM_BIGVAL
	movsd	Q XMM_TMP2, xmm4	;; Save integer part
	mulsd	xmm4, ZPAD_K1_LO	;; Compute remainder
	subsd	xmm0, xmm4

	mulsd	xmm0, ZPAD_SHIFT1	;; Shift previous zpad word
	addsd	xmm0, ZPAD0		;; Add in zpad data
	movsd	xmm4, ZPAD_INVERSE_K1	;; Load by 1/k
	mulsd	xmm4, xmm0		;; Mul ZPAD data by 1/k
	addsd	xmm4, Q XMM_BIGVAL	;; Round to integer	
	subsd	xmm4, Q XMM_BIGVAL
	movsd	Q XMM_TMP1, xmm4	;; Save integer part
	mulsd	xmm4, ZPAD_K1_LO	;; Compute remainder
	subsd	xmm0, xmm4
	movsd	ZPAD0, xmm0		;; Save remainder

	subsd	xmm1, xmm1		;; Zero words that other cases set
	movsd	Q XMM_TMP6, xmm1
	
div_k_done:

	;; Now normalize the data above the halfway point.  Remember that the
	;; column two-to-phi multiplier for the first value will be 1.0.

	movzx	rax, BYTE PTR [rdi]	;; First word 
	movsd	xmm0, ZPAD0		;; Load remainder of divide by k
	addsd	xmm0, Q XMM_BIGVAL
	single_rounding base2, xmm0, xmm2, xmm4, rax
	movsd	Q [rsi+32], xmm0	;; Save value1

	movzx	rax, BYTE PTR [rdi+1]	;; Load big vs. little flags
	single_rounding base2, xmm2, xmm0, xmm4, rax
	mulsd	xmm2, Q [rbp+48]	;; new value2 = val * two-to-phi
	movsd	Q [rsi+48], xmm2	;; Save value2

	movzx	rax, BYTE PTR [rdi+4]	;; Load big vs. little flags
	single_rounding base2, xmm0, xmm2, xmm4, rax
	mulsd	xmm0, Q [rbp+128+16]	;; new value3 = val * two-to-phi
	movsd	Q [rsi+64+32], xmm0	;; Save value3

	subsd	xmm2, Q XMM_BIGVAL	;; Remove integer rounding constant
	mulsd	xmm2, Q [rbp+128+48]	;; value4 = carry * two-to-phi
	movsd	Q [rsi+64+48], xmm2	;; Save new value4

	;; Mul the integer part of (ZPAD data divided by k) by -c in
	;; preparation for adding it into the lower FFT data area.
	;; Also add in the shifted high FFT carry at this time.

	;; Now add in and normalize the bottom FFT data.  Remember that the
	;; column two-to-phi multiplier for the first value will be 1.0.  We 
	;; must go 6 words deep in case k is 48-50 bits and c is 32 bits.

	movzx	rax, BYTE PTR [rdi]	;; First word 
	movsd	xmm0, Q XMM_TMP1	;; Load integer part of divide by k
	addsd	xmm0, xmm3		;; Add in shifted high FFT carry
	mulsd	xmm0, Q XMM_MINUS_C	;; Mul by -c
	addsd	xmm0, Q XMM_BIGVAL
	addsd	xmm0, Q [rsi]		;; Add in the FFT data
	single_rounding base2, xmm0, xmm2, xmm4, rax
	movsd	Q [rsi], xmm0		;; Save value1

	movzx	rax, BYTE PTR [rdi+1]	;; Load big vs. little flags
	movsd	xmm0, Q XMM_TMP2	;; Load integer part of divide by k
	mulsd	xmm0, Q XMM_MINUS_C	;; Mul by -c
	movsd	xmm1, Q [rsi+16]	;; Load FFT data
	mulsd	xmm1, Q [rbp+32]	;; Mul values2 by two-to-minus-phi
	mulsd	xmm1, Q XMM_NORM012_FF	;; Mul by FFTLEN/2
	addsd	xmm0, xmm1		;; Add in the FFT data
	addsd	xmm0, xmm2		;; x2 = value + carry
	single_rounding base2, xmm0, xmm2, xmm4, rax
	mulsd	xmm0, Q [rbp+48]	;; new value2 = val * two-to-phi
	movsd	Q [rsi+16], xmm0	;; Save value2

	movzx	rax, BYTE PTR [rdi+4]	;; Load big vs. little flags
	movsd	xmm0, Q XMM_TMP3	;; Load integer part of divide by k
	mulsd	xmm0, Q XMM_MINUS_C	;; Mul by -c
	movsd	xmm1, Q [rsi+64]	;; Load FFT data
	mulsd	xmm1, Q [rbp+128]	;; Mul values3 by two-to-minus-phi
	mulsd	xmm1, Q XMM_NORM012_FF	;; Mul by FFTLEN/2
	addsd	xmm0, xmm1		;; Add in the FFT data
	addsd	xmm0, xmm2		;; x3 = value + carry
	single_rounding base2, xmm0, xmm2, xmm4, rax
	mulsd	xmm0, Q [rbp+128+16]	;; new value3 = val * two-to-phi
	movsd	Q [rsi+64], xmm0	;; Save value3

	movzx	rax, BYTE PTR [rdi+5]	;; Load big vs. little flags
	movsd	xmm0, Q XMM_TMP4	;; Load integer part of divide by k
	mulsd	xmm0, Q XMM_MINUS_C	;; Mul by -c
	movsd	xmm1, Q [rsi+64+16]	;; Load FFT data
	mulsd	xmm1, Q [rbp+128+32]	;; Mul values4 by two-to-minus-phi
	mulsd	xmm1, Q XMM_NORM012_FF	;; Mul by FFTLEN/2
	addsd	xmm0, xmm1		;; Add in the FFT data
	addsd	xmm0, xmm2		;; x4 = value + carry
	single_rounding base2, xmm0, xmm2, xmm4, rax
	mulsd	xmm0, Q [rbp+128+48]	;; new value4 = val * two-to-phi
	movsd	Q [rsi+64+16], xmm0	;; Save value4

	cmp	FFTLEN, 80		;; Length 80 and 112 have different
	je	funnyaddr2		;; memory addresses for the fourth
	cmp	FFTLEN, 112		;; and higher data elements
	je	funnyaddr2

	movzx	rax, BYTE PTR [rdi+8]	;; Load big vs. little flags
	movsd	xmm0, Q XMM_TMP5	;; Load integer part of divide by k
	mulsd	xmm0, Q XMM_MINUS_C	;; Mul by -c
	movsd	xmm1, Q [rsi+128]	;; Load FFT data
	mulsd	xmm1, Q [rbp+256]	;; Mul values4 by two-to-minus-phi
	mulsd	xmm1, Q XMM_NORM012_FF	;; Mul by FFTLEN/2
	addsd	xmm0, xmm1		;; Add in the FFT data
	addsd	xmm0, xmm2		;; x5 = value + carry
	single_rounding base2, xmm0, xmm2, xmm4, rax
	mulsd	xmm0, Q [rbp+256+16]	;; new value5 = val * two-to-phi
	movsd	Q [rsi+128], xmm0	;; Save value5

	movzx	rax, BYTE PTR [rdi+9]	;; Load big vs. little flags
	movsd	xmm0, Q XMM_TMP6	;; Load integer part of divide by k
	mulsd	xmm0, Q XMM_MINUS_C	;; Mul by -c
	movsd	xmm1, Q [rsi+128+16]	;; Load FFT data
	mulsd	xmm1, Q [rbp+256+32]	;; Mul values6 by two-to-minus-phi
	mulsd	xmm1, Q XMM_NORM012_FF	;; Mul by FFTLEN/2
	addsd	xmm0, xmm1		;; Add in the FFT data
	addsd	xmm0, xmm2		;; x6 = value + carry
	single_rounding base2, xmm0, xmm2, xmm4, rax
	mulsd	xmm0, Q [rbp+256+48]	;; new value6 = val * two-to-phi
	movsd	Q [rsi+128+16], xmm0	;; Save value6

	movzx	rax, BYTE PTR [rdi+12]	;; Load big vs. little flags
	movsd	xmm0, Q [rsi+192]	;; Load FFT data
	mulsd	xmm0, Q [rbp+384]	;; Mul values7 by two-to-minus-phi
	mulsd	xmm0, Q XMM_NORM012_FF	;; Mul by FFTLEN/2
	addsd	xmm0, xmm2		;; x7 = value + carry
	single_rounding base2, xmm0, xmm2, xmm4, rax
	mulsd	xmm0, Q [rbp+384+16]	;; new value7 = val * two-to-phi
	movsd	Q [rsi+192], xmm0	;; Save value7

	subsd	xmm2, Q XMM_BIGVAL	;; Remove rounding constant
	mulsd	xmm2, Q [rbp+384+48]	;; new value8 = val * two-to-phi
	addsd	xmm2, Q [rsi+192+16]	;; Add in FFT data
	movsd	Q [rsi+192+16], xmm2	;; Save value8
	jmp	funny2done

	;; Same as the above but with different addresses required by
	;; the length 80 and 112 FFT lengths

funnyaddr2:
	movzx	rax, BYTE PTR [rdi]	;; Load big vs. little flags
	movsd	xmm0, Q XMM_TMP5	;; Load integer part of divide by k
	mulsd	xmm0, Q XMM_MINUS_C	;; Mul by -c
	movsd	xmm1, Q [rsi+8]		;; Load FFT data
	mulsd	xmm1, Q [rbp+8]		;; Mul values5 by two-to-minus-phi
	mulsd	xmm1, Q XMM_NORM012_FF	;; Mul by FFTLEN/2
	addsd	xmm0, xmm1		;; Add in the FFT data
	addsd	xmm0, xmm2		;; x5 = value + carry
	single_rounding base2, xmm0, xmm2, xmm4, rax+8
	mulsd	xmm0, Q [rbp+16+8]	;; new value5 = val * two-to-phi
	movsd	Q [rsi+8], xmm0		;; Save value5

	movzx	rax, BYTE PTR [rdi+1]	;; Load big vs. little flags
	movsd	xmm0, Q XMM_TMP6	;; Load integer part of divide by k
	mulsd	xmm0, Q XMM_MINUS_C	;; Mul by -c
	movsd	xmm1, Q [rsi+16+8]	;; Load FFT data
	mulsd	xmm1, Q [rbp+32+8]	;; Mul values6 by two-to-minus-phi
	mulsd	xmm1, Q XMM_NORM012_FF	;; Mul by FFTLEN/2
	addsd	xmm0, xmm1		;; Add in the FFT data
	addsd	xmm0, xmm2		;; x6 = value + carry
	single_rounding base2, xmm0, xmm2, xmm4, rax+8
	mulsd	xmm0, Q [rbp+48+8]	;; new value6 = val * two-to-phi
	movsd	Q [rsi+16+8], xmm0	;; Save value6

	movzx	rax, BYTE PTR [rdi+4]	;; Load big vs. little flags
	movsd	xmm0, Q [rsi+64+8]	;; Load FFT data
	mulsd	xmm0, Q [rbp+128+8]	;; Mul values5 by two-to-minus-phi
	mulsd	xmm0, Q XMM_NORM012_FF	;; Mul by FFTLEN/2
	addsd	xmm0, xmm2		;; x7 = value + carry
	single_rounding base2, xmm0, xmm2, xmm4, rax+8
	mulsd	xmm0, Q [rbp+128+16+8]	;; new value7 = val * two-to-phi
	movsd	Q [rsi+64+8], xmm0	;; Save value7

	subsd	xmm2, Q XMM_BIGVAL	;; Remove rounding constant
	mulsd	xmm2, Q [rbp+128+48+8]	;; new value8 = val * two-to-phi
	addsd	xmm2, Q [rsi+64+16+8]	;; Add in FFT data
	movsd	Q [rsi+64+16+8], xmm2	;; Save value8
funny2done:

	ENDM



; For 2D macros, these registers are set on input:
; xmm6 = maxerr
; rbp = pointer to carries
; rdi = pointer to big/little flags
; rsi = pointer to the FFT data
; rbx = pointer two-to-phi column multipliers
; rdx = pointer two-to-phi group multipliers
; ecx = big vs. little word flag #2
; eax = big vs. little word flag #1


; *************** 2D macro ******************
; A pipelined version of this code:
;	movzx	rax, BYTE PTR [rdi]	;; Load big vs. little flags
;	xload	xmm0, [rsi+0*dist1]	;; Load values1
;	xload	xmm2, [rbx]		;; col two-to-minus-phi
;	mulpd	xmm2, XMM_TTMP_FUDGE[rax];; Mul by fudge two-to-minus-phi
;	mulpd	xmm0, [rdx]		;; Mul by grp two-to-minus-phi
;	mulpd	xmm0, xmm2		;; Mul by fudged col two-to-minus-phi
;	addpd	xmm0, [rbp+0*16]	;; x1 = values + carry
;	xload	xmm2, XMM_LIMIT_BIGMAX[rax];; Load maximum * BIGVAL - BIGVAL
;	addpd	xmm2, xmm0		;; y1 = top bits of x
;	xload	xmm6, XMM_LIMIT_BIGMAX_NEG[rax];; Load -(maximum*BIGVAL-BIGVAL)
;	addpd	xmm6, xmm2		;; z1 = y1-(maximum * BIGVAL - BIGVAL)
;	subpd	xmm0, xmm6		;; rounded value = x1 - z1
;	mulpd	xmm2, XMM_LIMIT_INVERSE[rax];; next carry = shifted y1
;	xload	xmm4, [rbx]		;; col two-to-phi
;	mulpd	xmm4, XMM_TTP_FUDGE[rax];; mul by fudge two-to-phi
;	mulpd	xmm0, [rdx+0*32+16]	;; new value1 = val * grp two-to-phi
;	mulpd	xmm0, xmm4		;; new value1 *= fudged col two-to-phi
;	xstore	[rsi+0*dist1], xmm0	;; Save new value1
;	xstore	[rbp+0*16], xmm2	;; Save carry
;

xnorm_2d_setup MACRO ttp		;; Precompute FUDGE * col multipliers
ttp	lea	rbp, [rdi+256]		;; Create pointer for more one-byte offsets below
	xload	xmm0, [rbx]		;; Load col two-to-minus-phi
	xstore	[rdi-128], xmm0		;; Save ttmp * 1.0,1.0
ttp	xstore	[rdi-128+16], xmm0	;; Save ttmp * 1.0,1.0
ttp	xstore	[rdi-128+32], xmm0	;; Save ttmp * 1.0,1.0
ttp	xstore	[rdi-128+48], xmm0	;; Save ttmp * 1.0,1.0
ttp	mulsd	xmm0, Q XMM_TTMP_FUDGE[64];; Compute ttmp * 1.0,B
	xload	xmm1, [rbx+16]		;; Load col two-to-phi
no ttp	xstore	[rdi+256-128], xmm1	;; Save ttp * 1.0,1.0
ttp	xstore	[rbp-128], xmm1		;; Save ttp * 1.0,1.0
ttp	xstore	[rbp-128+16], xmm1	;; Save ttp * 1.0,1.0
ttp	xstore	[rbp-128+32], xmm1	;; Save ttp * 1.0,1.0
ttp	xstore	[rbp-128+48], xmm1	;; Save ttp * 1.0,1.0
ttp	mulsd	xmm1, Q XMM_TTP_FUDGE[64];; Compute ttp * 1.0,1/B
ttp	xstore	[rdi-128+64], xmm0	;; Save ttmp * 1.0,B
ttp	xstore	[rdi-128+64+16], xmm0	;; Save ttmp * 1.0,B
ttp	xstore	[rdi-128+64+32], xmm0	;; Save ttmp * 1.0,B
ttp	xstore	[rdi-128+64+48], xmm0	;; Save ttmp * 1.0,B
ttp	shufpd	xmm0, xmm0, 1		;; swizzle
ttp	xstore	[rbp-128+64], xmm1	;; Save ttp * 1.0,1/B
ttp	xstore	[rbp-128+64+16], xmm1	;; Save ttp * 1.0,1/B
ttp	xstore	[rbp-128+64+32], xmm1	;; Save ttp * 1.0,1/B
ttp	xstore	[rbp-128+64+48], xmm1	;; Save ttp * 1.0,1/B
ttp	shufpd	xmm1, xmm1, 1		;; swizzle
ttp	xstore	[rdi-128+128], xmm0	;; Save ttmp * B,1.0
ttp	xstore	[rdi-128+128+16], xmm0	;; Save ttmp * B,1.0
ttp	xstore	[rdi-128+128+32], xmm0	;; Save ttmp * B,1.0
ttp	xstore	[rdi-128+128+48], xmm0	;; Save ttmp * B,1.0
ttp	mulsd	xmm0, Q XMM_TTMP_FUDGE[64]
ttp	xstore	[rbp-128+128], xmm1	;; Save ttp * 1/B,1.0
ttp	xstore	[rbp-128+128+16], xmm1	;; Save ttp * 1/B,1.0
ttp	xstore	[rbp-128+128+32], xmm1	;; Save ttp * 1/B,1.0
ttp	xstore	[rbp-128+128+48], xmm1	;; Save ttp * 1/B,1.0
ttp	mulsd	xmm1, Q XMM_TTP_FUDGE[64]
ttp	xstore	[rdi-128+192], xmm0	;; Save ttmp * B,B
ttp	xstore	[rdi-128+192+16], xmm0	;; Save ttmp * B,B
ttp	xstore	[rdi-128+192+32], xmm0	;; Save ttmp * B,B
ttp	xstore	[rdi-128+192+48], xmm0	;; Save ttmp * B,B
ttp	xstore	[rbp-128+192], xmm1	;; Save ttp * 1/B,1/B
ttp	xstore	[rbp-128+192+16], xmm1	;; Save ttp * 1/B,1/B
ttp	xstore	[rbp-128+192+32], xmm1	;; Save ttp * 1/B,1/B
ttp	xstore	[rbp-128+192+48], xmm1	;; Save ttp * 1/B,1/B
	ENDM

xnorm_2d MACRO ttp, echk, const, base2, sse4
	xload	xmm0, [rsi+0*16]	;; Load values1
ttp	xload	xmm2, [rdx+0*32]	;; grp two-to-minus-phi
ttp	mulpd	xmm2, [rbx][rax]	;; Mul by col two-to-minus-phi
no ttp	xload	xmm2, [rbx]		;; two-to-minus-phi
	mulpd	xmm0, xmm2		;; Mul by fudged col two-to-minus-phi
	xload	xmm1, [rsi+1*16]	;; Load values2
ttp	xload	xmm3, [rdx+1*32]	;; grp two-to-minus-phi
ttp	mulpd	xmm3, [rbx][rcx]	;; Mul by col two-to-minus-phi
no ttp	xload	xmm3, [rbx]		;; two-to-minus-phi
	mulpd	xmm1, xmm3		;; Mul by fudged col two-to-minus-phi
no const echk error_check_interleaved sse4, xmm0, xmm4, xmm1, xmm5, xmm6
const	mul_by_const_interleaved ttp, base2, echk, sse4, xmm0, xmm4, xmm2, rax, xmm1, xmm5, xmm3, rcx, xmm6
	addpd	xmm0, [rbp+0*16]	;; x1 = values + carry
	addpd	xmm1, [rbp+1*16]	;; x2 = values + carry

no base2 rounding_interleaved ttp, base2, const, sse4, xmm0, xmm4, xmm2, rax, xmm1, xmm5, xmm3, rcx

base2	xload	xmm2, XMM_LIMIT_BIGMAX[rax];; Load maximum * BIGVAL - BIGVAL
base2	addpd	xmm2, xmm0		;; y1 = top bits of x
base2	xload	xmm3, XMM_LIMIT_BIGMAX[rcx];; Load maximum * BIGVAL - BIGVAL
base2	addpd	xmm3, xmm1		;; y2 = top bits of x
base2 const addpd xmm4, xmm2		;; Add in upper mul-by-const bits
base2 const mulpd xmm4, XMM_LIMIT_INVERSE[rax];; next carry = shifted y1
base2 no const xload xmm4, XMM_LIMIT_INVERSE[rax];; next carry = shifted y1
base2 no const mulpd xmm4, xmm2		;; next carry = shifted y1
	xstore	[rbp+0*16], xmm4	;; Save carry1
base2	subpd	xmm2, XMM_LIMIT_BIGMAX[rax];; z1 = y1 - (maximum*BIGVAL-BIGVAL)
ttp	xload	xmm4, [rbx+256][rax]	;; col two-to-phi
ttp	mulpd	xmm4, [rdx+0*32+16]	;; two-to-phi = col * grp
ttp	movzx	rax, BYTE PTR [rdi+2]	;; Load next big vs. little flags
base2 const addpd xmm5, xmm3		;; Add in upper mul-by-const bits
base2 const mulpd xmm5, XMM_LIMIT_INVERSE[rcx];; next carry = shifted y2
base2 no const xload	xmm5, XMM_LIMIT_INVERSE[rcx];; next carry = shifted y2
base2 no const mulpd	xmm5, xmm3		;; next carry = shifted y2
	xstore	[rbp+1*16], xmm5	;; Save carry2
base2	subpd	xmm3, XMM_LIMIT_BIGMAX[rcx];; z2 = y2 - (maximum*BIGVAL-BIGVAL)
ttp	xload	xmm5, [rbx+256][rcx]	;; col two-to-phi
ttp	mulpd	xmm5, [rdx+1*32+16]	;; two-to-phi = col * grp
ttp	movzx	rcx, BYTE PTR [rdi+3]	;; Load next big vs. little flags
base2	subpd	xmm0, xmm2		;; rounded value = x1 - z1
base2	subpd	xmm1, xmm3		;; rounded value = x2 - z2
ttp	mulpd	xmm0, xmm4		;; value1 = rounded value * two-to-phi
	xstore	[rsi+0*16], xmm0	;; Save new value1
ttp	mulpd	xmm1, xmm5		;; value2 = rounded value * two-to-phi
	xstore	[rsi+1*16], xmm1	;; Save new value2

	xload	xmm0, [rsi+2*16]	;; Load values1
ttp	xload	xmm2, [rdx+2*32]	;; grp two-to-minus-phi
ttp	mulpd	xmm2, [rbx][rax]	;; Mul by col two-to-minus-phi
no ttp	xload	xmm2, [rbx]		;; two-to-minus-phi
	mulpd	xmm0, xmm2		;; Mul by fudged col two-to-minus-phi
	xload	xmm1, [rsi+3*16]	;; Load values2
ttp	xload	xmm3, [rdx+3*32]	;; grp two-to-minus-phi
ttp	mulpd	xmm3, [rbx][rcx]	;; Mul by col two-to-minus-phi
no ttp	xload	xmm3, [rbx]		;; two-to-minus-phi
	mulpd	xmm1, xmm3		;; Mul by fudged col two-to-minus-phi
no const echk error_check_interleaved sse4, xmm0, xmm4, xmm1, xmm5, xmm6
const	mul_by_const_interleaved ttp, base2, echk, sse4, xmm0, xmm4, xmm2, rax, xmm1, xmm5, xmm3, rcx, xmm6
	addpd	xmm0, [rbp+2*16]	;; x1 = values + carry
	addpd	xmm1, [rbp+3*16]	;; x2 = values + carry

no base2 rounding_interleaved ttp, base2, const, sse4, xmm0, xmm4, xmm2, rax, xmm1, xmm5, xmm3, rcx

base2	xload	xmm2, XMM_LIMIT_BIGMAX[rax];; Load maximum * BIGVAL - BIGVAL
base2	addpd	xmm2, xmm0		;; y1 = top bits of x
base2	xload	xmm3, XMM_LIMIT_BIGMAX[rcx];; Load maximum * BIGVAL - BIGVAL
base2	addpd	xmm3, xmm1		;; y2 = top bits of x
base2 const addpd xmm4, xmm2		;; Add in upper mul-by-const bits
base2 const mulpd xmm4, XMM_LIMIT_INVERSE[rax];; next carry = shifted y1
base2 no const xload xmm4, XMM_LIMIT_INVERSE[rax];; next carry = shifted y1
base2 no const mulpd xmm4, xmm2		;; next carry = shifted y1
	xstore	[rbp+2*16], xmm4	;; Save carry1
base2	subpd	xmm2, XMM_LIMIT_BIGMAX[rax];; z1 = y1 - (maximum*BIGVAL-BIGVAL)
ttp	xload	xmm4, [rbx+256][rax]	;; col two-to-phi
ttp	mulpd	xmm4, [rdx+2*32+16]	;; two-to-phi = col * grp
ttp	movzx	rax, BYTE PTR [rdi+4]	;; Load next big vs. little flags
base2 const addpd xmm5, xmm3		;; Add in upper mul-by-const bits
base2 const mulpd xmm5, XMM_LIMIT_INVERSE[rcx];; next carry = shifted y2
base2 no const xload	xmm5, XMM_LIMIT_INVERSE[rcx];; next carry = shifted y2
base2 no const mulpd	xmm5, xmm3		;; next carry = shifted y2
	xstore	[rbp+3*16], xmm5	;; Save carry2
base2	subpd	xmm3, XMM_LIMIT_BIGMAX[rcx];; z2 = y2 - (maximum*BIGVAL-BIGVAL)
ttp	xload	xmm5, [rbx+256][rcx]	;; col two-to-phi
ttp	mulpd	xmm5, [rdx+3*32+16]	;; two-to-phi = col * grp
ttp	movzx	rcx, BYTE PTR [rdi+5]	;; Load next big vs. little flags
base2	subpd	xmm0, xmm2		;; rounded value = x1 - z1
base2	subpd	xmm1, xmm3		;; rounded value = x2 - z2
ttp	mulpd	xmm0, xmm4		;; value1 *= rounded value * two-to-phi
	xstore	[rsi+2*16], xmm0	;; Save new value1
ttp	mulpd	xmm1, xmm5		;; value2 = rounded value * two-to-phi
	xstore	[rsi+3*16], xmm1	;; Save new value2
	ENDM

IFDEF X86_64
xnorm_2d MACRO ttp, echk, const, base2, sse4
ttp	xload	xmm10, [rdx+0*32]	;; grp two-to-minus-phi			;P4	;Core2
ttp	mulpd	xmm10, [rbx][rax]	;; Mul by col two-to-minus-phi		;1-6	;1-5
no ttp	xload	xmm10, [rbx]		;; two-to-minus-phi
	xload	xmm8, [rsi+0*16]	;; Load values1
ttp	xload	xmm11, [rdx+1*32]	;; grp two-to-minus-phi
ttp	mulpd	xmm11, [rbx][rcx]	;; Mul by col two-to-minus-phi		;3-8	;2-6
no ttp	xload	xmm11, [rbx]		;; two-to-minus-phi
ttp	xload	xmm2, [rdx+2*32]	;; grp two-to-minus-phi
ttp	mulpd	xmm2, [rbx][r8]		;; Mul by col two-to-minus-phi		;5-10	;3-7
no ttp	xload	xmm2, [rbx]		;; two-to-minus-phi
	xload	xmm9, [rsi+1*16]	;; Load values2
ttp	xload	xmm3, [rdx+3*32]	;; grp two-to-minus-phi
ttp	mulpd	xmm3, [rbx][r9]		;; Mul by col two-to-minus-phi		;7-12	;4-8
no ttp	xload	xmm3, [rbx]		;; two-to-minus-phi
	mulpd	xmm8, xmm10		;; Mul by fudged col two-to-minus-phi	;9-14	;6-10
	xload	xmm1, [rsi+3*16]	;; Load values4
	mulpd	xmm9, xmm11		;; Mul by fudged col two-to-minus-phi	;11-16	;7-11
no const echk error_check_interleaved sse4, xmm8, xmm12, xmm9, xmm13, xmm6
const	mul_by_const_interleaved ttp, base2, echk, sse4, xmm8, xmm12, xmm10, rax, xmm9, xmm13, xmm11, rcx, xmm6
	xload	xmm0, [rsi+2*16]	;; Load values3
	mulpd	xmm0, xmm2		;; Mul by fudged col two-to-minus-phi	;13-18	;8-12
	mulpd	xmm1, xmm3		;; Mul by fudged col two-to-minus-phi	;15-20	;9-13
no const echk error_check_interleaved sse4, xmm0, xmm4, xmm1, xmm5, xmm6
const	mul_by_const_interleaved ttp, base2, echk, sse4, xmm0, xmm4, xmm2, r8, xmm1, xmm5, xmm3, r9, xmm6
	addpd	xmm8, [rbp+0*16]	;; x1 = values + carry			;16-19	;11-13
	addpd	xmm9, [rbp+1*16]	;; x2 = values + carry			;18-21	;12-14

no base2 rounding_interleaved ttp, base2, const, sse4, xmm8, xmm12, xmm10, rax, xmm9, xmm13, xmm11, rcx

ttp	xload	xmm14, [rbx+256][rax]	;; col two-to-phi
ttp	mulpd	xmm14, [rdx+0*32+16]	;; two-to-phi = col * grp		;19-24	;12-16
	addpd	xmm0, [rbp+2*16]	;; x3 = values + carry			;20-23	;13-15
ttp	xload	xmm15, [rbx+256][rcx]	;; col two-to-phi
ttp	mulpd	xmm15, [rdx+1*32+16]	;; two-to-phi = col * grp		;21-26	;13-17
base2	xload	xmm10, XMM_LIMIT_BIGMAX[rax];; Load maximum * BIGVAL - BIGVAL
base2	addpd	xmm10, xmm8		;; y1 = top bits of x			;22-25	;14-16
	addpd	xmm1, [rbp+3*16]	;; x4 = values + carry			;24-27	;15-17

no base2 rounding_interleaved ttp, base2, const, sse4, xmm0, xmm4, xmm2, r8, xmm1, xmm5, xmm3, r9

base2	xload	xmm11, XMM_LIMIT_BIGMAX[rcx];; Load maximum * BIGVAL - BIGVAL
base2	addpd	xmm11, xmm9		;; y2 = top bits of x			;26-29	;16-18
base2 const addpd xmm12, xmm10		;; Add in upper mul-by-const bits
base2 const mulpd xmm12, XMM_LIMIT_INVERSE[rax];; next carry = shifted y1
base2 no const xload xmm12, XMM_LIMIT_INVERSE[rax];; next carry = shifted y1
base2 no const mulpd xmm12, xmm10	;; next carry = shifted y1		;27-32	;17-21
base2	xload	xmm2, XMM_LIMIT_BIGMAX[r8];; Load maximum * BIGVAL - BIGVAL
base2	addpd	xmm2, xmm0		;; y3 = top bits of x			;28-31	;17-19
base2	xload	xmm3, XMM_LIMIT_BIGMAX[r9];; Load maximum * BIGVAL - BIGVAL
base2	addpd	xmm3, xmm1		;; y4 = top bits of x			;30-33	;18-20
base2 const addpd xmm13, xmm11		;; Add in upper mul-by-const bits
base2 const mulpd xmm13, XMM_LIMIT_INVERSE[rcx];; next carry = shifted y2
base2 no const xload xmm13, XMM_LIMIT_INVERSE[rcx];; next carry = shifted y2
base2 no const mulpd xmm13, xmm11	;; next carry = shifted y2		;31-36	;19-23
base2	subpd	xmm10, XMM_LIMIT_BIGMAX[rax];; z1 = y1 - (maximum*BIGVAL-BIGVAL) ;32-35	;19-22
ttp	movzx	rax, BYTE PTR [rdi+4]	;; Load next big vs. little flags
base2 const addpd xmm4, xmm2		;; Add in upper mul-by-const bits
base2 const mulpd xmm4, XMM_LIMIT_INVERSE[r8];; next carry = shifted y3
base2 no const xload xmm4, XMM_LIMIT_INVERSE[r8];; next carry = shifted y3
base2 no const mulpd xmm4, xmm2		;; next carry = shifted y3		;33-38	;20-24
base2	subpd	xmm11, XMM_LIMIT_BIGMAX[rcx];; z2 = y2 - (maximum*BIGVAL-BIGVAL) ;34-37	;20-22
ttp	movzx	rcx, BYTE PTR [rdi+5]	;; Load next big vs. little flags
base2 const addpd xmm5, xmm3		;; Add in upper mul-by-const bits
base2 const mulpd xmm5, XMM_LIMIT_INVERSE[r9];; next carry = shifted y4
base2 no const xload xmm5, XMM_LIMIT_INVERSE[r9];; next carry = shifted y4
base2 no const mulpd xmm5, xmm3		;; next carry = shifted y4		;35-40	;21-25
	xstore	[rbp+0*16], xmm12	;; Save carry1
base2	subpd	xmm2, XMM_LIMIT_BIGMAX[r8];; z3 = y3 - (maximum*BIGVAL-BIGVAL)	;36-39	;21-23
ttp	xload	xmm12, [rbx+256][r8]	;; col two-to-phi
ttp	mulpd	xmm12, [rdx+2*32+16]	;; two-to-phi = col * grp		;37-42	;22-26
ttp	movzx	r8, BYTE PTR [rdi+6]	;; Load next big vs. little flags
	xstore	[rbp+1*16], xmm13	;; Save carry2
base2	subpd	xmm3, XMM_LIMIT_BIGMAX[r9];; z4 = y4 - (maximum*BIGVAL-BIGVAL)	;38-41	;22-24
ttp	xload	xmm13, [rbx+256][r9]	;; col two-to-phi
ttp	mulpd	xmm13, [rdx+3*32+16]	;; two-to-phi = col * grp		;39-43	;23-27
ttp	movzx	r9, BYTE PTR [rdi+7]	;; Load next big vs. little flags
base2	subpd	xmm8, xmm10		;; rounded value = x1 - z1		;40-43	;23-25
	xstore	[rbp+2*16], xmm4	;; Save carry3
base2	subpd	xmm9, xmm11		;; rounded value = x2 - z2		;42-45	;24-26
	xstore	[rbp+3*16], xmm5	;; Save carry4
base2	subpd	xmm0, xmm2		;; rounded value = x3 - z3		;44-47	;25-27
ttp	mulpd	xmm8, xmm14		;; value1 = rounded value * two-to-phi	;45-50	;26-30
base2	subpd	xmm1, xmm3		;; rounded value = x4 - z4		;46-49	;26-28
ttp	mulpd	xmm9, xmm15		;; value2 = rounded value * two-to-phi	;47-52	;27-31
ttp	mulpd	xmm0, xmm12		;; value3 = rounded value * two-to-phi	;49-54	;28-32
ttp	mulpd	xmm1, xmm13		;; value4 = rounded value * two-to-phi	;51-56	;29-33
	xstore	[rsi+0*16], xmm8	;; Save new value1
	xstore	[rsi+1*16], xmm9	;; Save new value2
	xstore	[rsi+2*16], xmm0	;; Save new value3
	xstore	[rsi+3*16], xmm1	;; Save new value4
	ENDM
ENDIF

;; NOTE: We'd rather store the high FFT carry without the XMM_BIGVAL added in,
;; but there is too much code that expects this (like add and subtract).

xnorm_2d_zpad MACRO ttp, echk, const, base2, sse4, khi, c1, cm1
	xload	xmm2, [rbx][rax]	;; col two-to-minus-phi
ttp	mulpd	xmm2, [rdx+0*32]	;; Mul by grp two-to-minus-phi
	xload	xmm0, [rsi]		;; Load values1
	xload	xmm1, [rsi+2*16]	;; Load values2
	mulpd	xmm0, xmm2		;; Mul by fudged col two-to-minus-phi
	mulpd	xmm1, xmm2		;; Mul by fudged col two-to-minus-phi

	xload	xmm3, [rbp+2*16]	;; Add in previous high FFT data
	split_lower_zpad_word echk, base2, sse4, xmm0, xmm3, xmm4, rax
	xstore	[rbp+2*16], xmm3

no const	xload	xmm0, XMM_K_LO
const		xload	xmm0, XMM_K_TIMES_MULCONST_LO
		mulpd	xmm0, xmm4
khi no const	xload	xmm5, XMM_K_HI
khi const	xload	xmm5, XMM_K_TIMES_MULCONST_HI
khi no base2	mulpd	xmm5, XMM_LIMIT_INVERSE[rax] ;; Non-base2 rounding needs shifted carry
khi		mulpd	xmm5, xmm4
 
		addpd	xmm0, [rbp+0*16]	;; x1 = values + carry

c1		mulpd   xmm1, XMM_MINUS_C	;; Do one mul before split rather than two after split

	split_upper_zpad_word echk, base2, sse4, xmm1, xmm4, xmm2, rax

no const no c1 no cm1	mulpd	xmm2, XMM_MINUS_C
no const no c1 no cm1	mulpd	xmm4, XMM_MINUS_C
const			mulpd	xmm2, XMM_MINUS_C_TIMES_MULCONST
const			mulpd	xmm4, XMM_MINUS_C_TIMES_MULCONST

	addpd	xmm0, xmm2		;; Add upper FFT word to lower FFT word
khi	addpd	xmm4, xmm5		;; Add upper FFT word to lower FFT word

	rounding ttp, base2, exec, sse4, xmm0, xmm4, xmm2, rax

ttp	xload	xmm5, [rbx+256][rax]	;; col two-to-phi
ttp	mulpd	xmm5, [rdx+0*32+16]	;; new value1 = val * grp two-to-phi
ttp	movzx	rax, BYTE PTR [rdi+1]	;; Load next big vs. little flags
ttp	mulpd	xmm0, xmm5		;; new value1 *= fudged col two-to-phi
	xstore	[rbp+0*16], xmm4	;; Save carry
	xstore	[rsi], xmm0		;; Save new value1

	xload	xmm3, [rbx][rax]	;; col two-to-minus-phi
ttp	mulpd	xmm3, [rdx+1*32]	;; Mul by grp two-to-minus-phi
	xload	xmm5, [rsi+16]		;; Load high values1
	xload	xmm1, [rsi+3*16]	;; Load high values2
	mulpd	xmm5, xmm3		;; Mul by fudged col two-to-minus-phi
	mulpd	xmm1, xmm3		;; Mul by fudged col two-to-minus-phi

	xload	xmm3, [rbp+3*16]	;; Add in previous high FFT data
	split_lower_zpad_word echk, base2, sse4, xmm5, xmm3, xmm2, rax
	xstore	[rbp+3*16], xmm3

no const	xload	xmm0, XMM_K_LO
const		xload	xmm0, XMM_K_TIMES_MULCONST_LO
		mulpd	xmm0, xmm2
khi no const	xload	xmm5, XMM_K_HI
khi const	xload	xmm5, XMM_K_TIMES_MULCONST_HI
khi no base2	mulpd	xmm5, XMM_LIMIT_INVERSE[rax] ;; Non-base2 rounding needs shifted carry
khi		mulpd	xmm5, xmm2

		addpd	xmm0, [rbp+1*16]	;; x2 = values + carry

c1		mulpd   xmm1, XMM_MINUS_C	;; Do one mul before split rather than two after split

	split_upper_zpad_word echk, base2, sse4, xmm1, xmm2, xmm4, rax

no const no c1 no cm1	mulpd	xmm4, XMM_MINUS_C
no const no c1 no cm1	mulpd	xmm2, XMM_MINUS_C
const			mulpd	xmm4, XMM_MINUS_C_TIMES_MULCONST
const			mulpd	xmm2, XMM_MINUS_C_TIMES_MULCONST

	addpd	xmm0, xmm4		;; Add upper FFT word to lower FFT word
khi	addpd	xmm2, xmm5		;; Add upper FFT word to lower FFT word

	rounding ttp, base2, exec, sse4, xmm0, xmm2, xmm4, rax

ttp	xload	xmm3, [rbx+256][rax]	;; col two-to-phi
ttp	mulpd	xmm3, [rdx+1*32+16]	;; new value2 = val * grp two-to-phi
ttp	movzx	rax, BYTE PTR [rdi+4]	;; Load next big vs. little flags
ttp	mulpd	xmm0, xmm3		;; new value2 *= fudged col two-to-phi
	xstore	[rbp+1*16], xmm2	;; Save carry
	xstore	[rsi+1*16], xmm0	;; Save new value2

	subpd	xmm1, xmm1		;; new high values = zero
	xstore	[rsi+2*16], xmm1	;; Zero high value1
	xstore	[rsi+3*16], xmm1	;; Zero high value2
	ENDM


;; 64-bit implementation using extra registers

IFDEF X86_64

xnorm_2d_zpad MACRO ttp, echk, const, base2, sse4, khi, c1, cm1
	xload	xmm2, [rbx][rax]	;; col two-to-minus-phi
ttp	mulpd	xmm2, [rdx+0*32]	;; Mul by grp two-to-minus-phi
	xload	xmm0, [rsi]		;; Load values1
	xload	xmm1, [rsi+2*16]	;; Load values2
	mulpd	xmm0, xmm2		;; Mul by fudged col two-to-minus-phi
	mulpd	xmm1, xmm2		;; Mul by fudged col two-to-minus-phi

	split_lower_zpad_word echk, base2, sse4, xmm0, xmm3, xmm5, rax

no const	xload	xmm0, XMM_K_LO
const		xload	xmm0, XMM_K_TIMES_MULCONST_LO
		mulpd	xmm0, xmm5

		addpd	xmm0, xmm4	;; x1 = values + carry

khi no const	xload	xmm4, XMM_K_HI
khi const	xload	xmm4, XMM_K_TIMES_MULCONST_HI
khi no base2	mulpd	xmm4, XMM_LIMIT_INVERSE[rax] ;; Non-base2 rounding needs shifted carry
khi		mulpd	xmm5, xmm4
 
c1		mulpd   xmm1, XMM_MINUS_C	;; Do one mul before split rather than two after split

	split_upper_zpad_word echk, base2, sse4, xmm1, xmm4, xmm2, rax

no const no c1 no cm1	mulpd	xmm2, XMM_MINUS_C
no const no c1 no cm1	mulpd	xmm4, XMM_MINUS_C
const			mulpd	xmm2, XMM_MINUS_C_TIMES_MULCONST
const			mulpd	xmm4, XMM_MINUS_C_TIMES_MULCONST

	addpd	xmm0, xmm2		;; Add upper FFT word to lower FFT word
khi	addpd	xmm4, xmm5		;; Add upper FFT word to lower FFT word

ttp	movzx	rcx, BYTE PTR [rdi+1]	;; Load next big vs. little flags
	xload	xmm10, [rbx][rcx]	;; col two-to-minus-phi
ttp	mulpd	xmm10, [rdx+1*32]	;; Mul by grp two-to-minus-phi
	xload	xmm8, [rsi+16]		;; Load high values1
	xload	xmm9, [rsi+3*16]	;; Load high values2
	mulpd	xmm8, xmm10		;; Mul by fudged col two-to-minus-phi
	mulpd	xmm9, xmm10		;; Mul by fudged col two-to-minus-phi

	split_lower_zpad_word echk, base2, sse4, xmm8, xmm11, xmm13, rcx

no const	xload	xmm8, XMM_K_LO
const		xload	xmm8, XMM_K_TIMES_MULCONST_LO
		mulpd	xmm8, xmm13

		addpd	xmm8, xmm12	;; x2 = values + carry

khi no const	xload	xmm12, XMM_K_HI
khi const	xload	xmm12, XMM_K_TIMES_MULCONST_HI
khi no base2	mulpd	xmm12, XMM_LIMIT_INVERSE[rcx] ;; Non-base2 rounding needs shifted carry
khi		mulpd	xmm13, xmm12

c1		mulpd   xmm9, XMM_MINUS_C	;; Do one mul before split rather than two after split

	split_upper_zpad_word echk, base2, sse4, xmm9, xmm12, xmm10, rcx

no const no c1 no cm1	mulpd	xmm10, XMM_MINUS_C
no const no c1 no cm1	mulpd	xmm12, XMM_MINUS_C
const			mulpd	xmm10, XMM_MINUS_C_TIMES_MULCONST
const			mulpd	xmm12, XMM_MINUS_C_TIMES_MULCONST

	addpd	xmm8, xmm10		;; Add upper FFT word to lower FFT word
khi	addpd	xmm12, xmm13		;; Add upper FFT word to lower FFT word

	rounding_interleaved ttp, base2, exec, sse4, xmm0, xmm4, xmm2, rax, xmm8, xmm12, xmm10, rcx

ttp	xload	xmm5, [rbx+256][rax]	;; col two-to-phi
ttp	mulpd	xmm5, [rdx+0*32+16]	;; new value1 = val * grp two-to-phi
ttp	mulpd	xmm0, xmm5		;; new value1 *= fudged col two-to-phi
	xstore	[rsi], xmm0		;; Save new value1

ttp	xload	xmm13, [rbx+256][rcx]	;; col two-to-phi
ttp	mulpd	xmm13, [rdx+1*32+16]	;; new value2 = val * grp two-to-phi
ttp	movzx	rax, BYTE PTR [rdi+4]	;; Load next big vs. little flags
ttp	mulpd	xmm8, xmm13		;; new value2 *= fudged col two-to-phi
	xstore	[rsi+1*16], xmm8	;; Save new value2

	subpd	xmm1, xmm1		;; new high values = zero
	xstore	[rsi+2*16], xmm1	;; Zero high value1
	xstore	[rsi+3*16], xmm1	;; Zero high value2
	ENDM
ENDIF


; *************** Top carry adjust macro ******************
; This macro corrects the carry out of the topmost word when k is not 1.
; The problem is the top carry is from b^ceil(logb(k)+n) rather than at k*b^n.
; So we recompute the top carry by multiplying by b^ceil(logb(k)) and then
; dividing by k.  The integer part is the new carry and the remainder is
; added back to the top three words.

; The single-pass case, the top carry is in high word of xmm3
xnorm_top_carry_1d MACRO
	xnorm_top_carry_cmn rsi, xmm3, 0
	ENDM

; The multi-pass case.  The top carry is loaded into xmm7 from the
; carries array.
xnorm_top_carry MACRO
	xnorm_top_carry_cmn rsi, xmm7, 1
	ENDM

xnorm_top_carry_cmn MACRO srcreg, xreg, twopass
	LOCAL	kok
	cmp	TOP_CARRY_NEEDS_ADJUSTING, 1 ;; Does top carry need work?
	jne	kok			;; Skip this code if K is 1

	IF twopass EQ 1			;; Two pass case - load the carry
	mov	rdi, carries		;; Addr of the carries
	mov	eax, addcount1		;; Load count of carry rows
	shl	rax, 6			;; Compute addr of the high carries
	add	rdi, rax
	movsd	xreg, Q [rdi-8]		;; Load very last carry
	ENDIF

	subsd	xreg, Q XMM_BIGVAL	;; Convert top carry from int+BIGVAL state
	movsd	Q XMM_TMP6, xreg

	;; We want to calculate carry * b^ceil(logb(k)) / k and
	;; carry * b^ceil(logb(k)) % k.  This must be done very carefully as
	;; carry * b^ceil(logb(k)) may not fit in 53 bits.

	;; Here is a strategy that works for k values up to and including 34 bits.
	;; We do lots of modulo k operations along the way to insure all intermediate
	;; results are 51 bits or less.
	;; Calculate y = carry % k.  This will fit in 34 bits.
	;; Let z = b^ceil(logb(k)) % k.  Precalculate high_17_bits(z) and low_17_bits(z)
	;; Remainder is (high_17_bits(z) * y % k * 2^17 + low_17_bits(z) * y) % k

	movsd	xmm0, INVERSE_K
	mulsd	xmm0, xreg		;; Mul top carry by 1/k
	addsd	xmm0, Q XMM_BIGVAL	;; Integer part
	subsd	xmm0, Q XMM_BIGVAL
	mulsd	xmm0, K
	subsd	xreg, xmm0		;; y = carry % k

	movsd	xmm0, CARRY_ADJUST1_HI	;; high_17_bits(z)
	mulsd	xmm0, xreg		;; high_17_bits(z) * y
	movsd	xmm1, INVERSE_K
	mulsd	xmm1, xmm0		;; Mul high_17_bits(z) * y by 1/k
	addsd	xmm1, Q XMM_BIGVAL	;; Integer part
	subsd	xmm1, Q XMM_BIGVAL
	mulsd	xmm1, K
	subsd	xmm0, xmm1		;; high_17_bits(z) * y % k
	mulsd	xmm0, TWO_TO_17		;; high_17_bits(z) * y % k * 2^17
	mulsd	xreg, CARRY_ADJUST1_LO	;; low_17_bits(z) * y
	addsd	xmm0, xreg		;; high_17_bits(z) * y % k * 2^17 + low_17_bits(z) * y

	movsd	xmm1, INVERSE_K
	mulsd	xmm1, xmm0		;; Mul by 1/k
	addsd	xmm1, Q XMM_BIGVAL	;; Integer part
	subsd	xmm1, Q XMM_BIGVAL
	mulsd	xmm1, K
	subsd	xmm0, xmm1		;; Remainder!!!

	;; Finally calculate integer_part = (carry * b^ceil(logb(k)) - remainder) / k

	movsd	xreg, Q XMM_TMP6		;; Reload top carry
	mulsd	xreg, CARRY_ADJUST1	;; Mul by b^ceil(logb(k))
	subsd	xreg, xmm0		;; Subtract the remainder
	mulsd	xreg, INVERSE_K		;; Mul by 1/k
	addsd	xreg, Q XMM_BIGVAL	;; Integer part of top carry over k
	subsd	xreg, Q XMM_BIGVAL

	;; Now add the remainder to the top words

	mulsd	xmm0, CARRY_ADJUST2	;; Shift remainder
	movsd	xmm1, Q XMM_BIGVAL	;; Integer part of shifted remainder
	addsd	xmm1, xmm0
	subsd	xmm1, Q XMM_BIGVAL
	subsd	xmm0, xmm1		;; Fractional part of shifted remainder
	mulsd	xmm1, CARRY_ADJUST3	;; Weight integer part

	IF twopass EQ 1			;; Two pass scratch area case
	mov	eax, HIGH_SCRATCH1_OFFSET ;; Add integer part to top word
	addsd	xmm1, Q [srcreg][rax]
	movsd	Q [srcreg][rax], xmm1
	mulsd	xmm0, CARRY_ADJUST4	;; Shift fractional part
	addsd	xmm0, Q XMM_BIGVAL
	subsd	xmm0, Q XMM_BIGVAL
	mulsd	xmm0, CARRY_ADJUST5	;; Weight fractional part
	mov	eax, HIGH_SCRATCH2_OFFSET;; Add frac part to top-1 word
	addsd	xmm0, Q [srcreg][rax]
	movsd	Q [srcreg][rax], xmm0
	ENDIF

	IF twopass EQ 2			;; Two pass FFT data case
	mov	eax, HIGH_WORD1_OFFSET	;; Add integer part to top word
	addsd	xmm1, Q [srcreg][rax]
	movsd	Q [srcreg][rax], xmm1
	mulsd	xmm0, CARRY_ADJUST4	;; Shift fractional part
	addsd	xmm0, Q XMM_BIGVAL
	subsd	xmm0, Q XMM_BIGVAL
	mulsd	xmm0, CARRY_ADJUST5	;; Weight fractional part
	mov	eax, HIGH_WORD2_OFFSET	;; Add frac part to top-1 word
	addsd	xmm0, Q [srcreg][rax]
	movsd	Q [srcreg][rax], xmm0
	ENDIF

	IF twopass EQ 0			;; Single pass case
	mov	eax, HIGH_WORD1_OFFSET	;; Add integer part to top word
	addsd	xmm1, Q [srcreg][rax]
	movsd	Q [srcreg][rax], xmm1
	mulsd	xmm0, CARRY_ADJUST4	;; Shift fractional part
	movsd	xmm1, Q XMM_BIGVAL	;; Integer part of shifted fractional
	addsd	xmm1, xmm0
	subsd	xmm1, Q XMM_BIGVAL
	subsd	xmm0, xmm1		;; Fractional part
	mulsd	xmm1, CARRY_ADJUST5	;; Weight integer part
	mov	eax, HIGH_WORD2_OFFSET	;; Add frac part to top-1 word
	addsd	xmm1, Q [srcreg][rax]
	movsd	Q [srcreg][rax], xmm1
	mulsd	xmm0, CARRY_ADJUST6	;; Shift fractional part
	addsd	xmm0, Q XMM_BIGVAL
	subsd	xmm0, Q XMM_BIGVAL
	mulsd	xmm0, CARRY_ADJUST7	;; Weight fractional part
	mov	eax, HIGH_WORD3_OFFSET	;; Add frac part to top-2 word
	addsd	xmm0, Q [srcreg][rax]
	movsd	Q [srcreg][rax], xmm0
	ENDIF

	addsd	xreg, Q XMM_BIGVAL	;; Restore carry to int+BIGVAL state

	IF twopass EQ 1
	movsd	Q [rdi-8], xreg		;; Save very last carry
	ENDIF
kok:
	ENDM

; *************** 2D followup macro ******************
; This macro finishes the normalize process by adding the final carries
; back into the appropriate FFT values.
; rsi = pointer to carries
; rbp = pointer to FFT data
; rdi = pointer to big/little flags
; rbx = pointer two-to-phi column multipliers
; rdx = pointer two-to-phi group multipliers
; ecx = big vs. little word flag #2
; eax = big vs. little word flag #1

xnorm012_2d_part1 MACRO
	LOCAL	shufsec, shuflp, shufdn, done

	cmp	THIS_BLOCK, 0		;; Are we carrying into the first data block?
	jne	done			;; If not, skip wrapping carries and negating the last carry

	mov	eax, count3		;; Load 3 section counts

shufsec:
	mov	rdi, rsi		;; Save section start
	mov	rbx, rax		;; Form count for this section
	and	rbx, 07FFh
	jz	shufdn			;; No cache lines to do.  We're all done!
	shr	rax, 11			;; Move counts list along

shuflp:	movsd	xmm4, Q [rsi+0*16]	;; Load low carry word
	movsd	Q [rsi+0*16], xmm0	;; Save prev cache line's high carry in low word
	movsd	xmm0, Q [rsi+0*16+8]	;; Load high carry for next cache line
	movsd	Q [rsi+0*16+8], xmm4	;; Move this cache line's low carry to high word
	movsd	xmm4, Q [rsi+1*16]	;; Load low carry word
	movsd	Q [rsi+1*16], xmm1	;; Save prev cache line's high carry in low word
	movsd	xmm1, Q [rsi+1*16+8]	;; Load high carry for next cache line
	movsd	Q [rsi+1*16+8], xmm4	;; Move this cache line's low carry to high word
	movsd	xmm4, Q [rsi+2*16]	;; Load low carry word
	movsd	Q [rsi+2*16], xmm2	;; Save prev cache line's high carry in low word
	movsd	xmm2, Q [rsi+2*16+8]	;; Load high carry for next cache line
	movsd	Q [rsi+2*16+8], xmm4	;; Move this cache line's low carry to high word
	movsd	xmm4, Q [rsi+3*16]	;; Load low carry word
	movsd	Q [rsi+3*16], xmm3	;; Save prev cache line's high carry in low word
	movsd	xmm3, Q [rsi+3*16+8]	;; Load high carry for next cache line
	movsd	Q [rsi+3*16+8], xmm4	;; Move this cache line's low carry to high word
	bump	rsi, 64			;; Next carry cache line
	dec	rbx			;; Test loop counter
	jnz	short shuflp		;; Next carry row in section

	movsd	Q [rdi+1*16], xmm0	;; Do extra shuffling of section's last cache line
	movsd	xmm0, xmm1
	movsd	Q [rdi+3*16], xmm2
	movsd	xmm2, xmm3
	jmp	shufsec			;; Next section

shufdn:	subsd	xmm2, Q XMM_BIGVAL
	mulsd	xmm2, Q XMM_MINUS_C	;; Negate the very last carry
	addsd	xmm2, Q XMM_BIGVAL
	mov	rsi, carries		;; Reload carries array pointer
	movsd	Q [rsi+0*16], xmm2	;; Move last cache line's high carries into first cache line
	movsd	Q [rsi+2*16], xmm0

done:
	ENDM

xnorm012_2d MACRO base2
	LOCAL	hard, done

	;; If k or c is more than one, then there will be fewer bits-per-word.
	;; This means the carry may need to be spread over 4 words instead
	;; of just 2.

	cmp	SPREAD_CARRY_OVER_EXTRA_WORDS, 1;; Are there few bits per word?
	je	hard			;; Yes, go do it the hard way

	movzx	rax, BYTE PTR [rdi+0]	;; Load big vs. little flag
	movzx	rcx, BYTE PTR [rdi+4]
	xload	xmm0, [rbp+0*16]	;; FFT data
	xload	xmm6, [rbx+0*32]	;; col two-to-minus-phi
	mulpd	xmm6, XMM_NORM012_FF	;; Mul by FFTLEN/2
	mulpd	xmm0, [rdx+0*32]	;; mul by grp two-to-minus-phi
	mulpd	xmm0, XMM_TTMP_FUDGE[rax];; mul by fudge two-to-minus-phi
	mulpd	xmm0, xmm6		;; mul by col two-to-minus-phi
	addpd	xmm0, [rsi+0*16]	;; Add in the carry
	rounding exec, base2, noexec, noexec, xmm0, xmm5, xmm4, rax
	subpd	xmm5, XMM_BIGVAL
	xload	xmm7, [rdx+0*32+16]	;; grp two-to-phi
	mulpd	xmm0, xmm7		;; value *= grp two-to-phi
	mulpd	xmm0, XMM_TTP_FUDGE[rax];; value *= fudge two-to-phi
	xload	xmm2, [rbx+0*32+16]	;; col two-to-phi
	mulpd	xmm0, xmm2		;; value *= col two-to-phi
	xstore	[rbp+0*16], xmm0	;; Save FFT data
	mulpd	xmm5, xmm7		;; carry *= grp two-to-phi
	mulpd	xmm5, XMM_TTP_FUDGE[rcx];; carry *= fudge two-to-phi
	xload	xmm3, [rbx+1*32+16]	;; col two-to-phi
	mulpd	xmm5, xmm3		;; carry *= col two-to-phi
	addpd	xmm5, [rbp+4*16]	;; Add carry and FFT data
	xstore	[rbp+4*16], xmm5	;; Save FFT data

	movzx	rax, BYTE PTR [rdi+1]	;; Load big vs. little flag
	movzx	rcx, BYTE PTR [rdi+5]
	xload	xmm1, [rbp+1*16]	;; FFT data
	mulpd	xmm1, [rdx+1*32]	;; mul by grp two-to-minus-phi
	mulpd	xmm1, XMM_TTMP_FUDGE[rax];; mul by fudge two-to-minus-phi
	mulpd	xmm1, xmm6		;; mul by col two-to-minus-phi
	addpd	xmm1, [rsi+1*16]	;; Add in the carry
	rounding exec, base2, noexec, noexec, xmm1, xmm5, xmm4, rax
	subpd	xmm5, XMM_BIGVAL
	xload	xmm7, [rdx+1*32+16]	;; grp two-to-phi
	mulpd	xmm1, xmm7		;; value *= grp two-to-phi
	mulpd	xmm1, XMM_TTP_FUDGE[rax];; value *= fudge two-to-phi
	mulpd	xmm1, xmm2		;; value *= col two-to-phi
	xstore	[rbp+1*16], xmm1	;; Save FFT data
	mulpd	xmm5, xmm7		;; carry *= grp two-to-phi
	mulpd	xmm5, XMM_TTP_FUDGE[rcx];; carry *= fudge two-to-phi
	mulpd	xmm5, xmm3		;; carry *= col two-to-phi
	addpd	xmm5, [rbp+5*16]	;; Add carry and FFT data
	xstore	[rbp+5*16], xmm5	;; Save FFT data

	movzx	rax, BYTE PTR [rdi+2]	;; Load big vs. little flag
	movzx	rcx, BYTE PTR [rdi+6]
	xload	xmm0, [rbp+2*16]	;; FFT data
	mulpd	xmm0, [rdx+2*32]	;; mul by grp two-to-minus-phi
	mulpd	xmm0, XMM_TTMP_FUDGE[rax];; mul by fudge two-to-minus-phi
	mulpd	xmm0, xmm6		;; mul by col two-to-minus-phi
	addpd	xmm0, [rsi+2*16]	;; Add in the carry
	rounding exec, base2, noexec, noexec, xmm0, xmm5, xmm4, rax
	subpd	xmm5, XMM_BIGVAL
	xload	xmm7, [rdx+2*32+16]	;; grp two-to-phi
	mulpd	xmm0, xmm7		;; value *= grp two-to-phi
	mulpd	xmm0, XMM_TTP_FUDGE[rax];; value *= fudge two-to-phi
	mulpd	xmm0, xmm2		;; value *= col two-to-phi
	xstore	[rbp+2*16], xmm0	;; Save FFT data
	mulpd	xmm5, xmm7		;; carry *= grp two-to-phi
	mulpd	xmm5, XMM_TTP_FUDGE[rcx];; carry *= fudge two-to-phi
	mulpd	xmm5, xmm3		;; carry *= col two-to-phi
	addpd	xmm5, [rbp+6*16]	;; Add carry and FFT data
	xstore	[rbp+6*16], xmm5	;; Save FFT data

	movzx	rax, BYTE PTR [rdi+3]	;; Load big vs. little flag
	movzx	rcx, BYTE PTR [rdi+7]
	xload	xmm1, [rbp+3*16]	;; FFT data
	mulpd	xmm1, [rdx+3*32]	;; mul by grp two-to-minus-phi
	mulpd	xmm1, XMM_TTMP_FUDGE[rax];; mul by fudge two-to-minus-phi
	mulpd	xmm1, xmm6		;; mul by col two-to-minus-phi
	addpd	xmm1, [rsi+3*16]	;; Add in the carry
	rounding exec, base2, noexec, noexec, xmm1, xmm5, xmm4, rax
	subpd	xmm5, XMM_BIGVAL
	xload	xmm7, [rdx+3*32+16]	;; grp two-to-phi
	mulpd	xmm1, xmm7		;; value *= grp two-to-phi
	mulpd	xmm1, XMM_TTP_FUDGE[rax];; value *= fudge two-to-phi
	mulpd	xmm1, xmm2		;; value *= col two-to-phi
	xstore	[rbp+3*16], xmm1	;; Save FFT data
	mulpd	xmm5, xmm7		;; carry *= grp two-to-phi
	mulpd	xmm5, XMM_TTP_FUDGE[rcx];; carry *= fudge two-to-phi
	mulpd	xmm5, xmm3	;; carry *= col two-to-phi
	addpd	xmm5, [rbp+7*16]	;; Add carry and FFT data
	xstore	[rbp+7*16], xmm5	;; Save FFT data
	jmp	done

;; Same as above, but spread carry over 6 words

hard:	movzx	rax, BYTE PTR [rdi+0]	;; Load big vs. little flag
	xload	xmm0, [rbp+0*16]	;; FFT data
	xload	xmm6, [rdx+0*32]	;; grp two-to-minus-phi
	mulpd	xmm6, XMM_NORM012_FF	;; Mul by FFTLEN/2
	mulpd	xmm0, XMM_TTMP_FUDGE[rax];; mul by fudge two-to-minus-phi
	mulpd	xmm0, xmm6		;; mul by grp two-to-minus-phi
	mulpd	xmm0, [rbx+0*32]	;; mul by col two-to-minus-phi
	addpd	xmm0, [rsi+0*16]	;; Add in the carry
	rounding exec, base2, noexec, noexec, xmm0, xmm5, xmm4, rax
	xload	xmm7, [rdx+0*32+16]	;; grp two-to-phi
	mulpd	xmm0, xmm7		;; value *= grp two-to-phi
	mulpd	xmm0, XMM_TTP_FUDGE[rax];; value *= fudge two-to-phi
	mulpd	xmm0, [rbx+0*32+16]	;; value *= col two-to-phi
	xstore	[rbp+0*16], xmm0	;; Save FFT data

	movzx	rax, BYTE PTR [rdi+4]	;; Load big vs. little flag
	xload	xmm0, [rbp+4*16]	;; FFT data
	mulpd	xmm0, xmm6		;; mul by grp two-to-minus-phi
	mulpd	xmm0, XMM_TTMP_FUDGE[rax];; mul by fudge two-to-minus-phi
	mulpd	xmm0, [rbx+1*32]	;; mul by col two-to-minus-phi
	addpd	xmm0, xmm5		;; Add in the carry
	rounding exec, base2, noexec, noexec, xmm0, xmm5, xmm4, rax
	mulpd	xmm0, xmm7		;; value *= grp two-to-phi
	mulpd	xmm0, XMM_TTP_FUDGE[rax];; value *= fudge two-to-phi
	mulpd	xmm0, [rbx+1*32+16]	;; value *= col two-to-phi
	xstore	[rbp+4*16], xmm0	;; Save FFT data

	mov	ecx, BIGLIT_INCR2	;; Different clm values step through
					;; big/lit array differently
	movzx	rax, BYTE PTR [rdi+rcx]	;; Load big vs. little flag
	xload	xmm0, [rbp+8*16]	;; FFT data
	mulpd	xmm0, xmm6		;; mul by grp two-to-minus-phi
	mulpd	xmm0, XMM_TTMP_FUDGE[rax];; mul by fudge two-to-minus-phi
	mulpd	xmm0, [rbx+2*32]	;; mul by col two-to-minus-phi
	addpd	xmm0, xmm5		;; Add in the carry
	rounding exec, base2, noexec, noexec, xmm0, xmm5, xmm4, rax
	mulpd	xmm0, xmm7		;; value *= grp two-to-phi
	mulpd	xmm0, XMM_TTP_FUDGE[rax];; value *= fudge two-to-phi
	mulpd	xmm0, [rbx+2*32+16]	;; value *= col two-to-phi
	xstore	[rbp+8*16], xmm0	;; Save FFT data

	movzx	rax, BYTE PTR [rdi+rcx+4] ;; Load big vs. little flag
	xload	xmm0, [rbp+12*16]	;; FFT data
	mulpd	xmm0, xmm6		;; mul by grp two-to-minus-phi
	mulpd	xmm0, XMM_TTMP_FUDGE[rax];; mul by fudge two-to-minus-phi
	mulpd	xmm0, [rbx+3*32]	;; mul by col two-to-minus-phi
	addpd	xmm0, xmm5		;; Add in the carry
	rounding exec, base2, noexec, noexec, xmm0, xmm5, xmm4, rax
	mulpd	xmm0, xmm7		;; value *= grp two-to-phi
	mulpd	xmm0, XMM_TTP_FUDGE[rax];; value *= fudge two-to-phi
	mulpd	xmm0, [rbx+3*32+16]	;; value *= col two-to-phi
	xstore	[rbp+12*16], xmm0	;; Save FFT data

	mov	ecx, BIGLIT_INCR4	;; Different clm values step through
					;; big/lit array differently
	movzx	rax, BYTE PTR [rdi+rcx]	;; Load big vs. little flag
	xload	xmm0, [rbp+16*16]	;; FFT data
	mulpd	xmm0, xmm6		;; mul by grp two-to-minus-phi
	mulpd	xmm0, XMM_TTMP_FUDGE[rax];; mul by fudge two-to-minus-phi
	mulpd	xmm0, [rbx+4*32]	;; mul by col two-to-minus-phi
	addpd	xmm0, xmm5		;; Add in the carry
	rounding exec, base2, noexec, noexec, xmm0, xmm5, xmm4, rax
	mulpd	xmm0, xmm7		;; value *= grp two-to-phi
	mulpd	xmm0, XMM_TTP_FUDGE[rax];; value *= fudge two-to-phi
	mulpd	xmm0, [rbx+4*32+16]	;; value *= col two-to-phi
	xstore	[rbp+16*16], xmm0	;; Save FFT data

	movzx	rax, BYTE PTR [rdi+rcx+4]
	subpd	xmm5, XMM_BIGVAL
	mulpd	xmm5, xmm7		;; carry *= grp two-to-phi
	mulpd	xmm5, XMM_TTP_FUDGE[rax];; carry *= fudge two-to-phi
	mulpd	xmm5, [rbx+5*32+16]	;; carry *= col two-to-phi
	addpd	xmm5, [rbp+20*16]	;; Add carry and FFT data
	xstore	[rbp+20*16], xmm5	;; Save FFT data


	movzx	rax, BYTE PTR [rdi+1]	;; Load big vs. little flag
	xload	xmm1, [rbp+1*16]	;; FFT data
	xload	xmm6, [rdx+1*32]	;; grp two-to-minus-phi
	mulpd	xmm6, XMM_NORM012_FF	;; Mul by FFTLEN/2
	mulpd	xmm1, XMM_TTMP_FUDGE[rax];; mul by fudge two-to-minus-phi
	mulpd	xmm1, xmm6		;; mul by grp two-to-minus-phi
	mulpd	xmm1, [rbx+0*32]	;; mul by col two-to-minus-phi
	addpd	xmm1, [rsi+1*16]	;; Add in the carry
	rounding exec, base2, noexec, noexec, xmm1, xmm5, xmm4, rax
	xload	xmm7, [rdx+1*32+16]	;; grp two-to-phi
	mulpd	xmm1, xmm7		;; value *= grp two-to-phi
	mulpd	xmm1, XMM_TTP_FUDGE[rax];; value *= fudge two-to-phi
	mulpd	xmm1, [rbx+0*32+16]	;; value *= col two-to-phi
	xstore	[rbp+1*16], xmm1	;; Save FFT data

	movzx	rax, BYTE PTR [rdi+5]	;; Load big vs. little flag
	xload	xmm1, [rbp+5*16]	;; FFT data
	mulpd	xmm1, xmm6		;; mul by grp two-to-minus-phi
	mulpd	xmm1, XMM_TTMP_FUDGE[rax];; mul by fudge two-to-minus-phi
	mulpd	xmm1, [rbx+1*32]	;; mul by col two-to-minus-phi
	addpd	xmm1, xmm5		;; Add in the carry
	rounding exec, base2, noexec, noexec, xmm1, xmm5, xmm4, rax
	mulpd	xmm1, xmm7		;; value *= grp two-to-phi
	mulpd	xmm1, XMM_TTP_FUDGE[rax];; value *= fudge two-to-phi
	mulpd	xmm1, [rbx+1*32+16]	;; value *= col two-to-phi
	xstore	[rbp+5*16], xmm1	;; Save FFT data

	mov	ecx, BIGLIT_INCR2	;; Different clm values step through
					;; big/lit array differently
	movzx	rax, BYTE PTR [rdi+rcx+1] ;; Load big vs. little flag
	xload	xmm1, [rbp+9*16]	;; FFT data
	mulpd	xmm1, xmm6		;; mul by grp two-to-minus-phi
	mulpd	xmm1, XMM_TTMP_FUDGE[rax];; mul by fudge two-to-minus-phi
	mulpd	xmm1, [rbx+2*32]	;; mul by col two-to-minus-phi
	addpd	xmm1, xmm5		;; Add in the carry
	rounding exec, base2, noexec, noexec, xmm1, xmm5, xmm4, rax
	mulpd	xmm1, xmm7		;; value *= grp two-to-phi
	mulpd	xmm1, XMM_TTP_FUDGE[rax];; value *= fudge two-to-phi
	mulpd	xmm1, [rbx+2*32+16]	;; value *= col two-to-phi
	xstore	[rbp+9*16], xmm1	;; Save FFT data

	movzx	rax, BYTE PTR [rdi+rcx+5] ;; Load big vs. little flag
	xload	xmm1, [rbp+13*16]	;; FFT data
	mulpd	xmm1, xmm6		;; mul by grp two-to-minus-phi
	mulpd	xmm1, XMM_TTMP_FUDGE[rax];; mul by fudge two-to-minus-phi
	mulpd	xmm1, [rbx+3*32]	;; mul by col two-to-minus-phi
	addpd	xmm1, xmm5		;; Add in the carry
	rounding exec, base2, noexec, noexec, xmm1, xmm5, xmm4, rax
	mulpd	xmm1, xmm7		;; value *= grp two-to-phi
	mulpd	xmm1, XMM_TTP_FUDGE[rax];; value *= fudge two-to-phi
	mulpd	xmm1, [rbx+3*32+16]	;; value *= col two-to-phi
	xstore	[rbp+13*16], xmm1	;; Save FFT data

	mov	ecx, BIGLIT_INCR4	;; Different clm values step through
					;; big/lit array differently
	movzx	rax, BYTE PTR [rdi+rcx+1] ;; Load big vs. little flag
	xload	xmm1, [rbp+17*16]	;; FFT data
	mulpd	xmm1, xmm6		;; mul by grp two-to-minus-phi
	mulpd	xmm1, XMM_TTMP_FUDGE[rax];; mul by fudge two-to-minus-phi
	mulpd	xmm1, [rbx+4*32]	;; mul by col two-to-minus-phi
	addpd	xmm1, xmm5		;; Add in the carry
	rounding exec, base2, noexec, noexec, xmm1, xmm5, xmm4, rax
	mulpd	xmm1, xmm7		;; value *= grp two-to-phi
	mulpd	xmm1, XMM_TTP_FUDGE[rax];; value *= fudge two-to-phi
	mulpd	xmm1, [rbx+4*32+16]	;; value *= col two-to-phi
	xstore	[rbp+17*16], xmm1	;; Save FFT data

	movzx	rax, BYTE PTR [rdi+rcx+5]
	subpd	xmm5, XMM_BIGVAL
	mulpd	xmm5, xmm7		;; carry *= grp two-to-phi
	mulpd	xmm5, XMM_TTP_FUDGE[rax];; carry *= fudge two-to-phi
	mulpd	xmm5, [rbx+5*32+16]	;; carry *= col two-to-phi
	addpd	xmm5, [rbp+21*16]	;; Add carry and FFT data
	xstore	[rbp+21*16], xmm5	;; Save FFT data

	movzx	rax, BYTE PTR [rdi+2]	;; Load big vs. little flag
	xload	xmm2, [rbp+2*16]	;; FFT data
	xload	xmm6, [rdx+2*32]	;; grp two-to-minus-phi
	mulpd	xmm6, XMM_NORM012_FF	;; Mul by FFTLEN/2
	mulpd	xmm2, XMM_TTMP_FUDGE[rax];; mul by fudge two-to-minus-phi
	mulpd	xmm2, xmm6		;; mul by grp two-to-minus-phi
	mulpd	xmm2, [rbx+0*32]	;; mul by col two-to-minus-phi
	addpd	xmm2, [rsi+2*16]	;; Add in the carry
	rounding exec, base2, noexec, noexec, xmm2, xmm5, xmm4, rax
	xload	xmm7, [rdx+2*32+16]	;; grp two-to-phi
	mulpd	xmm2, xmm7		;; value *= grp two-to-phi
	mulpd	xmm2, XMM_TTP_FUDGE[rax];; value *= fudge two-to-phi
	mulpd	xmm2, [rbx+0*32+16]	;; value *= col two-to-phi
	xstore	[rbp+2*16], xmm2	;; Save FFT data

	movzx	rax, BYTE PTR [rdi+6]	;; Load big vs. little flag
	xload	xmm2, [rbp+6*16]	;; FFT data
	mulpd	xmm2, xmm6		;; mul by grp two-to-minus-phi
	mulpd	xmm2, XMM_TTMP_FUDGE[rax];; mul by fudge two-to-minus-phi
	mulpd	xmm2, [rbx+1*32]	;; mul by col two-to-minus-phi
	addpd	xmm2, xmm5		;; Add in the carry
	rounding exec, base2, noexec, noexec, xmm2, xmm5, xmm4, rax
	mulpd	xmm2, xmm7		;; value *= grp two-to-phi
	mulpd	xmm2, XMM_TTP_FUDGE[rax];; value *= fudge two-to-phi
	mulpd	xmm2, [rbx+1*32+16]	;; value *= col two-to-phi
	xstore	[rbp+6*16], xmm2	;; Save FFT data

	mov	ecx, BIGLIT_INCR2	;; Different clm values step through
					;; big/lit array differently
	movzx	rax, BYTE PTR [rdi+rcx+2] ;; Load big vs. little flag
	xload	xmm2, [rbp+10*16]	;; FFT data
	mulpd	xmm2, xmm6		;; mul by grp two-to-minus-phi
	mulpd	xmm2, XMM_TTMP_FUDGE[rax];; mul by fudge two-to-minus-phi
	mulpd	xmm2, [rbx+2*32]	;; mul by col two-to-minus-phi
	addpd	xmm2, xmm5		;; Add in the carry
	rounding exec, base2, noexec, noexec, xmm2, xmm5, xmm4, rax
	mulpd	xmm2, xmm7		;; value *= grp two-to-phi
	mulpd	xmm2, XMM_TTP_FUDGE[rax];; value *= fudge two-to-phi
	mulpd	xmm2, [rbx+2*32+16]	;; value *= col two-to-phi
	xstore	[rbp+10*16], xmm2	;; Save FFT data

	movzx	rax, BYTE PTR [rdi+rcx+6] ;; Load big vs. little flag
	xload	xmm2, [rbp+14*16]	;; FFT data
	mulpd	xmm2, xmm6		;; mul by grp two-to-minus-phi
	mulpd	xmm2, XMM_TTMP_FUDGE[rax];; mul by fudge two-to-minus-phi
	mulpd	xmm2, [rbx+3*32]	;; mul by col two-to-minus-phi
	addpd	xmm2, xmm5		;; Add in the carry
	rounding exec, base2, noexec, noexec, xmm2, xmm5, xmm4, rax
	mulpd	xmm2, xmm7		;; value *= grp two-to-phi
	mulpd	xmm2, XMM_TTP_FUDGE[rax];; value *= fudge two-to-phi
	mulpd	xmm2, [rbx+3*32+16]	;; value *= col two-to-phi
	xstore	[rbp+14*16], xmm2	;; Save FFT data

	mov	ecx, BIGLIT_INCR4	;; Different clm values step through
					;; big/lit array differently
	movzx	rax, BYTE PTR [rdi+rcx+2] ;; Load big vs. little flag
	xload	xmm2, [rbp+18*16]	;; FFT data
	mulpd	xmm2, xmm6		;; mul by grp two-to-minus-phi
	mulpd	xmm2, XMM_TTMP_FUDGE[rax];; mul by fudge two-to-minus-phi
	mulpd	xmm2, [rbx+4*32]	;; mul by col two-to-minus-phi
	addpd	xmm2, xmm5		;; Add in the carry
	rounding exec, base2, noexec, noexec, xmm2, xmm5, xmm4, rax
	mulpd	xmm2, xmm7		;; value *= grp two-to-phi
	mulpd	xmm2, XMM_TTP_FUDGE[rax];; value *= fudge two-to-phi
	mulpd	xmm2, [rbx+4*32+16]	;; value *= col two-to-phi
	xstore	[rbp+18*16], xmm2	;; Save FFT data

	movzx	rax, BYTE PTR [rdi+rcx+6]
	subpd	xmm5, XMM_BIGVAL
	mulpd	xmm5, xmm7		;; carry *= grp two-to-phi
	mulpd	xmm5, XMM_TTP_FUDGE[rax];; carry *= fudge two-to-phi
	mulpd	xmm5, [rbx+5*32+16]	;; carry *= col two-to-phi
	addpd	xmm5, [rbp+22*16]	;; Add carry and FFT data
	xstore	[rbp+22*16], xmm5	;; Save FFT data


	movzx	rax, BYTE PTR [rdi+3]	;; Load big vs. little flag
	xload	xmm3, [rbp+3*16]	;; FFT data
	xload	xmm6, [rdx+3*32]	;; grp two-to-minus-phi
	mulpd	xmm6, XMM_NORM012_FF	;; Mul by FFTLEN/2
	mulpd	xmm3, XMM_TTMP_FUDGE[rax];; mul by fudge two-to-minus-phi
	mulpd	xmm3, xmm6		;; mul by grp two-to-minus-phi
	mulpd	xmm3, [rbx+0*32]	;; mul by col two-to-minus-phi
	addpd	xmm3, [rsi+3*16]	;; Add in the carry
	rounding exec, base2, noexec, noexec, xmm3, xmm5, xmm4, rax
	xload	xmm7, [rdx+3*32+16]	;; grp two-to-phi
	mulpd	xmm3, xmm7		;; value *= grp two-to-phi
	mulpd	xmm3, XMM_TTP_FUDGE[rax];; value *= fudge two-to-phi
	mulpd	xmm3, [rbx+0*32+16]	;; value *= col two-to-phi
	xstore	[rbp+3*16], xmm3	;; Save FFT data

	movzx	rax, BYTE PTR [rdi+7]	;; Load big vs. little flag
	xload	xmm3, [rbp+7*16]	;; FFT data
	mulpd	xmm3, xmm6		;; mul by grp two-to-minus-phi
	mulpd	xmm3, XMM_TTMP_FUDGE[rax];; mul by fudge two-to-minus-phi
	mulpd	xmm3, [rbx+1*32]	;; mul by col two-to-minus-phi
	addpd	xmm3, xmm5		;; Add in the carry
	rounding exec, base2, noexec, noexec, xmm3, xmm5, xmm4, rax
	mulpd	xmm3, xmm7		;; value *= grp two-to-phi
	mulpd	xmm3, XMM_TTP_FUDGE[rax];; value *= fudge two-to-phi
	mulpd	xmm3, [rbx+1*32+16]	;; value *= col two-to-phi
	xstore	[rbp+7*16], xmm3	;; Save FFT data

	mov	ecx, BIGLIT_INCR2	;; Different clm values step through
					;; big/lit array differently
	movzx	rax, BYTE PTR [rdi+rcx+3] ;; Load big vs. little flag
	xload	xmm3, [rbp+11*16]	;; FFT data
	mulpd	xmm3, xmm6		;; mul by grp two-to-minus-phi
	mulpd	xmm3, XMM_TTMP_FUDGE[rax];; mul by fudge two-to-minus-phi
	mulpd	xmm3, [rbx+2*32]	;; mul by col two-to-minus-phi
	addpd	xmm3, xmm5		;; Add in the carry
	rounding exec, base2, noexec, noexec, xmm3, xmm5, xmm4, rax
	mulpd	xmm3, xmm7		;; value *= grp two-to-phi
	mulpd	xmm3, XMM_TTP_FUDGE[rax];; value *= fudge two-to-phi
	mulpd	xmm3, [rbx+2*32+16]	;; value *= col two-to-phi
	xstore	[rbp+11*16], xmm3	;; Save FFT data

	movzx	rax, BYTE PTR [rdi+rcx+7] ;; Load big vs. little flag
	xload	xmm3, [rbp+15*16]	;; FFT data
	mulpd	xmm3, xmm6		;; mul by grp two-to-minus-phi
	mulpd	xmm3, XMM_TTMP_FUDGE[rax];; mul by fudge two-to-minus-phi
	mulpd	xmm3, [rbx+3*32]	;; mul by col two-to-minus-phi
	addpd	xmm3, xmm5		;; Add in the carry
	rounding exec, base2, noexec, noexec, xmm3, xmm5, xmm4, rax
	mulpd	xmm3, xmm7		;; value *= grp two-to-phi
	mulpd	xmm3, XMM_TTP_FUDGE[rax];; value *= fudge two-to-phi
	mulpd	xmm3, [rbx+3*32+16]	;; value *= col two-to-phi
	xstore	[rbp+15*16], xmm3	;; Save FFT data

	mov	ecx, BIGLIT_INCR4	;; Different clm values step through
					;; big/lit array differently
	movzx	rax, BYTE PTR [rdi+rcx+3] ;; Load big vs. little flag
	xload	xmm3, [rbp+19*16]	;; FFT data
	mulpd	xmm3, xmm6		;; mul by grp two-to-minus-phi
	mulpd	xmm3, XMM_TTMP_FUDGE[rax];; mul by fudge two-to-minus-phi
	mulpd	xmm3, [rbx+4*32]	;; mul by col two-to-minus-phi
	addpd	xmm3, xmm5		;; Add in the carry
	rounding exec, base2, noexec, noexec, xmm3, xmm5, xmm4, rax
	mulpd	xmm3, xmm7		;; value *= grp two-to-phi
	mulpd	xmm3, XMM_TTP_FUDGE[rax];; value *= fudge two-to-phi
	mulpd	xmm3, [rbx+4*32+16]	;; value *= col two-to-phi
	xstore	[rbp+19*16], xmm3	;; Save FFT data

	movzx	rax, BYTE PTR [rdi+rcx+7]
	subpd	xmm5, XMM_BIGVAL
	mulpd	xmm5, xmm7		;; carry *= grp two-to-phi
	mulpd	xmm5, XMM_TTP_FUDGE[rax];; carry *= fudge two-to-phi
	mulpd	xmm5, [rbx+5*32+16]	;; carry *= col two-to-phi
	addpd	xmm5, [rbp+23*16]	;; Add carry and FFT data
	xstore	[rbp+23*16], xmm5	;; Save FFT data

done:	xload	xmm4, XMM_BIGVAL
	xstore	[rsi+0*16], xmm4	;; Clear carry
	xstore	[rsi+1*16], xmm4
	xstore	[rsi+2*16], xmm4
	xstore	[rsi+3*16], xmm4
	ENDM


;; Significantly different cleanup code for zero-padded FFTs.
;; Note: The group multiplier should be 1.0 for the bottom FFT words and
;; the FFT words just above the half-way point.

xnorm012_2d_zpad_part1 MACRO
	LOCAL	shufsec, shuflp, shufdn, b2, zpaddn, done

	cmp	THIS_BLOCK, 0		;; Are we carrying into the first data block?
	jne	done			;; If not, skip wrapping carries and negating the last carry

	mov	eax, count3		;; Load 3 section counts

shufsec:
	mov	rdi, rsi		;; Save section start
	mov	rbx, rax		;; Form count for this section
	and	rbx, 07FFh
	jz	shufdn			;; No cache lines to do.  We're all done!
	shr	rax, 11			;; Move counts list along

shuflp:	movsd	xmm4, Q [rsi+0*16]	;; Load low carry word
	movsd	Q [rsi+0*16], xmm0	;; Save prev cache line's high carry in low word
	movsd	xmm0, Q [rsi+0*16+8]	;; Load high carry for next cache line
	movsd	Q [rsi+0*16+8], xmm4	;; Move this cache line's low carry to high word
	movsd	xmm4, Q [rsi+1*16]	;; Load low carry word
	movsd	Q [rsi+1*16], xmm2	;; Save prev cache line's high carry in low word
	movsd	xmm2, Q [rsi+1*16+8]	;; Load high carry for next cache line
	movsd	Q [rsi+1*16+8], xmm4	;; Move this cache line's low carry to high word
	movsd	xmm4, Q [rsi+2*16]	;; Load low carry word
	movsd	Q [rsi+2*16], xmm1	;; Save prev cache line's high carry in low word
	movsd	xmm1, Q [rsi+2*16+8]	;; Load high carry for next cache line
	movsd	Q [rsi+2*16+8], xmm4	;; Move this cache line's low carry to high word
	movsd	xmm4, Q [rsi+3*16]	;; Load low carry word
	movsd	Q [rsi+3*16], xmm3	;; Save prev cache line's high carry in low word
	movsd	xmm3, Q [rsi+3*16+8]	;; Load high carry for next cache line
	movsd	Q [rsi+3*16+8], xmm4	;; Move this cache line's low carry to high word
	bump	rsi, 64			;; Next carry cache line
	dec	rbx			;; Test loop counter
	jnz	short shuflp		;; Next carry row in section

	movsd	Q [rdi+1*16], xmm0	;; Do extra shuffling of section's last cache line
	movsd	xmm0, xmm2
	movsd	Q [rdi+3*16], xmm1
	movsd	xmm1, xmm3
	jmp	shufsec			;; Next section

shufdn:	mov	rsi, DESTARG		;; Load FFT data pointer
	mov	rdi, norm_biglit_array	;; Addr of the big/little flags array
	mov	rbp, norm_col_mults	;; Addr of the group multipliers
	cmp	B_IS_2, 0		;; Is b = 2?
	jne	b2			;; Yes, do simpler rounding
	xnorm012_2d_zpad_part1a noexec	;; No, do harder rounding
	jmp	zpaddn
b2:	xnorm012_2d_zpad_part1a exec

zpaddn:	mov	rsi, carries		;; Reload carries pointer
	movsd	xmm7, Q XMM_BIGVAL	;; Clear two carries just processed by xnorm012_2d_zpad_part1_cmn
	subsd	xmm6, xmm6
	movsd	Q [rsi], xmm7
	movsd	Q [rsi+32], xmm6
done:
	ENDM

xnorm012_2d_zpad_part1a MACRO base2
	LOCAL	noncon, done
	cmp	const_fft, 0		;; Are we also multiplying by a constant?
	je	noncon			;; Jump if not const
	xnorm012_2d_zpad_part1_cmn exec, base2
	jmp	done
noncon:	xnorm012_2d_zpad_part1_cmn noexec, base2
done:
	ENDM

xnorm012_2d_zpad_part1_cmn MACRO const, base2
	LOCAL	smallk, mediumk, div_k_done

	;; Strip BIGVAL from the traditional carry, we'll add the traditional
	;; carry in later when we are working on the ZPAD0 - ZPAD6 values.
	subsd	xmm2, Q XMM_BIGVAL	;; Integerize traditional carry

	;; Rather than calculate high FFT carry times k and then later dividing
	;; by k, we multiply FFT high carry by const and we'll add it
	;; to the lower FFT data later (after multiplying by -c).
const	mulsd	xmm3, Q XMM_MULCONST

	;; Work on zero-pad addin value.
	movsd	xmm4, Q ADDIN_VALUE		;; Load the add in value
const	mulsd	xmm4, Q XMM_MULCONST		;; Multiply the add in value by the small mul const
	addsd	xmm4, POSTADDIN_VALUE		;; Add the post-mul-by-const addin value
	;; when c = -1, 1 = b^n
	;; when c = 1, -1 = b^n, 1 = -b^n
	;; when c = -3, 3 = b^n, 1 = b^n - 2
	;; when c = 3, -3 = b^n, 3 = -b^n, 1 = -b^n - 2
	;; The "- 2" has been precomputed in ZPAD_LSW_ADJUST.  Add ADDIN_VALUE * mul-by-const * ZPAD_LSW_ADJUST into the least significant FFT word
	mov	rsi, DESTARG			;; Address of squared number
	movsd	xmm0, Q ZPAD_LSW_ADJUST
	mulsd	xmm0, xmm4
	addsd	xmm0, [rsi]
	movsd	[rsi], xmm0

	;; Multiply ZPAD0 through ZPAD6 by const * -C.  This, in essense,
	;; wraps this data from above the FFT data area to the halfway point.
	;; Later on we'll divide this by K to decide which data needs wrapping
	;; all the way down to the bottom of the FFT data.

	;; NOTE that ZPAD0's column multiplier is 1.0.  Also, ZPAD6 will not
	;; be bigger than a big word.  We must be careful to handle c's up
	;; to about 30 bits

	movzx	rax, BYTE PTR [rdi]	;; Load big vs. little flags
	movsd	xmm0, ZPAD0		;; Load values1
	subsd	xmm5, xmm5		;; Create a zero high FFT carry to add in
	single_split_lower_zpad_word base2, xmm0, xmm5, xmm1, rax
no const mulsd	xmm0, Q XMM_MINUS_C
const	mulsd	xmm0, Q XMM_MINUS_C_TIMES_MULCONST
	addsd	xmm0, xmm4		;; Apply rest of ADDIN_VALUE * mul-by-const here (the FFT half way point)
	round_zpad7_word base2, xmm0, xmm2, xmm1, rax
	movsd	ZPAD0, xmm0

	movzx	rax, BYTE PTR [rdi+4]	;; Load big vs. little flags
	movsd	xmm0, ZPAD1		;; Load values1
	mulsd	xmm0, Q [rbp+32]	;; Mul values1 by two-to-minus-phi
	mulsd	xmm0, Q XMM_NORM012_FF	;; Mul by FFTLEN/2
	single_split_lower_zpad_word base2, xmm0, xmm5, xmm1, rax
no const mulsd	xmm0, Q XMM_MINUS_C
const	mulsd	xmm0, Q XMM_MINUS_C_TIMES_MULCONST
	round_zpad7_word base2, xmm0, xmm2, xmm1, rax
	movsd	ZPAD1, xmm0

	mov	ecx, BIGLIT_INCR2	;; Different clm values step through
					;; big/lit array differently
	movzx	rax, BYTE PTR [rdi+rcx]	;; Load big vs. little flags
	movsd	xmm0, ZPAD2		;; Load values1
	mulsd	xmm0, Q [rbp+2*32]	;; Mul values1 by two-to-minus-phi
	mulsd	xmm0, Q XMM_NORM012_FF	;; Mul by FFTLEN/2
	single_split_lower_zpad_word base2, xmm0, xmm5, xmm1, rax
no const mulsd	xmm0, Q XMM_MINUS_C
const	mulsd	xmm0, Q XMM_MINUS_C_TIMES_MULCONST
	round_zpad7_word base2, xmm0, xmm2, xmm1, rax
	movsd	ZPAD2, xmm0

	movzx	rax, BYTE PTR [rdi+rcx+4] ;; Load big vs. little flags
	movsd	xmm0, ZPAD3		;; Load values1
	mulsd	xmm0, Q [rbp+3*32]	;; Mul values1 by two-to-minus-phi
	mulsd	xmm0, Q XMM_NORM012_FF	;; Mul by FFTLEN/2
	single_split_lower_zpad_word base2, xmm0, xmm5, xmm1, rax
no const mulsd	xmm0, Q XMM_MINUS_C
const	mulsd	xmm0, Q XMM_MINUS_C_TIMES_MULCONST
	round_zpad7_word base2, xmm0, xmm2, xmm1, rax
	movsd	ZPAD3, xmm0

	mov	ecx, BIGLIT_INCR4	;; Different clm values step through
					;; big/lit array differently
	movzx	rax, BYTE PTR [rdi+rcx]	;; Load big vs. little flags
	movsd	xmm0, ZPAD4		;; Load values1
	mulsd	xmm0, Q [rbp+4*32]	;; Mul values1 by two-to-minus-phi
	mulsd	xmm0, Q XMM_NORM012_FF	;; Mul by FFTLEN/2
	single_split_lower_zpad_word base2, xmm0, xmm5, xmm1, rax
no const mulsd	xmm0, Q XMM_MINUS_C
const	mulsd	xmm0, Q XMM_MINUS_C_TIMES_MULCONST
	round_zpad7_word base2, xmm0, xmm2, xmm1, rax
	movsd	ZPAD4, xmm0

	movzx	rax, BYTE PTR [rdi+rcx+4] ;; Load big vs. little flags
	movsd	xmm0, ZPAD5		;; Load values1
	mulsd	xmm0, Q [rbp+5*32]	;; Mul values1 by two-to-minus-phi
	mulsd	xmm0, Q XMM_NORM012_FF	;; Mul by FFTLEN/2
	single_split_lower_zpad_word base2, xmm0, xmm5, xmm1, rax
no const mulsd	xmm0, Q XMM_MINUS_C
const	mulsd	xmm0, Q XMM_MINUS_C_TIMES_MULCONST
	round_zpad7_word base2, xmm0, xmm2, xmm1, rax
	movsd	ZPAD5, xmm0

	movsd	xmm0, ZPAD6		;; Load values1
	mulsd	xmm0, Q [rbp+6*32]	;; Mul values1 by two-to-minus-phi
	mulsd	xmm0, Q XMM_NORM012_FF	;; Mul by FFTLEN/2
	addsd	xmm0, Q XMM_BIGVAL	;; Round to an integer
	subsd	xmm0, Q XMM_BIGVAL
	addsd	xmm0, xmm5		;; Add in shifted high ZPAD data
no const mulsd	xmm0, Q XMM_MINUS_C
const	mulsd	xmm0, Q XMM_MINUS_C_TIMES_MULCONST
	addsd	xmm0, xmm2		;; Add in high part of last calculation
	movsd	ZPAD6, xmm0

	;; Divide the zpad data by k.  Store the integer part in XMM_TMP
	;; and the remainder in ZPAD0.  Later we will wrap the integer part
	;; down to the bottom of the FFT data area (and multiply by -c).
	;; And we will store the remainder in the upper half of the FFT
	;; data area.

	;; Note there are three cases to handle.  K is smaller than a big word.
	;; K is between one and 2 big words in size.  And K is more than
	;; 2 big words in size.

	cmp	ZPAD_TYPE, 2		;; Are we dealing with case 1,2,or 3
	jl	smallk			;; One word case
	je	mediumk			;; Two word case

	;; This case does the divide by k where k is three words

	movsd	xmm0, ZPAD6		;; Load zpad word (high bits)
	movsd	xmm1, ZPAD5		;; Load zpad word (middle bits)
	movsd	xmm2, ZPAD4		;; Load zpad word (low bits)
	movsd	xmm4, ZPAD_INVERSE_K6	;; Load shifted 1/k
	mulsd	xmm4, xmm0		;; Mul ZPAD by shifted 1/k
	addsd	xmm4, Q XMM_BIGVAL	;; Round to integer	
	subsd	xmm4, Q XMM_BIGVAL
	movsd	xmm5, ZPAD_K6_HI	;; Load high bits of k
	mulsd	xmm5, xmm4
	subsd	xmm0, xmm5		;; Calculate high bits of remainder
	movsd	xmm5, ZPAD_K6_MID	;; Load middle bits of k
	mulsd	xmm5, xmm4
	subsd	xmm1, xmm5		;; Calculate middle bits of remainder
	movsd	xmm5, ZPAD_K6_LO	;; Load low bits of k
	mulsd	xmm5, xmm4
	subsd	xmm2, xmm5		;; Calculate low bits of remainder
	movsd	Q XMM_TMP5, xmm4	;; Save word of zpad / k

	mulsd	xmm0, ZPAD_SHIFT6	;; Shift previous zpad word
	addsd	xmm1, xmm0		;; Add to create new high zpad bits
	movsd	xmm0, ZPAD3		;; Load zpad word (new low bits)
	movsd	xmm5, ZPAD_SHIFT5	;; Combine high and medium bits
	mulsd	xmm5, xmm1
	addsd	xmm5, xmm2
	movsd	xmm4, ZPAD_INVERSE_K5	;; Load shifted 1/k
	mulsd	xmm4, xmm5		;; Mul ZPAD by shifted 1/k
	addsd	xmm4, Q XMM_BIGVAL	;; Round to integer
	subsd	xmm4, Q XMM_BIGVAL
	movsd	xmm5, ZPAD_K5_HI	;; Load high bits of k
	mulsd	xmm5, xmm4
	subsd	xmm1, xmm5		;; Calculate high bits of remainder
	movsd	xmm5, ZPAD_K5_MID	;; Load middle bits of k
	mulsd	xmm5, xmm4
	subsd	xmm2, xmm5		;; Calculate middle bits of remainder
	movsd	xmm5, ZPAD_K5_LO	;; Load low bits of k
	mulsd	xmm5, xmm4
	subsd	xmm0, xmm5		;; Calculate low bits of remainder
	movsd	Q XMM_TMP4, xmm4	;; Save word of zpad / k

	mulsd	xmm1, ZPAD_SHIFT5	;; Shift previous zpad word
	addsd	xmm2, xmm1		;; Add to create new high zpad bits
	movsd	xmm1, ZPAD2		;; Load zpad word (new low bits)
	movsd	xmm5, ZPAD_SHIFT4	;; Combine high and medium bits
	mulsd	xmm5, xmm2
	addsd	xmm5, xmm0
	movsd	xmm4, ZPAD_INVERSE_K4	;; Load shifted 1/k
	mulsd	xmm4, xmm5		;; Mul ZPAD by shifted 1/k
	addsd	xmm4, Q XMM_BIGVAL	;; Round to integer
	subsd	xmm4, Q XMM_BIGVAL
	movsd	xmm5, ZPAD_K4_HI	;; Load high bits of k
	mulsd	xmm5, xmm4
	subsd	xmm2, xmm5		;; Calculate high bits of remainder
	movsd	xmm5, ZPAD_K4_MID	;; Load middle bits of k
	mulsd	xmm5, xmm4
	subsd	xmm0, xmm5		;; Calculate middle bits of remainder
	movsd	xmm5, ZPAD_K4_LO	;; Load low bits of k
	mulsd	xmm5, xmm4
	subsd	xmm1, xmm5		;; Calculate low bits of remainder
	movsd	Q XMM_TMP3, xmm4	;; Save word of zpad / k

	mulsd	xmm2, ZPAD_SHIFT4	;; Shift previous zpad word
	addsd	xmm0, xmm2		;; Add to create new high zpad bits
	movsd	xmm2, ZPAD1		;; Load zpad word (new low bits)
	movsd	xmm5, ZPAD_SHIFT3	;; Combine high and medium bits
	mulsd	xmm5, xmm0
	addsd	xmm5, xmm1
	movsd	xmm4, ZPAD_INVERSE_K3	;; Load shifted 1/k
	mulsd	xmm4, xmm5		;; Mul ZPAD by shifted 1/k
	addsd	xmm4, Q XMM_BIGVAL	;; Round to integer
	subsd	xmm4, Q XMM_BIGVAL
	movsd	xmm5, ZPAD_K3_HI	;; Load high bits of k
	mulsd	xmm5, xmm4
	subsd	xmm0, xmm5		;; Calculate high bits of remainder
	movsd	xmm5, ZPAD_K3_MID	;; Load middle bits of k
	mulsd	xmm5, xmm4
	subsd	xmm1, xmm5		;; Calculate middle bits of remainder
	movsd	xmm5, ZPAD_K3_LO	;; Load low bits of k
	mulsd	xmm5, xmm4
	subsd	xmm2, xmm5		;; Calculate low bits of remainder
	movsd	Q XMM_TMP2, xmm4	;; Save word of zpad / k

	mulsd	xmm0, ZPAD_SHIFT3	;; Shift previous zpad word
	addsd	xmm1, xmm0		;; Add to create new high zpad bits
	movsd	xmm0, ZPAD0		;; Load zpad word (new low bits)
	movsd	xmm5, ZPAD_SHIFT2	;; Combine high and medium bits
	mulsd	xmm5, xmm1
	addsd	xmm5, xmm2
	movsd	xmm4, ZPAD_INVERSE_K2	;; Load shifted 1/k
	mulsd	xmm4, xmm5		;; Mul ZPAD by shifted 1/k
	addsd	xmm4, Q XMM_BIGVAL	;; Round to integer
	subsd	xmm4, Q XMM_BIGVAL
	movsd	xmm5, ZPAD_K2_HI	;; Load high bits of k
	mulsd	xmm5, xmm4
	subsd	xmm1, xmm5		;; Calculate high bits of remainder
	movsd	xmm5, ZPAD_K2_MID	;; Load middle bits of k
	mulsd	xmm5, xmm4
	subsd	xmm2, xmm5		;; Calculate middle bits of remainder
	movsd	xmm5, ZPAD_K2_LO	;; Load low bits of k
	mulsd	xmm5, xmm4
	subsd	xmm0, xmm5		;; Calculate low bits of remainder
	movsd	Q XMM_TMP1, xmm4	;; Save word of zpad / k

	mulsd	xmm1, ZPAD_SHIFT2	;; Shift previous zpad word
	addsd	xmm2, xmm1		;; Add to create new high zpad bits
	mulsd	xmm2, ZPAD_SHIFT1	;; Shift previous zpad word
	addsd	xmm0, xmm2		;; Add to create new high zpad bits
	movsd	ZPAD0, xmm0		;; Save remainder of zpad / k

	subsd	xmm1, xmm1		;; Zero words that other cases set
	movsd	Q XMM_TMP6, xmm1
	
	jmp	div_k_done

	;; This case does the divide by k where k is two words
mediumk:
	movsd	xmm0, ZPAD6		;; Load zpad word (high bits)
	movsd	xmm1, ZPAD5		;; Load zpad word (low bits)
	movsd	xmm4, ZPAD_INVERSE_K6	;; Load shifted 1/k
	mulsd	xmm4, xmm0		;; Mul ZPAD by shifted 1/k
	addsd	xmm4, Q XMM_BIGVAL	;; Round to integer	
	subsd	xmm4, Q XMM_BIGVAL
	movsd	xmm5, ZPAD_K6_HI	;; Load high bits of k
	mulsd	xmm5, xmm4
	subsd	xmm0, xmm5		;; Calculate high bits of remainder
	movsd	xmm5, ZPAD_K6_LO	;; Load low bits of k
	mulsd	xmm5, xmm4
	subsd	xmm1, xmm5		;; Calculate low bits of remainder
	movsd	Q XMM_TMP6, xmm4	;; Save word of zpad / k

	mulsd	xmm0, ZPAD_SHIFT6	;; Shift previous zpad word
	addsd	xmm0, xmm1		;; Add to create new high zpad bits
	movsd	xmm1, ZPAD4		;; Load zpad word (new low bits)
	movsd	xmm4, ZPAD_INVERSE_K5	;; Load shifted 1/k
	mulsd	xmm4, xmm0		;; Mul ZPAD by shifted 1/k
	addsd	xmm4, Q XMM_BIGVAL	;; Round to integer
	subsd	xmm4, Q XMM_BIGVAL
	movsd	xmm5, ZPAD_K5_HI	;; Load high bits of k
	mulsd	xmm5, xmm4
	subsd	xmm0, xmm5		;; Calculate high bits of remainder
	movsd	xmm5, ZPAD_K5_LO	;; Load low bits of k
	mulsd	xmm5, xmm4
	subsd	xmm1, xmm5		;; Calculate low bits of remainder
	movsd	Q XMM_TMP5, xmm4	;; Save word of zpad / k

	mulsd	xmm0, ZPAD_SHIFT5	;; Shift previous zpad word
	addsd	xmm0, xmm1		;; Add to create new high zpad bits
	movsd	xmm1, ZPAD3		;; Load zpad word (new low bits)
	movsd	xmm4, ZPAD_INVERSE_K4	;; Load shifted 1/k
	mulsd	xmm4, xmm0		;; Mul ZPAD by shifted 1/k
	addsd	xmm4, Q XMM_BIGVAL	;; Round to integer
	subsd	xmm4, Q XMM_BIGVAL
	movsd	xmm5, ZPAD_K4_HI	;; Load high bits of k
	mulsd	xmm5, xmm4
	subsd	xmm0, xmm5		;; Calculate high bits of remainder
	movsd	xmm5, ZPAD_K4_LO	;; Load low bits of k
	mulsd	xmm5, xmm4
	subsd	xmm1, xmm5		;; Calculate low bits of remainder
	movsd	Q XMM_TMP4, xmm4	;; Save word of zpad / k

	mulsd	xmm0, ZPAD_SHIFT4	;; Shift previous zpad word
	addsd	xmm0, xmm1		;; Add to create new high zpad bits
	movsd	xmm1, ZPAD2		;; Load zpad word (new low bits)
	movsd	xmm4, ZPAD_INVERSE_K3	;; Load shifted 1/k
	mulsd	xmm4, xmm0		;; Mul ZPAD by shifted 1/k
	addsd	xmm4, Q XMM_BIGVAL	;; Round to integer
	subsd	xmm4, Q XMM_BIGVAL
	movsd	xmm5, ZPAD_K3_HI	;; Load high bits of k
	mulsd	xmm5, xmm4
	subsd	xmm0, xmm5		;; Calculate high bits of remainder
	movsd	xmm5, ZPAD_K3_LO	;; Load low bits of k
	mulsd	xmm5, xmm4
	subsd	xmm1, xmm5		;; Calculate low bits of remainder
	movsd	Q XMM_TMP3, xmm4	;; Save word of zpad / k

	mulsd	xmm0, ZPAD_SHIFT3	;; Shift previous zpad word
	addsd	xmm0, xmm1		;; Add to create new high zpad bits
	movsd	xmm1, ZPAD1		;; Load zpad word (new low bits)
	movsd	xmm4, ZPAD_INVERSE_K2	;; Load shifted 1/k
	mulsd	xmm4, xmm0		;; Mul ZPAD by shifted 1/k
	addsd	xmm4, Q XMM_BIGVAL	;; Round to integer
	subsd	xmm4, Q XMM_BIGVAL
	movsd	xmm5, ZPAD_K2_HI	;; Load high bits of k
	mulsd	xmm5, xmm4
	subsd	xmm0, xmm5		;; Calculate high bits of remainder
	movsd	xmm5, ZPAD_K2_LO	;; Load low bits of k
	mulsd	xmm5, xmm4
	subsd	xmm1, xmm5		;; Calculate low bits of remainder
	movsd	Q XMM_TMP2, xmm4	;; Save word of zpad / k

	mulsd	xmm0, ZPAD_SHIFT2	;; Shift previous zpad word
	addsd	xmm0, xmm1		;; Add to create new high zpad bits
	movsd	xmm1, ZPAD0		;; Load zpad word (new low bits)
	movsd	xmm4, ZPAD_INVERSE_K1	;; Load shifted 1/k
	mulsd	xmm4, xmm0		;; Mul ZPAD by shifted 1/k
	addsd	xmm4, Q XMM_BIGVAL	;; Round to integer
	subsd	xmm4, Q XMM_BIGVAL
	movsd	xmm5, ZPAD_K1_HI	;; Load high bits of k
	mulsd	xmm5, xmm4
	subsd	xmm0, xmm5		;; Calculate high bits of remainder
	movsd	xmm5, ZPAD_K1_LO	;; Load low bits of k
	mulsd	xmm5, xmm4
	subsd	xmm1, xmm5		;; Calculate low bits of remainder
	movsd	Q XMM_TMP1, xmm4	;; Save word of zpad / k

	mulsd	xmm0, ZPAD_SHIFT1	;; Shift previous zpad word
	addsd	xmm0, xmm1		;; Add to create new high zpad bits
	movsd	ZPAD0, xmm0		;; Save remainder of zpad / k

	jmp	div_k_done

	;; This case does the divide by k where k is one word
	;; Assume ZPAD5 and ZPAD6 are zero.
smallk:	movsd	xmm0, ZPAD4		;; Load zpad data
	movsd	xmm4, ZPAD_INVERSE_K1	;; Load by 1/k
	mulsd	xmm4, xmm0		;; Mul ZPAD data by 1/k
	addsd	xmm4, Q XMM_BIGVAL	;; Round to integer	
	subsd	xmm4, Q XMM_BIGVAL
	movsd	Q XMM_TMP5, xmm4	;; Save integer part
	mulsd	xmm4, ZPAD_K1_LO	;; Compute remainder
	subsd	xmm0, xmm4

	mulsd	xmm0, ZPAD_SHIFT4	;; Shift previous zpad word
	addsd	xmm0, ZPAD3		;; Add in zpad data
	movsd	xmm4, ZPAD_INVERSE_K1	;; Load by 1/k
	mulsd	xmm4, xmm0		;; Mul ZPAD data by 1/k
	addsd	xmm4, Q XMM_BIGVAL	;; Round to integer	
	subsd	xmm4, Q XMM_BIGVAL
	movsd	Q XMM_TMP4, xmm4	;; Save integer part
	mulsd	xmm4, ZPAD_K1_LO	;; Compute remainder
	subsd	xmm0, xmm4

	mulsd	xmm0, ZPAD_SHIFT3	;; Shift previous zpad word
	addsd	xmm0, ZPAD2		;; Add in zpad data
	movsd	xmm4, ZPAD_INVERSE_K1	;; Load by 1/k
	mulsd	xmm4, xmm0		;; Mul ZPAD data by 1/k
	addsd	xmm4, Q XMM_BIGVAL	;; Round to integer	
	subsd	xmm4, Q XMM_BIGVAL
	movsd	Q XMM_TMP3, xmm4	;; Save integer part
	mulsd	xmm4, ZPAD_K1_LO	;; Compute remainder
	subsd	xmm0, xmm4

	mulsd	xmm0, ZPAD_SHIFT2	;; Shift previous zpad word
	addsd	xmm0, ZPAD1		;; Add in zpad data
	movsd	xmm4, ZPAD_INVERSE_K1	;; Load by 1/k
	mulsd	xmm4, xmm0		;; Mul ZPAD data by 1/k
	addsd	xmm4, Q XMM_BIGVAL	;; Round to integer	
	subsd	xmm4, Q XMM_BIGVAL
	movsd	Q XMM_TMP2, xmm4	;; Save integer part
	mulsd	xmm4, ZPAD_K1_LO	;; Compute remainder
	subsd	xmm0, xmm4

	mulsd	xmm0, ZPAD_SHIFT1	;; Shift previous zpad word
	addsd	xmm0, ZPAD0		;; Add in zpad data
	movsd	xmm4, ZPAD_INVERSE_K1	;; Load by 1/k
	mulsd	xmm4, xmm0		;; Mul ZPAD data by 1/k
	addsd	xmm4, Q XMM_BIGVAL	;; Round to integer	
	subsd	xmm4, Q XMM_BIGVAL
	movsd	Q XMM_TMP1, xmm4	;; Save integer part
	mulsd	xmm4, ZPAD_K1_LO	;; Compute remainder
	subsd	xmm0, xmm4
	movsd	ZPAD0, xmm0		;; Save remainder

	subsd	xmm1, xmm1		;; Zero words that other cases set
	movsd	Q XMM_TMP6, xmm1
	
div_k_done:

	;; Now normalize the data above the halfway point.  Remember that the
	;; column two-to-phi multiplier for the first value will be 1.0.

	movzx	rax, BYTE PTR [rdi]	;; First word 
	movsd	xmm0, ZPAD0		;; Load remainder of divide by k
	addsd	xmm0, Q XMM_BIGVAL
	single_rounding base2, xmm0, xmm2, xmm4, rax
	movsd	Q [rsi+0*64+32], xmm0	;; Save value1

	movzx	rax, BYTE PTR [rdi+4]	;; Load big vs. little flags
	single_rounding base2, xmm2, xmm0, xmm4, rax
	mulsd	xmm2, Q [rbp+1*32+16]	;; new value2 = val * two-to-phi
	movsd	Q [rsi+1*64+32], xmm2	;; Save value2

	mov	ecx, BIGLIT_INCR2	;; Different clm values step through
					;; big/lit array differently
	movzx	rax, BYTE PTR [rdi+rcx]	;; Load big vs. little flags
	single_rounding base2, xmm0, xmm2, xmm4, rax
	mulsd	xmm0, Q [rbp+2*32+16]	;; new value3 = val * two-to-phi
	movsd	Q [rsi+2*64+32], xmm0	;; Save value3

	subsd	xmm2, Q XMM_BIGVAL	;; Remove integer rounding constant
	mulsd	xmm2, Q [rbp+3*32+16]	;; value4 = carry * two-to-phi
	movsd	Q [rsi+3*64+32], xmm2	;; Save new value4

	;; Mul the integer part of (ZPAD data divided by k) by -c in
	;; preparation for adding it into the lower FFT data area.
	;; Also add in the shifted high FFT carry at this time.

	;; Now add in and normalize the bottom FFT data.  Remember that the
	;; column two-to-phi multiplier for the first value will be 1.0.  We 
	;; must go 6 words deep in case k is 48-50 bits and c is 32 bits.

	movzx	rax, BYTE PTR [rdi]	;; First word 
	movsd	xmm0, Q XMM_TMP1	;; Load integer part of divide by k
	addsd	xmm0, xmm3		;; Add in shifted high FFT carry
	mulsd	xmm0, Q XMM_MINUS_C	;; Mul by -c
	addsd	xmm0, Q XMM_BIGVAL
	addsd	xmm0, Q [rsi+0*64]	;; Add in the FFT data
	single_rounding base2, xmm0, xmm2, xmm4, rax
	movsd	Q [rsi+0*64], xmm0	;; Save value1

	movzx	rax, BYTE PTR [rdi+4]	;; Load big vs. little flags
	movsd	xmm0, Q XMM_TMP2	;; Load integer part of divide by k
	mulsd	xmm0, Q XMM_MINUS_C	;; Mul by -c
	movsd	xmm1, Q [rsi+1*64]	;; Load FFT data
	mulsd	xmm1, Q [rbp+1*32]	;; Mul values2 by two-to-minus-phi
	mulsd	xmm1, Q XMM_NORM012_FF	;; Mul by FFTLEN/2
	addsd	xmm0, xmm1		;; Add in the FFT data
	addsd	xmm0, xmm2		;; x2 = value + carry
	single_rounding base2, xmm0, xmm2, xmm4, rax
	mulsd	xmm0, Q [rbp+1*32+16]	;; new value2 = val * two-to-phi
	movsd	Q [rsi+1*64], xmm0	;; Save value2

	mov	ecx, BIGLIT_INCR2	;; Different clm values step through
					;; big/lit array differently
	movzx	rax, BYTE PTR [rdi+rcx] ;; Load big vs. little flags
	movsd	xmm0, Q XMM_TMP3	;; Load integer part of divide by k
	mulsd	xmm0, Q XMM_MINUS_C	;; Mul by -c
	movsd	xmm1, Q [rsi+2*64]	;; Load FFT data
	mulsd	xmm1, Q [rbp+2*32]	;; Mul values3 by two-to-minus-phi
	mulsd	xmm1, Q XMM_NORM012_FF	;; Mul by FFTLEN/2
	addsd	xmm0, xmm1		;; Add in the FFT data
	addsd	xmm0, xmm2		;; x3 = value + carry
	single_rounding base2, xmm0, xmm2, xmm4, rax
	mulsd	xmm0, Q [rbp+2*32+16]	;; new value3 = val * two-to-phi
	movsd	Q [rsi+2*64], xmm0	;; Save value3

	movzx	rax, BYTE PTR [rdi+rcx+4] ;; Load big vs. little flags
	movsd	xmm0, Q XMM_TMP4	;; Load integer part of divide by k
	mulsd	xmm0, Q XMM_MINUS_C	;; Mul by -c
	movsd	xmm1, Q [rsi+3*64]	;; Load FFT data
	mulsd	xmm1, Q [rbp+3*32]	;; Mul values4 by two-to-minus-phi
	mulsd	xmm1, Q XMM_NORM012_FF	;; Mul by FFTLEN/2
	addsd	xmm0, xmm1		;; Add in the FFT data
	addsd	xmm0, xmm2		;; x4 = value + carry
	single_rounding base2, xmm0, xmm2, xmm4, rax
	mulsd	xmm0, Q [rbp+3*32+16]	;; new value4 = val * two-to-phi
	movsd	Q [rsi+3*64], xmm0	;; Save value4

	mov	ecx, BIGLIT_INCR4	;; Different clm values step through
					;; big/lit array differently
	movzx	rax, BYTE PTR [rdi+rcx]	;; Load big vs. little flags
	movsd	xmm0, Q XMM_TMP5	;; Load integer part of divide by k
	mulsd	xmm0, Q XMM_MINUS_C	;; Mul by -c
	movsd	xmm1, Q [rsi+4*64]	;; Load FFT data
	mulsd	xmm1, Q [rbp+4*32]	;; Mul values4 by two-to-minus-phi
	mulsd	xmm1, Q XMM_NORM012_FF	;; Mul by FFTLEN/2
	addsd	xmm0, xmm1		;; Add in the FFT data
	addsd	xmm0, xmm2		;; x4 = value + carry
	single_rounding base2, xmm0, xmm2, xmm4, rax
	mulsd	xmm0, Q [rbp+4*32+16]	;; new value4 = val * two-to-phi
	movsd	Q [rsi+4*64], xmm0	;; Save value4

	movzx	rax, BYTE PTR [rdi+rcx+4] ;; Load big vs. little flags
	movsd	xmm0, Q XMM_TMP6	;; Load integer part of divide by k
	mulsd	xmm0, Q XMM_MINUS_C	;; Mul by -c
	movsd	xmm1, Q [rsi+5*64]	;; Load FFT data
	mulsd	xmm1, Q [rbp+5*32]	;; Mul values5 by two-to-minus-phi
	mulsd	xmm1, Q XMM_NORM012_FF	;; Mul by FFTLEN/2
	addsd	xmm0, xmm1		;; Add in the FFT data
	addsd	xmm0, xmm2		;; x5 = value + carry
	single_rounding base2, xmm0, xmm2, xmm4, rax
	mulsd	xmm0, Q [rbp+5*32+16]	;; new value5 = val * two-to-phi
	movsd	Q [rsi+5*64], xmm0	;; Save value5

	add	ecx, BIGLIT_INCR2	;; Different clm values step through
					;; big/lit array differently
	movzx	rax, BYTE PTR [rdi+rcx]	;; Load big vs. little flags
	movsd	xmm0, Q [rsi+6*64]	;; Load FFT data
	mulsd	xmm0, Q [rbp+6*32]	;; Mul values3 by two-to-minus-phi
	mulsd	xmm0, Q XMM_NORM012_FF	;; Mul by FFTLEN/2
	addsd	xmm0, xmm2		;; x6 = value + carry
	single_rounding base2, xmm0, xmm2, xmm4, rax
	mulsd	xmm0, Q [rbp+6*32+16]	;; new value6 = val * two-to-phi
	movsd	Q [rsi+6*64], xmm0	;; Save value6

	subsd	xmm2, Q XMM_BIGVAL	;; Remove rounding constant
	mulsd	xmm2, Q [rbp+7*32+16]	;; new value7 = val * two-to-phi
	addsd	xmm2, Q [rsi+7*64]	;; Add in FFT data
	movsd	Q [rsi+7*64], xmm2	;; Save value7
	ENDM

xnorm012_2d_zpad MACRO
	LOCAL	b2, done
	cmp	B_IS_2, 0		;; Is b = 2?
	jne	b2			;; Yes, do simpler rounding
	xnorm012_2d_zpada noexec	;; No, do harder rounding
	jmp	done
b2:	xnorm012_2d_zpada exec
done:
	ENDM
xnorm012_2d_zpada MACRO base2
	LOCAL	noncon, done
	cmp	const_fft, 0		;; Are we also multiplying by a constant?
	je	noncon			;; Jump if not const
	xnorm012_2d_zpad_cmn exec, base2
	jmp	done
noncon:	xnorm012_2d_zpad_cmn noexec, base2
done:
	ENDM

xnorm012_2d_zpad_cmn MACRO const, base2
	movzx	rax, BYTE PTR [rdi+0]	;; Load big vs. little flag
	xload	xmm0, [rsi+0*16]	;; Load carry
	xload	xmm2, [rsi+2*16]	;; Load FFT hi data carry
	xload	xmm4, [rbp+0*64]	;; Load FFT data
	mulpd	xmm4, [rdx+0*32]	;; mul by grp two-to-minus-phi
	mulpd	xmm4, [rbx+0*32]	;; mul by col two-to-minus-phi
	mulpd	xmm4, XMM_TTMP_FUDGE[rax];; mul by fudge two-to-minus-phi
	mulpd	xmm4, XMM_NORM012_FF	;; Mul by FFTLEN/2
	addpd	xmm0, xmm4		;; x1 = values1 + carry
	split_carry_zpad_word base2, xmm2, xmm6, xmm4, rax
no const xload	xmm4, XMM_K_LO		;; Calc high FFT carry times k
const	xload	xmm4, XMM_K_TIMES_MULCONST_LO
	mulpd	xmm4, xmm2		;; high_FFT_carry * k_lo
	addpd	xmm0, xmm4		;; x1 = x1 + high_FFT_carry * k_lo
no const xload	xmm5, XMM_K_HI
const	xload	xmm5, XMM_K_TIMES_MULCONST_HI
	mulpd	xmm5, XMM_LIMIT_INVERSE[rax] ;; shift k_hi
	mulpd	xmm5, xmm2
	rounding exec, base2, noexec, noexec, xmm0, xmm2, xmm4, rax
	addpd	xmm2, xmm5		;; Carry += high_FFT_carry * shifted k_hi
	mulpd	xmm0, [rdx+0*32+16]	;; mul by grp two-to-phi
	mulpd	xmm0, [rbx+0*32+16]	;; mul by col two-to-phi
	mulpd	xmm0, XMM_TTP_FUDGE[rax];; mul by fudge two-to-phi
	xstore	[rbp+0*64], xmm0	;; Save FFT data

	movzx	rax, BYTE PTR [rdi+4]	;; Load big vs. little flag
	xload	xmm0, [rbp+1*64]	;; Load FFT data
	mulpd	xmm0, [rdx+0*32]	;; mul by grp two-to-minus-phi
	xload	xmm4, [rbx+1*32]	;; col two-to-minus-phi
	mulpd	xmm4, XMM_TTMP_FUDGE[rax];; mul by fudge two-to-minus-phi
	mulpd	xmm0, xmm4		;; data *= fudged col two-to-minus-phi
	mulpd	xmm0, XMM_NORM012_FF	;; Mul by FFTLEN/2
	addpd	xmm0, xmm2		;; x2 = values1 + carry
	split_carry_zpad_word base2, xmm6, xmm5, xmm4, rax
no const xload	xmm4, XMM_K_LO		;; Calc high FFT carry times k
const	xload	xmm4, XMM_K_TIMES_MULCONST_LO
	mulpd	xmm4, xmm6		;; high_FFT_carry * k_lo
	addpd	xmm0, xmm4		;; x2 = x2 + high_FFT_carry * k_lo
no const xload	xmm4, XMM_K_HI
const	xload	xmm4, XMM_K_TIMES_MULCONST_HI
	mulpd	xmm4, XMM_LIMIT_INVERSE[rax] ;; shift k_hi
	mulpd	xmm6, xmm4
	rounding exec, base2, noexec, noexec, xmm0, xmm2, xmm4, rax
	addpd	xmm2, xmm6		;; Carry += high_FFT_carry * shifted k_hi
	mulpd	xmm0, [rdx+0*32+16]	;; mul by grp two-to-phi
	xload	xmm4, [rbx+1*32+16]	;; col two-to-phi
	mulpd	xmm4, XMM_TTP_FUDGE[rax];; mul by fudge two-to-phi
	mulpd	xmm0, xmm4		;; data *= fudged col two-to-phi
	xstore	[rbp+1*64], xmm0	;; Save FFT data

	mov	ecx, BIGLIT_INCR2	;; Different clm values step through
					;; big/lit array differently
	movzx	rax, BYTE PTR [rdi+rcx]	;; Load big vs. little flag
	xload	xmm0, [rbp+2*64]	;; Load FFT data
	mulpd	xmm0, [rdx+0*32]	;; mul by grp two-to-minus-phi
	xload	xmm4, [rbx+2*32]	;; col two-to-minus-phi
	mulpd	xmm4, XMM_TTMP_FUDGE[rax];; mul by fudge two-to-minus-phi
	mulpd	xmm0, xmm4		;; data *= fudged col two-to-minus-phi
	mulpd	xmm0, XMM_NORM012_FF	;; Mul by FFTLEN/2
	addpd	xmm0, xmm2		;; x3 = values1 + carry
	split_carry_zpad_word base2, xmm5, xmm6, xmm4, rax
no const xload	xmm4, XMM_K_LO		;; Calc high FFT carry times k
const	xload	xmm4, XMM_K_TIMES_MULCONST_LO
	mulpd	xmm4, xmm5		;; high_FFT_carry * k_lo
	addpd	xmm0, xmm4		;; x3 = x3 + high_FFT_carry * k_lo
no const xload	xmm4, XMM_K_HI
const	xload	xmm4, XMM_K_TIMES_MULCONST_HI
	mulpd	xmm4, XMM_LIMIT_INVERSE[rax] ;; shift k_hi
	mulpd	xmm5, xmm4
	rounding exec, base2, noexec, noexec, xmm0, xmm2, xmm4, rax
	addpd	xmm2, xmm5		;; Carry += high_FFT_carry * shifted k_hi
	mulpd	xmm0, [rdx+0*32+16]	;; mul by grp two-to-phi
	xload	xmm4, [rbx+2*32+16]	;; col two-to-phi
	mulpd	xmm4, XMM_TTP_FUDGE[rax];; mul by fudge two-to-phi
	mulpd	xmm0, xmm4		;; data *= fudged col two-to-phi
	xstore	[rbp+2*64], xmm0	;; Save FFT data

	movzx	rax, BYTE PTR [rdi+rcx+4] ;; Load big vs. little flag
	xload	xmm0, [rbp+3*64]	;; Load FFT data
	mulpd	xmm0, [rdx+0*32]	;; mul by grp two-to-minus-phi
	xload	xmm4, [rbx+3*32]	;; col two-to-minus-phi
	mulpd	xmm4, XMM_TTMP_FUDGE[rax];; mul by fudge two-to-minus-phi
	mulpd	xmm0, xmm4		;; data *= fudged col two-to-minus-phi
	mulpd	xmm0, XMM_NORM012_FF	;; Mul by FFTLEN/2
	addpd	xmm0, xmm2		;; x4 = values1 + carry
	split_carry_zpad_word base2, xmm6, xmm5, xmm4, rax
no const xload	xmm4, XMM_K_LO		;; Calc high FFT carry times k
const	xload	xmm4, XMM_K_TIMES_MULCONST_LO
	mulpd	xmm4, xmm6		;; high_FFT_carry * k_lo
	addpd	xmm0, xmm4		;; x4 = x4 + high_FFT_carry * k_lo
no const xload	xmm4, XMM_K_HI
const	xload	xmm4, XMM_K_TIMES_MULCONST_HI
	mulpd	xmm4, XMM_LIMIT_INVERSE[rax] ;; shift k_hi
	mulpd	xmm6, xmm4
	rounding exec, base2, noexec, noexec, xmm0, xmm2, xmm4, rax
	addpd	xmm2, xmm6		;; Carry += high_FFT_carry * shifted k_hi
	mulpd	xmm0, [rdx+0*32+16]	;; mul by grp two-to-phi
	xload	xmm4, [rbx+3*32+16]	;; col two-to-phi
	mulpd	xmm4, XMM_TTP_FUDGE[rax];; mul by fudge two-to-phi
	mulpd	xmm0, xmm4		;; data *= fudged col two-to-phi
	xstore	[rbp+3*64], xmm0	;; Save FFT data

	mov	ecx, BIGLIT_INCR4	;; Different clm values step through
					;; big/lit array differently
	movzx	rax, BYTE PTR [rdi+rcx]	;; Load big vs. little flag
	xload	xmm0, [rbp+4*64]	;; Load FFT data
	mulpd	xmm0, [rdx+0*32]	;; mul by grp two-to-minus-phi
	xload	xmm4, [rbx+4*32]	;; col two-to-minus-phi
	mulpd	xmm4, XMM_TTMP_FUDGE[rax];; mul by fudge two-to-minus-phi
	mulpd	xmm0, xmm4		;; data *= fudged col two-to-minus-phi
	mulpd	xmm0, XMM_NORM012_FF	;; Mul by FFTLEN/2
	addpd	xmm0, xmm2		;; x5 = values1 + carry
no const xload	xmm4, XMM_K_LO		;; Calc high FFT carry times k
const	xload	xmm4, XMM_K_TIMES_MULCONST_LO
	mulpd	xmm4, xmm5		;; high_FFT_carry * k_lo
	addpd	xmm0, xmm4		;; x5 = x5 + high_FFT_carry * k_lo
no const xload	xmm4, XMM_K_HI
const	xload	xmm4, XMM_K_TIMES_MULCONST_HI
	mulpd	xmm4, XMM_LIMIT_INVERSE[rax] ;; shift k_hi
	mulpd	xmm5, xmm4
	rounding exec, base2, noexec, noexec, xmm0, xmm2, xmm4, rax
	addpd	xmm2, xmm5		;; Carry += high_FFT_carry * shifted k_hi
	mulpd	xmm0, [rdx+0*32+16]	;; mul by grp two-to-phi
	xload	xmm4, [rbx+4*32+16]	;; col two-to-phi
	mulpd	xmm4, XMM_TTP_FUDGE[rax];; mul by fudge two-to-phi
	mulpd	xmm0, xmm4		;; data *= fudged col two-to-phi
	xstore	[rbp+4*64], xmm0	;; Save FFT data

	movzx	rax, BYTE PTR [rdi+rcx+4] ;; Load big vs. little flag
	subpd	xmm2, XMM_BIGVAL	;; Remove rounding const from carry
	mulpd	xmm2, [rdx+0*32+16]	;; mul by grp two-to-phi
	xload	xmm4, [rbx+5*32+16]	;; col two-to-phi
	mulpd	xmm4, XMM_TTP_FUDGE[rax];; mul by fudge two-to-phi
	mulpd	xmm2, xmm4		;; data *= fudged col two-to-phi
	addpd	xmm2, [rbp+5*64]	;; Load FFT data
	xstore	[rbp+5*64], xmm2	;; Save FFT data

	movzx	rax, BYTE PTR [rdi+1]	;; Load big vs. little flag
	xload	xmm1, [rsi+1*16]	;; Load carry
	xload	xmm3, [rsi+3*16]	;; Load FFT hi data carry
	xload	xmm4, [rbp+0*64+16]	;; Load FFT data
	mulpd	xmm4, [rdx+1*32]	;; mul by grp two-to-minus-phi
	mulpd	xmm4, [rbx+0*32]	;; mul by col two-to-minus-phi
	mulpd	xmm4, XMM_TTMP_FUDGE[rax];; mul by fudge two-to-minus-phi
	mulpd	xmm4, XMM_NORM012_FF	;; Mul by FFTLEN/2
	addpd	xmm1, xmm4		;; x1 = values1 + carry
	split_carry_zpad_word base2, xmm3, xmm6, xmm4, rax
no const xload	xmm4, XMM_K_LO		;; Calc high FFT carry times k
const	xload	xmm4, XMM_K_TIMES_MULCONST_LO
	mulpd	xmm4, xmm3		;; high_FFT_carry * k_lo
	addpd	xmm1, xmm4		;; x1 = x1 + high_FFT_carry * k_lo
no const xload	xmm5, XMM_K_HI
const	xload	xmm5, XMM_K_TIMES_MULCONST_HI
	mulpd	xmm5, XMM_LIMIT_INVERSE[rax] ;; shift k_hi
	mulpd	xmm5, xmm3
	rounding exec, base2, noexec, noexec, xmm1, xmm3, xmm4, rax
	addpd	xmm3, xmm5		;; Carry += high_FFT_carry * shifted k_hi
	mulpd	xmm1, [rdx+1*32+16]	;; mul by grp two-to-phi
	mulpd	xmm1, [rbx+0*32+16]	;; mul by col two-to-phi
	mulpd	xmm1, XMM_TTP_FUDGE[rax];; mul by fudge two-to-phi
	xstore	[rbp+0*64+16], xmm1	;; Save FFT data

	movzx	rax, BYTE PTR [rdi+5]	;; Load big vs. little flag
	xload	xmm1, [rbp+1*64+16]	;; Load FFT data
	mulpd	xmm1, [rdx+1*32]	;; mul by grp two-to-minus-phi
	xload	xmm4, [rbx+1*32]	;; col two-to-minus-phi
	mulpd	xmm4, XMM_TTMP_FUDGE[rax];; mul by fudge two-to-minus-phi
	mulpd	xmm1, xmm4		;; data *= fudged col two-to-minus-phi
	mulpd	xmm1, XMM_NORM012_FF	;; Mul by FFTLEN/2
	addpd	xmm1, xmm3		;; x2 = values1 + carry
	split_carry_zpad_word base2, xmm6, xmm5, xmm4, rax
no const xload	xmm4, XMM_K_LO		;; Calc high FFT carry times k
const	xload	xmm4, XMM_K_TIMES_MULCONST_LO
	mulpd	xmm4, xmm6		;; high_FFT_carry * k_lo
	addpd	xmm1, xmm4		;; x2 = x2 + high_FFT_carry * k_lo
no const xload	xmm4, XMM_K_HI
const	xload	xmm4, XMM_K_TIMES_MULCONST_HI
	mulpd	xmm4, XMM_LIMIT_INVERSE[rax] ;; shift k_hi
	mulpd	xmm6, xmm4
	rounding exec, base2, noexec, noexec, xmm1, xmm3, xmm4, rax
	addpd	xmm3, xmm6		;; Carry += high_FFT_carry * shifted k_hi
	mulpd	xmm1, [rdx+1*32+16]	;; mul by grp two-to-phi
	xload	xmm4, [rbx+1*32+16]	;; col two-to-phi
	mulpd	xmm4, XMM_TTP_FUDGE[rax];; mul by fudge two-to-phi
	mulpd	xmm1, xmm4		;; data *= fudged col two-to-phi
	xstore	[rbp+1*64+16], xmm1	;; Save FFT data

	mov	ecx, BIGLIT_INCR2	;; Different clm values step through
					;; big/lit array differently
	movzx	rax, BYTE PTR [rdi+rcx+1] ;; Load big vs. little flag
	xload	xmm1, [rbp+2*64+16]	;; Load FFT data
	mulpd	xmm1, [rdx+1*32]	;; mul by grp two-to-minus-phi
	xload	xmm4, [rbx+2*32]	;; col two-to-minus-phi
	mulpd	xmm4, XMM_TTMP_FUDGE[rax];; mul by fudge two-to-minus-phi
	mulpd	xmm1, xmm4		;; data *= fudged col two-to-minus-phi
	mulpd	xmm1, XMM_NORM012_FF	;; Mul by FFTLEN/2
	addpd	xmm1, xmm3		;; x3 = values1 + carry
	split_carry_zpad_word base2, xmm5, xmm6, xmm4, rax
no const xload	xmm4, XMM_K_LO		;; Calc high FFT carry times k
const	xload	xmm4, XMM_K_TIMES_MULCONST_LO
	mulpd	xmm4, xmm5		;; high_FFT_carry * k_lo
	addpd	xmm1, xmm4		;; x3 = x3 + high_FFT_carry * k_lo
no const xload	xmm4, XMM_K_HI
const	xload	xmm4, XMM_K_TIMES_MULCONST_HI
	mulpd	xmm4, XMM_LIMIT_INVERSE[rax] ;; shift k_hi
	mulpd	xmm5, xmm4
	rounding exec, base2, noexec, noexec, xmm1, xmm3, xmm4, rax
	addpd	xmm3, xmm5		;; Carry += high_FFT_carry * shifted k_hi
	mulpd	xmm1, [rdx+1*32+16]	;; mul by grp two-to-phi
	xload	xmm4, [rbx+2*32+16]	;; col two-to-phi
	mulpd	xmm4, XMM_TTP_FUDGE[rax];; mul by fudge two-to-phi
	mulpd	xmm1, xmm4		;; data *= fudged col two-to-phi
	xstore	[rbp+2*64+16], xmm1	;; Save FFT data

	movzx	rax, BYTE PTR [rdi+rcx+5] ;; Load big vs. little flag
	xload	xmm1, [rbp+3*64+16]	;; Load FFT data
	mulpd	xmm1, [rdx+1*32]	;; mul by grp two-to-minus-phi
	xload	xmm4, [rbx+3*32]	;; col two-to-minus-phi
	mulpd	xmm4, XMM_TTMP_FUDGE[rax];; mul by fudge two-to-minus-phi
	mulpd	xmm1, xmm4		;; data *= fudged col two-to-minus-phi
	mulpd	xmm1, XMM_NORM012_FF	;; Mul by FFTLEN/2
	addpd	xmm1, xmm3		;; x4 = values1 + carry
	split_carry_zpad_word base2, xmm6, xmm5, xmm4, rax
no const xload	xmm4, XMM_K_LO		;; Calc high FFT carry times k
const	xload	xmm4, XMM_K_TIMES_MULCONST_LO
	mulpd	xmm4, xmm6		;; high_FFT_carry * k_lo
	addpd	xmm1, xmm4		;; x4 = x4 + high_FFT_carry * k_lo
no const xload	xmm4, XMM_K_HI
const	xload	xmm4, XMM_K_TIMES_MULCONST_HI
	mulpd	xmm4, XMM_LIMIT_INVERSE[rax] ;; shift k_hi
	mulpd	xmm6, xmm4
	rounding exec, base2, noexec, noexec, xmm1, xmm3, xmm4, rax
	addpd	xmm3, xmm6		;; Carry += high_FFT_carry * shifted k_hi
	mulpd	xmm1, [rdx+1*32+16]	;; mul by grp two-to-phi
	xload	xmm4, [rbx+3*32+16]	;; col two-to-phi
	mulpd	xmm4, XMM_TTP_FUDGE[rax];; mul by fudge two-to-phi
	mulpd	xmm1, xmm4		;; data *= fudged col two-to-phi
	xstore	[rbp+3*64+16], xmm1	;; Save FFT data

	mov	ecx, BIGLIT_INCR4	;; Different clm values step through
					;; big/lit array differently
	movzx	rax, BYTE PTR [rdi+rcx+1] ;; Load big vs. little flag
	xload	xmm1, [rbp+4*64+16]	;; Load FFT data
	mulpd	xmm1, [rdx+1*32]	;; mul by grp two-to-minus-phi
	xload	xmm4, [rbx+4*32]	;; col two-to-minus-phi
	mulpd	xmm4, XMM_TTMP_FUDGE[rax];; mul by fudge two-to-minus-phi
	mulpd	xmm1, xmm4		;; data *= fudged col two-to-minus-phi
	mulpd	xmm1, XMM_NORM012_FF	;; Mul by FFTLEN/2
	addpd	xmm1, xmm3		;; x5 = values1 + carry
no const xload	xmm4, XMM_K_LO		;; Calc high FFT carry times k
const	xload	xmm4, XMM_K_TIMES_MULCONST_LO
	mulpd	xmm4, xmm5		;; high_FFT_carry * k_lo
	addpd	xmm1, xmm4		;; x5 = x5 + high_FFT_carry * k_lo
no const xload	xmm4, XMM_K_HI
const	xload	xmm4, XMM_K_TIMES_MULCONST_HI
	mulpd	xmm4, XMM_LIMIT_INVERSE[rax] ;; shift k_hi
	mulpd	xmm5, xmm4
	rounding exec, base2, noexec, noexec, xmm1, xmm3, xmm4, rax
	addpd	xmm3, xmm5		;; Carry += high_FFT_carry * shifted k_hi
	mulpd	xmm1, [rdx+1*32+16]	;; mul by grp two-to-phi
	xload	xmm4, [rbx+4*32+16]	;; col two-to-phi
	mulpd	xmm4, XMM_TTP_FUDGE[rax];; mul by fudge two-to-phi
	mulpd	xmm1, xmm4		;; data *= fudged col two-to-phi
	xstore	[rbp+4*64+16], xmm1	;; Save FFT data

	movzx	rax, BYTE PTR [rdi+rcx+5] ;; Load big vs. little flag
	subpd	xmm3, XMM_BIGVAL	;; Remove rounding const from carry
	mulpd	xmm3, [rdx+1*32+16]	;; mul by grp two-to-phi
	xload	xmm4, [rbx+5*32+16]	;; col two-to-phi
	mulpd	xmm4, XMM_TTP_FUDGE[rax];; mul by fudge two-to-phi
	mulpd	xmm3, xmm4		;; data *= fudged col two-to-phi
	addpd	xmm3, [rbp+5*64+16]	;; Load FFT data
	xstore	[rbp+5*64+16], xmm3	;; Save FFT data

	xload	xmm4, XMM_BIGVAL
	xstore	[rsi+0*16], xmm4	;; Clear carry
	xstore	[rsi+1*16], xmm4
	xstore	[rsi+2*16], xmm4
	xstore	[rsi+3*16], xmm4
	ENDM


; For WPN macros, these registers are set on input:
; xmm6 = maxerr
; rbp = pointer to carries
; rdi = pointer to big/little flags
; rsi = pointer to the FFT data
; rdx = pointer two-to-phi group multipliers
; ebx = big vs. little & fudge flags
; ecx = big vs. little word flag #2
; eax = big vs. little word flag #1

; *************** WPN macro ******************
; A pipelined version of this code:
;	movzx	rax, BYTE PTR [rdi]	;; Load big vs. little flags
;	xload	xmm0, [rsi+0*dist1]	;; Load values1
;	mulpd	xmm0, [rdx+0*XMM_GMD][rax] ;; Mul by fudged grp two-to-minus-phi
;	addpd	xmm0, [rbp+0*16]	;; x1 = values + carry
;	xload	xmm2, XMM_LIMIT_BIGMAX[rax];; Load maximum * BIGVAL - BIGVAL
;	addpd	xmm2, xmm0		;; y1 = top bits of x
;	xload	xmm6, XMM_LIMIT_BIGMAX_NEG[rax];; Load -(maximum*BIGVAL-BIGVAL)
;	addpd	xmm6, xmm2		;; z1 = y1-(maximum * BIGVAL - BIGVAL)
;	subpd	xmm0, xmm6		;; rounded value = x1 - z1
;	mulpd	xmm2, XMM_LIMIT_INVERSE[rax];; next carry = shifted y1
;	mulpd	xmm0, [rdx+0*XMM_GMD+XMM_GMD/2][rax];; new value1 = val * fudged grp two-to-phi
;	xstore	[rsi+0*dist1], xmm0	;; Save new value1
;	xstore	[rbp+0*16], xmm2	;; Save carry
;

xnorm_wpn_preload MACRO ttp, echk, const, base2, sse4
	ENDM

xnorm_wpn MACRO ttp, echk, const, base2, sse4
ttp		movzx	ecx, bl				;; Fudge flags 1-2
ttp		and	rcx, 0f0h
		xload	xmm0, [rsi+0*16]		;; Load values1
ttp		mulpd	xmm0, [rdx+0*XMM_GMD][rcx]	;; Mul by fudged grp two-to-minus-phi
		xload	xmm1, [rsi+1*16]		;; Load values2
ttp		mulpd	xmm1, [rdx+1*XMM_GMD][rcx]	;; Mul by fudged grp two-to-minus-phi
ttp		movzx	eax, bh				;; Big/lit flags 1-4
no ttp base2	sub	rax, rax			;; --We should clean up base 2 rational FFT so that this isn't needed
no const echk	error_check_interleaved sse4, xmm0, xmm4, xmm1, xmm5, xmm6
const		mul_by_const_interleaved ttp, base2, echk, sse4, xmm0, xmm4, xmm2, rax*4, xmm1, xmm5, xmm3, rax*4+16, xmm6
		addpd	xmm0, [rbp+0*16]		;; x1 = values + carry
		addpd	xmm1, [rbp+1*16]		;; x2 = values + carry

no base2	rounding_interleaved ttp, base2, const, sse4, xmm0, xmm4, xmm2, rax*4, xmm1, xmm5, xmm3, rax*4+16

base2		xload	xmm2, XMM_LIMIT_BIGMAX[rax*4]	;; Load maximum * BIGVAL - BIGVAL
base2 no const	xcopy	xmm5, xmm2			;; Copy maximum * BIGVAL - BIGVAL
base2 const no echk xcopy xmm6, xmm2			;; Copy maximum * BIGVAL - BIGVAL
base2		addpd	xmm2, xmm0			;; y1 = top bits of x
base2		xload	xmm3, XMM_LIMIT_BIGMAX[rax*4+16];; Load maximum * BIGVAL - BIGVAL
base2		xcopy	xmm7, xmm3			;; Copy maximum * BIGVAL - BIGVAL
base2		addpd	xmm3, xmm1			;; y2 = top bits of x
base2 const	addpd	xmm4, xmm2			;; Add in upper mul-by-const bits
base2 const	mulpd	xmm4, XMM_LIMIT_INVERSE[rax*4]	;; next carry = shifted y1
base2 no const	xload	xmm4, XMM_LIMIT_INVERSE[rax*4]	;; next carry = shifted y1
base2 no const	mulpd	xmm4, xmm2			;; next carry = shifted y1
		xstore	[rbp+0*16], xmm4		;; Save carry1
base2 const echk subpd	xmm2, XMM_LIMIT_BIGMAX[rax*4]	;; z1 = y1 - (maximum*BIGVAL-BIGVAL)
base2 const no echk subpd xmm2, xmm6			;; z1 = y1 - (maximum*BIGVAL-BIGVAL)
base2 no const	subpd	xmm2, xmm5			;; z1 = y1 - (maximum*BIGVAL-BIGVAL)
base2 const	addpd	xmm5, xmm3			;; Add in upper mul-by-const bits
base2 const	mulpd	xmm5, XMM_LIMIT_INVERSE[rax*4+16];; next carry = shifted y2
base2 no const	xload	xmm5, XMM_LIMIT_INVERSE[rax*4+16];; next carry = shifted y2
base2 no const	mulpd	xmm5, xmm3			;; next carry = shifted y2
		xstore	[rbp+1*16], xmm5		;; Save carry2
base2		subpd	xmm3, xmm7			;; z2 = y2 - (maximum*BIGVAL-BIGVAL)
base2		subpd	xmm0, xmm2			;; rounded value = x1 - z1
base2		subpd	xmm1, xmm3			;; rounded value = x2 - z2
ttp		mulpd	xmm0, [rdx+0*XMM_GMD+XMM_GMD/2][rcx] ;; value1 = rounded value * fudged grp two-to-phi
		xstore	[rsi+0*16], xmm0		;; Save new value1
ttp		mulpd	xmm1, [rdx+1*XMM_GMD+XMM_GMD/2][rcx] ;; value2 = rounded value * fudged grp two-to-phi
		xstore	[rsi+1*16], xmm1		;; Save new value2

ttp		and	rbx, 0fh			;; Fudge flags 3-4
		xload	xmm0, [rsi+2*16]		;; Load values1
ttp		mulpd	xmm0, [rdx+2*XMM_GMD][rbx*8]	;; Mul by fudged grp two-to-minus-phi
		xload	xmm1, [rsi+3*16]		;; Load values2
ttp		mulpd	xmm1, [rdx+3*XMM_GMD][rbx*8]	;; Mul by fudged grp two-to-minus-phi
no const echk	error_check_interleaved sse4, xmm0, xmm4, xmm1, xmm5, xmm6
const		mul_by_const_interleaved ttp, base2, echk, sse4, xmm0, xmm4, xmm2, rax*4+32, xmm1, xmm5, xmm3, rax*4+48, xmm6
		addpd	xmm0, [rbp+2*16]		;; x1 = values + carry
		addpd	xmm1, [rbp+3*16]		;; x2 = values + carry

no base2	rounding_interleaved ttp, base2, const, sse4, xmm0, xmm4, xmm2, rax*4+32, xmm1, xmm5, xmm3, rax*4+48

base2		xload	xmm2, XMM_LIMIT_BIGMAX[rax*4+32];; Load maximum * BIGVAL - BIGVAL
base2 no const	xcopy	xmm5, xmm2			;; Copy maximum * BIGVAL - BIGVAL
base2 const no echk xcopy xmm6, xmm2			;; Copy maximum * BIGVAL - BIGVAL
base2		addpd	xmm2, xmm0			;; y1 = top bits of x
base2		xload	xmm3, XMM_LIMIT_BIGMAX[rax*4+48];; Load maximum * BIGVAL - BIGVAL
base2		xcopy	xmm7, xmm3			;; Copy maximum * BIGVAL - BIGVAL
base2		addpd	xmm3, xmm1			;; y2 = top bits of x
base2 const	addpd	xmm4, xmm2			;; Add in upper mul-by-const bits
base2 const	mulpd	xmm4, XMM_LIMIT_INVERSE[rax*4+32];; next carry = shifted y1
base2 no const	xload	xmm4, XMM_LIMIT_INVERSE[rax*4+32];; next carry = shifted y1
base2 no const	mulpd	xmm4, xmm2			;; next carry = shifted y1
		xstore	[rbp+2*16], xmm4		;; Save carry1
base2 const echk subpd	xmm2, XMM_LIMIT_BIGMAX[rax*4+32];; z1 = y1 - (maximum*BIGVAL-BIGVAL)
base2 const no echk subpd xmm2, xmm6			;; z1 = y1 - (maximum*BIGVAL-BIGVAL)
base2 no const	subpd	xmm2, xmm5			;; z1 = y1 - (maximum*BIGVAL-BIGVAL)
base2 const	addpd	xmm5, xmm3			;; Add in upper mul-by-const bits
base2 const	mulpd	xmm5, XMM_LIMIT_INVERSE[rax*4+48];; next carry = shifted y2
base2 no const	xload	xmm5, XMM_LIMIT_INVERSE[rax*4+48];; next carry = shifted y2
base2 no const	mulpd	xmm5, xmm3			;; next carry = shifted y2
		xstore	[rbp+3*16], xmm5		;; Save carry2
base2		subpd	xmm3, xmm7			;; z2 = y2 - (maximum*BIGVAL-BIGVAL)
base2		subpd	xmm0, xmm2			;; rounded value = x1 - z1
base2		subpd	xmm1, xmm3			;; rounded value = x2 - z2
ttp		mulpd	xmm0, [rdx+2*XMM_GMD+XMM_GMD/2][rbx*8] ;; value1 *= rounded value * fudged grp two-to-phi
ttp		mulpd	xmm1, [rdx+3*XMM_GMD+XMM_GMD/2][rbx*8] ;; value2 = rounded value * fudged grp two-to-phi
ttp		movzx	rbx, WORD PTR [rdi+2]		;; Load next 4 big vs. little & fudge flags
		xstore	[rsi+2*16], xmm0		;; Save new value1
		xstore	[rsi+3*16], xmm1		;; Save new value2
	ENDM

IFDEF X86_64

xnorm_wpn_preload MACRO ttp, echk, const, base2, sse4
echk sse4	movapd	xmm15, XMM_ABSVAL
echk no sse4	movapd	xmm15, XMM_BIGVAL
	ENDM

xnorm_wpn MACRO ttp, echk, const, base2, sse4
ttp		movzx	rcx, bl				;; Fudge flags 1,2
ttp		and	rcx, 0f0h
		xload	xmm8, [rsi+0*16]		;; Load values1				;P4	;Core2
ttp		mulpd	xmm8, [rdx+0*XMM_GMD][rcx]	;; Mul by fudged grp two-to-minus-phi	;1-5	;1-4
		xload	xmm9, [rsi+1*16]		;; Load values2
ttp		mulpd	xmm9, [rdx+1*XMM_GMD][rcx]	;; Mul by fudged grp two-to-minus-phi	;3-7	;2-5
ttp		movzx	eax, bh				;; Big/lit flags 1-4
no ttp base2	sub	rax, rax			;; --We should clean up base 2 rational FFT so that this isn't needed
no const echk	error_check_interleaved_64 sse4, xmm8, xmm10, xmm9, xmm11, xmm6
const		mul_by_const_interleaved ttp, base2, echk, sse4, xmm8, xmm12, xmm10, rax*4, xmm9, xmm13, xmm11, rax*4+16, xmm6
ttp		and	rbx, 0fh			;; Fudge flags 3,4
		xload	xmm0, [rsi+2*16]		;; Load values3
ttp		mulpd	xmm0, [rdx+2*XMM_GMD][rbx*8]	;; Mul by fudged grp two-to-minus-phi	;5-9	;3-6
		xload	xmm1, [rsi+3*16]		;; Load values4
ttp		mulpd	xmm1, [rdx+3*XMM_GMD][rbx*8]	;; Mul by fudged grp two-to-minus-phi	;7-11	;4-7
no const echk	error_check_interleaved_64 sse4, xmm0, xmm2, xmm1, xmm3, xmm6
const		mul_by_const_interleaved ttp, base2, echk, sse4, xmm0, xmm4, xmm2, rax*4+32, xmm1, xmm5, xmm3, rax*4+48, xmm6
const		addpd	xmm8, [rbp+0*16]		;; x1 = values + carry
no const	addpd	xmm8, xmm12			;; x1 = values + carry			;6-9	;5-7
const		addpd	xmm9, [rbp+1*16]		;; x2 = values + carry
no const	addpd	xmm9, xmm13			;; x2 = values + carry			;8-11	;6-8

no base2	rounding_interleaved ttp, base2, const, sse4, xmm8, xmm12, xmm10, rax*4, xmm9, xmm13, xmm11, rax*4+16

const		addpd	xmm0, [rbp+2*16]		;; x3 = values + carry
no const	addpd	xmm0, xmm4			;; x3 = values + carry			;10-13	;7-9
base2		xload	xmm10, XMM_LIMIT_BIGMAX[rax*4]	;; Load maximum * BIGVAL - BIGVAL
base2 no const	xcopy	xmm4, xmm10			;; Copy maximum * BIGVAL - BIGVAL
base2 const no echk xcopy xmm15, xmm10			;; Copy maximum * BIGVAL - BIGVAL
base2		addpd	xmm10, xmm8			;; y1 = top bits of x			;12-15	;8-10
const		addpd	xmm1, [rbp+3*16]		;; x4 = values + carry
no const	addpd	xmm1, xmm5			;; x4 = values + carry			;14-17	;9-11

no base2	rounding_interleaved ttp, base2, const, sse4, xmm0, xmm4, xmm2, rax*4+32, xmm1, xmm5, xmm3, rax*4+48

base2		xload	xmm11, XMM_LIMIT_BIGMAX[rax*4+16] ;; Load maximum * BIGVAL - BIGVAL
base2 no const	xcopy	xmm5, xmm11			;; Copy maximum * BIGVAL - BIGVAL
base2 const no echk xcopy xmm6, xmm11			;; Copy maximum * BIGVAL - BIGVAL
base2		addpd	xmm11, xmm9			;; y2 = top bits of x			;16-19	;10-12
base2 const	addpd	xmm12, xmm10			;; Add in upper mul-by-const bits
base2 const	mulpd	xmm12, XMM_LIMIT_INVERSE[rax*4]	;; next carry = shifted y1
base2 no const	xload	xmm12, XMM_LIMIT_INVERSE[rax*4]	;; next carry = shifted y1
base2 no const	mulpd	xmm12, xmm10			;; next carry = shifted y1		;17-22	;11-15
base2		xload	xmm2, XMM_LIMIT_BIGMAX[rax*4+32];; Load maximum * BIGVAL - BIGVAL
base2		xcopy	xmm7, xmm2			;; Copy maximum * BIGVAL - BIGVAL
base2		addpd	xmm2, xmm0			;; y3 = top bits of x			;18-21	;11-13
base2		xload	xmm3, XMM_LIMIT_BIGMAX[rax*4+48];; Load maximum * BIGVAL - BIGVAL
base2		xcopy	xmm14, xmm3			;; Copy maximum * BIGVAL - BIGVAL
base2		addpd	xmm3, xmm1			;; y4 = top bits of x			;20-23	;12-14
base2 const	addpd	xmm13, xmm11			;; Add in upper mul-by-const bits
base2 const	mulpd	xmm13, XMM_LIMIT_INVERSE[rax*4+16];; next carry = shifted y2
base2 no const	xload	xmm13, XMM_LIMIT_INVERSE[rax*4+16];; next carry = shifted y2
base2 no const	mulpd	xmm13, xmm11			;; next carry = shifted y2		;21-26	;13-17
base2 const echk subpd	xmm10, XMM_LIMIT_BIGMAX[rax*4]	;; z1 = y1 - (maximum*BIGVAL-BIGVAL)
base2 const no echk subpd xmm10, xmm15			;; z1 = y1 - (maximum*BIGVAL-BIGVAL)
base2 no const	subpd	xmm10, xmm4			;; z1 = y1 - (maximum*BIGVAL-BIGVAL)	;22-25	;13-15
base2 const	addpd	xmm4, xmm2			;; Add in upper mul-by-const bits
base2 const	mulpd	xmm4, XMM_LIMIT_INVERSE[rax*4+32];; next carry = shifted y3
base2 no const	xload	xmm4, XMM_LIMIT_INVERSE[rax*4+32];; next carry = shifted y3
base2 no const	mulpd	xmm4, xmm2			;; next carry = shifted y3		;23-28	;14-18
base2 const echk subpd	xmm11, XMM_LIMIT_BIGMAX[rax*4+16];; z2 = y2 - (maximum*BIGVAL-BIGVAL)
base2 const no echk subpd xmm11, xmm6			;; z2 = y2 - (maximum*BIGVAL-BIGVAL)
base2 no const	subpd	xmm11, xmm5			;; z2 = y2 - (maximum*BIGVAL-BIGVAL)	;24-27	;14-16
base2 const	addpd	xmm5, xmm3			;; Add in upper mul-by-const bits
base2 const	mulpd	xmm5, XMM_LIMIT_INVERSE[rax*4+48];; next carry = shifted y4
base2 no const	xload	xmm5, XMM_LIMIT_INVERSE[rax*4+48];; next carry = shifted y4
base2 no const	mulpd	xmm5, xmm3			;; next carry = shifted y4		;25-30	;15-19
const		xstore	[rbp+0*16], xmm12		;; Save carry1
base2		subpd	xmm2, xmm7			;; z3 = y3 - (maximum*BIGVAL-BIGVAL)	;26-29	;15-17
const		xstore	[rbp+1*16], xmm13		;; Save carry2
base2		subpd	xmm3, xmm14			;; z4 = y4 - (maximum*BIGVAL-BIGVAL)	;28-31	;16-18
base2		subpd	xmm8, xmm10			;; rounded value = x1 - z1		;30-33	;17-19
const		xstore	[rbp+2*16], xmm4		;; Save carry3
base2		subpd	xmm9, xmm11			;; rounded value = x2 - z2		;32-35	;18-20
const		xstore	[rbp+3*16], xmm5		;; Save carry4
base2		subpd	xmm0, xmm2			;; rounded value = x3 - z3		;34-37	;19-21
ttp		mulpd	xmm8, [rdx+0*XMM_GMD+XMM_GMD/2][rcx] ;; value1 *= fudged grp two-to-phi	;35-40	;20-24
base2		subpd	xmm1, xmm3			;; rounded value = x4 - z4		;36-39	;20-22
ttp		mulpd	xmm9, [rdx+1*XMM_GMD+XMM_GMD/2][rcx] ;; value2 *= fudged grp two-to-phi	;37-42	;21-25
ttp		mulpd	xmm0, [rdx+2*XMM_GMD+XMM_GMD/2][rbx*8] ;; value3 *= fudged grp two-to-phi;38-41	;22-26
ttp		mulpd	xmm1, [rdx+3*XMM_GMD+XMM_GMD/2][rbx*8] ;; value4 *= fudged grp two-to-phi;40-43	;23-27
ttp		movzx	rbx, WORD PTR [rdi+2]		;; Load next 4 big vs. little & fudge flags
		xstore	[rsi+0*16], xmm8		;; Save new value1
		xstore	[rsi+1*16], xmm9		;; Save new value2
		xstore	[rsi+2*16], xmm0		;; Save new value3
		xstore	[rsi+3*16], xmm1		;; Save new value4
	ENDM
ENDIF

;; NOTE: We'd rather store the high FFT carry without the XMM_BIGVAL added in,
;; but there is too much code that expects this (like add and subtract).
;; ALSO NOTE:  In the zero pad case, big/lit & fudge flags 1,2 and the
;; same as big/lit & fudge flags 3,4.

xnorm_wpn_zpad_preload MACRO ttp, echk, const, base2, sse4, khi, c1, cm1
	ENDM

xnorm_wpn_zpad MACRO ttp, echk, const, base2, sse4, khi, c1, cm1
ttp	movzx	eax, bh			;; Big/little flags 1-4
ttp	and	rbx, 0f0h		;; Fudge flags 1,2
ttp	xload	xmm2, [rdx+0*XMM_GMD][rbx] ;; Fudged grp two-to-minus-phi
	xload	xmm0, [rsi]		;; Load values1
	xload	xmm1, [rsi+2*16]	;; Load values2
ttp	mulpd	xmm0, xmm2		;; Mul by fudged grp two-to-minus-phi
ttp	mulpd	xmm1, xmm2		;; Mul by fudged grp two-to-minus-phi

	xload	xmm3, [rbp+2*16]	;; Add in previous high FFT data
	split_lower_zpad_word echk, base2, sse4, xmm0, xmm3, xmm4, rax*4
	xstore	[rbp+2*16], xmm3

no const	xload	xmm0, XMM_K_LO
const		xload	xmm0, XMM_K_TIMES_MULCONST_LO
		mulpd	xmm0, xmm4
khi no const	xload	xmm5, XMM_K_HI
khi const	xload	xmm5, XMM_K_TIMES_MULCONST_HI
khi no base2	mulpd	xmm5, XMM_LIMIT_INVERSE[rax*4] ;; Non-base2 rounding needs shifted carry
khi		mulpd	xmm5, xmm4
 
		addpd	xmm0, [rbp+0*16]	;; x1 = values + carry

c1		mulpd   xmm1, XMM_MINUS_C	;; Do one mul before split rather than two after split

	split_upper_zpad_word echk, base2, sse4, xmm1, xmm4, xmm2, rax*4

no const no c1 no cm1	mulpd	xmm2, XMM_MINUS_C
no const no c1 no cm1	mulpd	xmm4, XMM_MINUS_C
const			mulpd	xmm2, XMM_MINUS_C_TIMES_MULCONST
const			mulpd	xmm4, XMM_MINUS_C_TIMES_MULCONST

	addpd	xmm0, xmm2		;; Add upper FFT word to lower FFT word
khi	addpd	xmm4, xmm5		;; Add upper FFT word to lower FFT word

	rounding ttp, base2, exec, sse4, xmm0, xmm4, xmm2, rax*4

ttp	mulpd	xmm0, [rdx+0*XMM_GMD+XMM_GMD/2][rbx] ;; new value1 *= fudged grp two-to-phi
	xstore	[rbp+0*16], xmm4	;; Save carry
	xstore	[rsi], xmm0		;; Save new value1

ttp	xload	xmm3, [rdx+1*XMM_GMD][rbx] ;; Fudged grp two-to-minus-phi
	xload	xmm5, [rsi+16]		;; Load high values1
	xload	xmm1, [rsi+3*16]	;; Load high values2
ttp	mulpd	xmm5, xmm3		;; Mul by fudged grp two-to-minus-phi
ttp	mulpd	xmm1, xmm3		;; Mul by fudged grp two-to-minus-phi

	xload	xmm3, [rbp+3*16]	;; Add in previous high FFT data
	split_lower_zpad_word echk, base2, sse4, xmm5, xmm3, xmm2, rax*4+16
	xstore	[rbp+3*16], xmm3

no const	xload	xmm0, XMM_K_LO
const		xload	xmm0, XMM_K_TIMES_MULCONST_LO
		mulpd	xmm0, xmm2
khi no const	xload	xmm5, XMM_K_HI
khi const	xload	xmm5, XMM_K_TIMES_MULCONST_HI
khi no base2	mulpd	xmm5, XMM_LIMIT_INVERSE[rax*4+16] ;; Non-base2 rounding needs shifted carry
khi		mulpd	xmm5, xmm2

		addpd	xmm0, [rbp+1*16]	;; x2 = values + carry

c1		mulpd   xmm1, XMM_MINUS_C	;; Do one mul before split rather than two after split

	split_upper_zpad_word echk, base2, sse4, xmm1, xmm2, xmm4, rax*4+16

no const no c1 no cm1	mulpd	xmm4, XMM_MINUS_C
no const no c1 no cm1	mulpd	xmm2, XMM_MINUS_C
const			mulpd	xmm4, XMM_MINUS_C_TIMES_MULCONST
const			mulpd	xmm2, XMM_MINUS_C_TIMES_MULCONST

	addpd	xmm0, xmm4		;; Add upper FFT word to lower FFT word
khi	addpd	xmm2, xmm5		;; Add upper FFT word to lower FFT word

	rounding ttp, base2, exec, sse4, xmm0, xmm2, xmm4, rax*4+16

ttp	mulpd	xmm0, [rdx+1*XMM_GMD+XMM_GMD/2][rbx] ;; new value2 *= fudged grp two-to-phi
ttp	movzx	rbx, WORD PTR [rdi+2]	;; Load next big vs. little & fudge flags
	xstore	[rbp+1*16], xmm2	;; Save carry
	xstore	[rsi+1*16], xmm0	;; Save new value2

	subpd	xmm1, xmm1		;; new high values = zero
	xstore	[rsi+2*16], xmm1	;; Zero high value1
	xstore	[rsi+3*16], xmm1	;; Zero high value2
	ENDM

;; 64-bit version using extra registers

IFDEF X86_64

xnorm_wpn_zpad_preload MACRO ttp, echk, const, base2, sse4, khi, c1, cm1
no const			xload	xmm15, XMM_K_LO
const				xload	xmm15, XMM_K_TIMES_MULCONST_LO
no const			xload	xmm14, XMM_MINUS_C
const				xload	xmm14, XMM_MINUS_C_TIMES_MULCONST
base2				xload	xmm13, XMM_BIGBIGVAL
base2 no echk no sse4		xload	xmm6, XMM_BIGVAL
base2 no echk sse4 khi no const	xload	xmm6, XMM_K_HI
base2 no echk sse4 khi const	xload	xmm6, XMM_K_TIMES_MULCONST_HI
no base2 no sse4		xload	xmm13, XMM_BIGVAL
no base2 sse4 echk		xload	xmm13, XMM_ABSVAL
no base2 no echk khi no const	xload	xmm6, XMM_K_HI
no base2 no echk khi const	xload	xmm6, XMM_K_TIMES_MULCONST_HI
	ENDM

xnorm_wpn_zpad MACRO ttp, echk, const, base2, sse4, khi, c1, cm1
ttp	movzx	eax, bh			;; Big/little flags 1-4
ttp	and	rbx, 0f0h		;; Fudge flags 1,2

ttp	xload	xmm2, [rdx+0*XMM_GMD][rbx] ;; Fudged grp two-to-minus-phi
	xload	xmm0, [rsi]		;; Load values1
	xload	xmm1, [rsi+2*16]	;; Load values2
ttp	mulpd	xmm0, xmm2		;; Mul by fudged grp two-to-minus-phi
ttp	mulpd	xmm1, xmm2		;; Mul by fudged grp two-to-minus-phi

ttp	xload	xmm9, [rdx+1*XMM_GMD][rbx] ;; Fudged grp two-to-minus-phi
	xload	xmm7, [rsi+16]		;; Load high values1
	xload	xmm8, [rsi+3*16]	;; Load high values2
ttp	mulpd	xmm7, xmm9		;; Mul by fudged grp two-to-minus-phi
ttp	mulpd	xmm8, xmm9		;; Mul by fudged grp two-to-minus-phi

					;; Split lower word adding in previous high FFT data
	split_lower_zpad_word_interleaved echk, base2, sse4, xmm0, xmm3, xmm5, rax*4, xmm7, xmm10, xmm12, rax*4+16

no const	xcopy	xmm0, xmm15	;; XMM_K_LO
const		xcopy	xmm0, xmm15	;; XMM_K_TIMES_MULCONST_LO
		mulpd	xmm0, xmm5
no const	xcopy	xmm7, xmm15	;; XMM_K_LO
const		xcopy	xmm7, xmm15	;; XMM_K_TIMES_MULCONST_LO
		mulpd	xmm7, xmm12

		addpd	xmm0, xmm4	;; x1 = values + carry
		addpd	xmm7, xmm11	;; x2 = values + carry

khi no const no base2 echk		xload	xmm4, XMM_K_HI
khi no const no base2 no echk		xcopy	xmm4, xmm6	;; XMM_K_HI
khi no const base2 echk			xload	xmm4, XMM_K_HI
khi no const base2 no echk no sse4	xload	xmm4, XMM_K_HI
khi no const base2 no echk sse4		xcopy	xmm4, xmm6	;; XMM_K_HI
khi const no base2 echk			xload	xmm4, XMM_K_TIMES_MULCONST_HI
khi const no base2 no echk		xcopy	xmm4, xmm6	;; XMM_K_TIMES_MULCONST_HI
khi const base2 echk			xload	xmm4, XMM_K_TIMES_MULCONST_HI
khi const base2 no echk no sse4		xload	xmm4, XMM_K_TIMES_MULCONST_HI
khi const base2 no echk sse4		xcopy	xmm4, xmm6	;; XMM_K_TIMES_MULCONST_HI
khi no base2				mulpd	xmm4, XMM_LIMIT_INVERSE[rax*4] ;; Non-base2 rounding needs shifted carry
khi					mulpd	xmm5, xmm4

khi no const no base2 echk		xload	xmm11, XMM_K_HI
khi no const no base2 no echk		xcopy	xmm11, xmm6	;; XMM_K_HI
khi no const base2 echk			xload	xmm11, XMM_K_HI
khi no const base2 no echk no sse4	xload	xmm11, XMM_K_HI
khi no const base2 no echk sse4		xcopy	xmm11, xmm6	;; XMM_K_HI
khi const no base2 echk			xload	xmm11, XMM_K_TIMES_MULCONST_HI
khi const no base2 no echk		xcopy	xmm11, xmm6	;; XMM_K_TIMES_MULCONST_HI
khi const base2 echk			xload	xmm11, XMM_K_TIMES_MULCONST_HI
khi const base2 no echk no sse4		xload	xmm11, XMM_K_TIMES_MULCONST_HI
khi const base2 no echk sse4		xcopy	xmm11, xmm6	;; XMM_K_TIMES_MULCONST_HI
khi no base2				mulpd	xmm11, XMM_LIMIT_INVERSE[rax*4+16] ;; Non-base2 rounding needs shifted carry
khi					mulpd	xmm12, xmm11

c1	mulpd   xmm1, xmm14		;; XMM_MINUS_C	;; Do one mul before split rather than two after split
c1	mulpd   xmm8, xmm14		;; XMM_MINUS_C	;; Do one mul before split rather than two after split

	split_upper_zpad_word_interleaved echk, base2, sse4, xmm1, xmm4, xmm2, rax*4, xmm8, xmm11, xmm9, rax*4+16

no const no c1 no cm1	mulpd	xmm2, xmm14	;; XMM_MINUS_C
no const no c1 no cm1	mulpd	xmm4, xmm14	;; XMM_MINUS_C
const			mulpd	xmm2, xmm14	;; XMM_MINUS_C_TIMES_MULCONST
const			mulpd	xmm4, xmm14	;; XMM_MINUS_C_TIMES_MULCONST
no const no c1 no cm1	mulpd	xmm9, xmm14	;; XMM_MINUS_C
no const no c1 no cm1	mulpd	xmm11, xmm14	;; XMM_MINUS_C
const			mulpd	xmm9, xmm14	;; XMM_MINUS_C_TIMES_MULCONST
const			mulpd	xmm11, xmm14	;; XMM_MINUS_C_TIMES_MULCONST

	addpd	xmm0, xmm2		;; Add upper FFT word to lower FFT word
khi	addpd	xmm4, xmm5		;; Add upper FFT word to lower FFT word
	addpd	xmm7, xmm9		;; Add upper FFT word to lower FFT word
khi	addpd	xmm11, xmm12		;; Add upper FFT word to lower FFT word

	rounding_interleaved ttp, base2, exec, sse4, xmm0, xmm4, xmm2, rax*4, xmm7, xmm11, xmm9, rax*4+16

ttp	mulpd	xmm0, [rdx+0*XMM_GMD+XMM_GMD/2][rbx] ;; new value1 *= fudged grp two-to-phi
ttp	mulpd	xmm7, [rdx+1*XMM_GMD+XMM_GMD/2][rbx] ;; new value2 *= fudged grp two-to-phi

ttp	movzx	rbx, WORD PTR [rdi+2]	;; Load next big vs. little & fudge flags

	xstore	[rsi], xmm0		;; Save new value1
	xstore	[rsi+1*16], xmm7	;; Save new value2

	subpd	xmm1, xmm1		;; new high values = zero
	xstore	[rsi+2*16], xmm1	;; Zero high value1
	xstore	[rsi+3*16], xmm1	;; Zero high value2
	ENDM
ENDIF


; *************** WPN followup macro ******************
; This macro finishes the normalize process by adding the final carries
; back into the appropriate FFT values.
; rsi = pointer to carries
; rbp = pointer to FFT data
; rdi = pointer to big/little flags
; rdx = pointer two-to-phi group multipliers
; ecx = big vs. little word flag #2
; eax = big vs. little word flag #1

; Rotate the carries array
xnorm012_wpn_part1 MACRO
	LOCAL	shuflp, done
	cmp	THIS_BLOCK, 0		;; Are we carrying into the first data block?
	jne	done			;; If not, skip wrapping carries and negating the last carry
	mov	eax, addcount1		;; Load count of cache lines in the carries array
shuflp:	movsd	xmm4, Q [rsi+0*16]	;; Load low carry word
	movsd	Q [rsi+0*16], xmm0	;; Save prev cache line's high carry in low word
	movsd	xmm0, Q [rsi+0*16+8]	;; Load high carry for next cache line
	movsd	Q [rsi+0*16+8], xmm4	;; Move this cache line's low carry to high word
	movsd	xmm4, Q [rsi+1*16]	;; Load low carry word
	movsd	Q [rsi+1*16], xmm1	;; Save prev cache line's high carry in low word
	movsd	xmm1, Q [rsi+1*16+8]	;; Load high carry for next cache line
	movsd	Q [rsi+1*16+8], xmm4	;; Move this cache line's low carry to high word
	movsd	xmm4, Q [rsi+2*16]	;; Load low carry word
	movsd	Q [rsi+2*16], xmm2	;; Save prev cache line's high carry in low word
	movsd	xmm2, Q [rsi+2*16+8]	;; Load high carry for next cache line
	movsd	Q [rsi+2*16+8], xmm4	;; Move this cache line's low carry to high word
	movsd	xmm4, Q [rsi+3*16]	;; Load low carry word
	movsd	Q [rsi+3*16], xmm3	;; Save prev cache line's high carry in low word
	movsd	xmm3, Q [rsi+3*16+8]	;; Load high carry for next cache line
	movsd	Q [rsi+3*16+8], xmm4	;; Move this cache line's low carry to high word
	bump	rsi, 64			;; Next carry cache line
	dec	rax			;; Decrement count of cache lines
	jnz	short shuflp
	subsd	xmm3, Q XMM_BIGVAL
	mulsd	xmm3, Q XMM_MINUS_C	;; Negate the very last carry
	addsd	xmm3, Q XMM_BIGVAL
	mov	rsi, carries		;; Reload carries array pointer
	movsd	Q [rsi+0*16], xmm3	;; Move last cache line's high carries into first cache line
	movsd	Q [rsi+1*16], xmm0
	movsd	Q [rsi+2*16], xmm1
	movsd	Q [rsi+3*16], xmm2
done:
	ENDM

xnorm012_wpn MACRO base2
	LOCAL	hard, done

	;; If k or c is more than one, then there will be fewer bits-per-word.
	;; This means the carry may need to be spread over 4 words instead
	;; of just 2.

	cmp	SPREAD_CARRY_OVER_EXTRA_WORDS, 1;; Are there few bits per word?
	je	hard			;; Yes, go do it the hard way

	movzx	rax, BYTE PTR [rdi+1]	;; Load 4 big vs. little flags
	movzx	rcx, BYTE PTR [rdi]	;; Load fudge factor flags 1,2
	and	rcx, 0f0h
	xload	xmm4, [rbp+0*16]	;; FFT data
	mulpd	xmm4, [rdx+0*XMM_GMD][rcx] ;; mul by fudged grp two-to-minus-phi
	xload	xmm0, [rsi+0*16]	;; Load carry
	addpd	xmm0, xmm4		;; carry + FFT data
	rounding exec, base2, noexec, noexec, xmm0, xmm5, xmm4, rax*4
	subpd	xmm5, XMM_BIGVAL
	mulpd	xmm0, [rdx+0*XMM_GMD+XMM_GMD/2][rcx] ;; Value *= fudged grp two-to-phi
	movzx	rbx, BYTE PTR [rdi+2]	;; Load fudge factor flags 5,6
	and	rbx, 0f0h
	mulpd	xmm5, [rdx+0*XMM_GMD+XMM_GMD/2][rbx] ;; high carry *= fudged grp two-to-phi
	addpd	xmm5, [rbp+4*16]	;; Add high carry and FFT data
	xstore	[rbp+0*16], xmm0	;; Save FFT data
	xstore	[rbp+4*16], xmm5	;; Save FFT data

	xload	xmm4, [rbp+1*16]	;; FFT data
	mulpd	xmm4, [rdx+1*XMM_GMD][rcx] ;; mul by fudged grp two-to-minus-phi
	xload	xmm1, [rsi+1*16]	;; Load carry
	addpd	xmm1, xmm4		;; carry + FFT data
	rounding exec, base2, noexec, noexec, xmm1, xmm5, xmm4, rax*4+16
	subpd	xmm5, XMM_BIGVAL
	mulpd	xmm1, [rdx+1*XMM_GMD+XMM_GMD/2][rcx] ;; value *= fudged grp two-to-phi
	mulpd	xmm5, [rdx+1*XMM_GMD+XMM_GMD/2][rbx] ;; high carry *= fudged grp two-to-phi
	addpd	xmm5, [rbp+5*16]	;; Add high carry and FFT data
	xstore	[rbp+1*16], xmm1	;; Save FFT data
	xstore	[rbp+5*16], xmm5	;; Save FFT data

	movzx	rcx, BYTE PTR [rdi]	;; Load fudge factor flags 3,4
	and	rcx, 0fh
	xload	xmm4, [rbp+2*16]	;; FFT data
	mulpd	xmm4, [rdx+2*XMM_GMD][rcx*8] ;; mul by fudged grp two-to-minus-phi
	xload	xmm2, [rsi+2*16]	;; Load carry
	addpd	xmm2, xmm4		;; carry + FFT data
	rounding exec, base2, noexec, noexec, xmm2, xmm5, xmm4, rax*4+32
	subpd	xmm5, XMM_BIGVAL
	mulpd	xmm2, [rdx+2*XMM_GMD+XMM_GMD/2][rcx*8] ;; value *= fudged grp two-to-phi
	movzx	rbx, BYTE PTR [rdi+2]	;; Load fudge factor flags 7,8
	and	rbx, 0fh
	mulpd	xmm5, [rdx+2*XMM_GMD+XMM_GMD/2][rbx*8] ;; high carry *= fudged grp two-to-phi
	addpd	xmm5, [rbp+6*16]	;; Add high carry and FFT data
	xstore	[rbp+2*16], xmm2	;; Save FFT data
	xstore	[rbp+6*16], xmm5	;; Save FFT data

	xload	xmm4, [rbp+3*16]	;; FFT data
	mulpd	xmm4, [rdx+3*XMM_GMD][rcx*8] ;; mul by fudged grp two-to-minus-phi
	xload	xmm3, [rsi+3*16]	;; Load carry
	addpd	xmm3, xmm4		;; carry + FFT data
	rounding exec, base2, noexec, noexec, xmm3, xmm5, xmm4, rax*4+48
	subpd	xmm5, XMM_BIGVAL
	mulpd	xmm3, [rdx+3*XMM_GMD+XMM_GMD/2][rcx*8] ;; value *= fudged grp two-to-phi
	mulpd	xmm5, [rdx+3*XMM_GMD+XMM_GMD/2][rbx*8] ;; high carry *= fudged grp two-to-phi
	addpd	xmm5, [rbp+7*16]	;; Add high carry and FFT data
	xstore	[rbp+3*16], xmm3	;; Save FFT data
	xstore	[rbp+7*16], xmm5	;; Save FFT data
	jmp	done

;; Same as above, but spread carry over 6 words

hard:	movzx	rcx, BYTE PTR [rdi]	;; Load fudge factor flags 1,2
	and	rcx, 0f0h
	movzx	rax, BYTE PTR [rdi+1]	;; Load 4 big vs. little flags
	xload	xmm4, [rbp+0*16]	;; FFT data
	mulpd	xmm4, [rdx+0*XMM_GMD][rcx] ;; mul by fudged grp two-to-minus-phi
	xload	xmm0, [rsi+0*16]	;; Load carry
	addpd	xmm0, xmm4		;; carry + FFT data
	rounding exec, base2, noexec, noexec, xmm0, xmm5, xmm4, rax*4
	mulpd	xmm0, [rdx+0*XMM_GMD+XMM_GMD/2][rcx] ;; mul by fudged grp two-to-phi
	xstore	[rbp+0*16], xmm0	;; Save FFT data

	movzx	rcx, BYTE PTR [rdi+2]	;; Load fudge factor flags
	and	rcx, 0f0h
	movzx	rax, BYTE PTR [rdi+3]	;; Load big vs. little flag
	xload	xmm4, [rbp+4*16]	;; FFT data
	mulpd	xmm4, [rdx+0*XMM_GMD][rcx] ;; mul by fudged grp two-to-minus-phi
	addpd	xmm5, xmm4		;; carry + FFT data
	rounding exec, base2, noexec, noexec, xmm5, xmm0, xmm4, rax*4
	mulpd	xmm5, [rdx+0*XMM_GMD+XMM_GMD/2][rcx] ;; mul by fudged grp two-to-phi
	xstore	[rbp+4*16], xmm5	;; Save FFT data

	mov	ebx, BIGLIT_INCR2	;; Different clm values step through
					;; big/lit array differently
	movzx	rcx, BYTE PTR [rdi+rbx]	;; Load fudge factor flags
	and	rcx, 0f0h
	movzx	rax, BYTE PTR [rdi+rbx+1] ;; Load 4 big vs. little flags
	xload	xmm4, [rbp+8*16]	;; FFT data
	mulpd	xmm4, [rdx+0*XMM_GMD][rcx] ;; mul by fudged grp two-to-minus-phi
	addpd	xmm0, xmm4		;; carry + FFT data
	rounding exec, base2, noexec, noexec, xmm0, xmm5, xmm4, rax*4
	mulpd	xmm0, [rdx+0*XMM_GMD+XMM_GMD/2][rcx] ;; mul by fudged grp two-to-phi
	xstore	[rbp+8*16], xmm0	;; Save FFT data

	movzx	rcx, BYTE PTR [rdi+rbx+2] ;; Load fudge factor flags
	and	rcx, 0f0h
	movzx	rax, BYTE PTR [rdi+rbx+3] ;; Load big vs. little flag
	xload	xmm4, [rbp+12*16]	;; FFT data
	mulpd	xmm4, [rdx+0*XMM_GMD][rcx] ;; mul by fudged grp two-to-minus-phi
	addpd	xmm5, xmm4		;; carry + FFT data
	rounding exec, base2, noexec, noexec, xmm5, xmm0, xmm4, rax*4
	mulpd	xmm5, [rdx+0*XMM_GMD+XMM_GMD/2][rcx] ;; mul by fudged grp two-to-phi
	xstore	[rbp+12*16], xmm5	;; Save FFT data

	mov	ebx, BIGLIT_INCR4	;; Different clm values step through
					;; big/lit array differently
	movzx	rcx, BYTE PTR [rdi+rbx]	;; Load fudge factor flags
	and	rcx, 0f0h
	movzx	rax, BYTE PTR [rdi+rbx+1] ;; Load 4 big vs. little flags
	xload	xmm4, [rbp+16*16]	;; FFT data
	mulpd	xmm4, [rdx+0*XMM_GMD][rcx] ;; mul by fudged grp two-to-minus-phi
	addpd	xmm0, xmm4		;; carry + FFT data
	rounding exec, base2, noexec, noexec, xmm0, xmm5, xmm4, rax*4
	mulpd	xmm0, [rdx+0*XMM_GMD+XMM_GMD/2][rcx] ;; mul by fudged grp two-to-phi
	xstore	[rbp+16*16], xmm0	;; Save FFT data

	movzx	rcx, BYTE PTR [rdi+rbx+2] ;; Load fudge factor flags
	and	rcx, 0f0h
	subpd	xmm5, XMM_BIGVAL
	mulpd	xmm5, [rdx+0*XMM_GMD+XMM_GMD/2][rcx] ;; high carry *= fudged grp two-to-phi
	addpd	xmm5, [rbp+20*16]	;; Add high carry and FFT data
	xstore	[rbp+20*16], xmm5	;; Save FFT data


	movzx	rcx, BYTE PTR [rdi]	;; Load fudge factor flags 1,2
	and	rcx, 0f0h
	movzx	rax, BYTE PTR [rdi+1]	;; Load 4 big vs. little flags
	xload	xmm4, [rbp+1*16]	;; FFT data
	mulpd	xmm4, [rdx+1*XMM_GMD][rcx] ;; mul by fudged grp two-to-minus-phi
	xload	xmm1, [rsi+1*16]	;; Load carry
	addpd	xmm1, xmm4		;; carry + FFT data
	rounding exec, base2, noexec, noexec, xmm1, xmm5, xmm4, rax*4+16
	mulpd	xmm1, [rdx+1*XMM_GMD+XMM_GMD/2][rcx] ;; mul by fudged grp two-to-phi
	xstore	[rbp+1*16], xmm1	;; Save FFT data

	movzx	rcx, BYTE PTR [rdi+2]	;; Load fudge factor flags
	and	rcx, 0f0h
	movzx	rax, BYTE PTR [rdi+3]	;; Load big vs. little flag
	xload	xmm4, [rbp+5*16]	;; FFT data
	mulpd	xmm4, [rdx+1*XMM_GMD][rcx] ;; mul by fudged grp two-to-minus-phi
	addpd	xmm5, xmm4		;; carry + FFT data
	rounding exec, base2, noexec, noexec, xmm5, xmm1, xmm4, rax*4+16
	mulpd	xmm5, [rdx+1*XMM_GMD+XMM_GMD/2][rcx] ;; mul by fudged grp two-to-phi
	xstore	[rbp+5*16], xmm5	;; Save FFT data

	mov	ebx, BIGLIT_INCR2	;; Different clm values step through
					;; big/lit array differently
	movzx	rcx, BYTE PTR [rdi+rbx]	;; Load fudge factor flags
	and	rcx, 0f0h
	movzx	rax, BYTE PTR [rdi+rbx+1] ;; Load 4 big vs. little flags
	xload	xmm4, [rbp+9*16]	;; FFT data
	mulpd	xmm4, [rdx+1*XMM_GMD][rcx] ;; mul by grp two-to-minus-phi
	addpd	xmm1, xmm4		;; carry + FFT data
	rounding exec, base2, noexec, noexec, xmm1, xmm5, xmm4, rax*4+16
	mulpd	xmm1, [rdx+1*XMM_GMD+XMM_GMD/2][rcx] ;; mul by fudged grp two-to-phi
	xstore	[rbp+9*16], xmm1	;; Save FFT data

	movzx	rcx, BYTE PTR [rdi+rbx+2] ;; Load fudge factor flags
	and	rcx, 0f0h
	movzx	rax, BYTE PTR [rdi+rbx+3] ;; Load big vs. little flag
	xload	xmm4, [rbp+13*16]	;; FFT data
	mulpd	xmm4, [rdx+1*XMM_GMD][rcx] ;; mul by fudged grp two-to-minus-phi
	addpd	xmm5, xmm4		;; carry + FFT data
	rounding exec, base2, noexec, noexec, xmm5, xmm1, xmm4, rax*4+16
	mulpd	xmm5, [rdx+1*XMM_GMD+XMM_GMD/2][rcx] ;; mul by fudged grp two-to-phi
	xstore	[rbp+13*16], xmm5	;; Save FFT data

	mov	ebx, BIGLIT_INCR4	;; Different clm values step through
					;; big/lit array differently
	movzx	rcx, BYTE PTR [rdi+rbx]	;; Load fudge factor flags
	and	rcx, 0f0h
	movzx	rax, BYTE PTR [rdi+rbx+1] ;; Load 4 big vs. little flags
	xload	xmm4, [rbp+17*16]	;; FFT data
	mulpd	xmm4, [rdx+1*XMM_GMD][rcx] ;; mul by grp two-to-minus-phi
	addpd	xmm1, xmm4		;; carry + FFT data
	rounding exec, base2, noexec, noexec, xmm1, xmm5, xmm4, rax*4+16
	mulpd	xmm1, [rdx+1*XMM_GMD+XMM_GMD/2][rcx] ;; mul by fudged grp two-to-phi
	xstore	[rbp+17*16], xmm1	;; Save FFT data

	movzx	rcx, BYTE PTR [rdi+rbx+2] ;; Load fudge factor flags
	and	rcx, 0f0h
	subpd	xmm5, XMM_BIGVAL
	mulpd	xmm5, [rdx+1*XMM_GMD+XMM_GMD/2][rcx] ;; high carry *= fudged grp two-to-phi
	addpd	xmm5, [rbp+21*16]	;; Add high carry and FFT data
	xstore	[rbp+21*16], xmm5	;; Save FFT data

	movzx	rcx, BYTE PTR [rdi]	;; Load fudge factor flags 3,4
	and	rcx, 0fh
	movzx	rax, BYTE PTR [rdi+1]	;; Load 4 big vs. little flags
	xload	xmm4, [rbp+2*16]	;; FFT data
	mulpd	xmm4, [rdx+2*XMM_GMD][rcx*8] ;; mul by fudged grp two-to-minus-phi
	xload	xmm2, [rsi+2*16]	;; Load carry
	addpd	xmm2, xmm4		;; carry + FFT data
	rounding exec, base2, noexec, noexec, xmm2, xmm5, xmm4, rax*4+32
	mulpd	xmm2, [rdx+2*XMM_GMD+XMM_GMD/2][rcx*8] ;; mul by fudged grp two-to-phi
	xstore	[rbp+2*16], xmm2	;; Save FFT data

	movzx	rcx, BYTE PTR [rdi+2]	;; Load fudge factor flags
	and	rcx, 0fh
	movzx	rax, BYTE PTR [rdi+3]	;; Load big vs. little flag
	xload	xmm4, [rbp+6*16]	;; FFT data
	mulpd	xmm4, [rdx+2*XMM_GMD][rcx*8] ;; mul by fudged grp two-to-minus-phi
	addpd	xmm5, xmm4		;; carry + FFT data
	rounding exec, base2, noexec, noexec, xmm5, xmm2, xmm4, rax*4+32
	mulpd	xmm5, [rdx+2*XMM_GMD+XMM_GMD/2][rcx*8] ;; mul by fudged grp two-to-phi
	xstore	[rbp+6*16], xmm5	;; Save FFT data

	mov	ebx, BIGLIT_INCR2	;; Different clm values step through
					;; big/lit array differently
	movzx	rcx, BYTE PTR [rdi+rbx]	;; Load fudge factor flags
	and	rcx, 0fh
	movzx	rax, BYTE PTR [rdi+rbx+1] ;; Load 4 big vs. little flags
	xload	xmm4, [rbp+10*16]	;; FFT data
	mulpd	xmm4, [rdx+2*XMM_GMD][rcx*8] ;; mul by fudged grp two-to-minus-phi
	addpd	xmm2, xmm4		;; carry + FFT data
	rounding exec, base2, noexec, noexec, xmm2, xmm5, xmm4, rax*4+32
	mulpd	xmm2, [rdx+2*XMM_GMD+XMM_GMD/2][rcx*8] ;; mul by fudged grp two-to-phi
	xstore	[rbp+10*16], xmm2	;; Save FFT data

	movzx	rcx, BYTE PTR [rdi+rbx+2] ;; Load fudge factor flags
	and	rcx, 0fh
	movzx	rax, BYTE PTR [rdi+rbx+3] ;; Load big vs. little flag
	xload	xmm4, [rbp+14*16]	;; FFT data
	mulpd	xmm4, [rdx+2*XMM_GMD][rcx*8] ;; mul by fudged grp two-to-minus-phi
	addpd	xmm5, xmm4		;; carry + FFT data
	rounding exec, base2, noexec, noexec, xmm5, xmm2, xmm4, rax*4+32
	mulpd	xmm5, [rdx+2*XMM_GMD+XMM_GMD/2][rcx*8] ;; mul by fudged grp two-to-phi
	xstore	[rbp+14*16], xmm5	;; Save FFT data

	mov	ebx, BIGLIT_INCR4	;; Different clm values step through
					;; big/lit array differently
	movzx	rcx, BYTE PTR [rdi+rbx]	;; Load fudge factor flags
	and	rcx, 0fh
	movzx	rax, BYTE PTR [rdi+rbx+1] ;; Load 4 big vs. little flags
	xload	xmm4, [rbp+18*16]	;; FFT data
	mulpd	xmm4, [rdx+2*XMM_GMD][rcx*8] ;; mul by fudged grp two-to-minus-phi
	addpd	xmm2, xmm4		;; carry + FFT data
	rounding exec, base2, noexec, noexec, xmm2, xmm5, xmm4, rax*4+32
	mulpd	xmm2, [rdx+2*XMM_GMD+XMM_GMD/2][rcx*8] ;; mul by fudged grp two-to-phi
	xstore	[rbp+18*16], xmm2	;; Save FFT data

	movzx	rcx, BYTE PTR [rdi+rbx+2] ;; Load fudge factor flags
	and	rcx, 0fh
	subpd	xmm5, XMM_BIGVAL
	mulpd	xmm5, [rdx+2*XMM_GMD+XMM_GMD/2][rcx*8] ;; high carry *= fudged grp two-to-phi
	addpd	xmm5, [rbp+22*16]	;; Add high carry and FFT data
	xstore	[rbp+22*16], xmm5	;; Save FFT data


	movzx	rcx, BYTE PTR [rdi]	;; Load fudge factor flags 3,4
	and	rcx, 0fh
	movzx	rax, BYTE PTR [rdi+1]	;; Load 4 big vs. little flags
	xload	xmm4, [rbp+3*16]	;; FFT data
	mulpd	xmm4, [rdx+3*XMM_GMD][rcx*8] ;; mul by fudged grp two-to-minus-phi
	xload	xmm3, [rsi+3*16]	;; Load carry
	addpd	xmm3, xmm4		;; carry + FFT data
	rounding exec, base2, noexec, noexec, xmm3, xmm5, xmm4, rax*4+48
	mulpd	xmm3, [rdx+3*XMM_GMD+XMM_GMD/2][rcx*8] ;; mul by fudged grp two-to-phi
	xstore	[rbp+3*16], xmm3	;; Save FFT data

	movzx	rcx, BYTE PTR [rdi+2]	;; Load fudge factor flags
	and	rcx, 0fh
	movzx	rax, BYTE PTR [rdi+3]	;; Load big vs. little flag
	xload	xmm4, [rbp+7*16]	;; FFT data
	mulpd	xmm4, [rdx+3*XMM_GMD][rcx*8] ;; mul by fudged grp two-to-minus-phi
	addpd	xmm5, xmm4		;; carry + FFT data
	rounding exec, base2, noexec, noexec, xmm5, xmm3, xmm4, rax*4+48
	mulpd	xmm5, [rdx+3*XMM_GMD+XMM_GMD/2][rcx*8] ;; mul by fudged grp two-to-phi
	xstore	[rbp+7*16], xmm5	;; Save FFT data

	mov	ebx, BIGLIT_INCR2	;; Different clm values step through
					;; big/lit array differently
	movzx	rcx, BYTE PTR [rdi+rbx]	;; Load fudge factor flags
	and	rcx, 0fh
	movzx	rax, BYTE PTR [rdi+rbx+1] ;; Load 4 big vs. little flags
	xload	xmm4, [rbp+11*16]	;; FFT data
	mulpd	xmm4, [rdx+3*XMM_GMD][rcx*8] ;; mul by fudged grp two-to-minus-phi
	addpd	xmm3, xmm4		;; carry + FFT data
	rounding exec, base2, noexec, noexec, xmm3, xmm5, xmm4, rax*4+48
	mulpd	xmm3, [rdx+3*XMM_GMD+XMM_GMD/2][rcx*8] ;; mul by fudged grp two-to-phi
	xstore	[rbp+11*16], xmm3	;; Save FFT data

	movzx	rcx, BYTE PTR [rdi+rbx+2] ;; Load fudge factor flags
	and	rcx, 0fh
	movzx	rax, BYTE PTR [rdi+rbx+3] ;; Load big vs. little flag
	xload	xmm4, [rbp+15*16]	;; FFT data
	mulpd	xmm4, [rdx+3*XMM_GMD][rcx*8] ;; mul by fudged grp two-to-minus-phi
	addpd	xmm5, xmm4		;; carry + FFT data
	rounding exec, base2, noexec, noexec, xmm5, xmm3, xmm4, rax*4+48
	mulpd	xmm5, [rdx+3*XMM_GMD+XMM_GMD/2][rcx*8] ;; mul by fudged grp two-to-phi
	xstore	[rbp+15*16], xmm5	;; Save FFT data

	mov	ebx, BIGLIT_INCR4	;; Different clm values step through
					;; big/lit array differently
	movzx	rcx, BYTE PTR [rdi+rbx]	;; Load fudge factor flags
	and	rcx, 0fh
	movzx	rax, BYTE PTR [rdi+rbx+1] ;; Load 4 big vs. little flags
	xload	xmm4, [rbp+19*16]	;; FFT data
	mulpd	xmm4, [rdx+3*XMM_GMD][rcx*8] ;; mul by fudged grp two-to-minus-phi
	addpd	xmm3, xmm4		;; carry + FFT data
	rounding exec, base2, noexec, noexec, xmm3, xmm5, xmm4, rax*4+48
	mulpd	xmm3, [rdx+3*XMM_GMD+XMM_GMD/2][rcx*8] ;; mul by fudged grp two-to-phi
	xstore	[rbp+19*16], xmm3	;; Save FFT data

	movzx	rcx, BYTE PTR [rdi+rbx+2] ;; Load fudge factor flags
	and	rcx, 0fh
	subpd	xmm5, XMM_BIGVAL
	mulpd	xmm5, [rdx+3*XMM_GMD+XMM_GMD/2][rcx*8] ;; high carry *= fudged grp two-to-phi
	addpd	xmm5, [rbp+23*16]	;; Add high carry and FFT data
	xstore	[rbp+23*16], xmm5	;; Save FFT data

done:	xload	xmm4, XMM_BIGVAL
	xstore	[rsi+0*16], xmm4	;; Clear carry
	xstore	[rsi+1*16], xmm4
	xstore	[rsi+2*16], xmm4
	xstore	[rsi+3*16], xmm4
	ENDM


;; Significantly different cleanup code for zero-padded FFTs.
;; Note: The group multiplier should be 1.0 for the bottom FFT words and
;; the FFT words just above the half-way point.

xnorm012_wpn_zpad_part1 MACRO
	LOCAL	shuflp, b2, zpaddn, done
	cmp	THIS_BLOCK, 0		;; Are we carrying into the first data block?
	jne	done			;; If not, skip wrapping carries and negating the last carry
	mov	eax, addcount1		;; Load count of cache lines in the carries array
shuflp:	movsd	xmm4, Q [rsi+0*16]	;; Load low carry word
	movsd	Q [rsi+0*16], xmm0	;; Save prev cache line's high carry in low word
	movsd	xmm0, Q [rsi+0*16+8]	;; Load high carry for next cache line
	movsd	Q [rsi+0*16+8], xmm4	;; Move this cache line's low carry to high word
	movsd	xmm4, Q [rsi+1*16]	;; Load low carry word
	movsd	Q [rsi+1*16], xmm2	;; Save prev cache line's high carry in low word
	movsd	xmm2, Q [rsi+1*16+8]	;; Load high carry for next cache line
	movsd	Q [rsi+1*16+8], xmm4	;; Move this cache line's low carry to high word
	movsd	xmm4, Q [rsi+2*16]	;; Load low carry word
	movsd	Q [rsi+2*16], xmm1	;; Save prev cache line's high carry in low word
	movsd	xmm1, Q [rsi+2*16+8]	;; Load high carry for next cache line
	movsd	Q [rsi+2*16+8], xmm4	;; Move this cache line's low carry to high word
	movsd	xmm4, Q [rsi+3*16]	;; Load low carry word
	movsd	Q [rsi+3*16], xmm3	;; Save prev cache line's high carry in low word
	movsd	xmm3, Q [rsi+3*16+8]	;; Load high carry for next cache line
	movsd	Q [rsi+3*16+8], xmm4	;; Move this cache line's low carry to high word
	bump	rsi, 64			;; Next carry cache line
	dec	rax			;; Decrement count of cache lines
	jnz	short shuflp
	mov	rsi, carries		;; Reload carries array pointer
;;	movsd	Q [rsi+0*16], xmm3	;; Move last cache line's high carries into first cache line
	movsd	Q [rsi+1*16], xmm0
;;	movsd	Q [rsi+2*16], xmm2
	movsd	Q [rsi+3*16], xmm1

	mov	rsi, DESTARG		;; Addr of the FFT data
	mov	rdi, norm_biglit_array	;; Addr of the big/little flags array

	cmp	B_IS_2, 0		;; Is b = 2?
	jne	b2			;; Yes, do simpler rounding
	xnorm012_wpn_zpad_part1a noexec	;; No, do harder rounding
	jmp	zpaddn
b2:	xnorm012_wpn_zpad_part1a exec

zpaddn:	mov	rsi, carries		;; Reload carries pointer
	movsd	xmm0, Q XMM_BIGVAL	;; Clear two carries just processed by xnorm012_wpn_zpad_part1_cmn
	subsd	xmm2, xmm2
	movsd	Q [rsi], xmm0
	movsd	Q [rsi+32], xmm2
done:
	ENDM

xnorm012_wpn_zpad_part1a MACRO base2
	LOCAL	noncon, done
	cmp	const_fft, 0		;; Are we also multiplying by a constant?
	je	noncon			;; Jump if not const
	xnorm012_wpn_zpad_part1_cmn exec, base2
	jmp	done
noncon:	xnorm012_wpn_zpad_part1_cmn noexec, base2
done:
	ENDM

;; On input, xmm2 and xmm3 contain high carries from the last carries array row
xnorm012_wpn_zpad_part1_cmn MACRO const, base2
	LOCAL	smallk, mediumk, div_k_done

	;; Strip BIGVAL from the traditional carry, we'll add the traditional
	;; carry in later when we are working on the ZPAD0 - ZPAD6 values.
	subsd	xmm2, Q XMM_BIGVAL	;; Integerize traditional carry

	;; Rather than calculate high FFT carry times k and then later dividing
	;; by k, we multiply FFT high carry by const and we'll add it
	;; to the lower FFT data later (after multiplying by -c).
const	mulsd	xmm3, Q XMM_MULCONST

	;; Work on zero-pad addin value.
	movsd	xmm4, Q ADDIN_VALUE		;; Load the add in value
const	mulsd	xmm4, Q XMM_MULCONST		;; Multiply the add in value by the small mul const
	addsd	xmm4, POSTADDIN_VALUE		;; Add the post-mul-by-const addin value
	;; when c = -1, 1 = b^n
	;; when c = 1, -1 = b^n, 1 = -b^n
	;; when c = -3, 3 = b^n, 1 = b^n - 2
	;; when c = 3, -3 = b^n, 3 = -b^n, 1 = -b^n - 2
	;; The "- 2" has been precomputed in ZPAD_LSW_ADJUST.  Add ADDIN_VALUE * mul-by-const * ZPAD_LSW_ADJUST into the least significant FFT word
	mov	rsi, DESTARG			;; Address of squared number
	movsd	xmm0, Q ZPAD_LSW_ADJUST
	mulsd	xmm0, xmm4
	addsd	xmm0, [rsi]
	movsd	[rsi], xmm0

	;; Multiply ZPAD0 through ZPAD6 by const * -C.  This, in essense,
	;; wraps this data from above the FFT data area to the halfway point.
	;; Later on we'll divide this by K to decide which data needs wrapping
	;; all the way down to the bottom of the FFT data.

	;; NOTE: ZPAD0's grp multiplier is 1.0.  Also, ZPAD6 will not
	;; be bigger than a big word.  We must be careful to handle c's up
	;; to about 30 bits

	movzx	rax, BYTE PTR [rdi+1]	;; Load big vs. little flags
	movsd	xmm0, ZPAD0		;; Load values1
	subsd	xmm5, xmm5		;; Create a zero high FFT carry to add in
	single_split_lower_zpad_word base2, xmm0, xmm5, xmm1, rax*4
no const mulsd	xmm0, Q XMM_MINUS_C
const	mulsd	xmm0, Q XMM_MINUS_C_TIMES_MULCONST
	addsd	xmm0, xmm4		;; Apply rest of ADDIN_VALUE * mul-by-const here (the FFT half way point)
	round_zpad7_word base2, xmm0, xmm2, xmm1, rax*4
	movsd	ZPAD0, xmm0

	movzx	rax, BYTE PTR [rdi+3]	;; Load big vs. little flags
	movsd	xmm0, ZPAD1		;; Load values1
	single_split_lower_zpad_word base2, xmm0, xmm5, xmm1, rax*4
no const mulsd	xmm0, Q XMM_MINUS_C
const	mulsd	xmm0, Q XMM_MINUS_C_TIMES_MULCONST
	round_zpad7_word base2, xmm0, xmm2, xmm1, rax*4
	movsd	ZPAD1, xmm0

	mov	ecx, BIGLIT_INCR2	;; Different clm values step through
					;; big/lit array differently
	movzx	rax, BYTE PTR [rdi+rcx+1] ;; Load big vs. little flags
	movsd	xmm0, ZPAD2		;; Load values1
	single_split_lower_zpad_word base2, xmm0, xmm5, xmm1, rax*4
no const mulsd	xmm0, Q XMM_MINUS_C
const	mulsd	xmm0, Q XMM_MINUS_C_TIMES_MULCONST
	round_zpad7_word base2, xmm0, xmm2, xmm1, rax*4
	movsd	ZPAD2, xmm0

	movzx	rax, BYTE PTR [rdi+rcx+3] ;; Load big vs. little flags
	movsd	xmm0, ZPAD3		;; Load values1
	single_split_lower_zpad_word base2, xmm0, xmm5, xmm1, rax*4
no const mulsd	xmm0, Q XMM_MINUS_C
const	mulsd	xmm0, Q XMM_MINUS_C_TIMES_MULCONST
	round_zpad7_word base2, xmm0, xmm2, xmm1, rax*4
	movsd	ZPAD3, xmm0

	mov	ecx, BIGLIT_INCR4	;; Different clm values step through
					;; big/lit array differently
	movzx	rax, BYTE PTR [rdi+rcx+1] ;; Load big vs. little flags
	movsd	xmm0, ZPAD4		;; Load values1
	single_split_lower_zpad_word base2, xmm0, xmm5, xmm1, rax*4
no const mulsd	xmm0, Q XMM_MINUS_C
const	mulsd	xmm0, Q XMM_MINUS_C_TIMES_MULCONST
	round_zpad7_word base2, xmm0, xmm2, xmm1, rax*4
	movsd	ZPAD4, xmm0

	movzx	rax, BYTE PTR [rdi+rcx+3] ;; Load big vs. little flags
	movsd	xmm0, ZPAD5		;; Load values1
	single_split_lower_zpad_word base2, xmm0, xmm5, xmm1, rax*4
no const mulsd	xmm0, Q XMM_MINUS_C
const	mulsd	xmm0, Q XMM_MINUS_C_TIMES_MULCONST
	round_zpad7_word base2, xmm0, xmm2, xmm1, rax*4
	movsd	ZPAD5, xmm0

	movsd	xmm0, ZPAD6		;; Load values1
	addsd	xmm0, Q XMM_BIGVAL	;; Round to an integer
	subsd	xmm0, Q XMM_BIGVAL
	addsd	xmm0, xmm5		;; Add in shifted high ZPAD data
no const mulsd	xmm0, Q XMM_MINUS_C
const	mulsd	xmm0, Q XMM_MINUS_C_TIMES_MULCONST
	addsd	xmm0, xmm2		;; Add in high part of last calculation
	movsd	ZPAD6, xmm0

	;; Divide the zpad data by k.  Store the integer part in XMM_TMP
	;; and the remainder in ZPAD0.  Later we will wrap the integer part
	;; down to the bottom of the FFT data area (and multiply by -c).
	;; And we will store the remainder in the upper half of the FFT
	;; data area.

	;; Note there are three cases to handle.  K is smaller than a big word.
	;; K is between one and 2 big words in size.  And K is more than
	;; 2 big words in size.

	cmp	ZPAD_TYPE, 2		;; Are we dealing with case 1,2,or 3
	jl	smallk			;; One word case
	je	mediumk			;; Two word case

	;; This case does the divide by k where k is three words

	movsd	xmm0, ZPAD6		;; Load zpad word (high bits)
	movsd	xmm1, ZPAD5		;; Load zpad word (middle bits)
	movsd	xmm2, ZPAD4		;; Load zpad word (low bits)
	movsd	xmm4, ZPAD_INVERSE_K6	;; Load shifted 1/k
	mulsd	xmm4, xmm0		;; Mul ZPAD by shifted 1/k
	addsd	xmm4, Q XMM_BIGVAL	;; Round to integer	
	subsd	xmm4, Q XMM_BIGVAL
	movsd	xmm5, ZPAD_K6_HI	;; Load high bits of k
	mulsd	xmm5, xmm4
	subsd	xmm0, xmm5		;; Calculate high bits of remainder
	movsd	xmm5, ZPAD_K6_MID	;; Load middle bits of k
	mulsd	xmm5, xmm4
	subsd	xmm1, xmm5		;; Calculate middle bits of remainder
	movsd	xmm5, ZPAD_K6_LO	;; Load low bits of k
	mulsd	xmm5, xmm4
	subsd	xmm2, xmm5		;; Calculate low bits of remainder
	movsd	Q XMM_TMP5, xmm4	;; Save word of zpad / k

	mulsd	xmm0, ZPAD_SHIFT6	;; Shift previous zpad word
	addsd	xmm1, xmm0		;; Add to create new high zpad bits
	movsd	xmm0, ZPAD3		;; Load zpad word (new low bits)
	movsd	xmm5, ZPAD_SHIFT5	;; Combine high and medium bits
	mulsd	xmm5, xmm1
	addsd	xmm5, xmm2
	movsd	xmm4, ZPAD_INVERSE_K5	;; Load shifted 1/k
	mulsd	xmm4, xmm5		;; Mul ZPAD by shifted 1/k
	addsd	xmm4, Q XMM_BIGVAL	;; Round to integer
	subsd	xmm4, Q XMM_BIGVAL
	movsd	xmm5, ZPAD_K5_HI	;; Load high bits of k
	mulsd	xmm5, xmm4
	subsd	xmm1, xmm5		;; Calculate high bits of remainder
	movsd	xmm5, ZPAD_K5_MID	;; Load middle bits of k
	mulsd	xmm5, xmm4
	subsd	xmm2, xmm5		;; Calculate middle bits of remainder
	movsd	xmm5, ZPAD_K5_LO	;; Load low bits of k
	mulsd	xmm5, xmm4
	subsd	xmm0, xmm5		;; Calculate low bits of remainder
	movsd	Q XMM_TMP4, xmm4	;; Save word of zpad / k

	mulsd	xmm1, ZPAD_SHIFT5	;; Shift previous zpad word
	addsd	xmm2, xmm1		;; Add to create new high zpad bits
	movsd	xmm1, ZPAD2		;; Load zpad word (new low bits)
	movsd	xmm5, ZPAD_SHIFT4	;; Combine high and medium bits
	mulsd	xmm5, xmm2
	addsd	xmm5, xmm0
	movsd	xmm4, ZPAD_INVERSE_K4	;; Load shifted 1/k
	mulsd	xmm4, xmm5		;; Mul ZPAD by shifted 1/k
	addsd	xmm4, Q XMM_BIGVAL	;; Round to integer
	subsd	xmm4, Q XMM_BIGVAL
	movsd	xmm5, ZPAD_K4_HI	;; Load high bits of k
	mulsd	xmm5, xmm4
	subsd	xmm2, xmm5		;; Calculate high bits of remainder
	movsd	xmm5, ZPAD_K4_MID	;; Load middle bits of k
	mulsd	xmm5, xmm4
	subsd	xmm0, xmm5		;; Calculate middle bits of remainder
	movsd	xmm5, ZPAD_K4_LO	;; Load low bits of k
	mulsd	xmm5, xmm4
	subsd	xmm1, xmm5		;; Calculate low bits of remainder
	movsd	Q XMM_TMP3, xmm4	;; Save word of zpad / k

	mulsd	xmm2, ZPAD_SHIFT4	;; Shift previous zpad word
	addsd	xmm0, xmm2		;; Add to create new high zpad bits
	movsd	xmm2, ZPAD1		;; Load zpad word (new low bits)
	movsd	xmm5, ZPAD_SHIFT3	;; Combine high and medium bits
	mulsd	xmm5, xmm0
	addsd	xmm5, xmm1
	movsd	xmm4, ZPAD_INVERSE_K3	;; Load shifted 1/k
	mulsd	xmm4, xmm5		;; Mul ZPAD by shifted 1/k
	addsd	xmm4, Q XMM_BIGVAL	;; Round to integer
	subsd	xmm4, Q XMM_BIGVAL
	movsd	xmm5, ZPAD_K3_HI	;; Load high bits of k
	mulsd	xmm5, xmm4
	subsd	xmm0, xmm5		;; Calculate high bits of remainder
	movsd	xmm5, ZPAD_K3_MID	;; Load middle bits of k
	mulsd	xmm5, xmm4
	subsd	xmm1, xmm5		;; Calculate middle bits of remainder
	movsd	xmm5, ZPAD_K3_LO	;; Load low bits of k
	mulsd	xmm5, xmm4
	subsd	xmm2, xmm5		;; Calculate low bits of remainder
	movsd	Q XMM_TMP2, xmm4	;; Save word of zpad / k

	mulsd	xmm0, ZPAD_SHIFT3	;; Shift previous zpad word
	addsd	xmm1, xmm0		;; Add to create new high zpad bits
	movsd	xmm0, ZPAD0		;; Load zpad word (new low bits)
	movsd	xmm5, ZPAD_SHIFT2	;; Combine high and medium bits
	mulsd	xmm5, xmm1
	addsd	xmm5, xmm2
	movsd	xmm4, ZPAD_INVERSE_K2	;; Load shifted 1/k
	mulsd	xmm4, xmm5		;; Mul ZPAD by shifted 1/k
	addsd	xmm4, Q XMM_BIGVAL	;; Round to integer
	subsd	xmm4, Q XMM_BIGVAL
	movsd	xmm5, ZPAD_K2_HI	;; Load high bits of k
	mulsd	xmm5, xmm4
	subsd	xmm1, xmm5		;; Calculate high bits of remainder
	movsd	xmm5, ZPAD_K2_MID	;; Load middle bits of k
	mulsd	xmm5, xmm4
	subsd	xmm2, xmm5		;; Calculate middle bits of remainder
	movsd	xmm5, ZPAD_K2_LO	;; Load low bits of k
	mulsd	xmm5, xmm4
	subsd	xmm0, xmm5		;; Calculate low bits of remainder
	movsd	Q XMM_TMP1, xmm4	;; Save word of zpad / k

	mulsd	xmm1, ZPAD_SHIFT2	;; Shift previous zpad word
	addsd	xmm2, xmm1		;; Add to create new high zpad bits
	mulsd	xmm2, ZPAD_SHIFT1	;; Shift previous zpad word
	addsd	xmm0, xmm2		;; Add to create new high zpad bits
	movsd	ZPAD0, xmm0		;; Save remainder of zpad / k

	subsd	xmm1, xmm1		;; Zero words that other cases set
	movsd	Q XMM_TMP6, xmm1
	
	jmp	div_k_done

	;; This case does the divide by k where k is two words
mediumk:
	movsd	xmm0, ZPAD6		;; Load zpad word (high bits)
	movsd	xmm1, ZPAD5		;; Load zpad word (low bits)
	movsd	xmm4, ZPAD_INVERSE_K6	;; Load shifted 1/k
	mulsd	xmm4, xmm0		;; Mul ZPAD by shifted 1/k
	addsd	xmm4, Q XMM_BIGVAL	;; Round to integer	
	subsd	xmm4, Q XMM_BIGVAL
	movsd	xmm5, ZPAD_K6_HI	;; Load high bits of k
	mulsd	xmm5, xmm4
	subsd	xmm0, xmm5		;; Calculate high bits of remainder
	movsd	xmm5, ZPAD_K6_LO	;; Load low bits of k
	mulsd	xmm5, xmm4
	subsd	xmm1, xmm5		;; Calculate low bits of remainder
	movsd	Q XMM_TMP6, xmm4	;; Save word of zpad / k

	mulsd	xmm0, ZPAD_SHIFT6	;; Shift previous zpad word
	addsd	xmm0, xmm1		;; Add to create new high zpad bits
	movsd	xmm1, ZPAD4		;; Load zpad word (new low bits)
	movsd	xmm4, ZPAD_INVERSE_K5	;; Load shifted 1/k
	mulsd	xmm4, xmm0		;; Mul ZPAD by shifted 1/k
	addsd	xmm4, Q XMM_BIGVAL	;; Round to integer
	subsd	xmm4, Q XMM_BIGVAL
	movsd	xmm5, ZPAD_K5_HI	;; Load high bits of k
	mulsd	xmm5, xmm4
	subsd	xmm0, xmm5		;; Calculate high bits of remainder
	movsd	xmm5, ZPAD_K5_LO	;; Load low bits of k
	mulsd	xmm5, xmm4
	subsd	xmm1, xmm5		;; Calculate low bits of remainder
	movsd	Q XMM_TMP5, xmm4	;; Save word of zpad / k

	mulsd	xmm0, ZPAD_SHIFT5	;; Shift previous zpad word
	addsd	xmm0, xmm1		;; Add to create new high zpad bits
	movsd	xmm1, ZPAD3		;; Load zpad word (new low bits)
	movsd	xmm4, ZPAD_INVERSE_K4	;; Load shifted 1/k
	mulsd	xmm4, xmm0		;; Mul ZPAD by shifted 1/k
	addsd	xmm4, Q XMM_BIGVAL	;; Round to integer
	subsd	xmm4, Q XMM_BIGVAL
	movsd	xmm5, ZPAD_K4_HI	;; Load high bits of k
	mulsd	xmm5, xmm4
	subsd	xmm0, xmm5		;; Calculate high bits of remainder
	movsd	xmm5, ZPAD_K4_LO	;; Load low bits of k
	mulsd	xmm5, xmm4
	subsd	xmm1, xmm5		;; Calculate low bits of remainder
	movsd	Q XMM_TMP4, xmm4	;; Save word of zpad / k

	mulsd	xmm0, ZPAD_SHIFT4	;; Shift previous zpad word
	addsd	xmm0, xmm1		;; Add to create new high zpad bits
	movsd	xmm1, ZPAD2		;; Load zpad word (new low bits)
	movsd	xmm4, ZPAD_INVERSE_K3	;; Load shifted 1/k
	mulsd	xmm4, xmm0		;; Mul ZPAD by shifted 1/k
	addsd	xmm4, Q XMM_BIGVAL	;; Round to integer
	subsd	xmm4, Q XMM_BIGVAL
	movsd	xmm5, ZPAD_K3_HI	;; Load high bits of k
	mulsd	xmm5, xmm4
	subsd	xmm0, xmm5		;; Calculate high bits of remainder
	movsd	xmm5, ZPAD_K3_LO	;; Load low bits of k
	mulsd	xmm5, xmm4
	subsd	xmm1, xmm5		;; Calculate low bits of remainder
	movsd	Q XMM_TMP3, xmm4	;; Save word of zpad / k

	mulsd	xmm0, ZPAD_SHIFT3	;; Shift previous zpad word
	addsd	xmm0, xmm1		;; Add to create new high zpad bits
	movsd	xmm1, ZPAD1		;; Load zpad word (new low bits)
	movsd	xmm4, ZPAD_INVERSE_K2	;; Load shifted 1/k
	mulsd	xmm4, xmm0		;; Mul ZPAD by shifted 1/k
	addsd	xmm4, Q XMM_BIGVAL	;; Round to integer
	subsd	xmm4, Q XMM_BIGVAL
	movsd	xmm5, ZPAD_K2_HI	;; Load high bits of k
	mulsd	xmm5, xmm4
	subsd	xmm0, xmm5		;; Calculate high bits of remainder
	movsd	xmm5, ZPAD_K2_LO	;; Load low bits of k
	mulsd	xmm5, xmm4
	subsd	xmm1, xmm5		;; Calculate low bits of remainder
	movsd	Q XMM_TMP2, xmm4	;; Save word of zpad / k

	mulsd	xmm0, ZPAD_SHIFT2	;; Shift previous zpad word
	addsd	xmm0, xmm1		;; Add to create new high zpad bits
	movsd	xmm1, ZPAD0		;; Load zpad word (new low bits)
	movsd	xmm4, ZPAD_INVERSE_K1	;; Load shifted 1/k
	mulsd	xmm4, xmm0		;; Mul ZPAD by shifted 1/k
	addsd	xmm4, Q XMM_BIGVAL	;; Round to integer
	subsd	xmm4, Q XMM_BIGVAL
	movsd	xmm5, ZPAD_K1_HI	;; Load high bits of k
	mulsd	xmm5, xmm4
	subsd	xmm0, xmm5		;; Calculate high bits of remainder
	movsd	xmm5, ZPAD_K1_LO	;; Load low bits of k
	mulsd	xmm5, xmm4
	subsd	xmm1, xmm5		;; Calculate low bits of remainder
	movsd	Q XMM_TMP1, xmm4	;; Save word of zpad / k

	mulsd	xmm0, ZPAD_SHIFT1	;; Shift previous zpad word
	addsd	xmm0, xmm1		;; Add to create new high zpad bits
	movsd	ZPAD0, xmm0		;; Save remainder of zpad / k

	jmp	div_k_done

	;; This case does the divide by k where k is one word
	;; Assume ZPAD5 and ZPAD6 are zero.
smallk:	movsd	xmm0, ZPAD4		;; Load zpad data
	movsd	xmm4, ZPAD_INVERSE_K1	;; Load by 1/k
	mulsd	xmm4, xmm0		;; Mul ZPAD data by 1/k
	addsd	xmm4, Q XMM_BIGVAL	;; Round to integer	
	subsd	xmm4, Q XMM_BIGVAL
	movsd	Q XMM_TMP5, xmm4	;; Save integer part
	mulsd	xmm4, ZPAD_K1_LO	;; Compute remainder
	subsd	xmm0, xmm4

	mulsd	xmm0, ZPAD_SHIFT4	;; Shift previous zpad word
	addsd	xmm0, ZPAD3		;; Add in zpad data
	movsd	xmm4, ZPAD_INVERSE_K1	;; Load by 1/k
	mulsd	xmm4, xmm0		;; Mul ZPAD data by 1/k
	addsd	xmm4, Q XMM_BIGVAL	;; Round to integer	
	subsd	xmm4, Q XMM_BIGVAL
	movsd	Q XMM_TMP4, xmm4	;; Save integer part
	mulsd	xmm4, ZPAD_K1_LO	;; Compute remainder
	subsd	xmm0, xmm4

	mulsd	xmm0, ZPAD_SHIFT3	;; Shift previous zpad word
	addsd	xmm0, ZPAD2		;; Add in zpad data
	movsd	xmm4, ZPAD_INVERSE_K1	;; Load by 1/k
	mulsd	xmm4, xmm0		;; Mul ZPAD data by 1/k
	addsd	xmm4, Q XMM_BIGVAL	;; Round to integer	
	subsd	xmm4, Q XMM_BIGVAL
	movsd	Q XMM_TMP3, xmm4	;; Save integer part
	mulsd	xmm4, ZPAD_K1_LO	;; Compute remainder
	subsd	xmm0, xmm4

	mulsd	xmm0, ZPAD_SHIFT2	;; Shift previous zpad word
	addsd	xmm0, ZPAD1		;; Add in zpad data
	movsd	xmm4, ZPAD_INVERSE_K1	;; Load by 1/k
	mulsd	xmm4, xmm0		;; Mul ZPAD data by 1/k
	addsd	xmm4, Q XMM_BIGVAL	;; Round to integer	
	subsd	xmm4, Q XMM_BIGVAL
	movsd	Q XMM_TMP2, xmm4	;; Save integer part
	mulsd	xmm4, ZPAD_K1_LO	;; Compute remainder
	subsd	xmm0, xmm4

	mulsd	xmm0, ZPAD_SHIFT1	;; Shift previous zpad word
	addsd	xmm0, ZPAD0		;; Add in zpad data
	movsd	xmm4, ZPAD_INVERSE_K1	;; Load by 1/k
	mulsd	xmm4, xmm0		;; Mul ZPAD data by 1/k
	addsd	xmm4, Q XMM_BIGVAL	;; Round to integer	
	subsd	xmm4, Q XMM_BIGVAL
	movsd	Q XMM_TMP1, xmm4	;; Save integer part
	mulsd	xmm4, ZPAD_K1_LO	;; Compute remainder
	subsd	xmm0, xmm4
	movsd	ZPAD0, xmm0		;; Save remainder

	subsd	xmm1, xmm1		;; Zero words that other cases set
	movsd	Q XMM_TMP6, xmm1
	
div_k_done:

	;; Now normalize the data above the halfway point.  Remember that the
	;; two-to-phi multipliers will be applied by the FFT.

	movzx	rax, BYTE PTR [rdi+1]	;; Load big vs. little flags
	movsd	xmm0, ZPAD0		;; Load remainder of divide by k
	addsd	xmm0, Q XMM_BIGVAL
	single_rounding base2, xmm0, xmm2, xmm4, rax*4
	movsd	Q [rsi+0*64+32], xmm0	;; Save value1

	movzx	rax, BYTE PTR [rdi+3]	;; Load big vs. little flags
	single_rounding base2, xmm2, xmm0, xmm4, rax*4
	movsd	Q [rsi+1*64+32], xmm2	;; Save value2

	mov	ecx, BIGLIT_INCR2	;; Different clm values step through
					;; big/lit array differently
	movzx	rax, BYTE PTR [rdi+rcx+1] ;; Load big vs. little flags
	single_rounding base2, xmm0, xmm2, xmm4, rax*4
	movsd	Q [rsi+2*64+32], xmm0	;; Save value3

	subsd	xmm2, Q XMM_BIGVAL	;; Remove integer rounding constant
	movsd	Q [rsi+3*64+32], xmm2	;; Save new value4

	;; Mul the integer part of (ZPAD data divided by k) by -c in
	;; preparation for adding it into the lower FFT data area.
	;; Also add in the shifted high FFT carry at this time.

	;; Now add in and normalize the bottom FFT data.  Remember that the
	;; column two-to-phi multiplier for the first value will be 1.0.  We 
	;; must go 6 words deep in case k is 48-50 bits and c is 32 bits.

	movzx	rax, BYTE PTR [rdi+1]	;; Load big vs. little flags
	movsd	xmm0, Q XMM_TMP1	;; Load integer part of divide by k
	addsd	xmm0, xmm3		;; Add in shifted high FFT carry
	mulsd	xmm0, Q XMM_MINUS_C	;; Mul by -c
	addsd	xmm0, Q XMM_BIGVAL
	addsd	xmm0, Q [rsi+0*64]	;; Add in the FFT data
	single_rounding base2, xmm0, xmm2, xmm4, rax*4
	movsd	Q [rsi+0*64], xmm0	;; Save value1

	movzx	rax, BYTE PTR [rdi+3]	;; Load big vs. little flags
	movsd	xmm0, Q XMM_TMP2	;; Load integer part of divide by k
	mulsd	xmm0, Q XMM_MINUS_C	;; Mul by -c
	addsd	xmm0, Q [rsi+1*64]	;; Add in the FFT data
	addsd	xmm0, xmm2		;; x2 = value + carry
	single_rounding base2, xmm0, xmm2, xmm4, rax*4
	movsd	Q [rsi+1*64], xmm0	;; Save value2

	mov	ecx, BIGLIT_INCR2	;; Different clm values step through
					;; big/lit array differently
	movzx	rax, BYTE PTR [rdi+rcx+1] ;; Load big vs. little flags
	movsd	xmm0, Q XMM_TMP3	;; Load integer part of divide by k
	mulsd	xmm0, Q XMM_MINUS_C	;; Mul by -c
	addsd	xmm0, Q [rsi+2*64]	;; Add in the FFT data
	addsd	xmm0, xmm2		;; x3 = value + carry
	single_rounding base2, xmm0, xmm2, xmm4, rax*4
	movsd	Q [rsi+2*64], xmm0	;; Save value3

	movzx	rax, BYTE PTR [rdi+rcx+3] ;; Load big vs. little flags
	movsd	xmm0, Q XMM_TMP4	;; Load integer part of divide by k
	mulsd	xmm0, Q XMM_MINUS_C	;; Mul by -c
	addsd	xmm0, Q [rsi+3*64]	;; Add in the FFT data
	addsd	xmm0, xmm2		;; x4 = value + carry
	single_rounding base2, xmm0, xmm2, xmm4, rax*4
	movsd	Q [rsi+3*64], xmm0	;; Save value4

	mov	ecx, BIGLIT_INCR4	;; Different clm values step through
					;; big/lit array differently
	movzx	rax, BYTE PTR [rdi+rcx+1] ;; Load big vs. little flags
	movsd	xmm0, Q XMM_TMP5	;; Load integer part of divide by k
	mulsd	xmm0, Q XMM_MINUS_C	;; Mul by -c
	addsd	xmm0, Q [rsi+4*64]	;; Add in the FFT data
	addsd	xmm0, xmm2		;; x4 = value + carry
	single_rounding base2, xmm0, xmm2, xmm4, rax*4
	movsd	Q [rsi+4*64], xmm0	;; Save value4

	movzx	rax, BYTE PTR [rdi+rcx+3] ;; Load big vs. little flags
	movsd	xmm0, Q XMM_TMP6	;; Load integer part of divide by k
	mulsd	xmm0, Q XMM_MINUS_C	;; Mul by -c
	addsd	xmm0, Q [rsi+5*64]	;; Add in the FFT data
	addsd	xmm0, xmm2		;; x5 = value + carry
	single_rounding base2, xmm0, xmm2, xmm4, rax*4
	movsd	Q [rsi+5*64], xmm0	;; Save value5

	add	ecx, BIGLIT_INCR2	;; Different clm values step through
					;; big/lit array differently
	movzx	rax, BYTE PTR [rdi+rcx+1] ;; Load big vs. little flags
	movsd	xmm0, Q [rsi+6*64]	;; Load FFT data
	addsd	xmm0, xmm2		;; x6 = value + carry
	single_rounding base2, xmm0, xmm2, xmm4, rax*4
	movsd	Q [rsi+6*64], xmm0	;; Save value6

	subsd	xmm2, Q XMM_BIGVAL	;; Remove rounding constant
	addsd	xmm2, Q [rsi+7*64]	;; Add in FFT data
	movsd	Q [rsi+7*64], xmm2	;; Save value7
	ENDM

xnorm012_wpn_zpad MACRO
	LOCAL	b2, done
	cmp	B_IS_2, 0		;; Is b = 2?
	jne	b2			;; Yes, do simpler rounding
	xnorm012_wpn_zpada noexec	;; No, do harder rounding
	jmp	done
b2:	xnorm012_wpn_zpada exec
done:
	ENDM
xnorm012_wpn_zpada MACRO base2
	LOCAL	noncon, done
	cmp	const_fft, 0		;; Are we also multiplying by a constant?
	je	noncon			;; Jump if not const
	xnorm012_wpn_zpad_cmn exec, base2
	jmp	done
noncon:	xnorm012_wpn_zpad_cmn noexec, base2
done:
	ENDM
xnorm012_wpn_zpad_cmn MACRO const, base2
	movzx	rcx, BYTE PTR [rdi]	;; Load fudge factor flag
	and	rcx, 0f0h
	xload	xmm0, [rsi+0*16]	;; Load carry
	xload	xmm2, [rsi+2*16]	;; Load hi data carry
	xload	xmm4, [rbp+0*64]	;; Load FFT data
	mulpd	xmm4, [rdx+0*XMM_GMD][rcx] ;; mul by fudged grp two-to-minus-phi
	addpd	xmm0, xmm4		;; x1 = values1 + carry
	movzx	rax, BYTE PTR [rdi+1]	;; Load big vs. little flag
	split_carry_zpad_word base2, xmm2, xmm6, xmm4, rax*4
no const xload	xmm4, XMM_K_LO		;; Calc high FFT carry times k
const	xload	xmm4, XMM_K_TIMES_MULCONST_LO
	mulpd	xmm4, xmm2		;; high_FFT_carry * k_lo
	addpd	xmm0, xmm4		;; x1 = x1 + high_FFT_carry * k_lo
no const xload	xmm5, XMM_K_HI
const	xload	xmm5, XMM_K_TIMES_MULCONST_HI
	mulpd	xmm5, XMM_LIMIT_INVERSE[rax*4] ;; shift k_hi
	mulpd	xmm5, xmm2
	rounding exec, base2, noexec, noexec, xmm0, xmm2, xmm4, rax*4
	addpd	xmm2, xmm5		;; Carry += high_FFT_carry * shifted k_hi
	mulpd	xmm0, [rdx+0*XMM_GMD+XMM_GMD/2][rcx] ;; mul by fudged grp two-to-phi
	xstore	[rbp+0*64], xmm0	;; Save FFT data

	movzx	rcx, BYTE PTR [rdi+2]	;; Load fudge factor flags
	and	rcx, 0f0h
	xload	xmm0, [rbp+1*64]	;; Load FFT data
	mulpd	xmm0, [rdx+0*XMM_GMD][rcx] ;; mul by fudged grp two-to-minus-phi
	addpd	xmm0, xmm2		;; x2 = values1 + carry
	movzx	rax, BYTE PTR [rdi+3]	;; Load big vs. little flag
	split_carry_zpad_word base2, xmm6, xmm5, xmm4, rax*4
no const xload	xmm4, XMM_K_LO		;; Calc high FFT carry times k
const	xload	xmm4, XMM_K_TIMES_MULCONST_LO
	mulpd	xmm4, xmm6		;; high_FFT_carry * k_lo
	addpd	xmm0, xmm4		;; x2 = x2 + high_FFT_carry * k_lo
no const xload	xmm4, XMM_K_HI
const	xload	xmm4, XMM_K_TIMES_MULCONST_HI
	mulpd	xmm4, XMM_LIMIT_INVERSE[rax*4] ;; shift k_hi
	mulpd	xmm6, xmm4
	rounding exec, base2, noexec, noexec, xmm0, xmm2, xmm4, rax*4
	addpd	xmm2, xmm6		;; Carry += high_FFT_carry * shifted k_hi
	mulpd	xmm0, [rdx+0*XMM_GMD+XMM_GMD/2][rcx] ;; mul by fudged grp two-to-phi
	xstore	[rbp+1*64], xmm0	;; Save FFT data

	mov	ecx, BIGLIT_INCR2	;; Different clm values step through
					;; big/lit array differently
	movzx	rax, BYTE PTR [rdi+rcx+1] ;; Load big vs. little flag
	movzx	rcx, BYTE PTR [rdi+rcx]	;; Load fudge factor flag
	and	rcx, 0f0h
	xload	xmm0, [rbp+2*64]	;; Load FFT data
	mulpd	xmm0, [rdx+0*XMM_GMD][rcx] ;; mul by fudged grp two-to-minus-phi
	addpd	xmm0, xmm2		;; x3 = values1 + carry
	split_carry_zpad_word base2, xmm5, xmm6, xmm4, rax*4
no const xload	xmm4, XMM_K_LO		;; Calc high FFT carry times k
const	xload	xmm4, XMM_K_TIMES_MULCONST_LO
	mulpd	xmm4, xmm5		;; high_FFT_carry * k_lo
	addpd	xmm0, xmm4		;; x3 = x3 + high_FFT_carry * k_lo
no const xload	xmm4, XMM_K_HI
const	xload	xmm4, XMM_K_TIMES_MULCONST_HI
	mulpd	xmm4, XMM_LIMIT_INVERSE[rax*4] ;; shift k_hi
	mulpd	xmm5, xmm4
	rounding exec, base2, noexec, noexec, xmm0, xmm2, xmm4, rax*4
	addpd	xmm2, xmm5		;; Carry += high_FFT_carry * shifted k_hi
	mulpd	xmm0, [rdx+0*XMM_GMD+XMM_GMD/2][rcx] ;; mul by fudged grp two-to-phi
	xstore	[rbp+2*64], xmm0	;; Save FFT data

	mov	ecx, BIGLIT_INCR2	;; Different clm values step through
					;; big/lit array differently
	movzx	rax, BYTE PTR [rdi+rcx+3] ;; Load big vs. little flag
	movzx	rcx, BYTE PTR [rdi+rcx+2] ;; Load fudge factor flag
	and	rcx, 0f0h
	xload	xmm0, [rbp+3*64]	;; Load FFT data
	mulpd	xmm0, [rdx+0*XMM_GMD][rcx] ;; mul by grp two-to-minus-phi
	addpd	xmm0, xmm2		;; x4 = values1 + carry
	split_carry_zpad_word base2, xmm6, xmm5, xmm4, rax*4
no const xload	xmm4, XMM_K_LO		;; Calc high FFT carry times k
const	xload	xmm4, XMM_K_TIMES_MULCONST_LO
	mulpd	xmm4, xmm6		;; high_FFT_carry * k_lo
	addpd	xmm0, xmm4		;; x4 = x4 + high_FFT_carry * k_lo
no const xload	xmm4, XMM_K_HI
const	xload	xmm4, XMM_K_TIMES_MULCONST_HI
	mulpd	xmm4, XMM_LIMIT_INVERSE[rax*4] ;; shift k_hi
	mulpd	xmm6, xmm4
	rounding exec, base2, noexec, noexec, xmm0, xmm2, xmm4, rax*4
	addpd	xmm2, xmm6		;; Carry += high_FFT_carry * shifted k_hi
	mulpd	xmm0, [rdx+0*XMM_GMD+XMM_GMD/2][rcx] ;; mul by fudged grp two-to-phi
	xstore	[rbp+3*64], xmm0	;; Save FFT data

	mov	ecx, BIGLIT_INCR4	;; Different clm values step through
					;; big/lit array differently
	movzx	rax, BYTE PTR [rdi+rcx+1] ;; Load big vs. little flag
	movzx	rcx, BYTE PTR [rdi+rcx]	;; Load fudge factor flag
	and	rcx, 0f0h
	xload	xmm0, [rbp+4*64]	;; Load FFT data
	mulpd	xmm0, [rdx+0*XMM_GMD][rcx] ;; mul by fudged grp two-to-minus-phi
	addpd	xmm0, xmm2		;; x5 = values1 + carry
no const xload	xmm4, XMM_K_LO		;; Calc high FFT carry times k
const	xload	xmm4, XMM_K_TIMES_MULCONST_LO
	mulpd	xmm4, xmm5		;; high_FFT_carry * k_lo
	addpd	xmm0, xmm4		;; x5 = x5 + high_FFT_carry * k_lo
no const xload	xmm4, XMM_K_HI
const	xload	xmm4, XMM_K_TIMES_MULCONST_HI
	mulpd	xmm4, XMM_LIMIT_INVERSE[rax*4] ;; shift k_hi
	mulpd	xmm5, xmm4
	rounding exec, base2, noexec, noexec, xmm0, xmm2, xmm4, rax*4
	addpd	xmm2, xmm5		;; Carry += high_FFT_carry * shifted k_hi
	mulpd	xmm0, [rdx+0*XMM_GMD+XMM_GMD/2][rcx] ;; mul by fudged grp two-to-phi
	xstore	[rbp+4*64], xmm0	;; Save FFT data

	mov	ecx, BIGLIT_INCR4	;; Different clm values step through
					;; big/lit array differently
	movzx	rcx, BYTE PTR [rdi+rcx+2] ;; Load fudge factor flag
	and	rcx, 0f0h
	subpd	xmm2, XMM_BIGVAL	;; Remove rounding const from carry
	mulpd	xmm2, [rdx+0*XMM_GMD+XMM_GMD/2][rcx] ;; mul by fudged grp two-to-phi
	addpd	xmm2, [rbp+5*64]	;; Load FFT data
	xstore	[rbp+5*64], xmm2	;; Save FFT data

	movzx	rax, BYTE PTR [rdi+1]	;; Load big vs. little flag
	movzx	rcx, BYTE PTR [rdi]	;; Load big vs. little flag
	and	rcx, 0f0h
	xload	xmm1, [rsi+1*16]	;; Load carry
	xload	xmm3, [rsi+3*16]	;; Load hi data carry
	xload	xmm4, [rbp+0*64+16]	;; Load FFT data
	mulpd	xmm4, [rdx+1*XMM_GMD][rcx] ;; mul by fudged grp two-to-minus-phi
	addpd	xmm1, xmm4		;; x1 = values1 + carry
	split_carry_zpad_word base2, xmm3, xmm6, xmm4, rax*4+16
no const xload	xmm4, XMM_K_LO		;; Calc high FFT carry times k
const	xload	xmm4, XMM_K_TIMES_MULCONST_LO
	mulpd	xmm4, xmm3		;; high_FFT_carry * k_lo
	addpd	xmm1, xmm4		;; x1 = x1 + high_FFT_carry * k_lo
no const xload	xmm5, XMM_K_HI
const	xload	xmm5, XMM_K_TIMES_MULCONST_HI
	mulpd	xmm5, XMM_LIMIT_INVERSE[rax*4+16] ;; shift k_hi
	mulpd	xmm5, xmm3
	rounding exec, base2, noexec, noexec, xmm1, xmm3, xmm4, rax*4+16
	addpd	xmm3, xmm5		;; Carry += high_FFT_carry * shifted k_hi
	mulpd	xmm1, [rdx+1*XMM_GMD+XMM_GMD/2][rcx] ;; mul by fudged grp two-to-phi
	xstore	[rbp+0*64+16], xmm1	;; Save FFT data

	movzx	rax, BYTE PTR [rdi+3]	;; Load big vs. little flag
	movzx	rcx, BYTE PTR [rdi+2]	;; Load fudge factor flag
	and	rcx, 0f0h
	xload	xmm1, [rbp+1*64+16]	;; Load FFT data
	mulpd	xmm1, [rdx+1*XMM_GMD][rcx] ;; mul by fudged grp two-to-minus-phi
	addpd	xmm1, xmm3		;; x2 = values1 + carry
	split_carry_zpad_word base2, xmm6, xmm5, xmm4, rax*4+16
no const xload	xmm4, XMM_K_LO		;; Calc high FFT carry times k
const	xload	xmm4, XMM_K_TIMES_MULCONST_LO
	mulpd	xmm4, xmm6		;; high_FFT_carry * k_lo
	addpd	xmm1, xmm4		;; x2 = x2 + high_FFT_carry * k_lo
no const xload	xmm4, XMM_K_HI
const	xload	xmm4, XMM_K_TIMES_MULCONST_HI
	mulpd	xmm4, XMM_LIMIT_INVERSE[rax*4+16] ;; shift k_hi
	mulpd	xmm6, xmm4
	rounding exec, base2, noexec, noexec, xmm1, xmm3, xmm4, rax*4+16
	addpd	xmm3, xmm6		;; Carry += high_FFT_carry * shifted k_hi
	mulpd	xmm1, [rdx+1*XMM_GMD+XMM_GMD/2][rcx] ;; mul by fudged grp two-to-phi
	xstore	[rbp+1*64+16], xmm1	;; Save FFT data

	mov	ecx, BIGLIT_INCR2	;; Different clm values step through
					;; big/lit array differently
	movzx	rax, BYTE PTR [rdi+rcx+1] ;; Load big vs. little flag
	movzx	rcx, BYTE PTR [rdi+rcx]	;; Load fudge factor flag
	and	rcx, 0f0h
	xload	xmm1, [rbp+2*64+16]	;; Load FFT data
	mulpd	xmm1, [rdx+1*XMM_GMD][rcx] ;; mul by fudged grp two-to-minus-phi
	addpd	xmm1, xmm3		;; x3 = values1 + carry
	split_carry_zpad_word base2, xmm5, xmm6, xmm4, rax*4+16
no const xload	xmm4, XMM_K_LO		;; Calc high FFT carry times k
const	xload	xmm4, XMM_K_TIMES_MULCONST_LO
	mulpd	xmm4, xmm5		;; high_FFT_carry * k_lo
	addpd	xmm1, xmm4		;; x3 = x3 + high_FFT_carry * k_lo
no const xload	xmm4, XMM_K_HI
const	xload	xmm4, XMM_K_TIMES_MULCONST_HI
	mulpd	xmm4, XMM_LIMIT_INVERSE[rax*4+16] ;; shift k_hi
	mulpd	xmm5, xmm4
	rounding exec, base2, noexec, noexec, xmm1, xmm3, xmm4, rax*4+16
	addpd	xmm3, xmm5		;; Carry += high_FFT_carry * shifted k_hi
	mulpd	xmm1, [rdx+1*XMM_GMD+XMM_GMD/2][rcx] ;; mul by fudged grp two-to-phi
	xstore	[rbp+2*64+16], xmm1	;; Save FFT data

	mov	ecx, BIGLIT_INCR2	;; Different clm values step through
					;; big/lit array differently
	movzx	rax, BYTE PTR [rdi+rcx+3] ;; Load big vs. little flag
	movzx	rcx, BYTE PTR [rdi+rcx+2] ;; Load fudge factor flag
	and	rcx, 0f0h
	xload	xmm1, [rbp+3*64+16]	;; Load FFT data
	mulpd	xmm1, [rdx+1*XMM_GMD][rcx] ;; mul by fudged grp two-to-minus-phi
	addpd	xmm1, xmm3		;; x4 = values1 + carry
	split_carry_zpad_word base2, xmm6, xmm5, xmm4, rax*4+16
no const xload	xmm4, XMM_K_LO		;; Calc high FFT carry times k
const	xload	xmm4, XMM_K_TIMES_MULCONST_LO
	mulpd	xmm4, xmm6		;; high_FFT_carry * k_lo
	addpd	xmm1, xmm4		;; x4 = x4 + high_FFT_carry * k_lo
no const xload	xmm4, XMM_K_HI
const	xload	xmm4, XMM_K_TIMES_MULCONST_HI
	mulpd	xmm4, XMM_LIMIT_INVERSE[rax*4+16] ;; shift k_hi
	mulpd	xmm6, xmm4
	rounding exec, base2, noexec, noexec, xmm1, xmm3, xmm4, rax*4+16
	addpd	xmm3, xmm6		;; Carry += high_FFT_carry * shifted k_hi
	mulpd	xmm1, [rdx+1*XMM_GMD+XMM_GMD/2][rcx] ;; mul by fudged grp two-to-phi
	xstore	[rbp+3*64+16], xmm1	;; Save FFT data

	mov	ecx, BIGLIT_INCR4	;; Different clm values step through
					;; big/lit array differently
	movzx	rax, BYTE PTR [rdi+rcx+1] ;; Load big vs. little flag
	movzx	rcx, BYTE PTR [rdi+rcx]	;; Load fudge factor flag
	and	rcx, 0f0h
	xload	xmm1, [rbp+4*64+16]	;; Load FFT data
	mulpd	xmm1, [rdx+1*XMM_GMD][rcx] ;; mul by fudged grp two-to-minus-phi
	addpd	xmm1, xmm3		;; x5 = values1 + carry
no const xload	xmm4, XMM_K_LO		;; Calc high FFT carry times k
const	xload	xmm4, XMM_K_TIMES_MULCONST_LO
	mulpd	xmm4, xmm5		;; high_FFT_carry * k_lo
	addpd	xmm1, xmm4		;; x5 = x5 + high_FFT_carry * k_lo
no const xload	xmm4, XMM_K_HI
const	xload	xmm4, XMM_K_TIMES_MULCONST_HI
	mulpd	xmm4, XMM_LIMIT_INVERSE[rax*4+16] ;; shift k_hi
	mulpd	xmm5, xmm4
	rounding exec, base2, noexec, noexec, xmm1, xmm3, xmm4, rax*4+16
	addpd	xmm3, xmm5		;; Carry += high_FFT_carry * shifted k_hi
	mulpd	xmm1, [rdx+1*XMM_GMD+XMM_GMD/2][rcx] ;; mul by fudged grp two-to-phi
	xstore	[rbp+4*64+16], xmm1	;; Save FFT data

	mov	ecx, BIGLIT_INCR4	;; Different clm values step through
					;; big/lit array differently
	movzx	rcx, BYTE PTR [rdi+rcx+2] ;; Load big vs. little flag
	and	rcx, 0f0h
	subpd	xmm3, XMM_BIGVAL	;; Remove rounding const from carry
	mulpd	xmm3, [rdx+1*XMM_GMD+XMM_GMD/2][rcx] ;; mul by fudged grp two-to-phi
	addpd	xmm3, [rbp+5*64+16]	;; Add FFT data
	xstore	[rbp+5*64+16], xmm3	;; Save FFT data

	xload	xmm4, XMM_BIGVAL
	xstore	[rsi+0*16], xmm4	;; Clear carry
	xstore	[rsi+1*16], xmm4
	xstore	[rsi+2*16], xmm4
	xstore	[rsi+3*16], xmm4
	ENDM


; *************** 1D normalized add/sub macro ******************
; This macro adds or subtracts, then "normalizes" eight FFT
; data values.  This involves multiplying the summed values by
; two-to-minus-phi.  Rounding the value to an integer.  Making sure
; the integer is smaller than the maximum allowable integer, generating
; a carry if necessary. Finally, the value is multiplied by two-to-phi
; and stored.
; xmm3 = carry #2
; xmm2 = carry #1
; rcx = pointer to the first number
; rdx = pointer to the second number
; rsi = pointer to destination
; rbp = pointer two-to-phi multipliers
; rdi = pointer to array of big vs. little flags
; rbx = big vs. little word flag #2
; eax = big vs. little word flag #1
; A pipelined version of this code:
;	movzx	rax, BYTE PTR [rdi]	;; Load big vs. little flags
;	xload	xmm0, [rdx+0*dist1]	;; Load second number
;	fop	xmm0, [rcx]		;; Add/sub first number
;	mulpd	xmm0, [rbp+0]		;; Mul values1 by two-to-minus-phi
;	addpd	xmm0, xmm4		;; x = values + carry
;	xload	xmm2, XMM_LIMIT_BIGMAX[rax];; Load maximum * BIGVAL - BIGVAL
;	addpd	xmm2, xmm0		;; y = top bits of x
;	xload	xmm6, XMM_LIMIT_BIGMAX_NEG[rax];; Load -(maximum*BIGVAL-BIGVAL)
;	addpd	xmm6, xmm2		;; z = y - (maximum * BIGVAL - BIGVAL)
;	subpd	xmm0, xmm6		;; rounded value = x - z
;	mulpd	xmm2, XMM_LIMIT_INVERSE[rax];; next carry = shifted y
;	mulpd	xmm0, [rbp+16]		;; new value = val * two-to-phi
;	xstore	[rsi+0*dist1], xmm0	;; Save new value

xnorm_op_1d MACRO fop, ttp, base2
ttp	movzx	rax, BYTE PTR [rdi]	;; Load big vs. little flags
ttp	movzx	rbx, BYTE PTR [rdi+2]
	xload	xmm0, [rdx]		;; Load second number
	fop	xmm0, [rcx]		;; Add/sub first number
ttp	mulpd	xmm0, [rbp]		;; Mul values1 by two-to-minus-phi
ttp	mulpd	xmm0, XMM_NORM012_FF	;; Mul by FFTLEN/2
	xload	xmm1, [rdx+32]		;; Load second number
	fop	xmm1, [rcx+32]		;; Add/sub first number
ttp	mulpd	xmm1, [rbp+64]		;; Mul values2 by two-to-minus-phi
ttp	mulpd	xmm1, XMM_NORM012_FF	;; Mul by FFTLEN/2
	addpd	xmm0, xmm2		;; x1 = values + carry
	addpd	xmm1, xmm3		;; x2 = values + carry
	rounding_interleaved ttp, base2, noexec, noexec, xmm0, xmm2, xmm4, rax, xmm1, xmm3, xmm5, rbx 
ttp	mulpd	xmm0, [rbp+16]		;; new value1 = val * two-to-phi
ttp	mulpd	xmm1, [rbp+80]		;; new value2 = val * two-to-phi
	xstore	[rsi], xmm0		;; Save value1
	xstore	[rsi+32], xmm1		;; Save value2
ttp	movzx	rax, BYTE PTR [rdi+1]	;; Load big vs. little flags
ttp	movzx	rbx, BYTE PTR [rdi+3]
	xload	xmm0, [rdx+16]		;; Load values1
	fop	xmm0, [rcx+16]		;; Add/sub first number
ttp	mulpd	xmm0, [rbp+32]		;; Mul values1 by two-to-minus-phi
ttp	mulpd	xmm0, XMM_NORM012_FF	;; Mul by FFTLEN/2
	xload	xmm1, [rdx+48]		;; Load values2
	fop	xmm1, [rcx+48]		;; Add/sub first number
ttp	mulpd	xmm1, [rbp+96]		;; Mul values2 by two-to-minus-phi
ttp	mulpd	xmm1, XMM_NORM012_FF	;; Mul by FFTLEN/2
	addpd	xmm0, xmm2		;; x1 = values + carry
	addpd	xmm1, xmm3		;; x2 = values + carry
	rounding_interleaved ttp, base2, noexec, noexec, xmm0, xmm2, xmm4, rax, xmm1, xmm3, xmm5, rbx 
ttp	mulpd	xmm0, [rbp+48]		;; new value1 = val * two-to-phi
ttp	mulpd	xmm1, [rbp+112]		;; new value2 = val * two-to-phi
	xstore	[rsi+16], xmm0		;; Save new value1
	xstore	[rsi+48], xmm1		;; Save new value2
ttp	bump	rdi, 4			;; Next flags ptr
	bump	rcx, 64			;; Next src ptr
	bump	rdx, 64			;; Next src ptr
ttp	bump	rbp, 128		;; Next two-to-phi ptr
	bump	rsi, 64			;; Next dest ptr
	ENDM


; This macro finishes the normalize process by adding the final
; carry from the first pass back into the lower two data values.
; xmm2,xmm3 = carries
; rax = pointer to the FFT data values
; rbx = pointer two-to-phi multipliers

xnorm_op_1d_mid_cleanup MACRO
	movsd	xmm0, Q [rax+8]		;; Load values1
	movsd	xmm1, Q [rax+40]	;; Load values2
	subsd	xmm2, Q XMM_BIGVAL	;; Remove BIGVAL from carries
	subsd	xmm3, Q XMM_BIGVAL
	mulsd	xmm2, Q [rbx+24]	;; carry1 *= two-to-phi
	mulsd	xmm3, Q [rbx+88]	;; carry2 *= two-to-phi
	addsd	xmm0, xmm2		;; x1 = values + carry
	addsd	xmm1, xmm3		;; x2 = values + carry
	movsd	Q [rax+8], xmm0		;; Save new value1
	movsd	Q [rax+40], xmm1	;; Save new value2
	shufpd	xmm2, xmm2, 1		;; Rotate carry
	movhpd	xmm2, Q XMM_BIGVAL
	shufpd	xmm3, xmm3, 1		;; Rotate carry
	movhpd	xmm3, Q XMM_BIGVAL
	ENDM

; *************** 1D normalized add/sub macro ******************
; This macro adds and subtracts, then "normalizes" eight FFT
; data values.  This involves multiplying the sum values by
; two-to-minus-phi.  Adding, subtracting and rounding the value to an
; integer.  Make sure the integer is smaller than the maximum allowable
; integer, generating carries if necessary.  Finally, the values are
; multiplied by two-to-phi and stored.
; xmm7 = sub carry #2
; xmm6 = sub carry #1
; xmm3 = add carry #2
; xmm2 = add carry #1
; rcx = pointer to the first number
; rdx = pointer to the second number
; rsi = pointer to destination #1
; rbp = pointer to destination #2
; rbx = pointer two-to-phi multipliers
; rdi = pointer to array of big vs. little flags
; eax = big vs. little word flag #1

xnorm_addsub_1d MACRO ttp, base2
ttp	movzx	rax, BYTE PTR [rdi]	;; Load big vs. little flags
	xload	xmm0, [rcx]		;; Load first number
	addpd	xmm0, [rdx]		;; Add second number
ttp	xload	xmm5, [rbx]		;; Load fudged two-to-minus-phi
ttp	mulpd	xmm5, XMM_NORM012_FF	;; Mul by FFTLEN/2
ttp	mulpd	xmm0, xmm5		;; Mul values1 by two-to-minus-phi
	xload	xmm1, [rcx]		;; Load first number
	subpd	xmm1, [rdx]		;; Sub second number
ttp	mulpd	xmm1, xmm5		;; Mul values2 by two-to-minus-phi
	addpd	xmm0, xmm2		;; x1 = values + carry
	addpd	xmm1, xmm6		;; x2 = values + carry
	rounding_interleaved ttp, base2, noexec, noexec, xmm0, xmm2, xmm4, rax, xmm1, xmm6, xmm5, rax
ttp	mulpd	xmm0, [rbx+16]		;; new value1 = val * two-to-phi
ttp	mulpd	xmm1, [rbx+16]		;; new value2 = val * two-to-phi
	xstore	[rsi], xmm0		;; Save value1
	xstore	[rbp], xmm1		;; Save value2

ttp	movzx	rax, BYTE PTR [rdi+2]	;; Load big vs. little flags
	xload	xmm0, [rcx+32]		;; Load first number
	addpd	xmm0, [rdx+32]		;; Add second number
ttp	xload	xmm5, [rbx+64]		;; Load fudged two-to-minus-phi
ttp	mulpd	xmm5, XMM_NORM012_FF	;; Mul by FFTLEN/2
ttp	mulpd	xmm0, xmm5		;; Mul values1 by two-to-minus-phi
	xload	xmm1, [rcx+32]		;; Load first number
	subpd	xmm1, [rdx+32]		;; Sub second number
ttp	mulpd	xmm1, xmm5		;; Mul values2 by two-to-minus-phi
	addpd	xmm0, xmm3		;; x1 = values + carry
	addpd	xmm1, xmm7		;; x2 = values + carry
	rounding_interleaved ttp, base2, noexec, noexec, xmm0, xmm3, xmm4, rax, xmm1, xmm7, xmm5, rax
ttp	mulpd	xmm0, [rbx+80]		;; new value1 = val * two-to-phi
ttp	mulpd	xmm1, [rbx+80]		;; new value2 = val * two-to-phi
	xstore	[rsi+32], xmm0		;; Save value1
	xstore	[rbp+32], xmm1		;; Save value2

ttp	movzx	rax, BYTE PTR [rdi+1]	;; Load big vs. little flags
	xload	xmm0, [rcx+16]		;; Load first number
	addpd	xmm0, [rdx+16]		;; Add second number
ttp	xload	xmm5, [rbx+32]		;; Load fudged two-to-minus-phi
ttp	mulpd	xmm5, XMM_NORM012_FF	;; Mul by FFTLEN/2
ttp	mulpd	xmm0, xmm5		;; Mul values1 by two-to-minus-phi
	xload	xmm1, [rcx+16]		;; Load first number
	subpd	xmm1, [rdx+16]		;; Sub second number
ttp	mulpd	xmm1, xmm5		;; Mul values2 by two-to-minus-phi
	addpd	xmm0, xmm2		;; x1 = values + carry
	addpd	xmm1, xmm6		;; x2 = values + carry
	rounding_interleaved ttp, base2, noexec, noexec, xmm0, xmm2, xmm4, rax, xmm1, xmm6, xmm5, rax
ttp	mulpd	xmm0, [rbx+48]		;; new value1 = val * two-to-phi
ttp	mulpd	xmm1, [rbx+48]		;; new value2 = val * two-to-phi
	xstore	[rsi+16], xmm0		;; Save value1
	xstore	[rbp+16], xmm1		;; Save value2

ttp	movzx	rax, BYTE PTR [rdi+3]	;; Load big vs. little flags
	xload	xmm0, [rcx+48]		;; Load first number
	addpd	xmm0, [rdx+48]		;; Add second number
ttp	xload	xmm5, [rbx+96]		;; Load fudged two-to-minus-phi
ttp	mulpd	xmm5, XMM_NORM012_FF	;; Mul by FFTLEN/2
ttp	mulpd	xmm0, xmm5		;; Mul values1 by two-to-minus-phi
	xload	xmm1, [rcx+48]		;; Load first number
	subpd	xmm1, [rdx+48]		;; Sub second number
ttp	mulpd	xmm1, xmm5		;; Mul values2 by two-to-minus-phi
	addpd	xmm0, xmm3		;; x1 = values + carry
	addpd	xmm1, xmm7		;; x2 = values + carry
	rounding_interleaved ttp, base2, noexec, noexec, xmm0, xmm3, xmm4, rax, xmm1, xmm7, xmm5, rax
ttp	mulpd	xmm0, [rbx+112]		;; new value1 = val * two-to-phi
ttp	mulpd	xmm1, [rbx+112]		;; new value2 = val * two-to-phi
	xstore	[rsi+48], xmm0		;; Save value1
	xstore	[rbp+48], xmm1		;; Save value2

ttp	bump	rdi, 4			;; Next flags ptr
	bump	rcx, 64			;; Next src ptr
	bump	rdx, 64			;; Next src ptr
ttp	bump	rbx, 128		;; Next two-to-phi ptr
	bump	rsi, 64			;; Next dest ptr
	bump	rbp, 64			;; Next dest ptr
	ENDM

; This macro finishes the normalize process by adding the final
; carry from the first pass back into the lower two data values.
; xmm2,xmm3 = carries #1
; xmm6,xmm7 = carries #2
; rax = pointer to the FFT data values #1
; top of stack = pointer to the FFT destination #1 and #2
; rbx = pointer two-to-phi multipliers

xnorm_addsub_1d_mid_cleanup MACRO dest1, dest2
	mov	rax, dest1		;; Restore dest #1 pointer
	movsd	xmm0, Q [rax+8]		;; Load values1
	movsd	xmm1, Q [rax+40]	;; Load values2
	subsd	xmm2, Q XMM_BIGVAL	;; Remove BIGVAL from carries
	subsd	xmm3, Q XMM_BIGVAL
	mulsd	xmm2, Q [rbx+24]	;; carry1 *= two-to-phi
	mulsd	xmm3, Q [rbx+88]	;; carry2 *= two-to-phi
	addsd	xmm0, xmm2		;; x1 = values + carry
	addsd	xmm1, xmm3		;; x2 = values + carry
	movsd	Q [rax+8], xmm0		;; Save new value1
	movsd	Q [rax+40], xmm1	;; Save new value2
	shufpd	xmm2, xmm2, 1		;; Rotate carry
	movhpd	xmm2, Q XMM_BIGVAL
	shufpd	xmm3, xmm3, 1		;; Rotate carry
	movhpd	xmm3, Q XMM_BIGVAL

	mov	rax, dest2		;; Get FFT data pointer #2
	movsd	xmm0, Q [rax+8]		;; Load values1
	movsd	xmm1, Q [rax+40]	;; Load values2
	subsd	xmm6, Q XMM_BIGVAL	;; Remove BIGVAL from carries
	subsd	xmm7, Q XMM_BIGVAL
	mulsd	xmm6, Q [rbx+24]	;; carry1 *= two-to-phi
	mulsd	xmm7, Q [rbx+88]	;; carry2 *= two-to-phi
	addsd	xmm0, xmm6		;; x1 = values + carry
	addsd	xmm1, xmm7		;; x2 = values + carry
	movsd	Q [rax+8], xmm0		;; Save new value1
	movsd	Q [rax+40], xmm1	;; Save new value2
	shufpd	xmm6, xmm6, 1		;; Rotate carry
	movhpd	xmm6, Q XMM_BIGVAL
	shufpd	xmm7, xmm7, 1		;; Rotate carry
	movhpd	xmm7, Q XMM_BIGVAL
	ENDM

; rsi = pointer to the FFT data values #1
; rbp = pointer to the FFT data values #2

xnorm_addsub_1d_cleanup MACRO
	xnorm_top_carry_cmn rsi, xmm3, 0;; Adjust top carry if necessary
	xnorm_top_carry_cmn rbp, xmm7, 0;; Adjust top carry if necessary

	movsd	xmm0, Q [rsi]		;; Load values1
	movsd	xmm1, Q [rsi+32]	;; Load values2
	subsd	xmm3, Q XMM_BIGVAL	;; Remove BIGVAL from carries
	subsd	xmm2, Q XMM_BIGVAL
	mulsd	xmm3, Q XMM_MINUS_C	;; Mul wrap-around carry by -c
	mulsd	xmm3, Q [rbx+16]	;; carry *= two-to-phi
	mulsd	xmm2, Q [rbx+80]	;; carry *= two-to-phi
	addsd	xmm0, xmm3		;; value1 = values + carry
	addsd	xmm1, xmm2		;; value2 = values + carry
	movsd	Q [rsi], xmm0		;; Save new value1
	movsd	Q [rsi+32], xmm1	;; Save new value2

	movsd	xmm0, Q [rbp]		;; Load values1
	movsd	xmm1, Q [rbp+32]	;; Load values2
	subsd	xmm7, Q XMM_BIGVAL	;; Remove BIGVAL from carries
	subsd	xmm6, Q XMM_BIGVAL
	mulsd	xmm7, Q XMM_MINUS_C	;; Mul wrap-around carry by -c
	mulsd	xmm7, Q [rbx+16]	;; carry *= two-to-phi
	mulsd	xmm6, Q [rbx+80]	;; carry *= two-to-phi
	addsd	xmm0, xmm7		;; value1 = values + carry
	addsd	xmm1, xmm6		;; value2 = values + carry
	movsd	Q [rbp], xmm0		;; Save new value1
	movsd	Q [rbp+32], xmm1	;; Save new value2
	ENDM

; *************** 1D normalized smallmul macro ******************
; This macro multiplies by a small constant, then "normalizes" eight FFT
; data values.
; xmm7 = small multiplier value
; xmm3 = carry #2
; xmm2 = carry #1
; rsi = pointer to destination
; rbp = pointer two-to-phi multipliers
; rdi = pointer to array of big vs. little flags
; rax = big vs. little word flag #1
; rcx = big vs. little word flag #2

xnorm_smallmul_1d MACRO ttp, base2
ttp	movzx	rax, BYTE PTR [rdi]	;; Load big vs. little flags
ttp	movzx	rcx, BYTE PTR [rdi+2]
	xload	xmm0, [rsi]		;; Load values1
	mulpd	xmm0, xmm7		;; Mul by small value * FFTLEN/2
ttp	mulpd	xmm0, [rbp]		;; Mul values1 by two-to-minus-phi
	xload	xmm1, [rsi+32]		;; Load values2
	mulpd	xmm1, xmm7		;; Mul by small value * FFTLEN/2
ttp	mulpd	xmm1, [rbp+64]		;; Mul values2 by two-to-minus-phi
	addpd	xmm0, xmm2		;; x1 = values + carry
	addpd	xmm1, xmm3		;; x2 = values + carry
	rounding_interleaved ttp, base2, noexec, noexec, xmm0, xmm2, xmm4, rax, xmm1, xmm3, xmm5, rcx
ttp	mulpd	xmm0, [rbp+16]		;; new value1 = val * two-to-phi
ttp	mulpd	xmm1, [rbp+80]		;; new value2 = val * two-to-phi
	xstore	[rsi], xmm0		;; Save value1
	xstore	[rsi+32], xmm1		;; Save value2
ttp	movzx	rax, BYTE PTR [rdi+1]	;; Load big vs. little flags
ttp	movzx	rcx, BYTE PTR [rdi+3]
	xload	xmm0, [rsi+16]		;; Load values1
	mulpd	xmm0, xmm7		;; Mul by small value * FFTLEN/2
ttp	mulpd	xmm0, [rbp+32]		;; Mul values1 by two-to-minus-phi
	xload	xmm1, [rsi+48]		;; Load values2
	mulpd	xmm1, xmm7		;; Mul by small value * FFTLEN/2
ttp	mulpd	xmm1, [rbp+96]		;; Mul values2 by two-to-minus-phi
	addpd	xmm0, xmm2		;; x1 = values + carry
	addpd	xmm1, xmm3		;; x2 = values + carry
	rounding_interleaved ttp, base2, noexec, noexec, xmm0, xmm2, xmm4, rax, xmm1, xmm3, xmm5, rcx
ttp	mulpd	xmm0, [rbp+48]		;; new value1 = val * two-to-phi
ttp	mulpd	xmm1, [rbp+112]		;; new value2 = val * two-to-phi
	xstore	[rsi+16], xmm0		;; Save new value1
	xstore	[rsi+48], xmm1		;; Save new value2
ttp	bump	rdi, 4			;; Next flags ptr
ttp	bump	rbp, 128		;; Next two-to-phi ptr
	bump	rsi, 64			;; Next dest ptr
	ENDM

; This macro finishes the smallmul normalize process by adding the final
; carry from the first pass back into the lower two data values.
; xmm2,xmm3 = carries
; rsi = pointer to the FFT data values
; rbp = pointer two-to-phi multipliers
; rdi = pointer to biglit array

xnorm_smallmul_1d_mid_cleanup MACRO base2
	xnorm012_1d_mid exec, base2
	ENDM

xnorm_smallmul_1d_cleanup MACRO base2
	LOCAL	zpad, done

	cmp	ZERO_PADDED_FFT, 0	;; Zero-padded FFT?
	jne	zpad			;; Yes, do special zpad carry
	xnorm_top_carry_1d		;; No, do a very standard carry
	sub	rax, rax
	xnorm012_1d base2
	jmp	done
zpad:	xnorm_smallmul_1d_zpad_cleanup base2	;; Do the special zpad carry
done:
	ENDM

; This macro is similar to xnorm012_1d_zpad for handling zpad carries
; rsi = pointer to the FFT data values
; rbp = pointer two-to-phi multipliers
; rdi = big vs. little array pointer

xnorm_smallmul_1d_zpad_cleanup MACRO base2
	LOCAL	smallk, mediumk, div_k_done, funnyaddr2, funny2done

	;; Copy and integerize data from 7 words above halfway point to ZPAD0-ZPAD6
	;; Clear words 5,6,7
	;; Then we can make an exact copy of most of the xnorm012_1d_zpad code

	movzx	rax, BYTE PTR [rdi]	;; Load big vs. little flags
	movsd	xmm0, Q [rsi+32]	;; Value1
	subsd	xmm2, Q XMM_BIGVAL	;; Remove XMM_BIGVAL from the carry
	single_split_lower_zpad_word base2, xmm0, xmm2, xmm4, rax
	movsd	ZPAD0, xmm0

	movzx	rax, BYTE PTR [rdi+1]	;; Load big vs. little flags
	movsd	xmm0, Q [rsi+48]	;; Value2
	mulsd	xmm0, Q [rbp+32]	;; Mul values2 by two-to-minus-phi
	mulsd	xmm0, Q XMM_NORM012_FF	;; Mul by FFTLEN/2
	single_split_lower_zpad_word base2, xmm0, xmm2, xmm4, rax
	movsd	ZPAD1, xmm0

	movzx	rax, BYTE PTR [rdi+4]	;; Load big vs. little flags
	movsd	xmm0, Q [rsi+64+32]	;; Value3
	mulsd	xmm0, Q [rbp+128]	;; Mul values3 by two-to-minus-phi
	mulsd	xmm0, Q XMM_NORM012_FF	;; Mul by FFTLEN/2
	single_split_lower_zpad_word base2, xmm0, xmm2, xmm4, rax
	movsd	ZPAD2, xmm0

	movsd	xmm0, Q [rsi+64+48]	;; Value4
	mulsd	xmm0, Q [rbp+128+32]	;; Mul values4 by two-to-minus-phi
	mulsd	xmm0, Q XMM_NORM012_FF	;; Mul by FFTLEN/2
	addsd	xmm0, Q XMM_BIGVAL	;; Round to an integer
	subsd	xmm0, Q XMM_BIGVAL
	addsd	xmm0, xmm2		;; Value4 + carry
	movsd	ZPAD3, xmm0

	add	rsi, ZPAD_WORD5_OFFSET
	add	rbp, ZPAD_WORD5_RBP_OFFSET
	movsd	xmm0, Q [rsi+32]	;; Value5
	mulsd	xmm0, Q [rbp]		;; Mul values5 by two-to-minus-phi
	mulsd	xmm0, Q XMM_NORM012_FF	;; Mul by FFTLEN/2
	addsd	xmm0, Q XMM_BIGVAL	;; Round to an integer
	subsd	xmm0, Q XMM_BIGVAL
	movsd	ZPAD4, xmm0
	movsd	xmm0, Q [rsi+48]	;; Value6
	mulsd	xmm0, Q [rbp+32]	;; Mul values6 by two-to-minus-phi
	mulsd	xmm0, Q XMM_NORM012_FF	;; Mul by FFTLEN/2
	addsd	xmm0, Q XMM_BIGVAL	;; Round to an integer
	subsd	xmm0, Q XMM_BIGVAL
	movsd	ZPAD5, xmm0
	movsd	xmm0, Q [rsi+64+32]	;; Value7
	mulsd	xmm0, Q [rbp+128]	;; Mul values7 by two-to-minus-phi
	mulsd	xmm0, Q XMM_NORM012_FF	;; Mul by FFTLEN/2
	addsd	xmm0, Q XMM_BIGVAL	;; Round to an integer
	subsd	xmm0, Q XMM_BIGVAL
	movsd	ZPAD6, xmm0
	subsd	xmm0, xmm0		;; Clear highest words
	movsd	Q [rsi+32], xmm0
	movsd	Q [rsi+48], xmm0
	movsd	Q [rsi+64+32], xmm0
	sub	rsi, ZPAD_WORD5_OFFSET
	sub	rbp, ZPAD_WORD5_RBP_OFFSET

	;; Divide the zpad data by k.  Store the integer part in XMM_TMP
	;; and the remainder in ZPAD0.  Later we will wrap the integer part
	;; down to the bottom of the FFT data area (and multiply by -c).
	;; And we will store the remainder in the upper half of the FFT
	;; data area.

	;; Note there are three cases to handle.  K is smaller than a big word.
	;; K is between one and 2 big words in size.  And K is more than
	;; 2 big words in size.

	cmp	ZPAD_TYPE, 2		;; Are we dealing with case 1,2,or 3
	jl	smallk			;; One word case
	je	mediumk			;; Two word case

	;; This case does the divide by k where k is three words

	movsd	xmm0, ZPAD6		;; Load zpad word (high bits)
	movsd	xmm1, ZPAD5		;; Load zpad word (middle bits)
	movsd	xmm2, ZPAD4		;; Load zpad word (low bits)
	movsd	xmm4, ZPAD_INVERSE_K6	;; Load shifted 1/k
	mulsd	xmm4, xmm0		;; Mul ZPAD by shifted 1/k
	addsd	xmm4, Q XMM_BIGVAL	;; Round to integer	
	subsd	xmm4, Q XMM_BIGVAL
	movsd	xmm5, ZPAD_K6_HI	;; Load high bits of k
	mulsd	xmm5, xmm4
	subsd	xmm0, xmm5		;; Calculate high bits of remainder
	movsd	xmm5, ZPAD_K6_MID	;; Load middle bits of k
	mulsd	xmm5, xmm4
	subsd	xmm1, xmm5		;; Calculate middle bits of remainder
	movsd	xmm5, ZPAD_K6_LO	;; Load low bits of k
	mulsd	xmm5, xmm4
	subsd	xmm2, xmm5		;; Calculate low bits of remainder
	movsd	Q XMM_TMP5, xmm4	;; Save word of zpad / k

	mulsd	xmm0, ZPAD_SHIFT6	;; Shift previous zpad word
	addsd	xmm1, xmm0		;; Add to create new high zpad bits
	movsd	xmm0, ZPAD3		;; Load zpad word (new low bits)
	movsd	xmm5, ZPAD_SHIFT5	;; Combine high and medium bits
	mulsd	xmm5, xmm1
	addsd	xmm5, xmm2
	movsd	xmm4, ZPAD_INVERSE_K5	;; Load shifted 1/k
	mulsd	xmm4, xmm5		;; Mul ZPAD by shifted 1/k
	addsd	xmm4, Q XMM_BIGVAL	;; Round to integer
	subsd	xmm4, Q XMM_BIGVAL
	movsd	xmm5, ZPAD_K5_HI	;; Load high bits of k
	mulsd	xmm5, xmm4
	subsd	xmm1, xmm5		;; Calculate high bits of remainder
	movsd	xmm5, ZPAD_K5_MID	;; Load middle bits of k
	mulsd	xmm5, xmm4
	subsd	xmm2, xmm5		;; Calculate middle bits of remainder
	movsd	xmm5, ZPAD_K5_LO	;; Load low bits of k
	mulsd	xmm5, xmm4
	subsd	xmm0, xmm5		;; Calculate low bits of remainder
	movsd	Q XMM_TMP4, xmm4	;; Save word of zpad / k

	mulsd	xmm1, ZPAD_SHIFT5	;; Shift previous zpad word
	addsd	xmm2, xmm1		;; Add to create new high zpad bits
	movsd	xmm1, ZPAD2		;; Load zpad word (new low bits)
	movsd	xmm5, ZPAD_SHIFT4	;; Combine high and medium bits
	mulsd	xmm5, xmm2
	addsd	xmm5, xmm0
	movsd	xmm4, ZPAD_INVERSE_K4	;; Load shifted 1/k
	mulsd	xmm4, xmm5		;; Mul ZPAD by shifted 1/k
	addsd	xmm4, Q XMM_BIGVAL	;; Round to integer
	subsd	xmm4, Q XMM_BIGVAL
	movsd	xmm5, ZPAD_K4_HI	;; Load high bits of k
	mulsd	xmm5, xmm4
	subsd	xmm2, xmm5		;; Calculate high bits of remainder
	movsd	xmm5, ZPAD_K4_MID	;; Load middle bits of k
	mulsd	xmm5, xmm4
	subsd	xmm0, xmm5		;; Calculate middle bits of remainder
	movsd	xmm5, ZPAD_K4_LO	;; Load low bits of k
	mulsd	xmm5, xmm4
	subsd	xmm1, xmm5		;; Calculate low bits of remainder
	movsd	Q XMM_TMP3, xmm4	;; Save word of zpad / k

	mulsd	xmm2, ZPAD_SHIFT4	;; Shift previous zpad word
	addsd	xmm0, xmm2		;; Add to create new high zpad bits
	movsd	xmm2, ZPAD1		;; Load zpad word (new low bits)
	movsd	xmm5, ZPAD_SHIFT3	;; Combine high and medium bits
	mulsd	xmm5, xmm0
	addsd	xmm5, xmm1
	movsd	xmm4, ZPAD_INVERSE_K3	;; Load shifted 1/k
	mulsd	xmm4, xmm5		;; Mul ZPAD by shifted 1/k
	addsd	xmm4, Q XMM_BIGVAL	;; Round to integer
	subsd	xmm4, Q XMM_BIGVAL
	movsd	xmm5, ZPAD_K3_HI	;; Load high bits of k
	mulsd	xmm5, xmm4
	subsd	xmm0, xmm5		;; Calculate high bits of remainder
	movsd	xmm5, ZPAD_K3_MID	;; Load middle bits of k
	mulsd	xmm5, xmm4
	subsd	xmm1, xmm5		;; Calculate middle bits of remainder
	movsd	xmm5, ZPAD_K3_LO	;; Load low bits of k
	mulsd	xmm5, xmm4
	subsd	xmm2, xmm5		;; Calculate low bits of remainder
	movsd	Q XMM_TMP2, xmm4	;; Save word of zpad / k

	mulsd	xmm0, ZPAD_SHIFT3	;; Shift previous zpad word
	addsd	xmm1, xmm0		;; Add to create new high zpad bits
	movsd	xmm0, ZPAD0		;; Load zpad word (new low bits)
	movsd	xmm5, ZPAD_SHIFT2	;; Combine high and medium bits
	mulsd	xmm5, xmm1
	addsd	xmm5, xmm2
	movsd	xmm4, ZPAD_INVERSE_K2	;; Load shifted 1/k
	mulsd	xmm4, xmm5		;; Mul ZPAD by shifted 1/k
	addsd	xmm4, Q XMM_BIGVAL	;; Round to integer
	subsd	xmm4, Q XMM_BIGVAL
	movsd	xmm5, ZPAD_K2_HI	;; Load high bits of k
	mulsd	xmm5, xmm4
	subsd	xmm1, xmm5		;; Calculate high bits of remainder
	movsd	xmm5, ZPAD_K2_MID	;; Load middle bits of k
	mulsd	xmm5, xmm4
	subsd	xmm2, xmm5		;; Calculate middle bits of remainder
	movsd	xmm5, ZPAD_K2_LO	;; Load low bits of k
	mulsd	xmm5, xmm4
	subsd	xmm0, xmm5		;; Calculate low bits of remainder
	movsd	Q XMM_TMP1, xmm4	;; Save word of zpad / k

	mulsd	xmm1, ZPAD_SHIFT2	;; Shift previous zpad word
	addsd	xmm2, xmm1		;; Add to create new high zpad bits
	mulsd	xmm2, ZPAD_SHIFT1	;; Shift previous zpad word
	addsd	xmm0, xmm2		;; Add to create new high zpad bits
	movsd	ZPAD0, xmm0		;; Save remainder of zpad / k

	subsd	xmm1, xmm1		;; Zero words that other cases set
	movsd	Q XMM_TMP6, xmm1
	
	jmp	div_k_done

	;; This case does the divide by k where k is two words
mediumk:
	movsd	xmm0, ZPAD6		;; Load zpad word (high bits)
	movsd	xmm1, ZPAD5		;; Load zpad word (low bits)
	movsd	xmm4, ZPAD_INVERSE_K6	;; Load shifted 1/k
	mulsd	xmm4, xmm0		;; Mul ZPAD by shifted 1/k
	addsd	xmm4, Q XMM_BIGVAL	;; Round to integer	
	subsd	xmm4, Q XMM_BIGVAL
	movsd	xmm5, ZPAD_K6_HI	;; Load high bits of k
	mulsd	xmm5, xmm4
	subsd	xmm0, xmm5		;; Calculate high bits of remainder
	movsd	xmm5, ZPAD_K6_LO	;; Load low bits of k
	mulsd	xmm5, xmm4
	subsd	xmm1, xmm5		;; Calculate low bits of remainder
	movsd	Q XMM_TMP6, xmm4	;; Save word of zpad / k

	mulsd	xmm0, ZPAD_SHIFT6	;; Shift previous zpad word
	addsd	xmm0, xmm1		;; Add to create new high zpad bits
	movsd	xmm1, ZPAD4		;; Load zpad word (new low bits)
	movsd	xmm4, ZPAD_INVERSE_K5	;; Load shifted 1/k
	mulsd	xmm4, xmm0		;; Mul ZPAD by shifted 1/k
	addsd	xmm4, Q XMM_BIGVAL	;; Round to integer
	subsd	xmm4, Q XMM_BIGVAL
	movsd	xmm5, ZPAD_K5_HI	;; Load high bits of k
	mulsd	xmm5, xmm4
	subsd	xmm0, xmm5		;; Calculate high bits of remainder
	movsd	xmm5, ZPAD_K5_LO	;; Load low bits of k
	mulsd	xmm5, xmm4
	subsd	xmm1, xmm5		;; Calculate low bits of remainder
	movsd	Q XMM_TMP5, xmm4	;; Save word of zpad / k

	mulsd	xmm0, ZPAD_SHIFT5	;; Shift previous zpad word
	addsd	xmm0, xmm1		;; Add to create new high zpad bits
	movsd	xmm1, ZPAD3		;; Load zpad word (new low bits)
	movsd	xmm4, ZPAD_INVERSE_K4	;; Load shifted 1/k
	mulsd	xmm4, xmm0		;; Mul ZPAD by shifted 1/k
	addsd	xmm4, Q XMM_BIGVAL	;; Round to integer
	subsd	xmm4, Q XMM_BIGVAL
	movsd	xmm5, ZPAD_K4_HI	;; Load high bits of k
	mulsd	xmm5, xmm4
	subsd	xmm0, xmm5		;; Calculate high bits of remainder
	movsd	xmm5, ZPAD_K4_LO	;; Load low bits of k
	mulsd	xmm5, xmm4
	subsd	xmm1, xmm5		;; Calculate low bits of remainder
	movsd	Q XMM_TMP4, xmm4	;; Save word of zpad / k

	mulsd	xmm0, ZPAD_SHIFT4	;; Shift previous zpad word
	addsd	xmm0, xmm1		;; Add to create new high zpad bits
	movsd	xmm1, ZPAD2		;; Load zpad word (new low bits)
	movsd	xmm4, ZPAD_INVERSE_K3	;; Load shifted 1/k
	mulsd	xmm4, xmm0		;; Mul ZPAD by shifted 1/k
	addsd	xmm4, Q XMM_BIGVAL	;; Round to integer
	subsd	xmm4, Q XMM_BIGVAL
	movsd	xmm5, ZPAD_K3_HI	;; Load high bits of k
	mulsd	xmm5, xmm4
	subsd	xmm0, xmm5		;; Calculate high bits of remainder
	movsd	xmm5, ZPAD_K3_LO	;; Load low bits of k
	mulsd	xmm5, xmm4
	subsd	xmm1, xmm5		;; Calculate low bits of remainder
	movsd	Q XMM_TMP3, xmm4	;; Save word of zpad / k

	mulsd	xmm0, ZPAD_SHIFT3	;; Shift previous zpad word
	addsd	xmm0, xmm1		;; Add to create new high zpad bits
	movsd	xmm1, ZPAD1		;; Load zpad word (new low bits)
	movsd	xmm4, ZPAD_INVERSE_K2	;; Load shifted 1/k
	mulsd	xmm4, xmm0		;; Mul ZPAD by shifted 1/k
	addsd	xmm4, Q XMM_BIGVAL	;; Round to integer
	subsd	xmm4, Q XMM_BIGVAL
	movsd	xmm5, ZPAD_K2_HI	;; Load high bits of k
	mulsd	xmm5, xmm4
	subsd	xmm0, xmm5		;; Calculate high bits of remainder
	movsd	xmm5, ZPAD_K2_LO	;; Load low bits of k
	mulsd	xmm5, xmm4
	subsd	xmm1, xmm5		;; Calculate low bits of remainder
	movsd	Q XMM_TMP2, xmm4	;; Save word of zpad / k

	mulsd	xmm0, ZPAD_SHIFT2	;; Shift previous zpad word
	addsd	xmm0, xmm1		;; Add to create new high zpad bits
	movsd	xmm1, ZPAD0		;; Load zpad word (new low bits)
	movsd	xmm4, ZPAD_INVERSE_K1	;; Load shifted 1/k
	mulsd	xmm4, xmm0		;; Mul ZPAD by shifted 1/k
	addsd	xmm4, Q XMM_BIGVAL	;; Round to integer
	subsd	xmm4, Q XMM_BIGVAL
	movsd	xmm5, ZPAD_K1_HI	;; Load high bits of k
	mulsd	xmm5, xmm4
	subsd	xmm0, xmm5		;; Calculate high bits of remainder
	movsd	xmm5, ZPAD_K1_LO	;; Load low bits of k
	mulsd	xmm5, xmm4
	subsd	xmm1, xmm5		;; Calculate low bits of remainder
	movsd	Q XMM_TMP1, xmm4	;; Save word of zpad / k

	mulsd	xmm0, ZPAD_SHIFT1	;; Shift previous zpad word
	addsd	xmm0, xmm1		;; Add to create new high zpad bits
	movsd	ZPAD0, xmm0		;; Save remainder of zpad / k

	jmp	div_k_done

	;; This case does the divide by k where k is one word
	;; Assume ZPAD5 and ZPAD6 are zero.
smallk:	movsd	xmm0, ZPAD4		;; Load zpad data
	movsd	xmm4, ZPAD_INVERSE_K1	;; Load by 1/k
	mulsd	xmm4, xmm0		;; Mul ZPAD data by 1/k
	addsd	xmm4, Q XMM_BIGVAL	;; Round to integer	
	subsd	xmm4, Q XMM_BIGVAL
	movsd	Q XMM_TMP5, xmm4	;; Save integer part
	mulsd	xmm4, ZPAD_K1_LO	;; Compute remainder
	subsd	xmm0, xmm4

	mulsd	xmm0, ZPAD_SHIFT4	;; Shift previous zpad word
	addsd	xmm0, ZPAD3		;; Add in zpad data
	movsd	xmm4, ZPAD_INVERSE_K1	;; Load by 1/k
	mulsd	xmm4, xmm0		;; Mul ZPAD data by 1/k
	addsd	xmm4, Q XMM_BIGVAL	;; Round to integer	
	subsd	xmm4, Q XMM_BIGVAL
	movsd	Q XMM_TMP4, xmm4	;; Save integer part
	mulsd	xmm4, ZPAD_K1_LO	;; Compute remainder
	subsd	xmm0, xmm4

	mulsd	xmm0, ZPAD_SHIFT3	;; Shift previous zpad word
	addsd	xmm0, ZPAD2		;; Add in zpad data
	movsd	xmm4, ZPAD_INVERSE_K1	;; Load by 1/k
	mulsd	xmm4, xmm0		;; Mul ZPAD data by 1/k
	addsd	xmm4, Q XMM_BIGVAL	;; Round to integer	
	subsd	xmm4, Q XMM_BIGVAL
	movsd	Q XMM_TMP3, xmm4	;; Save integer part
	mulsd	xmm4, ZPAD_K1_LO	;; Compute remainder
	subsd	xmm0, xmm4

	mulsd	xmm0, ZPAD_SHIFT2	;; Shift previous zpad word
	addsd	xmm0, ZPAD1		;; Add in zpad data
	movsd	xmm4, ZPAD_INVERSE_K1	;; Load by 1/k
	mulsd	xmm4, xmm0		;; Mul ZPAD data by 1/k
	addsd	xmm4, Q XMM_BIGVAL	;; Round to integer	
	subsd	xmm4, Q XMM_BIGVAL
	movsd	Q XMM_TMP2, xmm4	;; Save integer part
	mulsd	xmm4, ZPAD_K1_LO	;; Compute remainder
	subsd	xmm0, xmm4

	mulsd	xmm0, ZPAD_SHIFT1	;; Shift previous zpad word
	addsd	xmm0, ZPAD0		;; Add in zpad data
	movsd	xmm4, ZPAD_INVERSE_K1	;; Load by 1/k
	mulsd	xmm4, xmm0		;; Mul ZPAD data by 1/k
	addsd	xmm4, Q XMM_BIGVAL	;; Round to integer	
	subsd	xmm4, Q XMM_BIGVAL
	movsd	Q XMM_TMP1, xmm4	;; Save integer part
	mulsd	xmm4, ZPAD_K1_LO	;; Compute remainder
	subsd	xmm0, xmm4
	movsd	ZPAD0, xmm0		;; Save remainder

	subsd	xmm1, xmm1		;; Zero words that other cases set
	movsd	Q XMM_TMP6, xmm1
	
div_k_done:

	;; Now normalize the data above the halfway point.  Remember that the
	;; column two-to-phi multiplier for the first value will be 1.0.

	movzx	rax, BYTE PTR [rdi]	;; First word 
	movsd	xmm0, ZPAD0		;; Load remainder of divide by k
	addsd	xmm0, Q XMM_BIGVAL
	single_rounding base2, xmm0, xmm2, xmm4, rax
	movsd	Q [rsi+32], xmm0	;; Save value1

	movzx	rax, BYTE PTR [rdi+1]	;; Load big vs. little flags
	single_rounding base2, xmm2, xmm0, xmm4, rax
	mulsd	xmm2, Q [rbp+48]	;; new value2 = val * two-to-phi
	movsd	Q [rsi+48], xmm2	;; Save value2

	movzx	rax, BYTE PTR [rdi+4]	;; Load big vs. little flags
	single_rounding base2, xmm0, xmm2, xmm4, rax
	mulsd	xmm0, Q [rbp+128+16]	;; new value3 = val * two-to-phi
	movsd	Q [rsi+64+32], xmm0	;; Save value3

	subsd	xmm2, Q XMM_BIGVAL	;; Remove integer rounding constant
	mulsd	xmm2, Q [rbp+128+48]	;; value4 = carry * two-to-phi
	movsd	Q [rsi+64+48], xmm2	;; Save new value4

	;; Mul the integer part of (ZPAD data divided by k) by -c in
	;; preparation for adding it into the lower FFT data area.
	;; Also add in the shifted high FFT carry at this time.

	;; Now add in and normalize the bottom FFT data.  Remember that the
	;; column two-to-phi multiplier for the first value will be 1.0.  We 
	;; must go 6 words deep in case k is 48-50 bits and c is 32 bits.

	movzx	rax, BYTE PTR [rdi]	;; First word 
	movsd	xmm0, Q XMM_TMP1	;; Load integer part of divide by k
	mulsd	xmm0, Q XMM_MINUS_C	;; Mul by -c
	addsd	xmm0, Q XMM_BIGVAL
	addsd	xmm0, Q [rsi]		;; Add in the FFT data
	single_rounding base2, xmm0, xmm2, xmm4, rax
	movsd	Q [rsi], xmm0		;; Save value1

	movzx	rax, BYTE PTR [rdi+1]	;; Load big vs. little flags
	movsd	xmm0, Q XMM_TMP2	;; Load integer part of divide by k
	mulsd	xmm0, Q XMM_MINUS_C	;; Mul by -c
	movsd	xmm1, Q [rsi+16]	;; Load FFT data
	mulsd	xmm1, Q [rbp+32]	;; Mul values2 by two-to-minus-phi
	mulsd	xmm1, Q XMM_NORM012_FF	;; Mul by FFTLEN/2
	addsd	xmm0, xmm1		;; Add in the FFT data
	addsd	xmm0, xmm2		;; x2 = value + carry
	single_rounding base2, xmm0, xmm2, xmm4, rax
	mulsd	xmm0, Q [rbp+48]	;; new value2 = val * two-to-phi
	movsd	Q [rsi+16], xmm0	;; Save value2

	movzx	rax, BYTE PTR [rdi+4]	;; Load big vs. little flags
	movsd	xmm0, Q XMM_TMP3	;; Load integer part of divide by k
	mulsd	xmm0, Q XMM_MINUS_C	;; Mul by -c
	movsd	xmm1, Q [rsi+64]	;; Load FFT data
	mulsd	xmm1, Q [rbp+128]	;; Mul values3 by two-to-minus-phi
	mulsd	xmm1, Q XMM_NORM012_FF	;; Mul by FFTLEN/2
	addsd	xmm0, xmm1		;; Add in the FFT data
	addsd	xmm0, xmm2		;; x3 = value + carry
	single_rounding base2, xmm0, xmm2, xmm4, rax
	mulsd	xmm0, Q [rbp+128+16]	;; new value3 = val * two-to-phi
	movsd	Q [rsi+64], xmm0	;; Save value3

	movzx	rax, BYTE PTR [rdi+5]	;; Load big vs. little flags
	movsd	xmm0, Q XMM_TMP4	;; Load integer part of divide by k
	mulsd	xmm0, Q XMM_MINUS_C	;; Mul by -c
	movsd	xmm1, Q [rsi+64+16]	;; Load FFT data
	mulsd	xmm1, Q [rbp+128+32]	;; Mul values4 by two-to-minus-phi
	mulsd	xmm1, Q XMM_NORM012_FF	;; Mul by FFTLEN/2
	addsd	xmm0, xmm1		;; Add in the FFT data
	addsd	xmm0, xmm2		;; x4 = value + carry
	single_rounding base2, xmm0, xmm2, xmm4, rax
	mulsd	xmm0, Q [rbp+128+48]	;; new value4 = val * two-to-phi
	movsd	Q [rsi+64+16], xmm0	;; Save value4

	cmp	FFTLEN, 80		;; Length 80 and 112 have different
	je	funnyaddr2		;; memory addresses for the fourth
	cmp	FFTLEN, 112		;; and higher data elements
	je	funnyaddr2

	movzx	rax, BYTE PTR [rdi+8]	;; Load big vs. little flags
	movsd	xmm0, Q XMM_TMP5	;; Load integer part of divide by k
	mulsd	xmm0, Q XMM_MINUS_C	;; Mul by -c
	movsd	xmm1, Q [rsi+128]	;; Load FFT data
	mulsd	xmm1, Q [rbp+256]	;; Mul values4 by two-to-minus-phi
	mulsd	xmm1, Q XMM_NORM012_FF	;; Mul by FFTLEN/2
	addsd	xmm0, xmm1		;; Add in the FFT data
	addsd	xmm0, xmm2		;; x5 = value + carry
	single_rounding base2, xmm0, xmm2, xmm4, rax
	mulsd	xmm0, Q [rbp+256+16]	;; new value5 = val * two-to-phi
	movsd	Q [rsi+128], xmm0	;; Save value5

	movzx	rax, BYTE PTR [rdi+9]	;; Load big vs. little flags
	movsd	xmm0, Q XMM_TMP6	;; Load integer part of divide by k
	mulsd	xmm0, Q XMM_MINUS_C	;; Mul by -c
	movsd	xmm1, Q [rsi+128+16]	;; Load FFT data
	mulsd	xmm1, Q [rbp+256+32]	;; Mul values6 by two-to-minus-phi
	mulsd	xmm1, Q XMM_NORM012_FF	;; Mul by FFTLEN/2
	addsd	xmm0, xmm1		;; Add in the FFT data
	addsd	xmm0, xmm2		;; x6 = value + carry
	single_rounding base2, xmm0, xmm2, xmm4, rax
	mulsd	xmm0, Q [rbp+256+48]	;; new value6 = val * two-to-phi
	movsd	Q [rsi+128+16], xmm0	;; Save value6

	subsd	xmm2, Q XMM_BIGVAL	;; Remove rounding constant
	mulsd	xmm2, Q [rbp+384+16]	;; new value7 = val * two-to-phi
	addsd	xmm2, Q [rsi+192]	;; Add in FFT data
	movsd	Q [rsi+192], xmm2	;; Save value7
	jmp	funny2done

	;; Same as the above but with different addresses required by
	;; the length 80 and 112 FFT lengths

funnyaddr2:
	movzx	rax, BYTE PTR [rdi]	;; Load big vs. little flags
	movsd	xmm0, Q XMM_TMP5	;; Load integer part of divide by k
	mulsd	xmm0, Q XMM_MINUS_C	;; Mul by -c
	movsd	xmm1, Q [rsi+8]		;; Load FFT data
	mulsd	xmm1, Q [rbp+8]		;; Mul values5 by two-to-minus-phi
	mulsd	xmm1, Q XMM_NORM012_FF	;; Mul by FFTLEN/2
	addsd	xmm0, xmm1		;; Add in the FFT data
	addsd	xmm0, xmm2		;; x5 = value + carry
	single_rounding base2, xmm0, xmm2, xmm4, rax+8
	mulsd	xmm0, Q [rbp+16+8]	;; new value5 = val * two-to-phi
	movsd	Q [rsi+8], xmm0		;; Save value5

	movzx	rax, BYTE PTR [rdi+1]	;; Load big vs. little flags
	movsd	xmm0, Q XMM_TMP6	;; Load integer part of divide by k
	mulsd	xmm0, Q XMM_MINUS_C	;; Mul by -c
	movsd	xmm1, Q [rsi+16+8]	;; Load FFT data
	mulsd	xmm1, Q [rbp+32+8]	;; Mul values6 by two-to-minus-phi
	mulsd	xmm1, Q XMM_NORM012_FF	;; Mul by FFTLEN/2
	addsd	xmm0, xmm1		;; Add in the FFT data
	addsd	xmm0, xmm2		;; x6 = value + carry
	single_rounding base2, xmm0, xmm2, xmm4, rax+8
	mulsd	xmm0, Q [rbp+48+8]	;; new value6 = val * two-to-phi
	movsd	Q [rsi+16+8], xmm0	;; Save value6

	subsd	xmm2, Q XMM_BIGVAL	;; Remove rounding constant
	mulsd	xmm2, Q [rbp+128+16+8]	;; new value7 = val * two-to-phi
	addsd	xmm2, Q [rsi+64+8]	;; Add in FFT data
	movsd	Q [rsi+64+8], xmm2	;; Save value7
funny2done:

	ENDM



; *************** 2D normalized add/sub macro ******************
; This macro adds or subtracts, then "normalizes" eight FFT
; data values.  This involves multiplying the summed values by
; two-to-minus-phi.  Rounding the value to an integer.  Making sure
; the integer is smaller than the maximum allowable integer, generating
; a carry if necessary. Finally, the value is multiplied by two-to-phi
; and stored.
; rcx = pointer to the first number
; rdx = pointer to the second number
; rsi = pointer to destination
; rdi = pointer to array of big vs. little flags
; rbx = pointer to two-to-phi column multipliers
; rbp = pointer two-to-phi group multipliers
; XMM_TMP1,XMM_TMP2,XMM_TMP3,XMM_TMP4 = carries
; A pipelined version of this code:
;	xload	xmm0, [rdx]		;; Load second number
;	fop	xmm0, [rcx]		;; Add/sub first number
;	mov	mem_loc, ecx
;	movzx	rcx, BYTE PTR [rdi]	;; Load big vs. little flags
;	xload	xmm2, [rbx]		;; col two-to-minus-phi
;	mulpd	xmm2, XMM_TTMP_FUDGE[rcx];; Mul by fudge two-to-minus-phi
;	mulpd	xmm0, [rax]		;; Mul by grp two-to-minus-phi
;	mulpd	xmm0, xmm2		;; Mul by fudged col two-to-minus-phi
;	addpd	xmm0, [rbp+0*16]	;; x1 = values + carry
;	xload	xmm2, XMM_LIMIT_BIGMAX[rcx];; Load maximum * BIGVAL - BIGVAL
;	addpd	xmm2, xmm0		;; y1 = top bits of x
;	xload	xmm6, XMM_LIMIT_BIGMAX_NEG[rcx];; Load -(maximum*BIGVAL-BIGVAL)
;	addpd	xmm6, xmm2		;; z1 = y1-(maximum * BIGVAL - BIGVAL)
;	subpd	xmm0, xmm6		;; rounded value = x1 - z1
;	mulpd	xmm2, XMM_LIMIT_INVERSE[rcx];; next carry = shifted y1
;	xload	xmm4, [rbx]		;; col two-to-phi
;	mulpd	xmm4, XMM_TTP_FUDGE[rcx];; mul by fudge two-to-phi
;	mulpd	xmm0, [rax+0*32+16]	;; new value1 = val * grp two-to-phi
;	mulpd	xmm0, xmm4		;; new value1 *= fudged col two-to-phi
;	xstore	[rsi+0*dist1], xmm0	;; Save new value1
;	xstore	[rbp+0*16], xmm2	;; Save carry
;	mov	ecx, mem_loc

xnorm_op_2d MACRO fop, ttp, base2, mem_loc
	xload	xmm0, [rdx]		;; Load second number
	fop	xmm0, [rcx]		;; Add/sub first number
ttp	xload	xmm2, [rbx]		;; Col two-to-minus-phi
ttp	mulpd	xmm2, XMM_NORM012_FF	;; Mul by FFTLEN/2
ttp	mulpd	xmm0, xmm2		;; Mul values1 by col two-to-minus-phi
	xload	xmm1, [rdx+16]		;; Load second number
	fop	xmm1, [rcx+16]		;; Add/sub first number
ttp	mulpd	xmm1, xmm2		;; Mul values2 by col two-to-minus-phi
	xload	xmm6, [rdx+32]		;; Load second number
	fop	xmm6, [rcx+32]		;; Add/sub first number
ttp	mulpd	xmm6, xmm2		;; Mul values3 by col two-to-minus-phi
	xload	xmm7, [rdx+48]		;; Load second number
	fop	xmm7, [rcx+48]		;; Add/sub first number
ttp	mulpd	xmm7, xmm2		;; Mul values4 by col two-to-minus-phi
	mov	mem_loc, rdx		;; Save second src ptr
ttp	movzx	rax, BYTE PTR [rdi]	;; Load big vs. little flags
ttp	movzx	rdx, BYTE PTR [rdi+1]
ttp	mulpd	xmm0, [rbp+0*32]	;; Mul by grp two-to-minus-phi
ttp	mulpd	xmm0, XMM_TTMP_FUDGE[rax];; Mul by fudge two-to-minus-phi
	addpd	xmm0, XMM_TMP1		;; x1 = values + carry
ttp	mulpd	xmm1, [rbp+1*32]	;; Mul by grp two-to-minus-phi
ttp	mulpd	xmm1, XMM_TTMP_FUDGE[rdx];; Mul by fudge two-to-minus-phi
	addpd	xmm1, XMM_TMP2		;; x1 = values + carry
	rounding_interleaved ttp, base2, noexec, noexec, xmm0, xmm2, xmm4, rax, xmm1, xmm3, xmm5, rdx
	xstore	XMM_TMP1, xmm2		;; Save carry
	xstore	XMM_TMP2, xmm3		;; Save carry
ttp	xload	xmm2, [rbx+16]		;; col two-to-phi
ttp	mulpd	xmm2, XMM_TTP_FUDGE[rax];; mul by fudge two-to-phi
ttp	mulpd	xmm0, [rbp+0*32+16]	;; new value1 = val * grp two-to-phi
ttp	mulpd	xmm0, xmm2		;; new value1 *= fudged col two-to-phi
	xstore	[rsi+0*16], xmm0	;; Save new value1
ttp	xload	xmm3, [rbx+16]		;; col two-to-phi
ttp	mulpd	xmm3, XMM_TTP_FUDGE[rdx];; mul by fudge two-to-phi
ttp	mulpd	xmm1, [rbp+1*32+16]	;; new value1 = val * grp two-to-phi
ttp	mulpd	xmm1, xmm3		;; new value1 *= fudged col two-to-phi
	xstore	[rsi+1*16], xmm1	;; Save new value1

ttp	movzx	rax, BYTE PTR [rdi+2]	;; Load big vs. little flags
ttp	movzx	rdx, BYTE PTR [rdi+3]
ttp	mulpd	xmm6, [rbp+2*32]	;; Mul by grp two-to-minus-phi
ttp	mulpd	xmm6, XMM_TTMP_FUDGE[rax];; Mul by fudge two-to-minus-phi
	addpd	xmm6, XMM_TMP3		;; x1 = values + carry
ttp	mulpd	xmm7, [rbp+3*32]	;; Mul by grp two-to-minus-phi
ttp	mulpd	xmm7, XMM_TTMP_FUDGE[rdx];; Mul by fudge two-to-minus-phi
	addpd	xmm7, XMM_TMP4		;; x1 = values + carry
	rounding_interleaved ttp, base2, noexec, noexec, xmm6, xmm2, xmm4, rax, xmm7, xmm3, xmm5, rdx
	xstore	XMM_TMP3, xmm2		;; Save carry
	xstore	XMM_TMP4, xmm3		;; Save carry
ttp	xload	xmm2, [rbx+16]		;; col two-to-phi
ttp	mulpd	xmm2, XMM_TTP_FUDGE[rax];; mul by fudge two-to-phi
ttp	mulpd	xmm6, [rbp+2*32+16]	;; new value1 = val * grp two-to-phi
ttp	mulpd	xmm6, xmm2		;; new value1 *= fudged col two-to-phi
	xstore	[rsi+2*16], xmm6	;; Save new value1
ttp	xload	xmm3, [rbx+16]		;; col two-to-phi
ttp	mulpd	xmm3, XMM_TTP_FUDGE[rdx];; mul by fudge two-to-phi
ttp	mulpd	xmm7, [rbp+3*32+16]	;; new value1 = val * grp two-to-phi
ttp	mulpd	xmm7, xmm3		;; new value1 *= fudged col two-to-phi
	xstore	[rsi+3*16], xmm7	;; Save new value1
	mov	rdx, mem_loc		;; Restore second src ptr
ttp	bump	rdi, 4			;; Next flags ptr
ttp	bump	rbx, 32			;; Next column two-to-phi ptr
	bump	rsi, 64			;; Next dest ptr
	bump	rdx, 64			;; Next src ptr
	bump	rcx, 64			;; Next src ptr
	ENDM

; *************** 2D followup macros ******************
; This macro finishes the normalize add/sub process by adding four carries
; from the end of a block back to the start of the block.  The remaining
; four carries are rotated for starting the next block.

xnorm_op_2d_blk MACRO srcreg, screg, carry1, carry2, carry3, carry4
	xload	xmm2, XMM_BIGVAL

	xload	xmm0, carry1		;; Load carries
	subsd	xmm0, xmm2
	mulsd	xmm0, Q [screg+0*32+24]	;; mul by grp two-to-phi
	addsd	xmm0, Q [srcreg+0*16+8]	;; Add in FFT data
	movsd	Q [srcreg+0*16+8], xmm0	;; Save FFT word
	shufpd	xmm0, xmm2, 1
	xstore	carry1, xmm0		;; Save carries for start of next blk

	xload	xmm0, carry2		;; Load carries
	subsd	xmm0, xmm2
	mulsd	xmm0, Q [screg+1*32+24]	;; mul by grp two-to-phi
	addsd	xmm0, Q [srcreg+1*16+8]	;; Add in FFT data
	movsd	Q [srcreg+1*16+8], xmm0	;; Save FFT word
	shufpd	xmm0, xmm2, 1
	xstore	carry2, xmm0		;; Save carries for start of next blk

	xload	xmm0, carry3		;; Load carries
	subsd	xmm0, xmm2
	mulsd	xmm0, Q [screg+2*32+24]	;; mul by grp two-to-phi
	addsd	xmm0, Q [srcreg+2*16+8]	;; Add in FFT data
	movsd	Q [srcreg+2*16+8], xmm0	;; Save FFT word
	shufpd	xmm0, xmm2, 1
	xstore	carry3, xmm0		;; Save carries for start of next blk

	xload	xmm0, carry4		;; Load carries
	subsd	xmm0, xmm2
	mulsd	xmm0, Q [screg+3*32+24]	;; mul by grp two-to-phi
	addsd	xmm0, Q [srcreg+3*16+8]	;; Add in FFT data
	movsd	Q [srcreg+3*16+8], xmm0	;; Save FFT word
	shufpd	xmm0, xmm2, 1
	xstore	carry4, xmm0		;; Save carries for start of next blk
	ENDM

; This macro finishes the normalize add/sub process by adding two carries
; from the end of a section back to the start of the section.  The remaining
; two carries are rotated for starting the next section.
; rax = pointer to FFT data
; rbx = pointer two-to-phi group multipliers

xnorm_op_2d_sec MACRO carry1, carry2, carry3, carry4
	xload	xmm2, XMM_BIGVAL

	movsd	xmm0, Q carry1		;; Load carry
	subsd	xmm0, xmm2
	mulsd	xmm0, Q [rbx+1*32+16]	;; mul by grp two-to-phi
	addsd	xmm0, Q [rax+1*16]	;; Add in FFT data
	movsd	Q [rax+1*16], xmm0	;; Save FFT word

	xload	xmm1, carry2
	xstore	carry1, xmm1
	xstore	carry2, xmm2		;; Save carries for start of next sec

	movsd	xmm0, Q carry3		;; Load carry
	subsd	xmm0, xmm2
	mulsd	xmm0, Q [rbx+3*32+16]	;; mul by grp two-to-phi
	addsd	xmm0, Q [rax+3*16]	;; Add in FFT data
	movsd	Q [rax+3*16], xmm0	;; Save FFT word

	xload	xmm1, carry4
	xstore	carry3, xmm1
	xstore	carry4, xmm2		;; Save carries for start of next sec
	ENDM


; This macro finishes the normalize add/sub process by adding the final
; two carries back into the appropriate FFT values at the start of the fft.
; rsi = pointer to FFT data
; rbp = pointer two-to-phi group multipliers
; xmm6 = non-wraparound carry
; xmm7 = wraparound carry

xnorm_op_2d_fft MACRO
	movsd	xmm2, Q XMM_BIGVAL

	subsd	xmm7, xmm2		;; Remove XMM_BIGVAL
	mulsd	xmm7, Q XMM_MINUS_C	;; mul wrap around carry by -c
	mulsd	xmm7, Q [rbp+0*32+16]	;; mul by grp two-to-phi
	addsd	xmm7, Q [rsi+0*16]	;; Add in FFT data
	movsd	Q [rsi+0*16], xmm7	;; Save FFT word

	subsd	xmm6, xmm2		;; Remove XMM_BIGVAL
	mulsd	xmm6, Q [rbp+2*32+16]	;; mul by grp two-to-phi
	addsd	xmm6, Q [rsi+2*16]	;; Add in FFT data
	movsd	Q [rsi+2*16], xmm6	;; Save FFT word
	ENDM


; *************** 2D normalized add & sub macro ******************
; This macro adds and subtracts, then "normalizes" eight FFT
; data values.  This involves multiplying the summed values by
; two-to-minus-phi.  Rounding the value to an integer.  Making sure
; the integer is smaller than the maximum allowable integer, generating
; a carry if necessary. Finally, the value is multiplied by two-to-phi
; and stored.
; rcx = pointer to the first number
; rdx = pointer to the second number
; rsi = pointer to destination #1
; rbp = pointer to destination #2
; rbx = pointer two-to-phi group multipliers
; rdi = pointer to array of big vs. little flags
; rax = pointer to two-to-phi column multipliers
; XMM_TMP1,XMM_TMP2,XMM_TMP3,XMM_TMP4 = addition carries
; XMM_TMP5,XMM_TMP6,XMM_TMP7,XMM_TMP8 = subtraction carries

xnorm_addsub_2d MACRO ttp, base2, mem_loc
	xload	xmm1, [rcx+0*16]	;; Load first number
	xload	xmm0, [rdx+0*16]	;; Load second number
	subpd	xmm1, xmm0		;; first - second number
	addpd	xmm0, [rcx+0*16]	;; first + second number
ttp	xload	xmm2, [rax]		;; Col two-to-minus-phi
ttp	mulpd	xmm2, XMM_NORM012_FF	;; Mul by FFTLEN/2
ttp	mulpd	xmm0, xmm2		;; Mul values1 by col two-to-minus-phi
ttp	mulpd	xmm1, xmm2		;; Mul values1 by col two-to-minus-phi

	xload	xmm7, [rcx+1*16]	;; Load first number
	xload	xmm6, [rdx+1*16]	;; Load second number
	subpd	xmm7, xmm6		;; first - second number
	addpd	xmm6, [rcx+1*16]	;; first + second number
ttp	mulpd	xmm6, xmm2		;; Mul values1 by col two-to-minus-phi
ttp	mulpd	xmm7, xmm2		;; Mul values1 by col two-to-minus-phi

	mov	mem_loc, rcx		;; Save first src ptr

ttp	movzx	rcx, BYTE PTR [rdi+0]	;; Load big vs. little flags
ttp	mulpd	xmm0, [rbx+0*32]	;; Mul by grp two-to-minus-phi
ttp	mulpd	xmm0, XMM_TTMP_FUDGE[rcx];; Mul by fudge two-to-minus-phi
	addpd	xmm0, XMM_TMP1		;; x1 = values + carry
ttp	mulpd	xmm1, [rbx+0*32]	;; Mul by grp two-to-minus-phi
ttp	mulpd	xmm1, XMM_TTMP_FUDGE[rcx];; Mul by fudge two-to-minus-phi
	addpd	xmm1, XMM_TMP5		;; x1 = values + carry
	rounding_interleaved ttp, base2, noexec, noexec, xmm0, xmm2, xmm4, rcx, xmm1, xmm3, xmm5, rcx
	xstore	XMM_TMP1, xmm2		;; Save carry
	xstore	XMM_TMP5, xmm3		;; Save carry
ttp	xload	xmm2, [rax+16]		;; col two-to-phi
ttp	mulpd	xmm2, XMM_TTP_FUDGE[rcx];; mul by fudge two-to-phi
ttp	mulpd	xmm0, [rbx+0*32+16]	;; new value1 = val * grp two-to-phi
ttp	mulpd	xmm0, xmm2		;; new value1 *= fudged col two-to-phi
	xstore	[rsi+0*16], xmm0	;; Save new value1
ttp	xload	xmm3, [rax+16]		;; col two-to-phi
ttp	mulpd	xmm3, XMM_TTP_FUDGE[rcx];; mul by fudge two-to-phi
ttp	mulpd	xmm1, [rbx+0*32+16]	;; new value1 = val * grp two-to-phi
ttp	mulpd	xmm1, xmm3		;; new value1 *= fudged col two-to-phi
	xstore	[rbp+0*16], xmm1	;; Save new value1

ttp	movzx	rcx, BYTE PTR [rdi+1]	;; Load big vs. little flags
ttp	mulpd	xmm6, [rbx+1*32]	;; Mul by grp two-to-minus-phi
ttp	mulpd	xmm6, XMM_TTMP_FUDGE[rcx];; Mul by fudge two-to-minus-phi
	addpd	xmm6, XMM_TMP2		;; x1 = values + carry
ttp	mulpd	xmm7, [rbx+1*32]	;; Mul by grp two-to-minus-phi
ttp	mulpd	xmm7, XMM_TTMP_FUDGE[rcx];; Mul by fudge two-to-minus-phi
	addpd	xmm7, XMM_TMP6		;; x1 = values + carry
	rounding_interleaved ttp, base2, noexec, noexec, xmm6, xmm2, xmm4, rcx, xmm7, xmm3, xmm5, rcx
	xstore	XMM_TMP2, xmm2		;; Save carry
	xstore	XMM_TMP6, xmm3		;; Save carry
ttp	xload	xmm2, [rax+16]		;; col two-to-phi
ttp	mulpd	xmm2, XMM_TTP_FUDGE[rcx];; mul by fudge two-to-phi
ttp	mulpd	xmm6, [rbx+1*32+16]	;; new value1 = val * grp two-to-phi
ttp	mulpd	xmm6, xmm2		;; new value1 *= fudged col two-to-phi
	xstore	[rsi+1*16], xmm6	;; Save new value1
ttp	xload	xmm3, [rax+16]		;; col two-to-phi
ttp	mulpd	xmm3, XMM_TTP_FUDGE[rcx];; mul by fudge two-to-phi
ttp	mulpd	xmm7, [rbx+1*32+16]	;; new value1 = val * grp two-to-phi
ttp	mulpd	xmm7, xmm3		;; new value1 *= fudged col two-to-phi
	xstore	[rbp+1*16], xmm7	;; Save new value1

	mov	rcx, mem_loc		;; Restore first src ptr

	xload	xmm1, [rcx+2*16]	;; Load first number
	xload	xmm0, [rdx+2*16]	;; Load second number
	subpd	xmm1, xmm0		;; first - second number
	addpd	xmm0, [rcx+2*16]	;; first + second number
ttp	xload	xmm2, [rax]		;; Col two-to-minus-phi
ttp	mulpd	xmm2, XMM_NORM012_FF	;; Mul by FFTLEN/2
ttp	mulpd	xmm0, xmm2		;; Mul values1 by col two-to-minus-phi
ttp	mulpd	xmm1, xmm2		;; Mul values1 by col two-to-minus-phi

	xload	xmm7, [rcx+3*16]	;; Load first number
	xload	xmm6, [rdx+3*16]	;; Load second number
	subpd	xmm7, xmm6		;; first - second number
	addpd	xmm6, [rcx+3*16]	;; first + second number
ttp	mulpd	xmm6, xmm2		;; Mul values1 by col two-to-minus-phi
ttp	mulpd	xmm7, xmm2		;; Mul values1 by col two-to-minus-phi

	mov	mem_loc, rcx		;; Save first src ptr

ttp	movzx	rcx, BYTE PTR [rdi+2]	;; Load big vs. little flags
ttp	mulpd	xmm0, [rbx+2*32]	;; Mul by grp two-to-minus-phi
ttp	mulpd	xmm0, XMM_TTMP_FUDGE[rcx];; Mul by fudge two-to-minus-phi
	addpd	xmm0, XMM_TMP3		;; x1 = values + carry
ttp	mulpd	xmm1, [rbx+2*32]	;; Mul by grp two-to-minus-phi
ttp	mulpd	xmm1, XMM_TTMP_FUDGE[rcx];; Mul by fudge two-to-minus-phi
	addpd	xmm1, XMM_TMP7		;; x1 = values + carry
	rounding_interleaved ttp, base2, noexec, noexec, xmm0, xmm2, xmm4, rcx, xmm1, xmm3, xmm5, rcx
	xstore	XMM_TMP3, xmm2		;; Save carry
	xstore	XMM_TMP7, xmm3		;; Save carry
ttp	xload	xmm2, [rax+16]		;; col two-to-phi
ttp	mulpd	xmm2, XMM_TTP_FUDGE[rcx];; mul by fudge two-to-phi
ttp	mulpd	xmm0, [rbx+2*32+16]	;; new value1 = val * grp two-to-phi
ttp	mulpd	xmm0, xmm2		;; new value1 *= fudged col two-to-phi
	xstore	[rsi+2*16], xmm0	;; Save new value1
ttp	xload	xmm3, [rax+16]		;; col two-to-phi
ttp	mulpd	xmm3, XMM_TTP_FUDGE[rcx];; mul by fudge two-to-phi
ttp	mulpd	xmm1, [rbx+2*32+16]	;; new value1 = val * grp two-to-phi
ttp	mulpd	xmm1, xmm3		;; new value1 *= fudged col two-to-phi
	xstore	[rbp+2*16], xmm1	;; Save new value1

ttp	movzx	rcx, BYTE PTR [rdi+3]	;; Load big vs. little flags
ttp	mulpd	xmm6, [rbx+3*32]	;; Mul by grp two-to-minus-phi
ttp	mulpd	xmm6, XMM_TTMP_FUDGE[rcx];; Mul by fudge two-to-minus-phi
	addpd	xmm6, XMM_TMP4		;; x1 = values + carry
ttp	mulpd	xmm7, [rbx+3*32]	;; Mul by grp two-to-minus-phi
ttp	mulpd	xmm7, XMM_TTMP_FUDGE[rcx];; Mul by fudge two-to-minus-phi
	addpd	xmm7, XMM_TMP8		;; x1 = values + carry
	rounding_interleaved ttp, base2, noexec, noexec, xmm6, xmm2, xmm4, rcx, xmm7, xmm3, xmm5, rcx
	xstore	XMM_TMP4, xmm2		;; Save carry
	xstore	XMM_TMP8, xmm3		;; Save carry
ttp	xload	xmm2, [rax+16]		;; col two-to-phi
ttp	mulpd	xmm2, XMM_TTP_FUDGE[rcx];; mul by fudge two-to-phi
ttp	mulpd	xmm6, [rbx+3*32+16]	;; new value1 = val * grp two-to-phi
ttp	mulpd	xmm6, xmm2		;; new value1 *= fudged col two-to-phi
	xstore	[rsi+3*16], xmm6	;; Save new value1
ttp	xload	xmm3, [rax+16]		;; col two-to-phi
ttp	mulpd	xmm3, XMM_TTP_FUDGE[rcx];; mul by fudge two-to-phi
ttp	mulpd	xmm7, [rbx+3*32+16]	;; new value1 = val * grp two-to-phi
ttp	mulpd	xmm7, xmm3		;; new value1 *= fudged col two-to-phi
	xstore	[rbp+3*16], xmm7	;; Save new value1

	mov	rcx, mem_loc		;; Restore first src ptr

	bump	rdx, 64			;; Next src ptr
	bump	rcx, 64			;; Next src ptr
ttp	bump	rdi, 4			;; Next flags ptr
ttp	bump	rax, 32			;; Next column two-to-phi ptr
	bump	rsi, 64			;; Next dest ptr
	bump	rbp, 64			;; Next dest ptr
	ENDM

; *************** 2D normalized small mul macro ******************
; This macro multiplies by a small value, then "normalizes" eight FFT
; data values. 
; rsi = pointer to destination
; rdi = pointer to array of big vs. little flags
; rbx = pointer to two-to-phi column multipliers
; rbp = pointer two-to-phi group multipliers
; XMM_TMP1,XMM_TMP2,XMM_TMP3,XMM_TMP4 = carries
; XMM_TMP5 = small value * optional FFTLEN/2
; A pipelined version of this code:
;	xload	xmm0, [rsi]		;; Load second number
;	mulpd	xmm0, XMM_TMP5		;; Mul by small value
;	movzx	rcx, BYTE PTR [rdi]	;; Load big vs. little flags
;	xload	xmm2, [rbx]		;; col two-to-minus-phi
;	mulpd	xmm2, XMM_TTMP_FUDGE[rcx];; Mul by fudge two-to-minus-phi
;	mulpd	xmm0, [rax]		;; Mul by grp two-to-minus-phi
;	mulpd	xmm0, xmm2		;; Mul by fudged col two-to-minus-phi
;	addpd	xmm0, [rbp+0*16]	;; x1 = values + carry
;	xload	xmm2, XMM_LIMIT_BIGMAX[rcx];; Load maximum * BIGVAL - BIGVAL
;	addpd	xmm2, xmm0		;; y1 = top bits of x
;	xload	xmm6, XMM_LIMIT_BIGMAX_NEG[rcx];; Load -(maximum*BIGVAL-BIGVAL)
;	addpd	xmm6, xmm2		;; z1 = y1-(maximum * BIGVAL - BIGVAL)
;	subpd	xmm0, xmm6		;; rounded value = x1 - z1
;	mulpd	xmm2, XMM_LIMIT_INVERSE[rcx];; next carry = shifted y1
;	xload	xmm4, [rbx]		;; col two-to-phi
;	mulpd	xmm4, XMM_TTP_FUDGE[rcx];; mul by fudge two-to-phi
;	mulpd	xmm0, [rax+0*32+16]	;; new value1 = val * grp two-to-phi
;	mulpd	xmm0, xmm4		;; new value1 *= fudged col two-to-phi
;	xstore	[rsi], xmm0		;; Save new value1
;	xstore	[rbp+0*16], xmm2	;; Save carry

xnorm_smallmul_2d MACRO ttp, base2
	xload	xmm2, XMM_TMP5		;; Load small value * FFTLEN/2
ttp	mulpd	xmm2, [rbx]		;; Mul by col two-to-minus-phi
	xload	xmm0, [rsi]		;; Load values1
	mulpd	xmm0, xmm2		;; Mul values1 by small value * col two-to-minus-phi
	xload	xmm1, [rsi+16]		;; Load values2
	mulpd	xmm1, xmm2		;; Mul values2 by small value * col two-to-minus-phi
	xload	xmm6, [rsi+32]		;; Load values3
	mulpd	xmm6, xmm2		;; Mul values3 by small value * col two-to-minus-phi
	xload	xmm7, [rsi+48]		;; Load values4
	mulpd	xmm7, xmm2		;; Mul values4 by small value * col two-to-minus-phi
ttp	movzx	rax, BYTE PTR [rdi]	;; Load big vs. little flags
ttp	movzx	rcx, BYTE PTR [rdi+1]
ttp	mulpd	xmm0, [rbp+0*32]	;; Mul by grp two-to-minus-phi
ttp	mulpd	xmm0, XMM_TTMP_FUDGE[rax];; Mul by fudge two-to-minus-phi
	addpd	xmm0, XMM_TMP1		;; x1 = values + carry
ttp	mulpd	xmm1, [rbp+1*32]	;; Mul by grp two-to-minus-phi
ttp	mulpd	xmm1, XMM_TTMP_FUDGE[rcx];; Mul by fudge two-to-minus-phi
	addpd	xmm1, XMM_TMP2		;; x1 = values + carry
	rounding_interleaved ttp, base2, noexec, noexec, xmm0, xmm2, xmm4, rax, xmm1, xmm3, xmm5, rcx
	xstore	XMM_TMP1, xmm2		;; Save carry
	xstore	XMM_TMP2, xmm3		;; Save carry
ttp	xload	xmm2, [rbx+16]		;; col two-to-phi
ttp	mulpd	xmm2, XMM_TTP_FUDGE[rax];; mul by fudge two-to-phi
ttp	mulpd	xmm0, [rbp+0*32+16]	;; new value1 = val * grp two-to-phi
ttp	mulpd	xmm0, xmm2		;; new value1 *= fudged col two-to-phi
	xstore	[rsi+0*16], xmm0	;; Save new value1
ttp	xload	xmm3, [rbx+16]		;; col two-to-phi
ttp	mulpd	xmm3, XMM_TTP_FUDGE[rcx];; mul by fudge two-to-phi
ttp	mulpd	xmm1, [rbp+1*32+16]	;; new value1 = val * grp two-to-phi
ttp	mulpd	xmm1, xmm3		;; new value1 *= fudged col two-to-phi
	xstore	[rsi+1*16], xmm1	;; Save new value1

ttp	movzx	rax, BYTE PTR [rdi+2]	;; Load big vs. little flags
ttp	movzx	rcx, BYTE PTR [rdi+3]
ttp	mulpd	xmm6, [rbp+2*32]	;; Mul by grp two-to-minus-phi
ttp	mulpd	xmm6, XMM_TTMP_FUDGE[rax];; Mul by fudge two-to-minus-phi
	addpd	xmm6, XMM_TMP3		;; x1 = values + carry
ttp	mulpd	xmm7, [rbp+3*32]	;; Mul by grp two-to-minus-phi
ttp	mulpd	xmm7, XMM_TTMP_FUDGE[rcx];; Mul by fudge two-to-minus-phi
	addpd	xmm7, XMM_TMP4		;; x1 = values + carry
	rounding_interleaved ttp, base2, noexec, noexec, xmm6, xmm2, xmm4, rax, xmm7, xmm3, xmm5, rcx
	xstore	XMM_TMP3, xmm2		;; Save carry
	xstore	XMM_TMP4, xmm3		;; Save carry
ttp	xload	xmm2, [rbx+16]		;; col two-to-phi
ttp	mulpd	xmm2, XMM_TTP_FUDGE[rax];; mul by fudge two-to-phi
ttp	mulpd	xmm6, [rbp+2*32+16]	;; new value1 = val * grp two-to-phi
ttp	mulpd	xmm6, xmm2		;; new value1 *= fudged col two-to-phi
	xstore	[rsi+2*16], xmm6	;; Save new value1
ttp	xload	xmm3, [rbx+16]		;; col two-to-phi
ttp	mulpd	xmm3, XMM_TTP_FUDGE[rcx];; mul by fudge two-to-phi
ttp	mulpd	xmm7, [rbp+3*32+16]	;; new value1 = val * grp two-to-phi
ttp	mulpd	xmm7, xmm3		;; new value1 *= fudged col two-to-phi
	xstore	[rsi+3*16], xmm7	;; Save new value1
ttp	bump	rdi, 4			;; Next flags ptr
ttp	bump	rbx, 32			;; Next column two-to-phi ptr
	bump	rsi, 64			;; Next dest ptr
	ENDM

; This macro finishes the smallmul normalize process by adding four carries
; from the end of a block back to the start of the block.  The remaining
; four carries are rotated for starting the next block.
; rsi = Source ptr to start of block
; rbp = Group ptr of two-to-phi
; rbx = Col ptr of two-to-phi
; rdi = biglit array ptr
; rcx is destroyed

xnorm_smallmul_2d_blk MACRO base2
	movsd	xmm5, Q XMM_BIGVAL

	movsd	xmm1, Q XMM_TMP1	;; Load carry
	xnorm_smallmul_2d_prop4 base2, rsi+0*16, 8, rbp+0*32, rbx, rdi+0*1, xmm1, xmm2, xmm3
	xload	xmm1, XMM_TMP1		;; Load carries (prop4 will zero high word of xmm1)
	shufpd	xmm1, xmm5, 1
	xstore	XMM_TMP1, xmm1		;; Save carries for start of next blk

	movsd	xmm1, Q XMM_TMP2	;; Load carry
	xnorm_smallmul_2d_prop4 base2, rsi+1*16, 8, rbp+1*32, rbx, rdi+1*1, xmm1, xmm2, xmm3
	xload	xmm1, XMM_TMP2		;; Load carries
	shufpd	xmm1, xmm5, 1
	xstore	XMM_TMP2, xmm1		;; Save carries for start of next blk

	movsd	xmm1, Q XMM_TMP3	;; Load carry
	xnorm_smallmul_2d_prop4 base2, rsi+2*16, 8, rbp+2*32, rbx, rdi+2*1, xmm1, xmm2, xmm3
	xload	xmm1, XMM_TMP3		;; Load carries
	shufpd	xmm1, xmm5, 1
	xstore	XMM_TMP3, xmm1		;; Save carries for start of next blk

	movsd	xmm1, Q XMM_TMP4	;; Load carry
	xnorm_smallmul_2d_prop4 base2, rsi+3*16, 8, rbp+3*32, rbx, rdi+3*1, xmm1, xmm2, xmm3
	xload	xmm1, XMM_TMP4		;; Load carries
	shufpd	xmm1, xmm5, 1
	xstore	XMM_TMP4, xmm1		;; Save carries for start of next blk
	ENDM

; This macro finishes the smallmul normalize process by adding two carries
; from the end of a section back to the start of the section.  The remaining
; two carries are rotated for starting the next section.
; rsi = Source ptr to start of block
; rbp = Group ptr of two-to-phi
; rbx = Col ptr of two-to-phi
; rdi = biglit array ptr
; rcx is destroyed

xnorm_smallmul_2d_sec MACRO base2
	xload	xmm5, XMM_BIGVAL

	movsd	xmm1, Q XMM_TMP1	;; Load carry
	xnorm_smallmul_2d_prop4 base2, rsi+1*16, 0, rbp+1*32, rbx, rdi+1*1, xmm1, xmm2, xmm3
	xload	xmm1, XMM_TMP2
	xstore	XMM_TMP1, xmm1
	xstore	XMM_TMP2, xmm5		;; Save carries for start of next sec

	movsd	xmm1, Q XMM_TMP3	;; Load carry
	xnorm_smallmul_2d_prop4 base2, rsi+3*16, 0, rbp+3*32, rbx, rdi+3*1, xmm1, xmm2, xmm3
	xload	xmm1, XMM_TMP4
	xstore	XMM_TMP3, xmm1
	xstore	XMM_TMP4, xmm5		;; Save carries for start of next sec
	ENDM

; This macro finishes the smallmul normalize process by adding the final
; two carries back into the appropriate FFT values at the start of the fft.
; rsi = pointer to FFT data
; rbp = pointer two-to-phi group multipliers
; rdi = pointer to big/little flags
; xmm6 = non-wraparound carry
; xmm7 = wraparound carry

xnorm_smallmul_2d_fft MACRO base2
	subsd	xmm7, Q XMM_BIGVAL
	mulsd	xmm7, Q XMM_MINUS_C	;; Negate the carry
	addsd	xmm7, Q XMM_BIGVAL
	xnorm_smallmul_2d_prop6 base2, rsi+0*16, 0, rbp+0*32, rbx, rdi+0*1, xmm7, xmm2, xmm3
	xnorm_smallmul_2d_prop4 base2, rsi+2*16, 0, rbp+2*32, rbx, rdi+2*1, xmm6, xmm2, xmm3
	ENDM

;; Propagate a single gwsmallmul carry across 4 words.

xnorm_smallmul_2d_prop4 MACRO base2, srcptr, off8, grpptr, colptr, biglit, xcarry, xtmp, xaux
	movzx	rax, BYTE PTR [biglit]		;; First word
	movsd	xmm0, Q [srcptr+0*64+off8]	;; Load FFT data
	mulsd	xmm0, Q [colptr+0*32+off8]	;; Mul by col two-to-minus-phi
	movsd	xaux, Q [grpptr+off8]		;; Mul by grp two-to-minus-phi
	mulsd	xaux, Q XMM_NORM012_FF		;; Mul by FFTLEN/2
	mulsd	xmm0, Q XMM_TTMP_FUDGE[rax+off8]	;; Mul by fudge two-to-minus-phi
	mulsd	xmm0, xaux
	addsd	xmm0, xcarry			;; Add in the carry
	single_rounding base2, xmm0, xcarry, xtmp, rax+off8
	movsd	xtmp, Q [colptr+0*32+16+off8]	;; col two-to-phi
	mulsd	xtmp, Q XMM_TTP_FUDGE[rax+off8]	;; mul by fudge two-to-phi
	mulsd	xmm0, Q [grpptr+16+off8]	;; new value = val * grp two-to-phi
	mulsd	xmm0, xtmp			;; new value *= fudged col two-to-phi
	movsd	Q [srcptr+0*64+off8], xmm0	;; Save value1

	movzx	rax, BYTE PTR [biglit+4]	;; Second word
	movsd	xmm0, Q [srcptr+1*64+off8]	;; Load FFT data
	mulsd	xmm0, Q [colptr+1*32+off8]	;; Mul by col two-to-minus-phi
	movsd	xtmp, Q XMM_TTMP_FUDGE[rax+off8];; Mul by fudge two-to-minus-phi
	mulsd	xtmp, xaux			;; Mul by grp * FFTLEN/2
	mulsd	xmm0, xtmp
	addsd	xmm0, xcarry			;; Add in the carry
	single_rounding base2, xmm0, xcarry, xtmp, rax+off8
	movsd	xtmp, Q [colptr+1*32+16+off8]	;; col two-to-phi
	mulsd	xtmp, Q XMM_TTP_FUDGE[rax+off8]	;; mul by fudge two-to-phi
	mulsd	xmm0, Q [grpptr+16+off8]	;; new value = val * grp two-to-phi
	mulsd	xmm0, xtmp			;; new value *= fudged col two-to-phi
	movsd	Q [srcptr+1*64+off8], xmm0	;; Save value2

	mov	ecx, BIGLIT_INCR2		;; Different clm values step through
						;; big/lit array differently
	movzx	rax, BYTE PTR [biglit][rcx]	;; Third word
	movsd	xmm0, Q [srcptr+2*64+off8]	;; Load FFT data
	mulsd	xmm0, Q [colptr+2*32+off8]	;; Mul by col two-to-minus-phi
	movsd	xtmp, Q XMM_TTMP_FUDGE[rax+off8];; Mul by fudge two-to-minus-phi
	mulsd	xtmp, xaux			;; Mul by grp * FFTLEN/2
	mulsd	xmm0, xtmp
	addsd	xmm0, xcarry			;; Add in the carry
	single_rounding base2, xmm0, xcarry, xtmp, rax+off8
	movsd	xtmp, Q [colptr+2*32+16+off8]	;; col two-to-phi
	mulsd	xtmp, Q XMM_TTP_FUDGE[rax+off8]	;; mul by fudge two-to-phi
	mulsd	xmm0, Q [grpptr+16+off8]	;; new value = val * grp two-to-phi
	mulsd	xmm0, xtmp			;; new value *= fudged col two-to-phi
	movsd	Q [srcptr+2*64+off8], xmm0	;; Save value3

	movzx	rax, BYTE PTR [biglit][rcx+4]	;; Fourth word
	subsd	xcarry, Q XMM_BIGVAL		;; Make carry an integer
	movsd	xtmp, Q [colptr+3*32+16+off8]	;; col two-to-phi
	mulsd	xtmp, Q XMM_TTP_FUDGE[rax+off8]	;; mul by fudge two-to-phi
	mulsd	xcarry, Q [grpptr+16+off8]	;; carry *= grp two-to-phi
	mulsd	xcarry, xtmp			;; carry *= fudged col two-to-phi
	addsd	xcarry, Q [srcptr+3*64+off8]	;; Add FFT data
	movsd	Q [srcptr+3*64+off8], xcarry	;; Save value4
ENDM

;; Propagate a single gwsmallmul carry across 6 words.

xnorm_smallmul_2d_prop6 MACRO base2, srcptr, off8, grpptr, colptr, biglit, xcarry, xtmp, xaux
	movzx	rax, BYTE PTR [biglit]		;; First word
	movsd	xmm0, Q [srcptr+0*64+off8]	;; Load FFT data
	mulsd	xmm0, Q [colptr+0*32+off8]	;; Mul by col two-to-minus-phi
	movsd	xaux, Q [grpptr+off8]		;; Mul by grp two-to-minus-phi
	mulsd	xaux, Q XMM_NORM012_FF		;; Mul by FFTLEN/2
	mulsd	xmm0, Q XMM_TTMP_FUDGE[rax+off8];; Mul by fudge two-to-minus-phi
	mulsd	xmm0, xaux
	addsd	xmm0, xcarry			;; Add in the carry
	single_rounding base2, xmm0, xcarry, xtmp, rax+off8
	movsd	xtmp, Q [colptr+0*32+16+off8]	;; col two-to-phi
	mulsd	xtmp, Q XMM_TTP_FUDGE[rax+off8]	;; mul by fudge two-to-phi
	mulsd	xmm0, Q [grpptr+16+off8]	;; new value = val * grp two-to-phi
	mulsd	xmm0, xtmp			;; new value *= fudged col two-to-phi
	movsd	Q [srcptr+0*64+off8], xmm0	;; Save value1

	movzx	rax, BYTE PTR [biglit+4]	;; Second word
	movsd	xmm0, Q [srcptr+1*64+off8]	;; Load FFT data
	mulsd	xmm0, Q [colptr+1*32+off8]	;; Mul by col two-to-minus-phi
	movsd	xtmp, Q XMM_TTMP_FUDGE[rax+off8];; Mul by fudge two-to-minus-phi
	mulsd	xtmp, xaux			;; Mul by grp * FFTLEN/2
	mulsd	xmm0, xtmp
	addsd	xmm0, xcarry			;; Add in the carry
	single_rounding base2, xmm0, xcarry, xtmp, rax+off8
	movsd	xtmp, Q [colptr+1*32+16+off8]	;; col two-to-phi
	mulsd	xtmp, Q XMM_TTP_FUDGE[rax+off8]	;; mul by fudge two-to-phi
	mulsd	xmm0, Q [grpptr+16+off8]	;; new value = val * grp two-to-phi
	mulsd	xmm0, xtmp			;; new value *= fudged col two-to-phi
	movsd	Q [srcptr+1*64+off8], xmm0	;; Save value2

	mov	ecx, BIGLIT_INCR2		;; Different clm values step through
						;; big/lit array differently
	movzx	rax, BYTE PTR [biglit][rcx]	;; Third word
	movsd	xmm0, Q [srcptr+2*64+off8]	;; Load FFT data
	mulsd	xmm0, Q [colptr+2*32+off8]	;; Mul by col two-to-minus-phi
	movsd	xtmp, Q XMM_TTMP_FUDGE[rax+off8];; Mul by fudge two-to-minus-phi
	mulsd	xtmp, xaux			;; Mul by grp * FFTLEN/2
	mulsd	xmm0, xtmp
	addsd	xmm0, xcarry			;; Add in the carry
	single_rounding base2, xmm0, xcarry, xtmp, rax+off8
	movsd	xtmp, Q [colptr+2*32+16+off8]	;; col two-to-phi
	mulsd	xtmp, Q XMM_TTP_FUDGE[rax+off8]	;; mul by fudge two-to-phi
	mulsd	xmm0, Q [grpptr+16+off8]	;; new value = val * grp two-to-phi
	mulsd	xmm0, xtmp			;; new value *= fudged col two-to-phi
	movsd	Q [srcptr+2*64+off8], xmm0	;; Save value3

	movzx	rax, BYTE PTR [biglit][rcx+4]	;; Fourth word
	movsd	xmm0, Q [srcptr+3*64+off8]	;; Load FFT data
	mulsd	xmm0, Q [colptr+3*32+off8]	;; Mul by col two-to-minus-phi
	movsd	xtmp, Q XMM_TTMP_FUDGE[rax+off8];; Mul by fudge two-to-minus-phi
	mulsd	xtmp, xaux			;; Mul by grp * FFTLEN/2
	mulsd	xmm0, xtmp
	addsd	xmm0, xcarry			;; Add in the carry
	single_rounding base2, xmm0, xcarry, xtmp, rax+off8
	movsd	xtmp, Q [colptr+3*32+16+off8]	;; col two-to-phi
	mulsd	xtmp, Q XMM_TTP_FUDGE[rax+off8]	;; mul by fudge two-to-phi
	mulsd	xmm0, Q [grpptr+16+off8]	;; new value = val * grp two-to-phi
	mulsd	xmm0, xtmp			;; new value *= fudged col two-to-phi
	movsd	Q [srcptr+3*64+off8], xmm0	;; Save value4

	mov	ecx, BIGLIT_INCR4		;; Different clm values step through
						;; big/lit array differently
	movzx	rax, BYTE PTR [biglit][rcx]	;; Fifth word
	movsd	xmm0, Q [srcptr+4*64+off8]	;; Load FFT data
	mulsd	xmm0, Q [colptr+4*32+off8]	;; Mul by col two-to-minus-phi
	movsd	xtmp, Q XMM_TTMP_FUDGE[rax+off8];; Mul by fudge two-to-minus-phi
	mulsd	xtmp, xaux			;; Mul by grp * FFTLEN/2
	mulsd	xmm0, xtmp
	addsd	xmm0, xcarry			;; Add in the carry
	single_rounding base2, xmm0, xcarry, xtmp, rax+off8
	movsd	xtmp, Q [colptr+4*32+16+off8]	;; col two-to-phi
	mulsd	xtmp, Q XMM_TTP_FUDGE[rax+off8]	;; mul by fudge two-to-phi
	mulsd	xmm0, Q [grpptr+16+off8]	;; new value = val * grp two-to-phi
	mulsd	xmm0, xtmp			;; new value *= fudged col two-to-phi
	movsd	Q [srcptr+4*64+off8], xmm0	;; Save value5

	movzx	rax, BYTE PTR [biglit][rcx+4]	;; Sixth word
	subsd	xcarry, Q XMM_BIGVAL		;; Make carry an integer
	movsd	xtmp, Q [colptr+5*32+16+off8]	;; col two-to-phi
	mulsd	xtmp, Q XMM_TTP_FUDGE[rax+off8]	;; mul by fudge two-to-phi
	mulsd	xcarry, Q [grpptr+16+off8]	;; carry *= grp two-to-phi
	mulsd	xcarry, xtmp			;; carry *= fudged col two-to-phi
	addsd	xcarry, Q [srcptr+5*64+off8]	;; Add FFT data
	movsd	Q [srcptr+5*64+off8], xcarry	;; Save value6
ENDM

; This macro finishes the smallmul normalize process by adding the final
; two carries back into the appropriate FFT values at the start of the fft.
; rsi = pointer to FFT data
; rbp = pointer two-to-phi group multipliers
; rbx = pointer two-to-phi col multipliers
; rdi = pointer to big/little flags
; xmm6 = non-wraparound carry
; xmm7 = wraparound carry (this will be zero for smallmul of zero-padded number)

xnorm_smallmul_2d_fft_zpad MACRO base2
	LOCAL	smallk, mediumk, div_k_done

	;; Copy and integerize data from 7 words above halfway point to ZPAD0-ZPAD6
	;; Clear words 5,6,7
	;; Then we can make an exact copy of most of the xnorm012_2d_zpad code

	movzx	rax, BYTE PTR [rdi]	;; Load big vs. little flags
	movsd	xmm0, Q [rsi+0*64+32]	;; Value1
	subsd	xmm6, Q XMM_BIGVAL	;; Remove XMM_BIGVAL from carry
	single_split_lower_zpad_word base2, xmm0, xmm6, xmm4, rax
	movsd	ZPAD0, xmm0

	movzx	rax, BYTE PTR [rdi+4]	;; Load big vs. little flags
	movsd	xmm0, Q [rsi+1*64+32]	;; Value2
	mulsd	xmm0, Q [rbx+1*32]	;; Mul values2 by col two-to-minus-phi
	mulsd	xmm0, Q XMM_NORM012_FF	;; Mul by FFTLEN/2
	single_split_lower_zpad_word base2, xmm0, xmm6, xmm4, rax
	movsd	ZPAD1, xmm0

	mov	ecx, BIGLIT_INCR2	;; Different clm values step through
					;; big/lit array differently
	movzx	rax, BYTE PTR [rdi+rcx]	;; Load big vs. little flags
	movsd	xmm0, Q [rsi+2*64+32]	;; Value3
	mulsd	xmm0, Q [rbx+2*32]	;; Mul values3 by col two-to-minus-phi
	mulsd	xmm0, Q XMM_NORM012_FF	;; Mul by FFTLEN/2
	single_split_lower_zpad_word base2, xmm0, xmm6, xmm4, rax
	movsd	ZPAD2, xmm0

	movsd	xmm0, Q [rsi+3*64+32]	;; Value4
	mulsd	xmm0, Q [rbx+3*32]	;; Mul values4 by col two-to-minus-phi
	mulsd	xmm0, Q XMM_NORM012_FF	;; Mul by FFTLEN/2
	addsd	xmm0, Q XMM_BIGVAL	;; Round to an integer
	subsd	xmm0, Q XMM_BIGVAL
	addsd	xmm0, xmm6		;; Value4 + carry
	movsd	ZPAD3, xmm0

	movsd	xmm0, Q [rsi+4*64+32]	;; Value5
	mulsd	xmm0, Q [rbx+4*32]	;; Mul values5 by col two-to-minus-phi
	mulsd	xmm0, Q XMM_NORM012_FF	;; Mul by FFTLEN/2
	addsd	xmm0, Q XMM_BIGVAL	;; Round to an integer
	subsd	xmm0, Q XMM_BIGVAL
	movsd	ZPAD4, xmm0

	movsd	xmm0, Q [rsi+5*64+32]	;; Value6
	mulsd	xmm0, Q [rbx+5*32]	;; Mul values6 by col two-to-minus-phi
	mulsd	xmm0, Q XMM_NORM012_FF	;; Mul by FFTLEN/2
	addsd	xmm0, Q XMM_BIGVAL	;; Round to an integer
	subsd	xmm0, Q XMM_BIGVAL
	movsd	ZPAD5, xmm0

	movsd	xmm0, Q [rsi+6*64+32]	;; Value7
	mulsd	xmm0, Q [rbx+6*32]	;; Mul values7 by col two-to-minus-phi
	mulsd	xmm0, Q XMM_NORM012_FF	;; Mul by FFTLEN/2
	addsd	xmm0, Q XMM_BIGVAL	;; Round to an integer
	subsd	xmm0, Q XMM_BIGVAL
	movsd	ZPAD6, xmm0

	subsd	xmm0, xmm0		;; Clear highest words
	movsd	Q [rsi+4*64+32], xmm0
	movsd	Q [rsi+5*64+32], xmm0
	movsd	Q [rsi+6*64+32], xmm0

	;; Divide the zpad data by k.  Store the integer part in XMM_TMP
	;; and the remainder in ZPAD0.  Later we will wrap the integer part
	;; down to the bottom of the FFT data area (and multiply by -c).
	;; And we will store the remainder in the upper half of the FFT
	;; data area.

	;; Note there are three cases to handle.  K is smaller than a big word.
	;; K is between one and 2 big words in size.  And K is more than
	;; 2 big words in size.

	cmp	ZPAD_TYPE, 2		;; Are we dealing with case 1,2,or 3
	jl	smallk			;; One word case
	je	mediumk			;; Two word case

	;; This case does the divide by k where k is three words

	movsd	xmm0, ZPAD6		;; Load zpad word (high bits)
	movsd	xmm1, ZPAD5		;; Load zpad word (middle bits)
	movsd	xmm2, ZPAD4		;; Load zpad word (low bits)
	movsd	xmm4, ZPAD_INVERSE_K6	;; Load shifted 1/k
	mulsd	xmm4, xmm0		;; Mul ZPAD by shifted 1/k
	addsd	xmm4, Q XMM_BIGVAL	;; Round to integer	
	subsd	xmm4, Q XMM_BIGVAL
	movsd	xmm5, ZPAD_K6_HI	;; Load high bits of k
	mulsd	xmm5, xmm4
	subsd	xmm0, xmm5		;; Calculate high bits of remainder
	movsd	xmm5, ZPAD_K6_MID	;; Load middle bits of k
	mulsd	xmm5, xmm4
	subsd	xmm1, xmm5		;; Calculate middle bits of remainder
	movsd	xmm5, ZPAD_K6_LO	;; Load low bits of k
	mulsd	xmm5, xmm4
	subsd	xmm2, xmm5		;; Calculate low bits of remainder
	movsd	Q XMM_TMP5, xmm4	;; Save word of zpad / k

	mulsd	xmm0, ZPAD_SHIFT6	;; Shift previous zpad word
	addsd	xmm1, xmm0		;; Add to create new high zpad bits
	movsd	xmm0, ZPAD3		;; Load zpad word (new low bits)
	movsd	xmm5, ZPAD_SHIFT5	;; Combine high and medium bits
	mulsd	xmm5, xmm1
	addsd	xmm5, xmm2
	movsd	xmm4, ZPAD_INVERSE_K5	;; Load shifted 1/k
	mulsd	xmm4, xmm5		;; Mul ZPAD by shifted 1/k
	addsd	xmm4, Q XMM_BIGVAL	;; Round to integer
	subsd	xmm4, Q XMM_BIGVAL
	movsd	xmm5, ZPAD_K5_HI	;; Load high bits of k
	mulsd	xmm5, xmm4
	subsd	xmm1, xmm5		;; Calculate high bits of remainder
	movsd	xmm5, ZPAD_K5_MID	;; Load middle bits of k
	mulsd	xmm5, xmm4
	subsd	xmm2, xmm5		;; Calculate middle bits of remainder
	movsd	xmm5, ZPAD_K5_LO	;; Load low bits of k
	mulsd	xmm5, xmm4
	subsd	xmm0, xmm5		;; Calculate low bits of remainder
	movsd	Q XMM_TMP4, xmm4	;; Save word of zpad / k

	mulsd	xmm1, ZPAD_SHIFT5	;; Shift previous zpad word
	addsd	xmm2, xmm1		;; Add to create new high zpad bits
	movsd	xmm1, ZPAD2		;; Load zpad word (new low bits)
	movsd	xmm5, ZPAD_SHIFT4	;; Combine high and medium bits
	mulsd	xmm5, xmm2
	addsd	xmm5, xmm0
	movsd	xmm4, ZPAD_INVERSE_K4	;; Load shifted 1/k
	mulsd	xmm4, xmm5		;; Mul ZPAD by shifted 1/k
	addsd	xmm4, Q XMM_BIGVAL	;; Round to integer
	subsd	xmm4, Q XMM_BIGVAL
	movsd	xmm5, ZPAD_K4_HI	;; Load high bits of k
	mulsd	xmm5, xmm4
	subsd	xmm2, xmm5		;; Calculate high bits of remainder
	movsd	xmm5, ZPAD_K4_MID	;; Load middle bits of k
	mulsd	xmm5, xmm4
	subsd	xmm0, xmm5		;; Calculate middle bits of remainder
	movsd	xmm5, ZPAD_K4_LO	;; Load low bits of k
	mulsd	xmm5, xmm4
	subsd	xmm1, xmm5		;; Calculate low bits of remainder
	movsd	Q XMM_TMP3, xmm4	;; Save word of zpad / k

	mulsd	xmm2, ZPAD_SHIFT4	;; Shift previous zpad word
	addsd	xmm0, xmm2		;; Add to create new high zpad bits
	movsd	xmm2, ZPAD1		;; Load zpad word (new low bits)
	movsd	xmm5, ZPAD_SHIFT3	;; Combine high and medium bits
	mulsd	xmm5, xmm0
	addsd	xmm5, xmm1
	movsd	xmm4, ZPAD_INVERSE_K3	;; Load shifted 1/k
	mulsd	xmm4, xmm5		;; Mul ZPAD by shifted 1/k
	addsd	xmm4, Q XMM_BIGVAL	;; Round to integer
	subsd	xmm4, Q XMM_BIGVAL
	movsd	xmm5, ZPAD_K3_HI	;; Load high bits of k
	mulsd	xmm5, xmm4
	subsd	xmm0, xmm5		;; Calculate high bits of remainder
	movsd	xmm5, ZPAD_K3_MID	;; Load middle bits of k
	mulsd	xmm5, xmm4
	subsd	xmm1, xmm5		;; Calculate middle bits of remainder
	movsd	xmm5, ZPAD_K3_LO	;; Load low bits of k
	mulsd	xmm5, xmm4
	subsd	xmm2, xmm5		;; Calculate low bits of remainder
	movsd	Q XMM_TMP2, xmm4	;; Save word of zpad / k

	mulsd	xmm0, ZPAD_SHIFT3	;; Shift previous zpad word
	addsd	xmm1, xmm0		;; Add to create new high zpad bits
	movsd	xmm0, ZPAD0		;; Load zpad word (new low bits)
	movsd	xmm5, ZPAD_SHIFT2	;; Combine high and medium bits
	mulsd	xmm5, xmm1
	addsd	xmm5, xmm2
	movsd	xmm4, ZPAD_INVERSE_K2	;; Load shifted 1/k
	mulsd	xmm4, xmm5		;; Mul ZPAD by shifted 1/k
	addsd	xmm4, Q XMM_BIGVAL	;; Round to integer
	subsd	xmm4, Q XMM_BIGVAL
	movsd	xmm5, ZPAD_K2_HI	;; Load high bits of k
	mulsd	xmm5, xmm4
	subsd	xmm1, xmm5		;; Calculate high bits of remainder
	movsd	xmm5, ZPAD_K2_MID	;; Load middle bits of k
	mulsd	xmm5, xmm4
	subsd	xmm2, xmm5		;; Calculate middle bits of remainder
	movsd	xmm5, ZPAD_K2_LO	;; Load low bits of k
	mulsd	xmm5, xmm4
	subsd	xmm0, xmm5		;; Calculate low bits of remainder
	movsd	Q XMM_TMP1, xmm4	;; Save word of zpad / k

	mulsd	xmm1, ZPAD_SHIFT2	;; Shift previous zpad word
	addsd	xmm2, xmm1		;; Add to create new high zpad bits
	mulsd	xmm2, ZPAD_SHIFT1	;; Shift previous zpad word
	addsd	xmm0, xmm2		;; Add to create new high zpad bits
	movsd	ZPAD0, xmm0		;; Save remainder of zpad / k

	subsd	xmm1, xmm1		;; Zero words that other cases set
	movsd	Q XMM_TMP6, xmm1
	
	jmp	div_k_done

	;; This case does the divide by k where k is two words
mediumk:
	movsd	xmm0, ZPAD6		;; Load zpad word (high bits)
	movsd	xmm1, ZPAD5		;; Load zpad word (low bits)
	movsd	xmm4, ZPAD_INVERSE_K6	;; Load shifted 1/k
	mulsd	xmm4, xmm0		;; Mul ZPAD by shifted 1/k
	addsd	xmm4, Q XMM_BIGVAL	;; Round to integer	
	subsd	xmm4, Q XMM_BIGVAL
	movsd	xmm5, ZPAD_K6_HI	;; Load high bits of k
	mulsd	xmm5, xmm4
	subsd	xmm0, xmm5		;; Calculate high bits of remainder
	movsd	xmm5, ZPAD_K6_LO	;; Load low bits of k
	mulsd	xmm5, xmm4
	subsd	xmm1, xmm5		;; Calculate low bits of remainder
	movsd	Q XMM_TMP6, xmm4	;; Save word of zpad / k

	mulsd	xmm0, ZPAD_SHIFT6	;; Shift previous zpad word
	addsd	xmm0, xmm1		;; Add to create new high zpad bits
	movsd	xmm1, ZPAD4		;; Load zpad word (new low bits)
	movsd	xmm4, ZPAD_INVERSE_K5	;; Load shifted 1/k
	mulsd	xmm4, xmm0		;; Mul ZPAD by shifted 1/k
	addsd	xmm4, Q XMM_BIGVAL	;; Round to integer
	subsd	xmm4, Q XMM_BIGVAL
	movsd	xmm5, ZPAD_K5_HI	;; Load high bits of k
	mulsd	xmm5, xmm4
	subsd	xmm0, xmm5		;; Calculate high bits of remainder
	movsd	xmm5, ZPAD_K5_LO	;; Load low bits of k
	mulsd	xmm5, xmm4
	subsd	xmm1, xmm5		;; Calculate low bits of remainder
	movsd	Q XMM_TMP5, xmm4	;; Save word of zpad / k

	mulsd	xmm0, ZPAD_SHIFT5	;; Shift previous zpad word
	addsd	xmm0, xmm1		;; Add to create new high zpad bits
	movsd	xmm1, ZPAD3		;; Load zpad word (new low bits)
	movsd	xmm4, ZPAD_INVERSE_K4	;; Load shifted 1/k
	mulsd	xmm4, xmm0		;; Mul ZPAD by shifted 1/k
	addsd	xmm4, Q XMM_BIGVAL	;; Round to integer
	subsd	xmm4, Q XMM_BIGVAL
	movsd	xmm5, ZPAD_K4_HI	;; Load high bits of k
	mulsd	xmm5, xmm4
	subsd	xmm0, xmm5		;; Calculate high bits of remainder
	movsd	xmm5, ZPAD_K4_LO	;; Load low bits of k
	mulsd	xmm5, xmm4
	subsd	xmm1, xmm5		;; Calculate low bits of remainder
	movsd	Q XMM_TMP4, xmm4	;; Save word of zpad / k

	mulsd	xmm0, ZPAD_SHIFT4	;; Shift previous zpad word
	addsd	xmm0, xmm1		;; Add to create new high zpad bits
	movsd	xmm1, ZPAD2		;; Load zpad word (new low bits)
	movsd	xmm4, ZPAD_INVERSE_K3	;; Load shifted 1/k
	mulsd	xmm4, xmm0		;; Mul ZPAD by shifted 1/k
	addsd	xmm4, Q XMM_BIGVAL	;; Round to integer
	subsd	xmm4, Q XMM_BIGVAL
	movsd	xmm5, ZPAD_K3_HI	;; Load high bits of k
	mulsd	xmm5, xmm4
	subsd	xmm0, xmm5		;; Calculate high bits of remainder
	movsd	xmm5, ZPAD_K3_LO	;; Load low bits of k
	mulsd	xmm5, xmm4
	subsd	xmm1, xmm5		;; Calculate low bits of remainder
	movsd	Q XMM_TMP3, xmm4	;; Save word of zpad / k

	mulsd	xmm0, ZPAD_SHIFT3	;; Shift previous zpad word
	addsd	xmm0, xmm1		;; Add to create new high zpad bits
	movsd	xmm1, ZPAD1		;; Load zpad word (new low bits)
	movsd	xmm4, ZPAD_INVERSE_K2	;; Load shifted 1/k
	mulsd	xmm4, xmm0		;; Mul ZPAD by shifted 1/k
	addsd	xmm4, Q XMM_BIGVAL	;; Round to integer
	subsd	xmm4, Q XMM_BIGVAL
	movsd	xmm5, ZPAD_K2_HI	;; Load high bits of k
	mulsd	xmm5, xmm4
	subsd	xmm0, xmm5		;; Calculate high bits of remainder
	movsd	xmm5, ZPAD_K2_LO	;; Load low bits of k
	mulsd	xmm5, xmm4
	subsd	xmm1, xmm5		;; Calculate low bits of remainder
	movsd	Q XMM_TMP2, xmm4	;; Save word of zpad / k

	mulsd	xmm0, ZPAD_SHIFT2	;; Shift previous zpad word
	addsd	xmm0, xmm1		;; Add to create new high zpad bits
	movsd	xmm1, ZPAD0		;; Load zpad word (new low bits)
	movsd	xmm4, ZPAD_INVERSE_K1	;; Load shifted 1/k
	mulsd	xmm4, xmm0		;; Mul ZPAD by shifted 1/k
	addsd	xmm4, Q XMM_BIGVAL	;; Round to integer
	subsd	xmm4, Q XMM_BIGVAL
	movsd	xmm5, ZPAD_K1_HI	;; Load high bits of k
	mulsd	xmm5, xmm4
	subsd	xmm0, xmm5		;; Calculate high bits of remainder
	movsd	xmm5, ZPAD_K1_LO	;; Load low bits of k
	mulsd	xmm5, xmm4
	subsd	xmm1, xmm5		;; Calculate low bits of remainder
	movsd	Q XMM_TMP1, xmm4	;; Save word of zpad / k

	mulsd	xmm0, ZPAD_SHIFT1	;; Shift previous zpad word
	addsd	xmm0, xmm1		;; Add to create new high zpad bits
	movsd	ZPAD0, xmm0		;; Save remainder of zpad / k

	jmp	div_k_done

	;; This case does the divide by k where k is one word
	;; Assume ZPAD5 and ZPAD6 are zero.
smallk:	movsd	xmm0, ZPAD4		;; Load zpad data
	movsd	xmm4, ZPAD_INVERSE_K1	;; Load by 1/k
	mulsd	xmm4, xmm0		;; Mul ZPAD data by 1/k
	addsd	xmm4, Q XMM_BIGVAL	;; Round to integer	
	subsd	xmm4, Q XMM_BIGVAL
	movsd	Q XMM_TMP5, xmm4	;; Save integer part
	mulsd	xmm4, ZPAD_K1_LO	;; Compute remainder
	subsd	xmm0, xmm4

	mulsd	xmm0, ZPAD_SHIFT4	;; Shift previous zpad word
	addsd	xmm0, ZPAD3		;; Add in zpad data
	movsd	xmm4, ZPAD_INVERSE_K1	;; Load by 1/k
	mulsd	xmm4, xmm0		;; Mul ZPAD data by 1/k
	addsd	xmm4, Q XMM_BIGVAL	;; Round to integer	
	subsd	xmm4, Q XMM_BIGVAL
	movsd	Q XMM_TMP4, xmm4	;; Save integer part
	mulsd	xmm4, ZPAD_K1_LO	;; Compute remainder
	subsd	xmm0, xmm4

	mulsd	xmm0, ZPAD_SHIFT3	;; Shift previous zpad word
	addsd	xmm0, ZPAD2		;; Add in zpad data
	movsd	xmm4, ZPAD_INVERSE_K1	;; Load by 1/k
	mulsd	xmm4, xmm0		;; Mul ZPAD data by 1/k
	addsd	xmm4, Q XMM_BIGVAL	;; Round to integer	
	subsd	xmm4, Q XMM_BIGVAL
	movsd	Q XMM_TMP3, xmm4	;; Save integer part
	mulsd	xmm4, ZPAD_K1_LO	;; Compute remainder
	subsd	xmm0, xmm4

	mulsd	xmm0, ZPAD_SHIFT2	;; Shift previous zpad word
	addsd	xmm0, ZPAD1		;; Add in zpad data
	movsd	xmm4, ZPAD_INVERSE_K1	;; Load by 1/k
	mulsd	xmm4, xmm0		;; Mul ZPAD data by 1/k
	addsd	xmm4, Q XMM_BIGVAL	;; Round to integer	
	subsd	xmm4, Q XMM_BIGVAL
	movsd	Q XMM_TMP2, xmm4	;; Save integer part
	mulsd	xmm4, ZPAD_K1_LO	;; Compute remainder
	subsd	xmm0, xmm4

	mulsd	xmm0, ZPAD_SHIFT1	;; Shift previous zpad word
	addsd	xmm0, ZPAD0		;; Add in zpad data
	movsd	xmm4, ZPAD_INVERSE_K1	;; Load by 1/k
	mulsd	xmm4, xmm0		;; Mul ZPAD data by 1/k
	addsd	xmm4, Q XMM_BIGVAL	;; Round to integer	
	subsd	xmm4, Q XMM_BIGVAL
	movsd	Q XMM_TMP1, xmm4	;; Save integer part
	mulsd	xmm4, ZPAD_K1_LO	;; Compute remainder
	subsd	xmm0, xmm4
	movsd	ZPAD0, xmm0		;; Save remainder

	subsd	xmm1, xmm1		;; Zero words that other cases set
	movsd	Q XMM_TMP6, xmm1
	
div_k_done:

	;; Now normalize the data above the halfway point.  Remember that the
	;; column two-to-phi multiplier for the first value will be 1.0.

	movzx	rax, BYTE PTR [rdi]	;; First biglit flag 
	movsd	xmm0, ZPAD0		;; Load remainder of divide by k
	addsd	xmm0, Q XMM_BIGVAL
	single_rounding base2, xmm0, xmm2, xmm4, rax
	movsd	Q [rsi+0*64+32], xmm0	;; Save value1

	movzx	rax, BYTE PTR [rdi+4]	;; Load big vs. little flags
	single_rounding base2, xmm2, xmm0, xmm4, rax
	mulsd	xmm2, Q [rbx+1*32+16]	;; new value2 = val * col two-to-phi
	movsd	Q [rsi+1*64+32], xmm2	;; Save value2

	mov	ecx, BIGLIT_INCR2	;; Different clm values step through
					;; big/lit array differently
	movzx	rax, BYTE PTR [rdi+rcx]	;; Load big vs. little flags
	single_rounding base2, xmm0, xmm2, xmm4, rax
	mulsd	xmm0, Q [rbx+2*32+16]	;; new value3 = val * col two-to-phi
	movsd	Q [rsi+2*64+32], xmm0	;; Save value3

	subsd	xmm2, Q XMM_BIGVAL	;; Remove integer rounding constant
	mulsd	xmm2, Q [rbx+3*32+16]	;; value4 = carry * col two-to-phi
	movsd	Q [rsi+3*64+32], xmm2	;; Save new value4

	;; Mul the integer part of (ZPAD data divided by k) by -c in
	;; preparation for adding it into the lower FFT data area.
	;; Also add in the shifted high FFT carry at this time.

	;; Now add in and normalize the bottom FFT data.  Remember that the
	;; column two-to-phi multiplier for the first value will be 1.0.  We 
	;; must go 6 words deep in case k is 48-50 bits and c is 32 bits.

	movzx	rax, BYTE PTR [rdi]	;; First word 
	movsd	xmm0, Q XMM_TMP1	;; Load integer part of divide by k
	mulsd	xmm0, Q XMM_MINUS_C	;; Mul by -c
	addsd	xmm0, Q XMM_BIGVAL
	addsd	xmm0, Q [rsi+0*64]	;; Add in the FFT data
	single_rounding base2, xmm0, xmm2, xmm4, rax
	movsd	Q [rsi+0*64], xmm0	;; Save value1

	movzx	rax, BYTE PTR [rdi+4]	;; Load big vs. little flags
	movsd	xmm0, Q XMM_TMP2	;; Load integer part of divide by k
	mulsd	xmm0, Q XMM_MINUS_C	;; Mul by -c
	movsd	xmm1, Q [rsi+1*64]	;; Load FFT data
	mulsd	xmm1, Q [rbx+1*32]	;; Mul values2 by col two-to-minus-phi
	mulsd	xmm1, Q XMM_NORM012_FF	;; Mul by FFTLEN/2
	addsd	xmm0, xmm1		;; Add in the FFT data
	addsd	xmm0, xmm2		;; x2 = value + carry
	single_rounding base2, xmm0, xmm2, xmm4, rax
	mulsd	xmm0, Q [rbx+1*32+16]	;; new value2 = val * col two-to-phi
	movsd	Q [rsi+1*64], xmm0	;; Save value2

	mov	ecx, BIGLIT_INCR2	;; Different clm values step through
					;; big/lit array differently
	movzx	rax, BYTE PTR [rdi+rcx]	;; Load big vs. little flags
	movsd	xmm0, Q XMM_TMP3	;; Load integer part of divide by k
	mulsd	xmm0, Q XMM_MINUS_C	;; Mul by -c
	movsd	xmm1, Q [rsi+2*64]	;; Load FFT data
	mulsd	xmm1, Q [rbx+2*32]	;; Mul values3 by col two-to-minus-phi
	mulsd	xmm1, Q XMM_NORM012_FF	;; Mul by FFTLEN/2
	addsd	xmm0, xmm1		;; Add in the FFT data
	addsd	xmm0, xmm2		;; x3 = value + carry
	single_rounding base2, xmm0, xmm2, xmm4, rax
	mulsd	xmm0, Q [rbx+2*32+16]	;; new value3 = val * col two-to-phi
	movsd	Q [rsi+2*64], xmm0	;; Save value3

	movzx	rax, BYTE PTR [rdi+rcx+4] ;; Load big vs. little flags
	movsd	xmm0, Q XMM_TMP4	;; Load integer part of divide by k
	mulsd	xmm0, Q XMM_MINUS_C	;; Mul by -c
	movsd	xmm1, Q [rsi+3*64]	;; Load FFT data
	mulsd	xmm1, Q [rbx+3*32]	;; Mul values4 by col two-to-minus-phi
	mulsd	xmm1, Q XMM_NORM012_FF	;; Mul by FFTLEN/2
	addsd	xmm0, xmm1		;; Add in the FFT data
	addsd	xmm0, xmm2		;; x4 = value + carry
	single_rounding base2, xmm0, xmm2, xmm4, rax
	mulsd	xmm0, Q [rbx+3*32+16]	;; new value4 = val * col two-to-phi
	movsd	Q [rsi+3*64], xmm0	;; Save value4

	mov	ecx, BIGLIT_INCR4	;; Different clm values step through
					;; big/lit array differently
	movzx	rax, BYTE PTR [rdi+rcx]	;; Load big vs. little flags
	movsd	xmm0, Q XMM_TMP5	;; Load integer part of divide by k
	mulsd	xmm0, Q XMM_MINUS_C	;; Mul by -c
	movsd	xmm1, Q [rsi+4*64]	;; Load FFT data
	mulsd	xmm1, Q [rbx+4*32]	;; Mul values4 by col two-to-minus-phi
	mulsd	xmm1, Q XMM_NORM012_FF	;; Mul by FFTLEN/2
	addsd	xmm0, xmm1		;; Add in the FFT data
	addsd	xmm0, xmm2		;; x4 = value + carry
	single_rounding base2, xmm0, xmm2, xmm4, rax
	mulsd	xmm0, Q [rbx+4*32+16]	;; new value4 = val * col two-to-phi
	movsd	Q [rsi+4*64], xmm0	;; Save value4

	movzx	rax, BYTE PTR [rdi+rcx+4] ;; Load big vs. little flags
	movsd	xmm0, Q XMM_TMP6	;; Load integer part of divide by k
	mulsd	xmm0, Q XMM_MINUS_C	;; Mul by -c
	movsd	xmm1, Q [rsi+5*64]	;; Load FFT data
	mulsd	xmm1, Q [rbx+5*32]	;; Mul values5 by col two-to-minus-phi
	mulsd	xmm1, Q XMM_NORM012_FF	;; Mul by FFTLEN/2
	addsd	xmm0, xmm1		;; Add in the FFT data
	addsd	xmm0, xmm2		;; x5 = value + carry
	single_rounding base2, xmm0, xmm2, xmm4, rax
	mulsd	xmm0, Q [rbx+5*32+16]	;; new value5 = val * col two-to-phi
	movsd	Q [rsi+5*64], xmm0	;; Save value5

	subsd	xmm2, Q XMM_BIGVAL	;; Remove rounding constant
	mulsd	xmm2, Q [rbx+6*32+16]	;; new value6 = val * col two-to-phi
	addsd	xmm2, Q [rsi+6*64]	;; Add in FFT data
	movsd	Q [rsi+6*64], xmm2	;; Save value6
	ENDM


; *************** WPN normalized add/sub macro ******************
; This macro adds or subtracts, then "normalizes" eight FFT
; data values.  This involves multiplying the summed values by
; two-to-minus-phi.  Rounding the value to an integer.  Making sure
; the integer is smaller than the maximum allowable integer, generating
; a carry if necessary. Finally, the value is multiplied by two-to-phi
; and stored.
; rcx = pointer to the first number
; rdx = pointer to the second number
; rsi = pointer to destination
; rdi = pointer to array of big vs. little flags
; rbp = pointer two-to-phi group multipliers
; XMM_TMP1,XMM_TMP2,XMM_TMP3,XMM_TMP4 = carries
; A pipelined version of this code:
;	xload	xmm0, [rdx]		;; Load second number
;	fop	xmm0, [rcx]		;; Add/sub first number
;	movzx	rax, BYTE PTR [rdi]	;; Load big vs. little flags
;	mulpd	xmm0, [rbp+0*XMM_GMD][rax] ;; Mul by fudged grp two-to-minus-phi
;	addpd	xmm0, XMM_TMP1		;; x1 = values + carry
;	xload	xmm2, XMM_LIMIT_BIGMAX[rax];; Load maximum * BIGVAL - BIGVAL
;	addpd	xmm2, xmm0		;; y1 = top bits of x
;	xload	xmm6, XMM_LIMIT_BIGMAX_NEG[rax];; Load -(maximum*BIGVAL-BIGVAL)
;	addpd	xmm6, xmm2		;; z1 = y1-(maximum * BIGVAL - BIGVAL)
;	subpd	xmm0, xmm6		;; rounded value = x1 - z1
;	mulpd	xmm2, XMM_LIMIT_INVERSE[rax];; next carry = shifted y1
;	mulpd	xmm0, [rbp+0*XMM_GMD+XMM_GMD/2][rax] ;; new value1 = val * fudged grp two-to-phi
;	xstore	[rsi+0*dist1], xmm0	;; Save new value1
;	xstore	XMM_TMP1, xmm2		;; Save carry

xnorm_op_wpn MACRO fop, ttp, base2
	xload	xmm0, [rdx]		;; Load second number
	fop	xmm0, [rcx]		;; Add/sub first number
	xload	xmm1, [rdx+16]		;; Load second number
	fop	xmm1, [rcx+16]		;; Add/sub first number
	xload	xmm6, [rdx+32]		;; Load second number
	fop	xmm6, [rcx+32]		;; Add/sub first number
	xload	xmm7, [rdx+48]		;; Load second number
	fop	xmm7, [rcx+48]		;; Add/sub first number
ttp	movzx	rax, BYTE PTR [rdi+1]	;; Load big vs. little flags 1-4
ttp	movzx	rbx, BYTE PTR [rdi]	;; Load fudge factor flags 1,2
ttp	and	rbx, 0f0h
ttp	mulpd	xmm0, [rbp+0*XMM_GMD][rbx] ;; Mul by fudged grp two-to-minus-phi
	addpd	xmm0, XMM_TMP1		;; x1 = values + carry
ttp	mulpd	xmm1, [rbp+1*XMM_GMD][rbx] ;; Mul by fudged grp two-to-minus-phi
	addpd	xmm1, XMM_TMP2		;; x1 = values + carry
	rounding_interleaved ttp, base2, noexec, noexec, xmm0, xmm2, xmm4, rax*4, xmm1, xmm3, xmm5, rax*4+16
	xstore	XMM_TMP1, xmm2		;; Save carry
	xstore	XMM_TMP2, xmm3		;; Save carry
ttp	mulpd	xmm0, [rbp+0*XMM_GMD+XMM_GMD/2][rbx] ;; value1 *= fudged grp two-to-phi
	xstore	[rsi+0*16], xmm0	;; Save new value1
ttp	mulpd	xmm1, [rbp+1*XMM_GMD+XMM_GMD/2][rbx] ;; value2 *= fudged grp two-to-phi
	xstore	[rsi+1*16], xmm1	;; Save new value1

ttp	movzx	rbx, BYTE PTR [rdi]	;; Load fudge factor flags 3,4
ttp	and	rbx, 0fh
ttp	mulpd	xmm6, [rbp+2*XMM_GMD][rbx*8] ;; Mul by fudged grp two-to-minus-phi
	addpd	xmm6, XMM_TMP3		;; x1 = values + carry
ttp	mulpd	xmm7, [rbp+3*XMM_GMD][rbx*8] ;; Mul by fudged grp two-to-minus-phi
	addpd	xmm7, XMM_TMP4		;; x1 = values + carry
	rounding_interleaved ttp, base2, noexec, noexec, xmm6, xmm2, xmm4, rax*4+32, xmm7, xmm3, xmm5, rax*4+48
	xstore	XMM_TMP3, xmm2		;; Save carry
	xstore	XMM_TMP4, xmm3		;; Save carry
ttp	mulpd	xmm6, [rbp+2*XMM_GMD+XMM_GMD/2][rbx*8] ;; value1 *= fudged grp two-to-phi
	xstore	[rsi+2*16], xmm6	;; Save new value1
ttp	mulpd	xmm7, [rbp+3*XMM_GMD+XMM_GMD/2][rbx*8] ;; value2 *= fudged grp two-to-phi
	xstore	[rsi+3*16], xmm7	;; Save new value1
ttp	bump	rdi, 2			;; Next flags ptr
	bump	rsi, 64			;; Next dest ptr
	bump	rdx, 64			;; Next src ptr
	bump	rcx, 64			;; Next src ptr
	ENDM

; *************** WPN followup macros ******************
; This macro finishes the normalize add/sub process by adding four carries
; from the end of a block back to the start of the block.  The remaining
; four carries are rotated for starting the next block.

xnorm_op_wpn_blk MACRO srcreg, screg, carry1, carry2, carry3, carry4
	xload	xmm2, XMM_BIGVAL

	movzx	rax, BYTE PTR [rdi]	;; Load fudge factor flags 1,2
	and	rax, 0f0h
	xload	xmm0, carry1		;; Load carries
	subsd	xmm0, xmm2
	mulsd	xmm0, Q [screg+0*XMM_GMD+XMM_GMD/2+8][rax] ;; mul by fudged grp two-to-phi
	addsd	xmm0, Q [srcreg+0*16+8]	;; Add in FFT data
	movsd	Q [srcreg+0*16+8], xmm0	;; Save FFT word
	shufpd	xmm0, xmm2, 1
	xstore	carry1, xmm0		;; Save carries for start of next blk

	xload	xmm0, carry2		;; Load carries
	subsd	xmm0, xmm2
	mulsd	xmm0, Q [screg+1*XMM_GMD+XMM_GMD/2+8][rax] ;; mul by fudged grp two-to-phi
	addsd	xmm0, Q [srcreg+1*16+8]	;; Add in FFT data
	movsd	Q [srcreg+1*16+8], xmm0	;; Save FFT word
	shufpd	xmm0, xmm2, 1
	xstore	carry2, xmm0		;; Save carries for start of next blk

	movzx	rax, BYTE PTR [rdi]	;; Load fudge factor flags 3,4
	and	rax, 0fh
	xload	xmm0, carry3		;; Load carries
	subsd	xmm0, xmm2
	mulsd	xmm0, Q [screg+2*XMM_GMD+XMM_GMD/2+8][rax*8] ;; mul by fudged grp two-to-phi
	addsd	xmm0, Q [srcreg+2*16+8]	;; Add in FFT data
	movsd	Q [srcreg+2*16+8], xmm0	;; Save FFT word
	shufpd	xmm0, xmm2, 1
	xstore	carry3, xmm0		;; Save carries for start of next blk

	xload	xmm0, carry4		;; Load carries
	subsd	xmm0, xmm2
	mulsd	xmm0, Q [screg+3*XMM_GMD+XMM_GMD/2+8][rax*8] ;; mul by fudged grp two-to-phi
	addsd	xmm0, Q [srcreg+3*16+8]	;; Add in FFT data
	movsd	Q [srcreg+3*16+8], xmm0	;; Save FFT word
	shufpd	xmm0, xmm2, 1
	xstore	carry4, xmm0		;; Save carries for start of next blk
	ENDM

; This macro finishes the normalize add/sub process by adding two carries
; from the end of a section back to the start of the section.  The remaining
; two carries are rotated for starting the next section.
; rax = pointer to FFT data
; rbx = pointer two-to-phi group multipliers

xnorm_op_wpn_sec MACRO carry1, carry2, carry3, carry4
	xload	xmm2, XMM_BIGVAL

	movzx	rcx, BYTE PTR [rdi]	;; Load fudge factor flags 1,2
	and	rcx, 0f0h
	movsd	xmm0, Q carry1		;; Load carry
	subsd	xmm0, xmm2
	mulsd	xmm0, Q [rbx+1*XMM_GMD+XMM_GMD/2][rcx] ;; mul by fudged grp two-to-phi
	addsd	xmm0, Q [rax+1*16]	;; Add in FFT data
	movsd	Q [rax+1*16], xmm0	;; Save FFT word

	xload	xmm1, carry2
	xstore	carry1, xmm1
	xstore	carry2, xmm2		;; Save carries for start of next sec

	movzx	rcx, BYTE PTR [rdi]	;; Load fudge factor flags 3,4
	and	rcx, 0fh
	movsd	xmm0, Q carry3		;; Load carry
	subsd	xmm0, xmm2
	mulsd	xmm0, Q [rbx+3*XMM_GMD+XMM_GMD/2][rcx*8] ;; mul by fudged grp two-to-phi
	addsd	xmm0, Q [rax+3*16]	;; Add in FFT data
	movsd	Q [rax+3*16], xmm0	;; Save FFT word

	xload	xmm1, carry4
	xstore	carry3, xmm1
	xstore	carry4, xmm2		;; Save carries for start of next sec
	ENDM


; This macro finishes the normalize add/sub process by adding the final
; two carries back into the appropriate FFT values at the start of the fft.
; rsi = pointer to FFT data
; rbp = pointer two-to-phi group multipliers
; xmm6 = non-wraparound carry
; xmm7 = wraparound carry

xnorm_op_wpn_fft MACRO
	movsd	xmm2, Q XMM_BIGVAL

	movzx	rax, BYTE PTR [rdi]	;; Load fudge factor flags 1,2
	and	rax, 0f0h
	subsd	xmm7, xmm2		;; Remove XMM_BIGVAL
	mulsd	xmm7, Q XMM_MINUS_C	;; mul wrap around carry by -c
	mulsd	xmm7, Q [rbp+0*XMM_GMD+XMM_GMD/2][rax] ;; mul by fudged grp two-to-phi
	addsd	xmm7, Q [rsi+0*16]	;; Add in FFT data
	movsd	Q [rsi+0*16], xmm7	;; Save FFT word

	movzx	rax, BYTE PTR [rdi]	;; Load fudge factor flags 3,4
	and	rax, 0fh
	subsd	xmm6, xmm2		;; Remove XMM_BIGVAL
	mulsd	xmm6, Q [rbp+2*XMM_GMD+XMM_GMD/2][rax*8] ;; mul by fudged grp two-to-phi
	addsd	xmm6, Q [rsi+2*16]	;; Add in FFT data
	movsd	Q [rsi+2*16], xmm6	;; Save FFT word
	ENDM


; *************** WPN normalized add & sub macro ******************
; This macro adds and subtracts, then "normalizes" eight FFT
; data values.  This involves multiplying the summed values by
; two-to-minus-phi.  Rounding the value to an integer.  Making sure
; the integer is smaller than the maximum allowable integer, generating
; a carry if necessary. Finally, the value is multiplied by two-to-phi
; and stored.
; rcx = pointer to the first number
; rdx = pointer to the second number
; rsi = pointer to destination #1
; rbp = pointer to destination #2
; rbx = pointer two-to-phi group multipliers
; rdi = pointer to array of big vs. little flags
; rax = big/lit flag
; XMM_TMP1,XMM_TMP2,XMM_TMP3,XMM_TMP4 = addition carries
; XMM_TMP5,XMM_TMP6,XMM_TMP7,XMM_TMP8 = subtraction carries

xnorm_addsub_wpn MACRO ttp, base2
	xload	xmm1, [rcx+0*16]	;; Load first number
	xload	xmm0, [rdx+0*16]	;; Load second number
	subpd	xmm1, xmm0		;; first - second number
	addpd	xmm0, [rcx+0*16]	;; first + second number

	xload	xmm7, [rcx+1*16]	;; Load first number
	xload	xmm6, [rdx+1*16]	;; Load second number
	subpd	xmm7, xmm6		;; first - second number
	addpd	xmm6, [rcx+1*16]	;; first + second number

ttp	movzx	rax, BYTE PTR [rdi]	;; Load fudge factor flags 1,2
ttp	and	rax, 0f0h
ttp	mulpd	xmm0, [rbx+0*XMM_GMD][rax] ;; Mul by fudged grp two-to-minus-phi
	addpd	xmm0, XMM_TMP1		;; x1 = values + carry
ttp	mulpd	xmm1, [rbx+0*XMM_GMD][rax] ;; Mul by fudged grp two-to-minus-phi
	addpd	xmm1, XMM_TMP5		;; x1 = values + carry
ttp	mulpd	xmm6, [rbx+1*XMM_GMD][rax] ;; Mul by fudged grp two-to-minus-phi
	addpd	xmm6, XMM_TMP2		;; x1 = values + carry
ttp	mulpd	xmm7, [rbx+1*XMM_GMD][rax] ;; Mul by fudged grp two-to-minus-phi
	addpd	xmm7, XMM_TMP6		;; x1 = values + carry

ttp	movzx	rax, BYTE PTR [rdi+1]	;; Load big vs. little flags 1-4
	rounding_interleaved ttp, base2, noexec, noexec, xmm0, xmm2, xmm4, rax*4, xmm1, xmm3, xmm5, rax*4
	xstore	XMM_TMP1, xmm2		;; Save carry
	xstore	XMM_TMP5, xmm3		;; Save carry
	rounding_interleaved ttp, base2, noexec, noexec, xmm6, xmm2, xmm4, rax*4+16, xmm7, xmm3, xmm5, rax*4+16
	xstore	XMM_TMP2, xmm2		;; Save carry
	xstore	XMM_TMP6, xmm3		;; Save carry

ttp	movzx	rax, BYTE PTR [rdi]	;; Load fudge factor flags 1,2
ttp	and	rax, 0f0h
ttp	mulpd	xmm0, [rbx+0*XMM_GMD+XMM_GMD/2][rax] ;; value1 *= fudged grp two-to-phi
	xstore	[rsi+0*16], xmm0	;; Save new value1
ttp	mulpd	xmm1, [rbx+0*XMM_GMD+XMM_GMD/2][rax] ;; value2 *= fudged grp two-to-phi
	xstore	[rbp+0*16], xmm1	;; Save new value1
ttp	mulpd	xmm6, [rbx+1*XMM_GMD+XMM_GMD/2][rax] ;; value1 *= fudged grp two-to-phi
	xstore	[rsi+1*16], xmm6	;; Save new value1
ttp	mulpd	xmm7, [rbx+1*XMM_GMD+XMM_GMD/2][rax] ;; value2 *= fudged grp two-to-phi
	xstore	[rbp+1*16], xmm7	;; Save new value1

	xload	xmm1, [rcx+2*16]	;; Load first number
	xload	xmm0, [rdx+2*16]	;; Load second number
	subpd	xmm1, xmm0		;; first - second number
	addpd	xmm0, [rcx+2*16]	;; first + second number

	xload	xmm7, [rcx+3*16]	;; Load first number
	xload	xmm6, [rdx+3*16]	;; Load second number
	subpd	xmm7, xmm6		;; first - second number
	addpd	xmm6, [rcx+3*16]	;; first + second number

ttp	movzx	rax, BYTE PTR [rdi]	;; Load fudge factor flags 3,4
ttp	and	rax, 0fh
ttp	mulpd	xmm0, [rbx+2*XMM_GMD][rax*8] ;; Mul by fudged grp two-to-minus-phi
	addpd	xmm0, XMM_TMP3		;; x1 = values + carry
ttp	mulpd	xmm1, [rbx+2*XMM_GMD][rax*8] ;; Mul by fudged grp two-to-minus-phi
	addpd	xmm1, XMM_TMP7		;; x1 = values + carry
ttp	mulpd	xmm6, [rbx+3*XMM_GMD][rax*8] ;; Mul by fudged grp two-to-minus-phi
	addpd	xmm6, XMM_TMP4		;; x1 = values + carry
ttp	mulpd	xmm7, [rbx+3*XMM_GMD][rax*8] ;; Mul by fudged grp two-to-minus-phi
	addpd	xmm7, XMM_TMP8		;; x1 = values + carry

ttp	movzx	rax, BYTE PTR [rdi+1]	;; Load big vs. little flags 1-4
	rounding_interleaved ttp, base2, noexec, noexec, xmm0, xmm2, xmm4, rax*4+32, xmm1, xmm3, xmm5, rax*4+32
	xstore	XMM_TMP3, xmm2		;; Save carry
	xstore	XMM_TMP7, xmm3		;; Save carry
	rounding_interleaved ttp, base2, noexec, noexec, xmm6, xmm2, xmm4, rax*4+48, xmm7, xmm3, xmm5, rax*4+48
	xstore	XMM_TMP4, xmm2		;; Save carry
	xstore	XMM_TMP8, xmm3		;; Save carry

ttp	movzx	rax, BYTE PTR [rdi]	;; Load fudge factor flags 3,4
ttp	and	rax, 0fh
ttp	mulpd	xmm0, [rbx+2*XMM_GMD+XMM_GMD/2][rax*8] ;; value1 *= fudged grp two-to-phi
	xstore	[rsi+2*16], xmm0	;; Save new value1
ttp	mulpd	xmm1, [rbx+2*XMM_GMD+XMM_GMD/2][rax*8] ;; value2 *= fudged grp two-to-phi
	xstore	[rbp+2*16], xmm1	;; Save new value1
ttp	mulpd	xmm6, [rbx+3*XMM_GMD+XMM_GMD/2][rax*8] ;; value1 *= fudged grp two-to-phi
	xstore	[rsi+3*16], xmm6	;; Save new value1
ttp	mulpd	xmm7, [rbx+3*XMM_GMD+XMM_GMD/2][rax*8] ;; value2 *= fudged grp two-to-phi
	xstore	[rbp+3*16], xmm7	;; Save new value1

	bump	rdx, 64			;; Next src ptr
	bump	rcx, 64			;; Next src ptr
ttp	bump	rdi, 2			;; Next flags ptr
	bump	rsi, 64			;; Next dest ptr
	bump	rbp, 64			;; Next dest ptr
	ENDM

; *************** WPN normalized small mul macro ******************
; This macro multiplies by a small value, then "normalizes" eight FFT
; data values. 
; rsi = pointer to destination
; rdi = pointer to array of big vs. little flags
; rbp = pointer two-to-phi group multipliers
; XMM_TMP1,XMM_TMP2,XMM_TMP3,XMM_TMP4 = carries
; XMM_TMP5 = small value * optional FFTLEN/2
; A pipelined version of this code:
;	xload	xmm0, [rsi]		;; Load second number
;	mulpd	xmm0, XMM_TMP5		;; Mul by small value
;	movzx	rcx, BYTE PTR [rdi]	;; Load big vs. little flags
;	xload	xmm2, XMM_TTMP_FUDGE[rcx];; fudge two-to-minus-phi
;	mulpd	xmm0, [rax]		;; Mul by grp two-to-minus-phi
;	mulpd	xmm0, xmm2		;; Mul by fudge two-to-minus-phi
;	addpd	xmm0, [rbp+0*16]	;; x1 = values + carry
;	xload	xmm2, XMM_LIMIT_BIGMAX[rcx];; Load maximum * BIGVAL - BIGVAL
;	addpd	xmm2, xmm0		;; y1 = top bits of x
;	xload	xmm6, XMM_LIMIT_BIGMAX_NEG[rcx];; Load -(maximum*BIGVAL-BIGVAL)
;	addpd	xmm6, xmm2		;; z1 = y1-(maximum * BIGVAL - BIGVAL)
;	subpd	xmm0, xmm6		;; rounded value = x1 - z1
;	mulpd	xmm2, XMM_LIMIT_INVERSE[rcx];; next carry = shifted y1
;	xload	xmm4, XMM_TTP_FUDGE[rcx];; fudge two-to-phi
;	mulpd	xmm0, [rax+0*32+16]	;; new value1 = val * grp two-to-phi
;	mulpd	xmm0, xmm4		;; new value1 *= fudge two-to-phi
;	xstore	[rsi], xmm0		;; Save new value1
;	xstore	[rbp+0*16], xmm2	;; Save carry

xnorm_smallmul_wpn MACRO ttp, base2
	xload	xmm2, XMM_TMP5		;; Load small value
	xload	xmm0, [rsi]		;; Load values1
	mulpd	xmm0, xmm2		;; Mul values1 by small value
	xload	xmm1, [rsi+16]		;; Load values2
	mulpd	xmm1, xmm2		;; Mul values2 by small value
	xload	xmm6, [rsi+32]		;; Load values3
	mulpd	xmm6, xmm2		;; Mul values3 by small value
	xload	xmm7, [rsi+48]		;; Load values4
	mulpd	xmm7, xmm2		;; Mul values4 by small value
ttp	movzx	rax, BYTE PTR [rdi+1]	;; Load big vs. little flags 1-4
ttp	movzx	rcx, BYTE PTR [rdi]	;; Load fudge factor flags 1,2
ttp	and	rcx, 0f0h
ttp	mulpd	xmm0, [rbp+0*XMM_GMD][rcx] ;; Mul by fudged grp two-to-minus-phi
	addpd	xmm0, XMM_TMP1		;; x1 = values + carry
ttp	mulpd	xmm1, [rbp+1*XMM_GMD][rcx] ;; Mul by fudged grp two-to-minus-phi
	addpd	xmm1, XMM_TMP2		;; x1 = values + carry
	rounding_interleaved ttp, base2, noexec, noexec, xmm0, xmm2, xmm4, rax*4, xmm1, xmm3, xmm5, rax*4+16
	xstore	XMM_TMP1, xmm2		;; Save carry
	xstore	XMM_TMP2, xmm3		;; Save carry
ttp	mulpd	xmm0, [rbp+0*XMM_GMD+XMM_GMD/2][rcx] ;; value1 *= fudged two-to-phi
	xstore	[rsi+0*16], xmm0	;; Save new value1
ttp	mulpd	xmm1, [rbp+1*XMM_GMD+XMM_GMD/2][rcx] ;; value2 *= fudged two-to-phi
	xstore	[rsi+1*16], xmm1	;; Save new value2

ttp	movzx	rcx, BYTE PTR [rdi]	;; Load fudge factor flags 3,4
ttp	and	rcx, 0fh
ttp	mulpd	xmm6, [rbp+2*XMM_GMD][rcx*8] ;; Mul by fudged grp two-to-minus-phi
	addpd	xmm6, XMM_TMP3		;; x1 = values + carry
ttp	mulpd	xmm7, [rbp+3*XMM_GMD][rcx*8] ;; Mul by fudged grp two-to-minus-phi
	addpd	xmm7, XMM_TMP4		;; x1 = values + carry
	rounding_interleaved ttp, base2, noexec, noexec, xmm6, xmm2, xmm4, rax*4+32, xmm7, xmm3, xmm5, rax*4+48
	xstore	XMM_TMP3, xmm2		;; Save carry
	xstore	XMM_TMP4, xmm3		;; Save carry
ttp	mulpd	xmm6, [rbp+2*XMM_GMD+XMM_GMD/2][rcx*8] ;; value1 *= fudged two-to-phi
	xstore	[rsi+2*16], xmm6	;; Save new value1
ttp	mulpd	xmm7, [rbp+3*XMM_GMD+XMM_GMD/2][rcx*8] ;; value2 *= fudged two-to-phi
	xstore	[rsi+3*16], xmm7	;; Save new value2
ttp	bump	rdi, 2			;; Next flags ptr
	bump	rsi, 64			;; Next dest ptr
	ENDM

; This macro finishes the smallmul normalize process by adding four carries
; from the end of a block back to the start of the block.  The remaining
; four carries are rotated for starting the next block.
; rsi = Source ptr to start of block
; rbp = Group ptr of two-to-phi
; rdi = biglit array ptr
; rcx is destroyed

xnorm_smallmul_wpn_blk MACRO base2
	movsd	xmm5, Q XMM_BIGVAL

	movsd	xmm1, Q XMM_TMP1	;; Load carry
	xnorm_smallmul_wpn_prop4 base2, rsi+0*16, 8, rbp+0*XMM_GMD, rdi, 0, xmm1, xmm2, xmm3
	xload	xmm1, XMM_TMP1		;; Load carries (prop4 will zero high word of xmm1)
	shufpd	xmm1, xmm5, 1
	xstore	XMM_TMP1, xmm1		;; Save carries for start of next blk

	movsd	xmm1, Q XMM_TMP2	;; Load carry
	xnorm_smallmul_wpn_prop4 base2, rsi+1*16, 8, rbp+1*XMM_GMD, rdi, 1, xmm1, xmm2, xmm3
	xload	xmm1, XMM_TMP2		;; Load carries
	shufpd	xmm1, xmm5, 1
	xstore	XMM_TMP2, xmm1		;; Save carries for start of next blk

	movsd	xmm1, Q XMM_TMP3	;; Load carry
	xnorm_smallmul_wpn_prop4 base2, rsi+2*16, 8, rbp+2*XMM_GMD, rdi, 2, xmm1, xmm2, xmm3
	xload	xmm1, XMM_TMP3		;; Load carries
	shufpd	xmm1, xmm5, 1
	xstore	XMM_TMP3, xmm1		;; Save carries for start of next blk

	movsd	xmm1, Q XMM_TMP4	;; Load carry
	xnorm_smallmul_wpn_prop4 base2, rsi+3*16, 8, rbp+3*XMM_GMD, rdi, 3, xmm1, xmm2, xmm3
	xload	xmm1, XMM_TMP4		;; Load carries
	shufpd	xmm1, xmm5, 1
	xstore	XMM_TMP4, xmm1		;; Save carries for start of next blk
	ENDM

; This macro finishes the smallmul normalize process by adding two carries
; from the end of a section back to the start of the section.  The remaining
; two carries are rotated for starting the next section.
; rsi = Source ptr to start of block
; rbp = Group ptr of two-to-phi
; rdi = biglit array ptr
; rcx is destroyed

xnorm_smallmul_wpn_sec MACRO base2
	xload	xmm5, XMM_BIGVAL

	movsd	xmm1, Q XMM_TMP1	;; Load carry
	xnorm_smallmul_wpn_prop4 base2, rsi+1*16, 0, rbp+1*XMM_GMD, rdi, 1, xmm1, xmm2, xmm3
	xload	xmm1, XMM_TMP2
	xstore	XMM_TMP1, xmm1
	xstore	XMM_TMP2, xmm5		;; Save carries for start of next sec

	movsd	xmm1, Q XMM_TMP3	;; Load carry
	xnorm_smallmul_wpn_prop4 base2, rsi+3*16, 0, rbp+3*XMM_GMD, rdi, 3, xmm1, xmm2, xmm3
	xload	xmm1, XMM_TMP4
	xstore	XMM_TMP3, xmm1
	xstore	XMM_TMP4, xmm5		;; Save carries for start of next sec
	ENDM

; This macro finishes the smallmul normalize process by adding the final
; two carries back into the appropriate FFT values at the start of the fft.
; rsi = pointer to FFT data
; rbp = pointer two-to-phi group multipliers
; rdi = pointer to big/little flags
; xmm6 = non-wraparound carry
; xmm7 = wraparound carry

xnorm_smallmul_wpn_fft MACRO base2
	subsd	xmm7, Q XMM_BIGVAL
	mulsd	xmm7, Q XMM_MINUS_C	;; Negate the carry
	addsd	xmm7, Q XMM_BIGVAL
	xnorm_smallmul_wpn_prop6 base2, rsi+0*16, 0, rbp+0*XMM_GMD, rdi, 0, xmm7, xmm2, xmm3
	xnorm_smallmul_wpn_prop4 base2, rsi+2*16, 0, rbp+2*XMM_GMD, rdi, 2, xmm6, xmm2, xmm3
	ENDM

;; Propagate a single gwsmallmul carry across 4 words.

xnorm_smallmul_wpn_prop4 MACRO base2, srcptr, off8, grpptr, biglitreg, biglitword, xcarry, xtmp, xaux
	movzx	rax, BYTE PTR [biglitreg+1]	;; Big/lit flags
	movzx	rcx, BYTE PTR [biglitreg]	;; Fudge factor flags
	IF biglitword LE 1
	and	rcx, 0f0h
	ELSE
	and	rcx, 0fh
	ENDIF
	movsd	xmm0, Q [srcptr+0*64+off8]	;; Load FFT data
	IF biglitword LE 1
	mulsd	xmm0, Q [grpptr+off8][rcx]	;; Mul by fudged two-to-minus-phi
	ELSE
	mulsd	xmm0, Q [grpptr+off8][rcx*8]	;; Mul by fudged two-to-minus-phi
	ENDIF
	addsd	xmm0, xcarry			;; Add in the carry
	single_rounding base2, xmm0, xcarry, xtmp, rax*4+biglitword*16+off8
	IF biglitword LE 1
	mulsd	xmm0, Q [grpptr+XMM_GMD/2+off8][rcx] ;; value1 *= fudged two-to-phi
	ELSE
	mulsd	xmm0, Q [grpptr+XMM_GMD/2+off8][rcx*8] ;; value1 *= fudged two-to-phi
	ENDIF
	movsd	Q [srcptr+0*64+off8], xmm0	;; Save value1

	movzx	rax, BYTE PTR [biglitreg+3]	;; Second word big/lit flags
	movzx	rcx, BYTE PTR [biglitreg+2]	;; Second word fudge factor flags
	IF biglitword LE 1
	and	rcx, 0f0h
	ELSE
	and	rcx, 0fh
	ENDIF
	movsd	xmm0, Q [srcptr+1*64+off8]	;; Load FFT data
	IF biglitword LE 1
	mulsd	xmm0, Q [grpptr+off8][rcx]	;; Mul by fudged two-to-minus-phi
	ELSE
	mulsd	xmm0, Q [grpptr+off8][rcx*8]	;; Mul by fudged two-to-minus-phi
	ENDIF
	addsd	xmm0, xcarry			;; Add in the carry
	single_rounding base2, xmm0, xcarry, xtmp, rax*4+biglitword*16+off8
	IF biglitword LE 1
	mulsd	xmm0, Q [grpptr+XMM_GMD/2+off8][rcx] ;; value2 *= fudged two-to-phi
	ELSE
	mulsd	xmm0, Q [grpptr+XMM_GMD/2+off8][rcx*8] ;; value2 *= fudged two-to-phi
	ENDIF
	movsd	Q [srcptr+1*64+off8], xmm0	;; Save value2

	mov	ecx, BIGLIT_INCR2		;; Different clm values step through
						;; big/lit array differently
	movzx	rax, BYTE PTR [biglitreg+1][rcx] ;; Third word big/lit flags
	movzx	rcx, BYTE PTR [biglitreg][rcx]	;; Third word fudge factor flags
	IF biglitword LE 1
	and	rcx, 0f0h
	ELSE
	and	rcx, 0fh
	ENDIF
	movsd	xmm0, Q [srcptr+2*64+off8]	;; Load FFT data
	IF biglitword LE 1
	mulsd	xmm0, Q [grpptr+off8][rcx]	;; Mul by fudged two-to-minus-phi
	ELSE
	mulsd	xmm0, Q [grpptr+off8][rcx*8]	;; Mul by fudged two-to-minus-phi
	ENDIF
	addsd	xmm0, xcarry			;; Add in the carry
	single_rounding base2, xmm0, xcarry, xtmp, rax*4+biglitword*16+off8
	IF biglitword LE 1
	mulsd	xmm0, Q [grpptr+XMM_GMD/2+off8][rcx] ;; value3 *= fudged two-to-phi
	ELSE
	mulsd	xmm0, Q [grpptr+XMM_GMD/2+off8][rcx*8] ;; value3 *= fudged two-to-phi
	ENDIF
	movsd	Q [srcptr+2*64+off8], xmm0	;; Save value3

	mov	ecx, BIGLIT_INCR2		;; Different clm values step through
						;; big/lit array differently
	movzx	rcx, BYTE PTR [biglitreg][rcx+2] ;; Fourth word fudge factor flags
	IF biglitword LE 1
	and	rcx, 0f0h
	ELSE
	and	rcx, 0fh
	ENDIF
	subsd	xcarry, Q XMM_BIGVAL		;; Make carry an integer
	IF biglitword LE 1
	mulsd	xcarry, Q [grpptr+XMM_GMD/2+off8][rcx] ;; carry *= fudged two-to-phi
	ELSE
	mulsd	xcarry, Q [grpptr+XMM_GMD/2+off8][rcx*8] ;; carry *= fudged two-to-phi
	ENDIF
	addsd	xcarry, Q [srcptr+3*64+off8]	;; Add FFT data
	movsd	Q [srcptr+3*64+off8], xcarry	;; Save value4
ENDM

;; Propagate a single gwsmallmul carry across 6 words.

xnorm_smallmul_wpn_prop6 MACRO base2, srcptr, off8, grpptr, biglitreg, biglitword, xcarry, xtmp, xaux

	IF biglitword NE 0
	not supported
	ENDIF

	movzx	rax, BYTE PTR [biglitreg+1]	;; Big/lit flags
	movzx	rcx, BYTE PTR [biglitreg]	;; Fudge factor flags
	and	rcx, 0f0h
	movsd	xmm0, Q [srcptr+0*64+off8]	;; Load FFT data
	mulsd	xmm0, Q [grpptr+off8][rcx]	;; Mul by fudged two-to-minus-phi
	addsd	xmm0, xcarry			;; Add in the carry
	single_rounding base2, xmm0, xcarry, xtmp, rax*4+off8
	mulsd	xmm0, Q [grpptr+XMM_GMD/2+off8][rcx] ;; value1 *= fudged two-to-phi
	movsd	Q [srcptr+0*64+off8], xmm0	;; Save value1

	movzx	rax, BYTE PTR [biglitreg+3]	;; Big/lit flags
	movzx	rcx, BYTE PTR [biglitreg+2]	;; Fudge factor flags
	and	rcx, 0f0h
	movsd	xmm0, Q [srcptr+1*64+off8]	;; Load FFT data
	mulsd	xmm0, Q [grpptr+off8][rcx]	;; Mul by fudged two-to-minus-phi
	addsd	xmm0, xcarry			;; Add in the carry
	single_rounding base2, xmm0, xcarry, xtmp, rax*4+off8
	mulsd	xmm0, Q [grpptr+XMM_GMD/2+off8][rcx] ;; value2 *= fudged two-to-phi
	movsd	Q [srcptr+1*64+off8], xmm0	;; Save value2

	mov	ecx, BIGLIT_INCR2		;; Different clm values step through
						;; big/lit array differently
	movzx	rax, BYTE PTR [biglitreg+1][rcx] ;; Big/lit flags
	movzx	rcx, BYTE PTR [biglitreg][rcx]	;; Fudge factor flags
	and	rcx, 0f0h
	movsd	xmm0, Q [srcptr+2*64+off8]	;; Load FFT data
	mulsd	xmm0, Q [grpptr+off8][rcx]	;; Mul by fudged two-to-minus-phi
	addsd	xmm0, xcarry			;; Add in the carry
	single_rounding base2, xmm0, xcarry, xtmp, rax*4+off8
	mulsd	xmm0, Q [grpptr+XMM_GMD/2+off8][rcx] ;; value3 *= fudged two-to-phi
	movsd	Q [srcptr+2*64+off8], xmm0	;; Save value3

	mov	ecx, BIGLIT_INCR2		;; Different clm values step through
						;; big/lit array differently
	movzx	rax, BYTE PTR [biglitreg+3][rcx] ;; Big/lit flags
	movzx	rcx, BYTE PTR [biglitreg+2][rcx] ;; Fudge factor flags
	and	rcx, 0f0h
	movsd	xmm0, Q [srcptr+3*64+off8]	;; Load FFT data
	mulsd	xmm0, Q [grpptr+off8][rcx]	;; Mul by fudged two-to-minus-phi
	addsd	xmm0, xcarry			;; Add in the carry
	single_rounding base2, xmm0, xcarry, xtmp, rax*4+off8
	mulsd	xmm0, Q [grpptr+XMM_GMD/2+off8][rcx] ;; value4 *= fudged two-to-phi
	movsd	Q [srcptr+3*64+off8], xmm0	;; Save value4

	mov	ecx, BIGLIT_INCR4		;; Different clm values step through
						;; big/lit array differently
	movzx	rax, BYTE PTR [biglitreg+1][rcx] ;; Big/lit flags
	movzx	rcx, BYTE PTR [biglitreg][rcx]	;; Fudge factor flags
	and	rcx, 0f0h
	movsd	xmm0, Q [srcptr+4*64+off8]	;; Load FFT data
	mulsd	xmm0, Q [grpptr+off8][rcx]	;; Mul by fudged two-to-minus-phi
	addsd	xmm0, xcarry			;; Add in the carry
	single_rounding base2, xmm0, xcarry, xtmp, rax*4+off8
	mulsd	xmm0, Q [grpptr+XMM_GMD/2+off8][rcx] ;; value5 *= fudged two-to-phi
	movsd	Q [srcptr+4*64+off8], xmm0	;; Save value5

	mov	ecx, BIGLIT_INCR4		;; Different clm values step through
						;; big/lit array differently
	movzx	rcx, BYTE PTR [biglitreg+2][rcx] ;; Fudge factor flags
	and	rcx, 0f0h
	subsd	xcarry, Q XMM_BIGVAL		;; Make carry an integer
	mulsd	xcarry, Q [grpptr+XMM_GMD/2+off8][rcx] ;; carry *= fudged two-to-phi
	addsd	xcarry, Q [srcptr+5*64+off8]	;; Add FFT data
	movsd	Q [srcptr+5*64+off8], xcarry	;; Save value6
ENDM

; This macro finishes the smallmul normalize process by adding the final
; two carries back into the appropriate FFT values at the start of the fft.
; rsi = pointer to FFT data
; rbp = pointer two-to-phi group multipliers
; rdi = pointer to big/little flags
; xmm6 = non-wraparound carry
; xmm7 = wraparound carry (this will be zero for smallmul of zero-padded number)

xnorm_smallmul_wpn_fft_zpad MACRO base2
	LOCAL	smallk, mediumk, div_k_done

	;; Copy and integerize data from 7 words above halfway point to ZPAD0-ZPAD6
	;; Clear words 5,6,7
	;; Then we can make an exact copy of most of the xnorm012_wpn_zpad code

	movzx	rax, BYTE PTR [rdi+1]	;; Load big vs. little flags
	movsd	xmm0, Q [rsi+0*64+32]	;; Value1
	subsd	xmm6, Q XMM_BIGVAL	;; Remove XMM_BIGVAL from carry
	single_split_lower_zpad_word base2, xmm0, xmm6, xmm4, rax*4
	movsd	ZPAD0, xmm0

	movzx	rax, BYTE PTR [rdi+3]	;; Load big vs. little flags
	movsd	xmm0, Q [rsi+1*64+32]	;; Value2
	single_split_lower_zpad_word base2, xmm0, xmm6, xmm4, rax*4
	movsd	ZPAD1, xmm0

	mov	ecx, BIGLIT_INCR2	;; Different clm values step through
					;; big/lit array differently
	movzx	rax, BYTE PTR [rdi+rcx+1] ;; Load big vs. little flags
	movsd	xmm0, Q [rsi+2*64+32]	;; Value3
	single_split_lower_zpad_word base2, xmm0, xmm6, xmm4, rax*4
	movsd	ZPAD2, xmm0

	movsd	xmm0, Q [rsi+3*64+32]	;; Value4
	addsd	xmm0, Q XMM_BIGVAL	;; Round to an integer
	subsd	xmm0, Q XMM_BIGVAL
	addsd	xmm0, xmm6		;; Value4 + carry
	movsd	ZPAD3, xmm0

	movsd	xmm0, Q [rsi+4*64+32]	;; Value5
	addsd	xmm0, Q XMM_BIGVAL	;; Round to an integer
	subsd	xmm0, Q XMM_BIGVAL
	movsd	ZPAD4, xmm0

	movsd	xmm0, Q [rsi+5*64+32]	;; Value6
	addsd	xmm0, Q XMM_BIGVAL	;; Round to an integer
	subsd	xmm0, Q XMM_BIGVAL
	movsd	ZPAD5, xmm0

	movsd	xmm0, Q [rsi+6*64+32]	;; Value7
	addsd	xmm0, Q XMM_BIGVAL	;; Round to an integer
	subsd	xmm0, Q XMM_BIGVAL
	movsd	ZPAD6, xmm0

	subsd	xmm0, xmm0		;; Clear highest words
	movsd	Q [rsi+4*64+32], xmm0
	movsd	Q [rsi+5*64+32], xmm0
	movsd	Q [rsi+6*64+32], xmm0

	;; Divide the zpad data by k.  Store the integer part in XMM_TMP
	;; and the remainder in ZPAD0.  Later we will wrap the integer part
	;; down to the bottom of the FFT data area (and multiply by -c).
	;; And we will store the remainder in the upper half of the FFT
	;; data area.

	;; Note there are three cases to handle.  K is smaller than a big word.
	;; K is between one and 2 big words in size.  And K is more than
	;; 2 big words in size.

	cmp	ZPAD_TYPE, 2		;; Are we dealing with case 1,2,or 3
	jl	smallk			;; One word case
	je	mediumk			;; Two word case

	;; This case does the divide by k where k is three words

	movsd	xmm0, ZPAD6		;; Load zpad word (high bits)
	movsd	xmm1, ZPAD5		;; Load zpad word (middle bits)
	movsd	xmm2, ZPAD4		;; Load zpad word (low bits)
	movsd	xmm4, ZPAD_INVERSE_K6	;; Load shifted 1/k
	mulsd	xmm4, xmm0		;; Mul ZPAD by shifted 1/k
	addsd	xmm4, Q XMM_BIGVAL	;; Round to integer	
	subsd	xmm4, Q XMM_BIGVAL
	movsd	xmm5, ZPAD_K6_HI	;; Load high bits of k
	mulsd	xmm5, xmm4
	subsd	xmm0, xmm5		;; Calculate high bits of remainder
	movsd	xmm5, ZPAD_K6_MID	;; Load middle bits of k
	mulsd	xmm5, xmm4
	subsd	xmm1, xmm5		;; Calculate middle bits of remainder
	movsd	xmm5, ZPAD_K6_LO	;; Load low bits of k
	mulsd	xmm5, xmm4
	subsd	xmm2, xmm5		;; Calculate low bits of remainder
	movsd	Q XMM_TMP5, xmm4	;; Save word of zpad / k

	mulsd	xmm0, ZPAD_SHIFT6	;; Shift previous zpad word
	addsd	xmm1, xmm0		;; Add to create new high zpad bits
	movsd	xmm0, ZPAD3		;; Load zpad word (new low bits)
	movsd	xmm5, ZPAD_SHIFT5	;; Combine high and medium bits
	mulsd	xmm5, xmm1
	addsd	xmm5, xmm2
	movsd	xmm4, ZPAD_INVERSE_K5	;; Load shifted 1/k
	mulsd	xmm4, xmm5		;; Mul ZPAD by shifted 1/k
	addsd	xmm4, Q XMM_BIGVAL	;; Round to integer
	subsd	xmm4, Q XMM_BIGVAL
	movsd	xmm5, ZPAD_K5_HI	;; Load high bits of k
	mulsd	xmm5, xmm4
	subsd	xmm1, xmm5		;; Calculate high bits of remainder
	movsd	xmm5, ZPAD_K5_MID	;; Load middle bits of k
	mulsd	xmm5, xmm4
	subsd	xmm2, xmm5		;; Calculate middle bits of remainder
	movsd	xmm5, ZPAD_K5_LO	;; Load low bits of k
	mulsd	xmm5, xmm4
	subsd	xmm0, xmm5		;; Calculate low bits of remainder
	movsd	Q XMM_TMP4, xmm4	;; Save word of zpad / k

	mulsd	xmm1, ZPAD_SHIFT5	;; Shift previous zpad word
	addsd	xmm2, xmm1		;; Add to create new high zpad bits
	movsd	xmm1, ZPAD2		;; Load zpad word (new low bits)
	movsd	xmm5, ZPAD_SHIFT4	;; Combine high and medium bits
	mulsd	xmm5, xmm2
	addsd	xmm5, xmm0
	movsd	xmm4, ZPAD_INVERSE_K4	;; Load shifted 1/k
	mulsd	xmm4, xmm5		;; Mul ZPAD by shifted 1/k
	addsd	xmm4, Q XMM_BIGVAL	;; Round to integer
	subsd	xmm4, Q XMM_BIGVAL
	movsd	xmm5, ZPAD_K4_HI	;; Load high bits of k
	mulsd	xmm5, xmm4
	subsd	xmm2, xmm5		;; Calculate high bits of remainder
	movsd	xmm5, ZPAD_K4_MID	;; Load middle bits of k
	mulsd	xmm5, xmm4
	subsd	xmm0, xmm5		;; Calculate middle bits of remainder
	movsd	xmm5, ZPAD_K4_LO	;; Load low bits of k
	mulsd	xmm5, xmm4
	subsd	xmm1, xmm5		;; Calculate low bits of remainder
	movsd	Q XMM_TMP3, xmm4	;; Save word of zpad / k

	mulsd	xmm2, ZPAD_SHIFT4	;; Shift previous zpad word
	addsd	xmm0, xmm2		;; Add to create new high zpad bits
	movsd	xmm2, ZPAD1		;; Load zpad word (new low bits)
	movsd	xmm5, ZPAD_SHIFT3	;; Combine high and medium bits
	mulsd	xmm5, xmm0
	addsd	xmm5, xmm1
	movsd	xmm4, ZPAD_INVERSE_K3	;; Load shifted 1/k
	mulsd	xmm4, xmm5		;; Mul ZPAD by shifted 1/k
	addsd	xmm4, Q XMM_BIGVAL	;; Round to integer
	subsd	xmm4, Q XMM_BIGVAL
	movsd	xmm5, ZPAD_K3_HI	;; Load high bits of k
	mulsd	xmm5, xmm4
	subsd	xmm0, xmm5		;; Calculate high bits of remainder
	movsd	xmm5, ZPAD_K3_MID	;; Load middle bits of k
	mulsd	xmm5, xmm4
	subsd	xmm1, xmm5		;; Calculate middle bits of remainder
	movsd	xmm5, ZPAD_K3_LO	;; Load low bits of k
	mulsd	xmm5, xmm4
	subsd	xmm2, xmm5		;; Calculate low bits of remainder
	movsd	Q XMM_TMP2, xmm4	;; Save word of zpad / k

	mulsd	xmm0, ZPAD_SHIFT3	;; Shift previous zpad word
	addsd	xmm1, xmm0		;; Add to create new high zpad bits
	movsd	xmm0, ZPAD0		;; Load zpad word (new low bits)
	movsd	xmm5, ZPAD_SHIFT2	;; Combine high and medium bits
	mulsd	xmm5, xmm1
	addsd	xmm5, xmm2
	movsd	xmm4, ZPAD_INVERSE_K2	;; Load shifted 1/k
	mulsd	xmm4, xmm5		;; Mul ZPAD by shifted 1/k
	addsd	xmm4, Q XMM_BIGVAL	;; Round to integer
	subsd	xmm4, Q XMM_BIGVAL
	movsd	xmm5, ZPAD_K2_HI	;; Load high bits of k
	mulsd	xmm5, xmm4
	subsd	xmm1, xmm5		;; Calculate high bits of remainder
	movsd	xmm5, ZPAD_K2_MID	;; Load middle bits of k
	mulsd	xmm5, xmm4
	subsd	xmm2, xmm5		;; Calculate middle bits of remainder
	movsd	xmm5, ZPAD_K2_LO	;; Load low bits of k
	mulsd	xmm5, xmm4
	subsd	xmm0, xmm5		;; Calculate low bits of remainder
	movsd	Q XMM_TMP1, xmm4	;; Save word of zpad / k

	mulsd	xmm1, ZPAD_SHIFT2	;; Shift previous zpad word
	addsd	xmm2, xmm1		;; Add to create new high zpad bits
	mulsd	xmm2, ZPAD_SHIFT1	;; Shift previous zpad word
	addsd	xmm0, xmm2		;; Add to create new high zpad bits
	movsd	ZPAD0, xmm0		;; Save remainder of zpad / k

	subsd	xmm1, xmm1		;; Zero words that other cases set
	movsd	Q XMM_TMP6, xmm1
	
	jmp	div_k_done

	;; This case does the divide by k where k is two words
mediumk:
	movsd	xmm0, ZPAD6		;; Load zpad word (high bits)
	movsd	xmm1, ZPAD5		;; Load zpad word (low bits)
	movsd	xmm4, ZPAD_INVERSE_K6	;; Load shifted 1/k
	mulsd	xmm4, xmm0		;; Mul ZPAD by shifted 1/k
	addsd	xmm4, Q XMM_BIGVAL	;; Round to integer	
	subsd	xmm4, Q XMM_BIGVAL
	movsd	xmm5, ZPAD_K6_HI	;; Load high bits of k
	mulsd	xmm5, xmm4
	subsd	xmm0, xmm5		;; Calculate high bits of remainder
	movsd	xmm5, ZPAD_K6_LO	;; Load low bits of k
	mulsd	xmm5, xmm4
	subsd	xmm1, xmm5		;; Calculate low bits of remainder
	movsd	Q XMM_TMP6, xmm4	;; Save word of zpad / k

	mulsd	xmm0, ZPAD_SHIFT6	;; Shift previous zpad word
	addsd	xmm0, xmm1		;; Add to create new high zpad bits
	movsd	xmm1, ZPAD4		;; Load zpad word (new low bits)
	movsd	xmm4, ZPAD_INVERSE_K5	;; Load shifted 1/k
	mulsd	xmm4, xmm0		;; Mul ZPAD by shifted 1/k
	addsd	xmm4, Q XMM_BIGVAL	;; Round to integer
	subsd	xmm4, Q XMM_BIGVAL
	movsd	xmm5, ZPAD_K5_HI	;; Load high bits of k
	mulsd	xmm5, xmm4
	subsd	xmm0, xmm5		;; Calculate high bits of remainder
	movsd	xmm5, ZPAD_K5_LO	;; Load low bits of k
	mulsd	xmm5, xmm4
	subsd	xmm1, xmm5		;; Calculate low bits of remainder
	movsd	Q XMM_TMP5, xmm4	;; Save word of zpad / k

	mulsd	xmm0, ZPAD_SHIFT5	;; Shift previous zpad word
	addsd	xmm0, xmm1		;; Add to create new high zpad bits
	movsd	xmm1, ZPAD3		;; Load zpad word (new low bits)
	movsd	xmm4, ZPAD_INVERSE_K4	;; Load shifted 1/k
	mulsd	xmm4, xmm0		;; Mul ZPAD by shifted 1/k
	addsd	xmm4, Q XMM_BIGVAL	;; Round to integer
	subsd	xmm4, Q XMM_BIGVAL
	movsd	xmm5, ZPAD_K4_HI	;; Load high bits of k
	mulsd	xmm5, xmm4
	subsd	xmm0, xmm5		;; Calculate high bits of remainder
	movsd	xmm5, ZPAD_K4_LO	;; Load low bits of k
	mulsd	xmm5, xmm4
	subsd	xmm1, xmm5		;; Calculate low bits of remainder
	movsd	Q XMM_TMP4, xmm4	;; Save word of zpad / k

	mulsd	xmm0, ZPAD_SHIFT4	;; Shift previous zpad word
	addsd	xmm0, xmm1		;; Add to create new high zpad bits
	movsd	xmm1, ZPAD2		;; Load zpad word (new low bits)
	movsd	xmm4, ZPAD_INVERSE_K3	;; Load shifted 1/k
	mulsd	xmm4, xmm0		;; Mul ZPAD by shifted 1/k
	addsd	xmm4, Q XMM_BIGVAL	;; Round to integer
	subsd	xmm4, Q XMM_BIGVAL
	movsd	xmm5, ZPAD_K3_HI	;; Load high bits of k
	mulsd	xmm5, xmm4
	subsd	xmm0, xmm5		;; Calculate high bits of remainder
	movsd	xmm5, ZPAD_K3_LO	;; Load low bits of k
	mulsd	xmm5, xmm4
	subsd	xmm1, xmm5		;; Calculate low bits of remainder
	movsd	Q XMM_TMP3, xmm4	;; Save word of zpad / k

	mulsd	xmm0, ZPAD_SHIFT3	;; Shift previous zpad word
	addsd	xmm0, xmm1		;; Add to create new high zpad bits
	movsd	xmm1, ZPAD1		;; Load zpad word (new low bits)
	movsd	xmm4, ZPAD_INVERSE_K2	;; Load shifted 1/k
	mulsd	xmm4, xmm0		;; Mul ZPAD by shifted 1/k
	addsd	xmm4, Q XMM_BIGVAL	;; Round to integer
	subsd	xmm4, Q XMM_BIGVAL
	movsd	xmm5, ZPAD_K2_HI	;; Load high bits of k
	mulsd	xmm5, xmm4
	subsd	xmm0, xmm5		;; Calculate high bits of remainder
	movsd	xmm5, ZPAD_K2_LO	;; Load low bits of k
	mulsd	xmm5, xmm4
	subsd	xmm1, xmm5		;; Calculate low bits of remainder
	movsd	Q XMM_TMP2, xmm4	;; Save word of zpad / k

	mulsd	xmm0, ZPAD_SHIFT2	;; Shift previous zpad word
	addsd	xmm0, xmm1		;; Add to create new high zpad bits
	movsd	xmm1, ZPAD0		;; Load zpad word (new low bits)
	movsd	xmm4, ZPAD_INVERSE_K1	;; Load shifted 1/k
	mulsd	xmm4, xmm0		;; Mul ZPAD by shifted 1/k
	addsd	xmm4, Q XMM_BIGVAL	;; Round to integer
	subsd	xmm4, Q XMM_BIGVAL
	movsd	xmm5, ZPAD_K1_HI	;; Load high bits of k
	mulsd	xmm5, xmm4
	subsd	xmm0, xmm5		;; Calculate high bits of remainder
	movsd	xmm5, ZPAD_K1_LO	;; Load low bits of k
	mulsd	xmm5, xmm4
	subsd	xmm1, xmm5		;; Calculate low bits of remainder
	movsd	Q XMM_TMP1, xmm4	;; Save word of zpad / k

	mulsd	xmm0, ZPAD_SHIFT1	;; Shift previous zpad word
	addsd	xmm0, xmm1		;; Add to create new high zpad bits
	movsd	ZPAD0, xmm0		;; Save remainder of zpad / k

	jmp	div_k_done

	;; This case does the divide by k where k is one word
	;; Assume ZPAD5 and ZPAD6 are zero.
smallk:	movsd	xmm0, ZPAD4		;; Load zpad data
	movsd	xmm4, ZPAD_INVERSE_K1	;; Load by 1/k
	mulsd	xmm4, xmm0		;; Mul ZPAD data by 1/k
	addsd	xmm4, Q XMM_BIGVAL	;; Round to integer	
	subsd	xmm4, Q XMM_BIGVAL
	movsd	Q XMM_TMP5, xmm4	;; Save integer part
	mulsd	xmm4, ZPAD_K1_LO	;; Compute remainder
	subsd	xmm0, xmm4

	mulsd	xmm0, ZPAD_SHIFT4	;; Shift previous zpad word
	addsd	xmm0, ZPAD3		;; Add in zpad data
	movsd	xmm4, ZPAD_INVERSE_K1	;; Load by 1/k
	mulsd	xmm4, xmm0		;; Mul ZPAD data by 1/k
	addsd	xmm4, Q XMM_BIGVAL	;; Round to integer	
	subsd	xmm4, Q XMM_BIGVAL
	movsd	Q XMM_TMP4, xmm4	;; Save integer part
	mulsd	xmm4, ZPAD_K1_LO	;; Compute remainder
	subsd	xmm0, xmm4

	mulsd	xmm0, ZPAD_SHIFT3	;; Shift previous zpad word
	addsd	xmm0, ZPAD2		;; Add in zpad data
	movsd	xmm4, ZPAD_INVERSE_K1	;; Load by 1/k
	mulsd	xmm4, xmm0		;; Mul ZPAD data by 1/k
	addsd	xmm4, Q XMM_BIGVAL	;; Round to integer	
	subsd	xmm4, Q XMM_BIGVAL
	movsd	Q XMM_TMP3, xmm4	;; Save integer part
	mulsd	xmm4, ZPAD_K1_LO	;; Compute remainder
	subsd	xmm0, xmm4

	mulsd	xmm0, ZPAD_SHIFT2	;; Shift previous zpad word
	addsd	xmm0, ZPAD1		;; Add in zpad data
	movsd	xmm4, ZPAD_INVERSE_K1	;; Load by 1/k
	mulsd	xmm4, xmm0		;; Mul ZPAD data by 1/k
	addsd	xmm4, Q XMM_BIGVAL	;; Round to integer	
	subsd	xmm4, Q XMM_BIGVAL
	movsd	Q XMM_TMP2, xmm4	;; Save integer part
	mulsd	xmm4, ZPAD_K1_LO	;; Compute remainder
	subsd	xmm0, xmm4

	mulsd	xmm0, ZPAD_SHIFT1	;; Shift previous zpad word
	addsd	xmm0, ZPAD0		;; Add in zpad data
	movsd	xmm4, ZPAD_INVERSE_K1	;; Load by 1/k
	mulsd	xmm4, xmm0		;; Mul ZPAD data by 1/k
	addsd	xmm4, Q XMM_BIGVAL	;; Round to integer	
	subsd	xmm4, Q XMM_BIGVAL
	movsd	Q XMM_TMP1, xmm4	;; Save integer part
	mulsd	xmm4, ZPAD_K1_LO	;; Compute remainder
	subsd	xmm0, xmm4
	movsd	ZPAD0, xmm0		;; Save remainder

	subsd	xmm1, xmm1		;; Zero words that other cases set
	movsd	Q XMM_TMP6, xmm1

div_k_done:

	;; Now normalize the data above the halfway point.

	movzx	rax, BYTE PTR [rdi+1]	;; First biglit flag 
	movsd	xmm0, ZPAD0		;; Load remainder of divide by k
	addsd	xmm0, Q XMM_BIGVAL
	single_rounding base2, xmm0, xmm2, xmm4, rax*4
	movsd	Q [rsi+0*64+32], xmm0	;; Save value1

	movzx	rax, BYTE PTR [rdi+3]	;; Load big vs. little flags
	single_rounding base2, xmm2, xmm0, xmm4, rax*4
	movsd	Q [rsi+1*64+32], xmm2	;; Save value2

	mov	ecx, BIGLIT_INCR2	;; Different clm values step through
					;; big/lit array differently
	movzx	rax, BYTE PTR [rdi+rcx+1] ;; Load big vs. little flags
	single_rounding base2, xmm0, xmm2, xmm4, rax*4
	movsd	Q [rsi+2*64+32], xmm0	;; Save value3

	subsd	xmm2, Q XMM_BIGVAL	;; Remove integer rounding constant
	movsd	Q [rsi+3*64+32], xmm2	;; Save new value4

	;; Mul the integer part of (ZPAD data divided by k) by -c in
	;; preparation for adding it into the lower FFT data area.
	;; Also add in the shifted high FFT carry at this time.

	;; Now add in and normalize the bottom FFT data.  Remember that the
	;; column two-to-phi multiplier for the first value will be 1.0.  We 
	;; must go 6 words deep in case k is 48-50 bits and c is 32 bits.

	movzx	rax, BYTE PTR [rdi+1]	;; First word 
	movsd	xmm0, Q XMM_TMP1	;; Load integer part of divide by k
	mulsd	xmm0, Q XMM_MINUS_C	;; Mul by -c
	addsd	xmm0, Q XMM_BIGVAL
	addsd	xmm0, Q [rsi+0*64]	;; Add in the FFT data
	single_rounding base2, xmm0, xmm2, xmm4, rax*4
	movsd	Q [rsi+0*64], xmm0	;; Save value1

	movzx	rax, BYTE PTR [rdi+3]	;; Load big vs. little flags
	movsd	xmm0, Q XMM_TMP2	;; Load integer part of divide by k
	mulsd	xmm0, Q XMM_MINUS_C	;; Mul by -c
	movsd	xmm1, Q [rsi+1*64]	;; Load FFT data
	addsd	xmm0, xmm1		;; Add in the FFT data
	addsd	xmm0, xmm2		;; x2 = value + carry
	single_rounding base2, xmm0, xmm2, xmm4, rax*4
	movsd	Q [rsi+1*64], xmm0	;; Save value2

	mov	ecx, BIGLIT_INCR2	;; Different clm values step through
					;; big/lit array differently
	movzx	rax, BYTE PTR [rdi+rcx+1] ;; Load big vs. little flags
	movsd	xmm0, Q XMM_TMP3	;; Load integer part of divide by k
	mulsd	xmm0, Q XMM_MINUS_C	;; Mul by -c
	movsd	xmm1, Q [rsi+2*64]	;; Load FFT data
	addsd	xmm0, xmm1		;; Add in the FFT data
	addsd	xmm0, xmm2		;; x3 = value + carry
	single_rounding base2, xmm0, xmm2, xmm4, rax*4
	movsd	Q [rsi+2*64], xmm0	;; Save value3

	movzx	rax, BYTE PTR [rdi+rcx+3] ;; Load big vs. little flags
	movsd	xmm0, Q XMM_TMP4	;; Load integer part of divide by k
	mulsd	xmm0, Q XMM_MINUS_C	;; Mul by -c
	movsd	xmm1, Q [rsi+3*64]	;; Load FFT data
	addsd	xmm0, xmm1		;; Add in the FFT data
	addsd	xmm0, xmm2		;; x4 = value + carry
	single_rounding base2, xmm0, xmm2, xmm4, rax*4
	movsd	Q [rsi+3*64], xmm0	;; Save value4

	mov	ecx, BIGLIT_INCR4	;; Different clm values step through
					;; big/lit array differently
	movzx	rax, BYTE PTR [rdi+rcx+1] ;; Load big vs. little flags
	movsd	xmm0, Q XMM_TMP5	;; Load integer part of divide by k
	mulsd	xmm0, Q XMM_MINUS_C	;; Mul by -c
	movsd	xmm1, Q [rsi+4*64]	;; Load FFT data
	addsd	xmm0, xmm1		;; Add in the FFT data
	addsd	xmm0, xmm2		;; x4 = value + carry
	single_rounding base2, xmm0, xmm2, xmm4, rax*4
	movsd	Q [rsi+4*64], xmm0	;; Save value4

	movzx	rax, BYTE PTR [rdi+rcx+3] ;; Load big vs. little flags
	movsd	xmm0, Q XMM_TMP6	;; Load integer part of divide by k
	mulsd	xmm0, Q XMM_MINUS_C	;; Mul by -c
	movsd	xmm1, Q [rsi+5*64]	;; Load FFT data
	addsd	xmm0, xmm1		;; Add in the FFT data
	addsd	xmm0, xmm2		;; x5 = value + carry
	single_rounding base2, xmm0, xmm2, xmm4, rax*4
	movsd	Q [rsi+5*64], xmm0	;; Save value5

	subsd	xmm2, Q XMM_BIGVAL	;; Remove rounding constant
	addsd	xmm2, Q [rsi+6*64]	;; Add in FFT data
	movsd	Q [rsi+6*64], xmm2	;; Save value6
	ENDM
