; Copyright 2011-2025 - Mersenne Research, Inc.  All rights reserved.
; Author:  George Woltman
; Email: woltman@alum.mit.edu
;
; These macros efficiently implement the normalization to integers
; and multiplication by two-to-phi powers using AVX-512F instructions
; for zero-padded FFTs.
;


; See "carry philosophy" in znormal.mac.


; Zero padded FFTs can be used for calculations mod k*b^n+c where k and/or c are too large to use a traditional
; discrete weighted transform.  We use an FFT that calculates modulo b^(2n)-1.  The top half of the FFT is zero
; except for the lowest 4 words which represent a value between 0 and k.
;
; Note that to store a value x in the FFT, we actually store the value x/k mod k*b^n+c.  When we square or multiply
; a number we get a result that is maultiplied by 1/k^2.  We transform this back to 1/k format by doing a multiply
; by k in the normalization routines that round to integers and propagate carries.  Also note that result values in
; the upper half wrap to the lower half by multipling by -c and dividing by k -- the multiply by k and divide by k
; cancel out when normalizing upper half results!
;
; Before doing an FFT squaring or multiplication the 4 non-zero words in the upper half and the top 3 words in the
; lower half are saved for later special post-processing.  After the FFT squaring or multiplication, we must undo
; the 7 values that were in excess of b^(2n) that wrapped around and were added to the lowest values in the result.
; These 7 values in excess of b^(2n) are computed from the 7 words we saved prior to squaring or multiplication.
; Instead, those 7 values are wrapped to upper half by multiplying by -c.
;
; And finally, the 7 values in the upper half are divided by k.  The remainder is stored in 4 words of the upper half.
; The quotient is multiplied by -c and wraps to the lower half.

;;
;; During zero-padded normalization we must be wary about precision issues.  This table summarizes the maximum precisions
;; we are likely to run into during calculations.
;;	k is 51ish-bits
;;	c is 25ish-bits
;;	mul-by-const is 25ish-bits
;;	zpad0-6 are 43ish bits (zpad0-6 are the 7 values that are saved for post-processing after a multiply)
;;	low-carry is 36ish bits (low-carry is the not-yet-multipled-by-k and not-yet-multiplied-by-smallmul-const carry)
;;	high-carry is 52ish bits (is a traditional carry)
;;


; Utility macros used in normalization macros


	;; BUG/OPT - change ttp case to use +0 carry format
split_zpad7_word MACRO ttp, xmmval, xmmcarry
ttp	vsubsd	xmmval, xmmval, xmm30			;; val = val - RNDVAL
ttp	vblendmpd zmm13 {k1}, zmm28, zmm26		;; Create (1/base) constant
ttp	zfmaddsd xmmcarry, xmmval, xmm13, xmm30		;; val_hi+RNDVAL = val / base + RNDVAL (next carry)
ttp	vblendmpd zmm12 {k1}, zmm29, zmm27		;; Create (base) constant
ttp	vsubsd	xmm14, xmmcarry, xmm30			;; val_hi = val_hi+RNDVAL - RNDVAL
ttp	zfnmaddsd xmmval, xmm14, xmm12, xmmval		;; val_lo = val - val_hi * base = val%base
no ttp	zfmsubsd xmmcarry, xmmval, xmm28, xmm26		;; val_hi+RNDVAL = (val+RNDVAL) / base - (RNDVAL/base-RNDVAL) = rnd(val/base)+RNDVAL
no ttp	zfmsubsd xmm14, xmmcarry, xmm29, xmm27		;; tmp+RNDVAL = (val_hi+RNDVAL) * base - (RNDVAL*base-RNDVAL) = rnd(val/base)*base+RNDVAL
no ttp	vsubsd	xmmval, xmmval, xmm14			;; val_lo = val+RNDVAL - tmp+RNDVAL = val - rnd(val/base)*base = val%base
	ENDM

split_zpad7_again MACRO ttp, xmmval, xmmcarry		;; Split a second time, we've already built the ttp masks
ttp	vsubsd	xmmval, xmmval, xmm30			;; val = val - RNDVAL
ttp	zfmaddsd xmmcarry, xmmval, xmm13, xmm30		;; val_hi+RNDVAL = val / base + RNDVAL (next carry)
ttp	vsubsd	xmm14, xmmcarry, xmm30			;; val_hi = val_hi+RNDVAL - RNDVAL
ttp	zfnmaddsd xmmval, xmm14, xmm12, xmmval		;; val_lo = val - val_hi * base = val%base
no ttp	zfmsubsd xmmcarry, xmmval, xmm28, xmm26		;; val_hi+RNDVAL = (val+RNDVAL) / base - (RNDVAL/base-RNDVAL) = rnd(val/base)+RNDVAL
no ttp	zfmsubsd xmm14, xmmcarry, xmm29, xmm27		;; tmp+RNDVAL = (val_hi+RNDVAL) * base - (RNDVAL*base-RNDVAL) = rnd(val/base)*base+RNDVAL
no ttp	vsubsd	xmmval, xmmval, xmm14			;; val_lo = val+RNDVAL - tmp+RNDVAL = val - rnd(val/base)*base = val%base
	ENDM

;; Same as above except we know that first zpad word in an irrational FFT is a large word and the first zpad word in a rational FFT is a small word.
split_first_zpad7_word MACRO ttp, xmmval, xmmcarry
ttp	vsubsd	xmmval, xmmval, xmm30			;; val = val - RNDVAL
ttp	zfmaddsd xmmcarry, xmmval, xmm26, xmm30		;; val_hi+RNDVAL = val / base + RNDVAL (next carry)
ttp	vsubsd	xmm14, xmmcarry, xmm30			;; val_hi = val_hi+RNDVAL - RNDVAL
ttp	zfnmaddsd xmmval, xmm14, xmm27, xmmval		;; val_lo = val - val_hi * base = val%base
no ttp	zfmsubsd xmmcarry, xmmval, xmm28, xmm26		;; val_hi+RNDVAL = (val+RNDVAL) / base - (RNDVAL/base-RNDVAL) = rnd(val/base)+RNDVAL
no ttp	zfmsubsd xmm14, xmmcarry, xmm29, xmm27		;; tmp+RNDVAL = (val_hi+RNDVAL) * base - (RNDVAL*base-RNDVAL) = rnd(val/base)*base+RNDVAL
no ttp	vsubsd	xmmval, xmmval, xmm14			;; val_lo = val+RNDVAL - tmp+RNDVAL = val - rnd(val/base)*base = val%base
	ENDM


; These are the normalization macros to use when we are computing modulo k*b^n+c
; with a zero-padded b^2n FFT.  We do this by multiplying the lower FFT
; word by k and adding in the upper word times -c.  Of course, this is made
; very tedious because we have to carefully avoid any loss of precision.
;
; Here is the general algorithm.  Let val1 be the FFT word from the lower half.
; Let val2 be the corresponding FFT word from the upper half.  We want to calculate
; new val1 = val1 * k * mul-by-const + val2 * -c * mul-by-const, and new val2 = 0
;
; To do this without overflowing 53-bits of precision, we pre-calculate k * mul-by-const
; and -c * mul-by-const.  k * mul-by-const is split into k_hi and k_lo, -c * mul-by-const
; is not split WHICH PLACES CONSTRAINTS ON THE SIZE OF c and mul-by-const.  That is,
; c * mul-by-const * big-base must not exceed 51-ish bits.
;
; val1 += carry1
; Split val1 and val2 into val1_lo, val1_hi, val2_lo, and val2_hi.
;
; next carry1 = val1_hi 
; x = val1_lo * k_lo*mulconst + val2_lo * -c*mulconst + carry2
; y = val1_lo * k_hi*mulconst + val2_hi * -c*mulconst
;
; new val1 = x % base
; new carry2 = y + x / base
;
; Translating the above into actual instructions
;
; FMA 1-4	val1+RNDVAL = val1 * ttmp + (carry1 + RNDVAL)
; FMA 5-8	next carry1+RNDVAL = val1_hi+RNDVAL = (val1+RNDVAL) / base - (RNDVAL / base - RNDVAL) = rnd(val1/base)+RNDVAL
; FMA 9-12	tmp1+RNDVAL = (val1_hi+RNDVAL) * base - (RNDVAL * base - RNDVAL) = rnd(val1/base)*base+RNDVAL
; SUB 13-16	val1_lo = val1+RNDVAL - tmp1+RNDVAL = val1 - rnd(val1/base)*base = val1%base
;
; FMA 1-4	val2+RNDVAL = val2 * ttmp + RNDVAL
; FMA 5-8	val2_hi+RNDVAL = (val2+RNDVAL) / base - (RNDVAL / base - RNDVAL) = rnd(val2/base)+RNDVAL
; FMA 9-12	tmp2+RNDVAL = (val2_hi+RNDVAL) * base - (RNDVAL * base - RNDVAL) = rnd(val2/base)*base+RNDVAL
; SUB 10-13	val2_hi = val2_hi+RNDVAL - RNDVAL
; SUB 13-16	val2_lo = val2+RNDVAL - tmp2+RNDVAL = val2 - rnd(val2/base)*base = val2%base
;
; if echk err1 = val1+RNDVAL - carry+RNDVAL;;  err1 = val1 * ttp - err1; absval; maxval
; if echk err2 = val2+RNDVAL - RNDVAL;  FMA for -= val1*ttp; absval; maxval
;
; FMA 17-20	x+RNDVAL = val1_lo * k_lo*mulconst + carry2+RNDVAL
; FMA 21-24	x+RNDVAL += val2_lo * -c*mulconst
;
; FMA 25-28	x_hi+RNDVAL = (x+RNDVAL) / base - (RNDVAL / base - RNDVAL) = rnd(x/base)+RNDVAL
; FMA 29-32	tmp3+RNDVAL = (x_hi+RNDVAL) * base - (RNDVAL * base - RNDVAL) = rnd(x/base)*base+RNDVAL
; SUB 33-36	new val1 = x_lo = x+RNDVAL - tmp3+RNDVAL = x - rnd(x/base)*base = x%base
;
; FMA 29-32	next carry2 = val2_hi * -c*mulconst + x_hi+RNDVAL
; FMA 33-36	if khi	next carry2 = val1_lo * k_hi*mulconst/base + next carry2
;
; MUL 37-40	new val1 = new val1 * ttp


;;*******************************************************************************************
;;			    Macros for traditional one pass FFTs
;;*******************************************************************************************

; These macros are nearly identical to the two pass _wpn variants

; This is the normalization routine when we are computing modulo k*b^n+c
; with a zero-padded b^2n FFT.  We do this by multiplying the lower FFT
; word by k and adding in the upper word times -c.  Of course, this is made
; very tedious because we have to carefully avoid any loss of precision.
;
;; NOTE: In zero pad FFTs, big/lit and fudge factor flags for the low and high words are identical

; For 1D zpad macros, these registers are set on input:
; rsi = pointer to the FFT data (source #1)
; r13 = distance to second FFT section data
; r14 = distance to fourth FFT section data
; rdi = pointer to big/little flags
; r12 = compressed biglit table
; r10 = pointer into inverse weights table
; rdx = register to load biglit index
; zmm0-zmm3 = carry in +RNDVAL format for the lower FFT word (not yet multiplied by k)
; zmm4-zmm7 = true carry for adding into the next FFT word.  In +RNDVAL format for rational FFTs and +0 format for irrational FFTs.
; zmm26-29 = available for preload (preloads must be compatible with zadd_carry_1d_zpad macros)
; zmm30 = RNDVAL
; zmm31 = maxerr

znorm_1d_zpad_preload MACRO ttp, echk, const, khi
no echk no ttp	znorm_1d_zpad_noechk_nottp_preload const, khi
   echk no ttp	znorm_1d_zpad_echk_nottp_preload const, khi
no echk    ttp	znorm_1d_zpad_noechk_ttp_preload const, khi
   echk    ttp	znorm_1d_zpad_echk_ttp_preload const, khi
	ENDM

znorm_1d_zpad MACRO ttp, echk, const, khi
;	L1prefetchw rsi+128, L1PREFETCH_ALL
;	L1prefetchw rsi+r13+128, L1PREFETCH_ALL
;	L1prefetchw rsi+2*r13+128, L1PREFETCH_ALL
;	L1prefetchw rsi+r14+128, L1PREFETCH_ALL
no echk no ttp	znorm_1d_zpad_noechk_nottp const, khi
   echk no ttp	znorm_1d_zpad_echk_nottp const, khi
no echk    ttp	znorm_1d_zpad_noechk_ttp const, khi
   echk    ttp	znorm_1d_zpad_echk_ttp const, khi
	ENDM

;; Rational cases differ from znorm_wpn_zpad in that inverse weights (usually containing 2/FFTlen) must be applied

znorm_1d_zpad_noechk_nottp_preload MACRO const, khi
		vbroadcastsd zmm30, ZMM_RNDVAL			;; RNDVAL
		vbroadcastsd zmm29, ZMM_SMALL_BASE		;; small_word_base
		vbroadcastsd zmm28, ZMM_SMALL_BASE_INVERSE	;; 1 / small_word_base
		vbroadcastsd zmm27, ZMM_RNDVAL_TIMES_SMALL_BASE	;; RNDVAL * small_word_base - RNDVAL
		vbroadcastsd zmm26, ZMM_RNDVAL_OVER_SMALL_BASE	;; RNDVAL / small_word_base - RNDVAL
no const	vbroadcastsd zmm25, ZMM_MINUS_C
const		vbroadcastsd zmm25, ZMM_MINUS_C_TIMES_MULCONST
no const	vbroadcastsd zmm24, ZMM_K_LO			;; k_lo
const		vbroadcastsd zmm24, ZMM_K_TIMES_MULCONST_LO
khi no const	vbroadcastsd zmm31, ZMM_K_HI_OVER_SMALL_BASE	;; k_hi / small_word_base
khi const	vbroadcastsd zmm31, ZMM_K_TIMES_MULCONST_HI_OVER_SMALL_BASE
	ENDM
znorm_1d_zpad_noechk_nottp MACRO const, khi
	vmovapd zmm8, [r10+0*64]				;; Inverse weight
	zfmaddpd zmm12, [rsi+64], zmm8, zmm30		; 1-4	;; val5+RNDVAL = value5*inv_weight + RNDVAL
	vmovapd zmm9, [r10+1*64]				;; Inverse weight
	zfmaddpd zmm13, [rsi+r13+64], zmm9, zmm30	; 1-4	;; val6+RNDVAL = value6*inv_weight + RNDVAL
	vmovapd zmm10, [r10+2*64]				;; Inverse weight
	zfmaddpd zmm14, [rsi+2*r13+64], zmm10, zmm30	; 2-5	;; val7+RNDVAL = value7*inv_weight + RNDVAL
	vmovapd zmm11, [r10+3*64]				;; Inverse weight
	zfmaddpd zmm15, [rsi+r14+64], zmm11, zmm30	; 2-5	;; val8+RNDVAL = value8*inv_weight + RNDVAL
	zfmaddpd zmm8, [rsi], zmm8, zmm0		; 3-6	;; val1+RNDVAL = value1*inv_weight + carry1+RNDVAL
	zfmaddpd zmm9, [rsi+r13], zmm9, zmm1		; 3-6	;; val2+RNDVAL = value2*inv_weight + carry2+RNDVAL
	zfmaddpd zmm10, [rsi+2*r13], zmm10, zmm2	; 4-7	;; val3+RNDVAL = value3*inv_weight + carry3+RNDVAL
	zfmaddpd zmm11, [rsi+r14], zmm11, zmm3		; 4-7	;; val4+RNDVAL = value4*inv_weight + carry4+RNDVAL
	zfmsubpd zmm16, zmm12, zmm28, zmm26		; 5-8	;; val5_hi+RNDVAL = (val5+RNDVAL) / base - (RNDVAL/base-RNDVAL) = rnd(val5/base)+RNDVAL
	zfmsubpd zmm17, zmm13, zmm28, zmm26		; 5-8	;; val6_hi+RNDVAL = (val6+RNDVAL) / base - (RNDVAL/base-RNDVAL) = rnd(val6/base)+RNDVAL
	zfmsubpd zmm18, zmm14, zmm28, zmm26		; 6-9	;; val7_hi+RNDVAL = (val7+RNDVAL) / base - (RNDVAL/base-RNDVAL) = rnd(val7/base)+RNDVAL
	zfmsubpd zmm19, zmm15, zmm28, zmm26		; 6-9	;; val8_hi+RNDVAL = (val8+RNDVAL) / base - (RNDVAL/base-RNDVAL) = rnd(val8/base)+RNDVAL
	zfmsubpd zmm0, zmm8, zmm28, zmm26		; 7-10	;; val1_hi+RNDVAL = (val1+RNDVAL) / base - (RNDVAL/base-RNDVAL) = rnd(val1/base)+RNDVAL (next carry1)
	zfmsubpd zmm1, zmm9, zmm28, zmm26		; 7-10	;; val2_hi+RNDVAL = (val2+RNDVAL) / base - (RNDVAL/base-RNDVAL) = rnd(val2/base)+RNDVAL (next carry2)
	zfmsubpd zmm2, zmm10, zmm28, zmm26		; 8-11	;; val3_hi+RNDVAL = (val3+RNDVAL) / base - (RNDVAL/base-RNDVAL) = rnd(val3/base)+RNDVAL (next carry3)
	zfmsubpd zmm3, zmm11, zmm28, zmm26		; 8-11	;; val4_hi+RNDVAL = (val4+RNDVAL) / base - (RNDVAL/base-RNDVAL) = rnd(val4/base)+RNDVAL (next carry4)
	zfmsubpd zmm20, zmm16, zmm29, zmm27		; 9-12	;; tmp5+RNDVAL = (val5_hi+RNDVAL) * base - (RNDVAL*base-RNDVAL) = rnd(val5/base)*base+RNDVAL
	zfmsubpd zmm21, zmm17, zmm29, zmm27		; 9-12	;; tmp6+RNDVAL = (val6_hi+RNDVAL) * base - (RNDVAL*base-RNDVAL) = rnd(val6/base)*base+RNDVAL
	zfmsubpd zmm22, zmm18, zmm29, zmm27		; 10-13	;; tmp7+RNDVAL = (val7_hi+RNDVAL) * base - (RNDVAL*base-RNDVAL) = rnd(val7/base)*base+RNDVAL
	zfmsubpd zmm23, zmm19, zmm29, zmm27		; 10-13	;; tmp8+RNDVAL = (val8_hi+RNDVAL) * base - (RNDVAL*base-RNDVAL) = rnd(val8/base)*base+RNDVAL
	vsubpd	zmm16, zmm16, zmm30			; 11-14	;; val5_hi = val5_hi+RNDVAL - RNDVAL
	vsubpd	zmm17, zmm17, zmm30			; 11-14	;; val6_hi = val6_hi+RNDVAL - RNDVAL
	vsubpd	zmm18, zmm18, zmm30			; 12-15	;; val7_hi = val7_hi+RNDVAL - RNDVAL
	vsubpd	zmm19, zmm19, zmm30			; 12-15	;; val8_hi = val8_hi+RNDVAL - RNDVAL
	vsubpd	zmm12, zmm12, zmm20			; 13-16	;; val5_lo = val5+RNDVAL - tmp5+RNDVAL = val5 - rnd(val5/base)*base = val5%base
	vsubpd	zmm13, zmm13, zmm21			; 13-16	;; val6_lo = val6+RNDVAL - tmp6+RNDVAL = val6 - rnd(val6/base)*base = val6%base
	vsubpd	zmm14, zmm14, zmm22			; 14-17	;; val7_lo = val7+RNDVAL - tmp7+RNDVAL = val7 - rnd(val7/base)*base = val7%base
	vsubpd	zmm15, zmm15, zmm23			; 14-17	;; val8_lo = val8+RNDVAL - tmp8+RNDVAL = val8 - rnd(val8/base)*base = val8%base
	zfmsubpd zmm20, zmm0, zmm29, zmm27		; 15-18	;; tmp1+RNDVAL = (val1_hi+RNDVAL) * base - (RNDVAL*base-RNDVAL) = rnd(val1/base)*base+RNDVAL
	zfmsubpd zmm21, zmm1, zmm29, zmm27		; 15-18	;; tmp2+RNDVAL = (val2_hi+RNDVAL) * base - (RNDVAL*base-RNDVAL) = rnd(val2/base)*base+RNDVAL
	zfmsubpd zmm22, zmm2, zmm29, zmm27		; 16-19	;; tmp3+RNDVAL = (val3_hi+RNDVAL) * base - (RNDVAL*base-RNDVAL) = rnd(val3/base)*base+RNDVAL
	zfmsubpd zmm23, zmm3, zmm29, zmm27		; 16-19	;; tmp4+RNDVAL = (val4_hi+RNDVAL) * base - (RNDVAL*base-RNDVAL) = rnd(val4/base)*base+RNDVAL
	zfmaddpd zmm12, zmm12, zmm25, zmm4		; 17-20	;; x1+RNDVAL = val5_lo * -c*mulconst + carry5+RNDVAL
	zfmaddpd zmm13, zmm13, zmm25, zmm5		; 17-20	;; x2+RNDVAL = val6_lo * -c*mulconst + carry6+RNDVAL
	zfmaddpd zmm14, zmm14, zmm25, zmm6		; 18-21	;; x3+RNDVAL = val7_lo * -c*mulconst + carry7+RNDVAL
	zfmaddpd zmm15, zmm15, zmm25, zmm7		; 18-21	;; x4+RNDVAL = val8_lo * -c*mulconst + carry8+RNDVAL
	vsubpd	zmm8, zmm8, zmm20			; 19-22	;; val1_lo = val1+RNDVAL - tmp1+RNDVAL = val1 - rnd(val1/base)*base = val1%base
	vsubpd	zmm9, zmm9, zmm21			; 19-22	;; val2_lo = val2+RNDVAL - tmp2+RNDVAL = val2 - rnd(val2/base)*base = val2%base
	vsubpd	zmm10, zmm10, zmm22			; 20-23	;; val3_lo = val3+RNDVAL - tmp3+RNDVAL = val3 - rnd(val3/base)*base = val3%base
	vsubpd	zmm11, zmm11, zmm23			; 20-23	;; val4_lo = val4+RNDVAL - tmp4+RNDVAL = val4 - rnd(val4/base)*base = val4%base
	zfmaddpd zmm12, zmm8, zmm24, zmm12		; 23-26 ;; x1+RNDVAL += val1_lo * k_lo*mulconst
	zfmaddpd zmm13, zmm9, zmm24, zmm13		; 23-26 ;; x2+RNDVAL += val2_lo * k_lo*mulconst
	zfmaddpd zmm14, zmm10, zmm24, zmm14		; 24-27 ;; x3+RNDVAL += val3_lo * k_lo*mulconst
	zfmaddpd zmm15, zmm11, zmm24, zmm15		; 24-27 ;; x4+RNDVAL += val4_lo * k_lo*mulconst
	zfmsubpd zmm4, zmm12, zmm28, zmm26		; 27-30 ;; x1_hi+RNDVAL = (x1+RNDVAL) / base - (RNDVAL/base-RNDVAL) = rnd(x1/base)+RNDVAL
	zfmsubpd zmm5, zmm13, zmm28, zmm26		; 27-30 ;; x2_hi+RNDVAL = (x2+RNDVAL) / base - (RNDVAL/base-RNDVAL) = rnd(x2/base)+RNDVAL
	zfmsubpd zmm6, zmm14, zmm28, zmm26		; 28-31 ;; x3_hi+RNDVAL = (x3+RNDVAL) / base - (RNDVAL/base-RNDVAL) = rnd(x3/base)+RNDVAL
	zfmsubpd zmm7, zmm15, zmm28, zmm26		; 28-31 ;; x4_hi+RNDVAL = (x4+RNDVAL) / base - (RNDVAL/base-RNDVAL) = rnd(x4/base)+RNDVAL
	zfmsubpd zmm20, zmm4, zmm29, zmm27		; 31-34 ;; tmp9+RNDVAL = (x1_hi+RNDVAL) * base - (RNDVAL*base-RNDVAL) = rnd(x1/base)*base+RNDVAL
	zfmsubpd zmm21, zmm5, zmm29, zmm27		; 31-34 ;; tmp10+RNDVAL = (x2_hi+RNDVAL) * base - (RNDVAL*base-RNDVAL) = rnd(x2/base)*base+RNDVAL
	zfmsubpd zmm22, zmm6, zmm29, zmm27		; 32-35 ;; tmp11+RNDVAL = (x3_hi+RNDVAL) * base - (RNDVAL*base-RNDVAL) = rnd(x3/base)*base+RNDVAL
	zfmsubpd zmm23, zmm7, zmm29, zmm27		; 32-35 ;; tmp12+RNDVAL = (x4_hi+RNDVAL) * base - (RNDVAL*base-RNDVAL) = rnd(x4/base)*base+RNDVAL
	zfmaddpd zmm4, zmm16, zmm25, zmm4		; 33-36 ;; next carry5 = val5_hi * -c*mulconst + x1_hi+RNDVAL
	zfmaddpd zmm5, zmm17, zmm25, zmm5		; 33-36 ;; next carry6 = val6_hi * -c*mulconst + x2_hi+RNDVAL
	zfmaddpd zmm6, zmm18, zmm25, zmm6		; 34-37 ;; next carry7 = val7_hi * -c*mulconst + x3_hi+RNDVAL
	zfmaddpd zmm7, zmm19, zmm25, zmm7		; 34-37 ;; next carry8 = val8_hi * -c*mulconst + x4_hi+RNDVAL
	vsubpd	zmm12, zmm12, zmm20			; 35-38 ;; new val1 = x1_lo = x1+RNDVAL - tmp9+RNDVAL = x1 - rnd(x1/base)*base = x1%base
	vsubpd	zmm13, zmm13, zmm21			; 35-38 ;; new val2 = x2_lo = x2+RNDVAL - tmp10+RNDVAL = x2 - rnd(x2/base)*base = x2%base
	vsubpd	zmm14, zmm14, zmm22			; 36-39 ;; new val3 = x3_lo = x3+RNDVAL - tmp11+RNDVAL = x1 - rnd(x3/base)*base = x3%base
	vsubpd	zmm15, zmm15, zmm23			; 36-39 ;; new val4 = x4_lo = x4+RNDVAL - tmp12+RNDVAL = x2 - rnd(x4/base)*base = x4%base
khi	zfmaddpd zmm4, zmm8, zmm31, zmm4		; 37-40 ;; if khi next carry5 = val1_lo * k_hi*mulconst/base + next carry5
khi	zfmaddpd zmm5, zmm9, zmm31, zmm5		; 37-40 ;; if khi next carry6 = val2_lo * k_hi*mulconst/base + next carry6
khi	zfmaddpd zmm6, zmm10, zmm31, zmm6		; 38-41 ;; if khi next carry7 = val3_lo * k_hi*mulconst/base + next carry7
khi	zfmaddpd zmm7, zmm11, zmm31, zmm7		; 38-41 ;; if khi next carry8 = val4_lo * k_hi*mulconst/base + next carry8

	zstore	[rsi], zmm12					;; Save value1
	zstore	[rsi+r13], zmm13				;; Save value2
	zstore	[rsi+2*r13], zmm14				;; Save value3
	zstore	[rsi+r14], zmm15				;; Save value4
	vzero	zmm8
	zstore	[rsi+64], zmm8					;; Save value5
	zstore	[rsi+r13+64], zmm8				;; Save value6
	zstore	[rsi+2*r13+64], zmm8				;; Save value7
	zstore	[rsi+r14+64], zmm8				;; Save value8
	ENDM

znorm_1d_zpad_echk_nottp_preload MACRO const, khi
		vbroadcastsd zmm30, ZMM_RNDVAL			;; RNDVAL
		vbroadcastsd zmm29, ZMM_SMALL_BASE		;; small_word_base
		vbroadcastsd zmm28, ZMM_SMALL_BASE_INVERSE	;; 1 / small_word_base
		vbroadcastsd zmm27, ZMM_RNDVAL_TIMES_SMALL_BASE	;; RNDVAL * small_word_base - RNDVAL
		vbroadcastsd zmm26, ZMM_RNDVAL_OVER_SMALL_BASE	;; RNDVAL / small_word_base - RNDVAL
no const	vbroadcastsd zmm25, ZMM_MINUS_C
const		vbroadcastsd zmm25, ZMM_MINUS_C_TIMES_MULCONST
no const	vbroadcastsd zmm24, ZMM_K_LO
const		vbroadcastsd zmm24, ZMM_K_TIMES_MULCONST_LO
	ENDM
znorm_1d_zpad_echk_nottp MACRO const, khi
	vmovapd zmm20, [r10+0*64]				;; Inverse weight
	zfmaddpd zmm8, [rsi], zmm20, zmm0		; 1-4	;; val1+RNDVAL = value1*inv_weight + carry1+RNDVAL
	vmovapd zmm21, [r10+1*64]				;; Inverse weight
	zfmaddpd zmm9, [rsi+r13], zmm21, zmm1		; 1-4	;; val2+RNDVAL = value2*inv_weight + carry2+RNDVAL
	vmovapd zmm22, [r10+2*64]				;; Inverse weight
	zfmaddpd zmm10, [rsi+2*r13], zmm22, zmm2	; 2-5	;; val3+RNDVAL = value3*inv_weight + carry3+RNDVAL
	vmovapd zmm23, [r10+3*64]				;; Inverse weight
	zfmaddpd zmm11, [rsi+r14], zmm23, zmm3		; 2-5	;; val4+RNDVAL = value4*inv_weight + carry4+RNDVAL
	zfmaddpd zmm12, [rsi+64], zmm20, zmm30		; 3-6	;; val5+RNDVAL = value5*inv_weight + RNDVAL
	zfmaddpd zmm13, [rsi+r13+64], zmm21, zmm30	; 3-6	;; val6+RNDVAL = value6*inv_weight + RNDVAL
	zfmaddpd zmm14, [rsi+2*r13+64], zmm22, zmm30	; 4-7	;; val7+RNDVAL = value7*inv_weight + RNDVAL
	zfmaddpd zmm15, [rsi+r14+64], zmm23, zmm30	; 4-7	;; val8+RNDVAL = value8*inv_weight + RNDVAL

	vsubpd	zmm16, zmm8, zmm0			; 5-8	;; tmp = val1+RNDVAL - carry+RNDVAL = rnd(value*inv_weight)
	vsubpd	zmm17, zmm9, zmm1			; 5-8	;; tmp = val2+RNDVAL - carry+RNDVAL = rnd(value*inv_weight)
	vsubpd	zmm18, zmm10, zmm2			; 6-9	;; tmp = val3+RNDVAL - carry+RNDVAL = rnd(value*inv_weight)
	vsubpd	zmm19, zmm11, zmm3			; 6-9	;; tmp = val4+RNDVAL - carry+RNDVAL = rnd(value*inv_weight)
	vsubpd	zmm0, zmm12, zmm30			; 7-10	;; tmp = val5+RNDVAL - RNDVAL = rnd(value*inv_weight)
	vsubpd	zmm1, zmm13, zmm30			; 7-10	;; tmp = val6+RNDVAL - RNDVAL = rnd(value*inv_weight)
	vsubpd	zmm2, zmm14, zmm30			; 8-11	;; tmp = val7+RNDVAL - RNDVAL = rnd(value*inv_weight)
	vsubpd	zmm3, zmm15, zmm30			; 8-11	;; tmp = val8+RNDVAL - RNDVAL = rnd(value*inv_weight)

	zfnmaddpd zmm16, [rsi], zmm20, zmm16		; 9-12	;; err = tmp - value*inv_weight
	zfnmaddpd zmm17, [rsi+r13], zmm21, zmm17	; 9-12	;; err = tmp - value*inv_weight
	zfnmaddpd zmm18, [rsi+2*r13], zmm22, zmm18	; 10-13	;; err = tmp - value*inv_weight
	zfnmaddpd zmm19, [rsi+r14], zmm23, zmm19	; 10-13	;; err = tmp - value*inv_weight
	zfnmaddpd zmm20, [rsi+64], zmm20, zmm0		; 11-14	;; err = tmp - value*inv_weight
	zfnmaddpd zmm21, [rsi+r13+64], zmm21, zmm1	; 11-14	;; err = tmp - value*inv_weight
	zfnmaddpd zmm22, [rsi+2*r13+64], zmm22, zmm2	; 12-15	;; err = tmp - value*inv_weight
	zfnmaddpd zmm23, [rsi+r14+64], zmm23, zmm3	; 12-15	;; err = tmp - value*inv_weight

	vbroadcastsd zmm0, ZMM_ABSVAL
	vpandq	zmm16, zmm16, zmm0			; 13	;; err = abs(err)
	vpandq	zmm17, zmm17, zmm0			; 13	;; err = abs(err)
	vpandq	zmm18, zmm18, zmm0			; 14	;; err = abs(err)
	vpandq	zmm19, zmm19, zmm0			; 14	;; err = abs(err)
	vpandq	zmm20, zmm20, zmm0			; 15	;; err = abs(err)
	vpandq	zmm21, zmm21, zmm0			; 15	;; err = abs(err)
	vpandq	zmm22, zmm22, zmm0			; 16	;; err = abs(err)
	vpandq	zmm23, zmm23, zmm0			; 16	;; err = abs(err)

;; BUG - Is there a better grouping available (use regs zmm16 up and interleave with the / base FMA instructions
;; Use more maxerr accumulators?

	vmaxpd	zmm20, zmm20, zmm16			; 17-20	;; accumulate maxerr
	vmaxpd	zmm21, zmm21, zmm17			; 17-20	;; accumulate maxerr
	vmaxpd	zmm22, zmm22, zmm18			; 18-21	;; accumulate maxerr
	vmaxpd	zmm23, zmm23, zmm19			; 18-21	;; accumulate maxerr

	zfmsubpd zmm16, zmm12, zmm28, zmm26		; 19-22	;; val5_hi+RNDVAL = (val5+RNDVAL) / base - (RNDVAL/base-RNDVAL) = rnd(val5/base)+RNDVAL
	zfmsubpd zmm17, zmm13, zmm28, zmm26		; 19-22	;; val6_hi+RNDVAL = (val6+RNDVAL) / base - (RNDVAL/base-RNDVAL) = rnd(val6/base)+RNDVAL
	zfmsubpd zmm18, zmm14, zmm28, zmm26		; 20-23	;; val7_hi+RNDVAL = (val7+RNDVAL) / base - (RNDVAL/base-RNDVAL) = rnd(val7/base)+RNDVAL
	zfmsubpd zmm19, zmm15, zmm28, zmm26		; 20-23	;; val8_hi+RNDVAL = (val8+RNDVAL) / base - (RNDVAL/base-RNDVAL) = rnd(val8/base)+RNDVAL
	zfmsubpd zmm0, zmm8, zmm28, zmm26		; 21-24	;; val1_hi+RNDVAL = (val1+RNDVAL) / base - (RNDVAL/base-RNDVAL) = rnd(val1/base)+RNDVAL (next carry1)
	zfmsubpd zmm1, zmm9, zmm28, zmm26		; 21-24	;; val2_hi+RNDVAL = (val2+RNDVAL) / base - (RNDVAL/base-RNDVAL) = rnd(val2/base)+RNDVAL (next carry2)
	zfmsubpd zmm2, zmm10, zmm28, zmm26		; 22-25	;; val3_hi+RNDVAL = (val3+RNDVAL) / base - (RNDVAL/base-RNDVAL) = rnd(val3/base)+RNDVAL (next carry3)
	zfmsubpd zmm3, zmm11, zmm28, zmm26		; 22-25	;; val4_hi+RNDVAL = (val4+RNDVAL) / base - (RNDVAL/base-RNDVAL) = rnd(val4/base)+RNDVAL (next carry4)

	vmaxpd	zmm22, zmm22, zmm20			; 23-26	;; accumulate maxerr
	vmaxpd	zmm23, zmm23, zmm21			; 23-26	;; accumulate maxerr
	vmaxpd	zmm31, zmm31, zmm22			; 27-30	;; accumulate maxerr
	vmaxpd	zmm31, zmm31, zmm23			; 31-34	;; accumulate maxerr

	zfmsubpd zmm20, zmm16, zmm29, zmm27		; 24-27	;; tmp5+RNDVAL = (val5_hi+RNDVAL) * base - (RNDVAL*base-RNDVAL) = rnd(val5/base)*base+RNDVAL
	zfmsubpd zmm21, zmm17, zmm29, zmm27		; 24-27	;; tmp6+RNDVAL = (val6_hi+RNDVAL) * base - (RNDVAL*base-RNDVAL) = rnd(val6/base)*base+RNDVAL
	zfmsubpd zmm22, zmm18, zmm29, zmm27		; 25-28	;; tmp7+RNDVAL = (val7_hi+RNDVAL) * base - (RNDVAL*base-RNDVAL) = rnd(val7/base)*base+RNDVAL
	zfmsubpd zmm23, zmm19, zmm29, zmm27		; 25-28	;; tmp8+RNDVAL = (val8_hi+RNDVAL) * base - (RNDVAL*base-RNDVAL) = rnd(val8/base)*base+RNDVAL
	vsubpd	zmm16, zmm16, zmm30			; 26-29	;; val5_hi = val5_hi+RNDVAL - RNDVAL
	vsubpd	zmm17, zmm17, zmm30			; 26-29	;; val6_hi = val6_hi+RNDVAL - RNDVAL
	vsubpd	zmm18, zmm18, zmm30			; 27-30	;; val7_hi = val7_hi+RNDVAL - RNDVAL
	vsubpd	zmm19, zmm19, zmm30			; 28-31	;; val8_hi = val8_hi+RNDVAL - RNDVAL
	vsubpd	zmm12, zmm12, zmm20			; 28-31	;; val5_lo = val5+RNDVAL - tmp5+RNDVAL = val5 - rnd(val5/base)*base = val5%base
	vsubpd	zmm13, zmm13, zmm21			; 29-32	;; val6_lo = val6+RNDVAL - tmp6+RNDVAL = val6 - rnd(val6/base)*base = val6%base
	vsubpd	zmm14, zmm14, zmm22			; 29-32	;; val7_lo = val7+RNDVAL - tmp7+RNDVAL = val7 - rnd(val7/base)*base = val7%base
	vsubpd	zmm15, zmm15, zmm23			; 30-33	;; val8_lo = val8+RNDVAL - tmp8+RNDVAL = val8 - rnd(val8/base)*base = val8%base
	zfmsubpd zmm20, zmm0, zmm29, zmm27		; 30-33	;; tmp1+RNDVAL = (val1_hi+RNDVAL) * base - (RNDVAL*base-RNDVAL) = rnd(val1/base)*base+RNDVAL
	zfmsubpd zmm21, zmm1, zmm29, zmm27		; 31-34	;; tmp2+RNDVAL = (val2_hi+RNDVAL) * base - (RNDVAL*base-RNDVAL) = rnd(val2/base)*base+RNDVAL
	zfmsubpd zmm22, zmm2, zmm29, zmm27		; 32-35	;; tmp3+RNDVAL = (val3_hi+RNDVAL) * base - (RNDVAL*base-RNDVAL) = rnd(val3/base)*base+RNDVAL
	zfmsubpd zmm23, zmm3, zmm29, zmm27		; 32-35	;; tmp4+RNDVAL = (val4_hi+RNDVAL) * base - (RNDVAL*base-RNDVAL) = rnd(val4/base)*base+RNDVAL
	zfmaddpd zmm12, zmm12, zmm25, zmm4		; 33-36	;; x1+RNDVAL = val5_lo * -c*mulconst + carry5+RNDVAL
	zfmaddpd zmm13, zmm13, zmm25, zmm5		; 33-36	;; x2+RNDVAL = val6_lo * -c*mulconst + carry6+RNDVAL
	zfmaddpd zmm14, zmm14, zmm25, zmm6		; 34-37	;; x3+RNDVAL = val7_lo * -c*mulconst + carry7+RNDVAL
	zfmaddpd zmm15, zmm15, zmm25, zmm7		; 34-37	;; x4+RNDVAL = val8_lo * -c*mulconst + carry8+RNDVAL
	vsubpd	zmm8, zmm8, zmm20			; 35-38	;; val1_lo = val1+RNDVAL - tmp1+RNDVAL = val1 - rnd(val1/base)*base = val1%base
	vsubpd	zmm9, zmm9, zmm21			; 35-38	;; val2_lo = val2+RNDVAL - tmp2+RNDVAL = val2 - rnd(val2/base)*base = val2%base
	vsubpd	zmm10, zmm10, zmm22			; 36-39	;; val3_lo = val3+RNDVAL - tmp3+RNDVAL = val3 - rnd(val3/base)*base = val3%base
	vsubpd	zmm11, zmm11, zmm23			; 36-39	;; val4_lo = val4+RNDVAL - tmp4+RNDVAL = val4 - rnd(val4/base)*base = val4%base

	zfmaddpd zmm12, zmm8, zmm24, zmm12		; 39-42 ;; x1+RNDVAL += val1_lo * k_lo*mulconst
	zfmaddpd zmm13, zmm9, zmm24, zmm13		; 39-42 ;; x2+RNDVAL += val2_lo * k_lo*mulconst
	zfmaddpd zmm14, zmm10, zmm24, zmm14		; 40-43 ;; x3+RNDVAL += val3_lo * k_lo*mulconst
	zfmaddpd zmm15, zmm11, zmm24, zmm15		; 40-43 ;; x4+RNDVAL += val4_lo * k_lo*mulconst

	zfmsubpd zmm4, zmm12, zmm28, zmm26		; 43-46 ;; x1_hi+RNDVAL = (x1+RNDVAL) / base - (RNDVAL/base-RNDVAL) = rnd(x1/base)+RNDVAL
	zfmsubpd zmm5, zmm13, zmm28, zmm26		; 43-46 ;; x2_hi+RNDVAL = (x2+RNDVAL) / base - (RNDVAL/base-RNDVAL) = rnd(x2/base)+RNDVAL
	zfmsubpd zmm6, zmm14, zmm28, zmm26		; 44-47 ;; x3_hi+RNDVAL = (x3+RNDVAL) / base - (RNDVAL/base-RNDVAL) = rnd(x3/base)+RNDVAL
	zfmsubpd zmm7, zmm15, zmm28, zmm26		; 44-47 ;; x4_hi+RNDVAL = (x4+RNDVAL) / base - (RNDVAL/base-RNDVAL) = rnd(x4/base)+RNDVAL
	zfmsubpd zmm20, zmm4, zmm29, zmm27		; 47-50 ;; tmp9+RNDVAL = (x1_hi+RNDVAL) * base - (RNDVAL*base-RNDVAL) = rnd(x1/base)*base+RNDVAL
	zfmsubpd zmm21, zmm5, zmm29, zmm27		; 47-50 ;; tmp10+RNDVAL = (x2_hi+RNDVAL) * base - (RNDVAL*base-RNDVAL) = rnd(x2/base)*base+RNDVAL
	zfmsubpd zmm22, zmm6, zmm29, zmm27		; 48-51 ;; tmp11+RNDVAL = (x3_hi+RNDVAL) * base - (RNDVAL*base-RNDVAL) = rnd(x3/base)*base+RNDVAL
	zfmsubpd zmm23, zmm7, zmm29, zmm27		; 48-51 ;; tmp12+RNDVAL = (x4_hi+RNDVAL) * base - (RNDVAL*base-RNDVAL) = rnd(x4/base)*base+RNDVAL
	zfmaddpd zmm4, zmm16, zmm25, zmm4		; 49-52 ;; next carry5 = val5_hi * -c*mulconst + x1_hi+RNDVAL
	zfmaddpd zmm5, zmm17, zmm25, zmm5		; 49-52 ;; next carry6 = val6_hi * -c*mulconst + x2_hi+RNDVAL
	zfmaddpd zmm6, zmm18, zmm25, zmm6		; 50-53 ;; next carry7 = val7_hi * -c*mulconst + x3_hi+RNDVAL
	zfmaddpd zmm7, zmm19, zmm25, zmm7		; 50-53 ;; next carry8 = val8_hi * -c*mulconst + x4_hi+RNDVAL
	vsubpd	zmm12, zmm12, zmm20			; 51-54 ;; new val1 = x1_lo = x1+RNDVAL - tmp9+RNDVAL = x1 - rnd(x1/base)*base = x1%base
	vsubpd	zmm13, zmm13, zmm21			; 51-54 ;; new val2 = x2_lo = x2+RNDVAL - tmp10+RNDVAL = x2 - rnd(x2/base)*base = x2%base
	vsubpd	zmm14, zmm14, zmm22			; 52-55 ;; new val1 = x3_lo = x3+RNDVAL - tmp11+RNDVAL = x1 - rnd(x3/base)*base = x3%base
	vsubpd	zmm15, zmm15, zmm23			; 52-55 ;; new val2 = x4_lo = x4+RNDVAL - tmp12+RNDVAL = x2 - rnd(x4/base)*base = x4%base
khi no const	vbroadcastsd zmm23, ZMM_K_HI_OVER_SMALL_BASE	;; k_hi / small_word_base
khi const	vbroadcastsd zmm23, ZMM_K_TIMES_MULCONST_HI_OVER_SMALL_BASE
khi	zfmaddpd zmm4, zmm8, zmm23, zmm4		; 53-56 ;; if khi next carry5 = val1_lo * k_hi*mulconst/base + next carry5
khi	zfmaddpd zmm5, zmm9, zmm23, zmm5		; 53-56 ;; if khi next carry6 = val2_lo * k_hi*mulconst/base + next carry6
khi	zfmaddpd zmm6, zmm10, zmm23, zmm6		; 54-57 ;; if khi next carry7 = val3_lo * k_hi*mulconst/base + next carry7
khi	zfmaddpd zmm7, zmm11, zmm23, zmm7		; 54-57 ;; if khi next carry8 = val4_lo * k_hi*mulconst/base + next carry8

	zstore	[rsi], zmm12					;; Save value1
	zstore	[rsi+r13], zmm13				;; Save value2
	zstore	[rsi+2*r13], zmm14				;; Save value3
	zstore	[rsi+r14], zmm15				;; Save value4
	vzero	zmm8
	zstore	[rsi+64], zmm8					;; Save value5
	zstore	[rsi+r13+64], zmm8				;; Save value6
	zstore	[rsi+2*r13+64], zmm8				;; Save value7
	zstore	[rsi+r14+64], zmm8				;; Save value8
	ENDM

; Irrational versions - same as two-pass _wpn versions

znorm_1d_zpad_noechk_ttp_preload MACRO const, khi
	znorm_wpn_zpad_noechk_ttp_preload const, khi
	ENDM
znorm_1d_zpad_noechk_ttp MACRO const, khi
	znorm_wpn_zpad_noechk_ttp const, khi
	ENDM

znorm_1d_zpad_echk_ttp_preload MACRO const, khi
	znorm_wpn_zpad_echk_ttp_preload const, khi
	ENDM
znorm_1d_zpad_echk_ttp MACRO const, khi
	znorm_wpn_zpad_echk_ttp const, khi
	ENDM


; Process the eight carries from 4 sections of a one-pass FFT normalization.  There are two versions.  One used for multiply, the other for add/sub/addsub/smallmul.
; The reason for this is that add/sub/addsub/smallmul know that the low carries are zero.
; On input:
; zmm0-7 = input carries. nottp in +RNDVAL format, ttp in mixed +RNDVAL/+0 format.  Outputs carries for next section.
; zmm24-29 = must be preloaded and preserved (from znorm_1d_zpad_preload).  Well, zmm24 need not be preserved in the ttp case.
; zmm30 = RNDVAL
; zmm31 = must be preserved (MAXERR)
; r9 = section start FFT pointer
; r8 = section start big/lit pointer
; rbp = section start compressed biglit table pointer (ttp only)
; r12 = pointer to next sections compressed biglit table (must be preserved)
; r13 = distance to second FFT data source
; r14 = distance to fourth FFT data source
; Also used internally:
; rsi = pointer to the FFT data (source #1)
; rdi = pointer to next sections big/little flags (must be preserved)
; rdx = register to load compressed biglit index into
; rcx = loop counter
; rax, r10 = must be preserved
; r15 = must be preserved for addsub (znorm_addsub_save could be modified to save/restore r15)
; rbx = used as count of double cache lines in each one-pass section (addcount1)
; zmm8-zmm23 = trashed

zadd_carry_1d_zpad MACRO ttp
no ttp	zadd_carry_1d_zpad_nottp
ttp	zadd_carry_1d_zpad_ttp
	ENDM

;; Rotate 8 carries.  Save every top carry from in zmm23.  Carries are rotated up, least significant word is set to RNDVAL.
;; nottp carries are in +RNDVAL format, ttp carries have been converted to all +0 format
zrotate_carries_1d_zpad MACRO ttp
	vpmovzxbq zmm9, ZMM_PERMUTE3		;; zmm9 = 0+6 0+5 0+4 0+3 0+2 0+1 0+0 8+7
	vpermt2pd zmm8, zmm9, zmm0		;; Save top carry from zmm0 into zmm8
no ttp	vpermt2pd zmm0, zmm9, zmm30		;; Rotate carries in zmm0
ttp	vpermt2pd zmm0, zmm9, zmm22		;; Rotate carries in zmm0
	vpermt2pd zmm8, zmm9, zmm1		;; Save top carry from zmm1 into zmm8
no ttp	vpermt2pd zmm1, zmm9, zmm30		;; Rotate carries in zmm1
ttp	vpermt2pd zmm1, zmm9, zmm22		;; Rotate carries in zmm1
	vpermt2pd zmm8, zmm9, zmm2		;; Save top carry from zmm2 into zmm8
no ttp	vpermt2pd zmm2, zmm9, zmm30		;; Rotate carries in zmm2
ttp	vpermt2pd zmm2, zmm9, zmm22		;; Rotate carries in zmm2
	vpermt2pd zmm8, zmm9, zmm3		;; Save top carry from zmm3 into zmm8
no ttp	vpermt2pd zmm3, zmm9, zmm30		;; Rotate carries in zmm3
ttp	vpermt2pd zmm3, zmm9, zmm22		;; Rotate carries in zmm3
	vpermt2pd zmm8, zmm9, zmm4		;; Save top carry from zmm4 into zmm8
no ttp	vpermt2pd zmm4, zmm9, zmm30		;; Rotate carries in zmm4
ttp	vpermt2pd zmm4, zmm9, zmm22		;; Rotate carries in zmm4
	vpermt2pd zmm8, zmm9, zmm5		;; Save top carry from zmm5 into zmm8
no ttp	vpermt2pd zmm5, zmm9, zmm30		;; Rotate carries in zmm5
ttp	vpermt2pd zmm5, zmm9, zmm22		;; Rotate carries in zmm5
	vpermt2pd zmm8, zmm9, zmm6		;; Save top carry from zmm6 into zmm8
no ttp	vpermt2pd zmm6, zmm9, zmm30		;; Rotate carries in zmm6
ttp	vpermt2pd zmm6, zmm9, zmm22		;; Rotate carries in zmm6
	vpermt2pd zmm8, zmm9, zmm7		;; Save top carry from zmm7 into zmm8
no ttp	vpermt2pd zmm7, zmm9, zmm30		;; Rotate carries in zmm7
ttp	vpermt2pd zmm7, zmm9, zmm22		;; Rotate carries in zmm7
no ttp	vsubpd	zmm8, zmm8, zmm30		;; Remove RNDVAL from saved top carries
	vaddpd	zmm23, zmm23, zmm8		;; Accumulate saved top carries in zmm23
	ENDM

;; Initialize 8 carries for next section using the saved top carries.
;; nottp carries are in +RNDVAL format, ttp carries have been converted to all +0 format
zrestore_top_carries_1d_zpad MACRO ttp
	mov	edx, 11111110b			;; For vblendmpd during swizzle
	kmovw	k7, edx
	vpmovzxbq zmm9, ZMM_PERMUTE4		;; zmm9 = 0+7 0+7 0+6 0+5 0+4 0+3 0+2 0+1
no ttp	vblendmpd zmm7{k7}, zmm23, zmm30	;; Restore lowest carry from zmm23 and RNDVAL into zmm7
ttp	vblendmpd zmm7{k7}, zmm23, zmm22	;; Restore lowest carry from zmm23 and zero into zmm7
	vpermt2pd zmm23, zmm9, zmm23		;; Rotate saved top carries down
no ttp	vblendmpd zmm6{k7}, zmm23, zmm30	;; Restore lowest carry from zmm23 and RNDVAL into zmm6
ttp	vblendmpd zmm6{k7}, zmm23, zmm22	;; Restore lowest carry from zmm23 and zero into zmm6
	vpermt2pd zmm23, zmm9, zmm23		;; Rotate saved top carries down
no ttp	vblendmpd zmm5{k7}, zmm23, zmm30	;; Restore lowest carry from zmm23 and RNDVAL into zmm5
ttp	vblendmpd zmm5{k7}, zmm23, zmm22	;; Restore lowest carry from zmm23 and zero into zmm5
	vpermt2pd zmm23, zmm9, zmm23		;; Rotate saved top carries down
no ttp	vblendmpd zmm4{k7}, zmm23, zmm30	;; Restore lowest carry from zmm23 and RNDVAL into zmm4
ttp	vblendmpd zmm4{k7}, zmm23, zmm22	;; Restore lowest carry from zmm23 and zero into zmm4
	vpermt2pd zmm23, zmm9, zmm23		;; Rotate saved top carries down
no ttp	vblendmpd zmm3{k7}, zmm23, zmm30	;; Restore lowest carry from zmm23 and RNDVAL into zmm3
ttp	vblendmpd zmm3{k7}, zmm23, zmm22	;; Restore lowest carry from zmm23 and zero into zmm3
	vpermt2pd zmm23, zmm9, zmm23		;; Rotate saved top carries down
no ttp	vblendmpd zmm2{k7}, zmm23, zmm30	;; Restore lowest carry from zmm23 and RNDVAL into zmm2
ttp	vblendmpd zmm2{k7}, zmm23, zmm22	;; Restore lowest carry from zmm23 and zero into zmm2
	vpermt2pd zmm23, zmm9, zmm23		;; Rotate saved top carries down
no ttp	vblendmpd zmm1{k7}, zmm23, zmm30	;; Restore lowest carry from zmm23 and RNDVAL into zmm1
ttp	vblendmpd zmm1{k7}, zmm23, zmm22	;; Restore lowest carry from zmm23 and zero into zmm1
	vpermt2pd zmm23, zmm9, zmm23		;; Rotate saved top carries down
no ttp	vblendmpd zmm0{k7}, zmm23, zmm30	;; Restore lowest carry from zmm23 and RNDVAL into zmm0
ttp	vblendmpd zmm0{k7}, zmm23, zmm22	;; Restore lowest carry from zmm23 and zero into zmm0
	ENDM

; This zadd_carry if for results of an FFT multiplication
zadd_carry_1d_zpad_nottp MACRO
	LOCAL	nonconst, rloop, cloop, cloop_end, done

	vmovapd	zmm23, zmm30				;; Top carries are saved here for adding into next section
	vzero	zmm22					;; Constant zero
	;; vbroadcastsd zmm24, ZMM_K_LO			;; k_lo or k_lo*mulconst was preloaded into zmm24 by znorm_1d_zpad_preload
	vbroadcastsd zmm21, ZMM_K_HI_OVER_SMALL_BASE	;; k_hi / small_word_base
	cmp	const_fft, 0				;; Are we also multiplying by a constant?
	je	short nonconst				;; Jump if not const
	;; vbroadcastsd zmm24, ZMM_K_TIMES_MULCONST_LO	;; k_lo or k_lo*mulconst was preloaded into zmm24 by znorm_1d_zpad_preload
	vbroadcastsd zmm21, ZMM_K_TIMES_MULCONST_HI_OVER_SMALL_BASE
nonconst:

	mov	cl, 8					;; Spread carry over a max of 8 words
rloop:	mov	rsi, r9					;; Restore section start pointers
	zrotate_carries_1d_zpad noexec
	mov	ebx, addcount1				;; Count of double cache lines in a section

	;; Proper carry propagation requires splitting the lower carry that has not been mul'ed-by-k or mul-by-const
cloop:	dec	cl					;; Decrement propagation count
	jz	cloop_end				;; Exit carry propagation loop
	vmovapd	zmm12, zmm0				;; Copy carry1+RNDVAL
	zfmsubpd zmm0, zmm0, zmm28, zmm26	; 1-4	;; carry1_hi+RNDVAL = (carry1+RNDVAL) / base - (RNDVAL/base-RNDVAL) = rnd(carry1/base)+RNDVAL (next carry1)
	vaddpd	zmm8, zmm4, [rsi]		; 1-4	;; x1+RNDVAL = carry5 + values1
	vmovapd	zmm13, zmm1				;; Copy carry1+RNDVAL
	zfmsubpd zmm1, zmm1, zmm28, zmm26	; 2-5	;; carry2_hi+RNDVAL = (carry2+RNDVAL) / base - (RNDVAL/base-RNDVAL) = rnd(carry2/base)+RNDVAL (next carry2)
	vaddpd	zmm9, zmm5, [rsi+r13]		; 2-5	;; x2+RNDVAL = carry6 + values2
	vmovapd	zmm14, zmm2				;; Copy carry1+RNDVAL
	zfmsubpd zmm2, zmm2, zmm28, zmm26	; 3-6	;; carry3_hi+RNDVAL = (carry3+RNDVAL) / base - (RNDVAL/base-RNDVAL) = rnd(carry3/base)+RNDVAL (next carry3)
	vaddpd	zmm10, zmm6, [rsi+2*r13]	; 3-6	;; x3+RNDVAL = carry7 + values3
	vmovapd	zmm15, zmm3				;; Copy carry1+RNDVAL
	zfmsubpd zmm3, zmm3, zmm28, zmm26	; 4-7	;; carry4_hi+RNDVAL = (carry4+RNDVAL) / base - (RNDVAL/base-RNDVAL) = rnd(carry4/base)+RNDVAL (next carry4)
	vaddpd	zmm11, zmm7, [rsi+r14]		; 4-7	;; x4+RNDVAL = carry8 + values4

	zfmsubpd zmm16, zmm0, zmm29, zmm27	; 5-8	;; tmp1+RNDVAL = (carry1_hi+RNDVAL) * base - (RNDVAL*base-RNDVAL) = rnd(carry1/base)*base+RNDVAL
	zfmsubpd zmm17, zmm1, zmm29, zmm27	; 6-9	;; tmp2+RNDVAL = (carry2_hi+RNDVAL) * base - (RNDVAL*base-RNDVAL) = rnd(carry2/base)*base+RNDVAL
	zfmsubpd zmm18, zmm2, zmm29, zmm27	; 7-10	;; tmp3+RNDVAL = (carry3_hi+RNDVAL) * base - (RNDVAL*base-RNDVAL) = rnd(carry3/base)*base+RNDVAL
	zfmsubpd zmm19, zmm3, zmm29, zmm27	; 8-11	;; tmp4+RNDVAL = (carry4_hi+RNDVAL) * base - (RNDVAL*base-RNDVAL) = rnd(carry4/base)*base+RNDVAL

	vsubpd	zmm12, zmm12, zmm16		; 9-12	;; carry1_lo = carry1+RNDVAL - tmp1+RNDVAL = carry1 - rnd(carry1/base)*base = carry1%base
	vsubpd	zmm13, zmm13, zmm17		; 10-13	;; carry2_lo = carry2+RNDVAL - tmp2+RNDVAL = carry2 - rnd(carry2/base)*base = carry2%base
	vsubpd	zmm14, zmm14, zmm18		; 11-14	;; carry3_lo = carry3+RNDVAL - tmp3+RNDVAL = carry3 - rnd(carry3/base)*base = carry3%base
	vsubpd	zmm15, zmm15, zmm19		; 12-15	;; carry4_lo = carry4+RNDVAL - tmp4+RNDVAL = carry4 - rnd(carry4/base)*base = carry4%base

	zfmaddpd zmm8, zmm12, zmm24, zmm8	; 13-16 ;; x1+RNDVAL += carry1_lo * k_lo*mulconst
	zfmaddpd zmm9, zmm13, zmm24, zmm9	; 14-17 ;; x2+RNDVAL += carry2_lo * k_lo*mulconst
	zfmaddpd zmm10, zmm14, zmm24, zmm10	; 15-18 ;; x3+RNDVAL += carry3_lo * k_lo*mulconst
	zfmaddpd zmm11, zmm15, zmm24, zmm11	; 16-19 ;; x4+RNDVAL += carry4_lo * k_lo*mulconst

	zfmsubpd zmm4, zmm8, zmm28, zmm26	; 17-20 ;; x1_hi+RNDVAL = (x1+RNDVAL) / base - (RNDVAL/base-RNDVAL) = rnd(x1/base)+RNDVAL (next carry5)
	zfmsubpd zmm5, zmm9, zmm28, zmm26	; 18-21 ;; x2_hi+RNDVAL = (x2+RNDVAL) / base - (RNDVAL/base-RNDVAL) = rnd(x2/base)+RNDVAL (next carry6)
	zfmsubpd zmm6, zmm10, zmm28, zmm26	; 19-22 ;; x3_hi+RNDVAL = (x3+RNDVAL) / base - (RNDVAL/base-RNDVAL) = rnd(x3/base)+RNDVAL (next carry7)
	zfmsubpd zmm7, zmm11, zmm28, zmm26	; 20-23 ;; x4_hi+RNDVAL = (x4+RNDVAL) / base - (RNDVAL/base-RNDVAL) = rnd(x4/base)+RNDVAL (next carry8)

	zfmsubpd zmm16, zmm4, zmm29, zmm27	; 21-24 ;; tmp5+RNDVAL = (x1_hi+RNDVAL) * base - (RNDVAL*base-RNDVAL) = rnd(x1/base)*base+RNDVAL
	zfmaddpd zmm4, zmm12, zmm21, zmm4	; 21-24 ;; next carry5 = carry1_lo * k_hi*mulconst/base + next carry5
	zfmsubpd zmm17, zmm5, zmm29, zmm27	; 22-25 ;; tmp6+RNDVAL = (x2_hi+RNDVAL) * base - (RNDVAL*base-RNDVAL) = rnd(x2/base)*base+RNDVAL
	zfmaddpd zmm5, zmm13, zmm21, zmm5	; 22-25 ;; next carry6 = carry2_lo * k_hi*mulconst/base + next carry6
	zfmsubpd zmm18, zmm6, zmm29, zmm27	; 23-26 ;; tmp7+RNDVAL = (x3_hi+RNDVAL) * base - (RNDVAL*base-RNDVAL) = rnd(x3/base)*base+RNDVAL
	zfmaddpd zmm6, zmm14, zmm21, zmm6	; 23-26 ;; next carry7 = carry3_lo * k_hi*mulconst/base + next carry7
	zfmsubpd zmm19, zmm7, zmm29, zmm27	; 24-27 ;; tmp8+RNDVAL = (x4_hi+RNDVAL) * base - (RNDVAL*base-RNDVAL) = rnd(x4/base)*base+RNDVAL
	zfmaddpd zmm7, zmm15, zmm21, zmm7	; 24-27 ;; next carry8 = carry4_lo * k_hi*mulconst/base + next carry8

	vsubpd	zmm8, zmm8, zmm16		; 25-28 ;; new val1 = x1_lo = x1+RNDVAL - tmp5+RNDVAL = x1 - rnd(x1/base)*base = x1%base
	vsubpd	zmm9, zmm9, zmm17		; 26-29 ;; new val2 = x2_lo = x2+RNDVAL - tmp6+RNDVAL = x2 - rnd(x2/base)*base = x2%base
	vsubpd	zmm10, zmm10, zmm18		; 27-30 ;; new val3 = x3_lo = x3+RNDVAL - tmp7+RNDVAL = x3 - rnd(x3/base)*base = x3%base
	vsubpd	zmm11, zmm11, zmm19		; 28-31 ;; new val4 = x4_lo = x4+RNDVAL - tmp8+RNDVAL = x4 - rnd(x4/base)*base = x4%base

	zstore	[rsi], zmm8			;; Save new val1
	zstore	[rsi+r13], zmm9			;; Save new val2
	zstore	[rsi+2*r13], zmm10		;; Save new val3
	zstore	[rsi+r14], zmm11		;; Save new val4

	vcmpneqpd k1, zmm0, zmm30		;; Are any non-zero carries remaining to propagate?
	vcmpneqpd k2, zmm1, zmm30
	vcmpneqpd k3, zmm2, zmm30
	vcmpneqpd k4, zmm3, zmm30
	korw	k5, k1, k2
	korw	k6, k3, k4
	vcmpneqpd k1, zmm4, zmm30		;; Are any non-zero carries remaining to propagate?
	vcmpneqpd k2, zmm5, zmm30
	vcmpneqpd k3, zmm6, zmm30
	vcmpneqpd k4, zmm7, zmm30
	korw	k1, k1, k2
	korw	k2, k3, k4
	korw	k3, k5, k6
	korw	k4, k1, k2
	kortestw k3, k4
	jz	done				;; If carries not found then we're done

	bump	rsi, 128			;; Next source pointer
	dec	ebx				;; Test for end of section
	jnz	cloop				;; Loop to next double cache lines within section
	jmp	rloop				;; Loop to rotate carries and go back to section start

	;; Do eighth and last iteration without propagating carries out
cloop_end:
	vsubpd	zmm8, zmm4, zmm30		;; Remove RNDVAL from traditional carry
	vsubpd	zmm9, zmm5, zmm30
	vsubpd	zmm10, zmm6, zmm30
	vsubpd	zmm11, zmm7, zmm30
	vaddpd	zmm8, zmm8, [rsi]		;; Values1 += carry
	vaddpd	zmm9, zmm9, [rsi+r13]		;; Values2 += carry
	vaddpd	zmm10, zmm10, [rsi+2*r13]	;; Values3 += carry
	vaddpd	zmm11, zmm11, [rsi+r14]		;; Values4 += carry
	zstore	[rsi], zmm8			;; Save value1
	zstore	[rsi+r13], zmm9			;; Save value2
	zstore	[rsi+2*r13], zmm10		;; Save value3
	zstore	[rsi+r14], zmm11		;; Save value4

done:	zrestore_top_carries_1d_zpad noexec	;; Load carries for next section
	ENDM

zadd_carry_1d_zpad_ttp MACRO
	LOCAL	nonconst, rloop, cloop, cloop_end, done

	push	rdi					;; Save next section's biglit pointer
	push	r12					;; Save next section' compressed biglit table pointer

	vzero	zmm23					;; Top carries are saved here for adding into next section
	vmovapd	zmm22, zmm23				;; Constant zero
	vbroadcastsd zmm24, ZMM_K_LO
	vbroadcastsd zmm21, ZMM_K_HI_OVER_SMALL_BASE
	vbroadcastsd zmm20, ZMM_K_HI_OVER_LARGE_BASE
	cmp	const_fft, 0				;; Are we also multiplying by a constant?
	je	short nonconst				;; Jump if not const
	vbroadcastsd zmm24, ZMM_K_TIMES_MULCONST_LO
	vbroadcastsd zmm21, ZMM_K_TIMES_MULCONST_HI_OVER_SMALL_BASE
	vbroadcastsd zmm20, ZMM_K_TIMES_MULCONST_HI_OVER_LARGE_BASE
nonconst:

	vsubpd	zmm0, zmm0, zmm30			;; Remove RNDVAL from low carries (high carries already in +0 format)
	vsubpd	zmm1, zmm1, zmm30
	vsubpd	zmm2, zmm2, zmm30
	vsubpd	zmm3, zmm3, zmm30

	mov	cl, 8					;; Spread carry over a max of 8 words
rloop:	mov	rsi, r9					;; Restore section start pointers
	mov	rdi, r8
	mov	r12, rbp
	zrotate_carries_1d_zpad exec
	mov	ebx, addcount1				;; Count of double cache lines in a section

cloop:	dec	cl					;; Decrement propagation count
	jz	cloop_end				;; Exit carry propagation loop
	movzx	rdx, BYTE PTR [rdi]			;; Load index into compressed biglit table
	mov	edx, [r12+rdx*4]			;; Load 4 big vs. little flags

	kmovw	k1, edx				; 1	;; Load 2 big vs. little flags (16-bit load - we use 8-bits)
	vblendmpd zmm16 {k1}, zmm28, zmm26	; 2	;; Create (1/base) constant
	kshiftrw k2, k1, 8			; 2
	vblendmpd zmm17 {k2}, zmm28, zmm26	; 3	;; Create (1/base) constant
	shr	edx, 16
	kmovw	k3, edx				; 3	;; Load 2 big vs. little flags (16-bit load - we use 8-bits)
	vblendmpd zmm18 {k3}, zmm28, zmm26	; 4	;; Create (1/base) constant
	kshiftrw k4, k3, 8			; 4
	vblendmpd zmm19 {k4}, zmm28, zmm26	; 5	;; Create (1/base) constant

	vmovapd	zmm12, zmm0				;; Copy input carry1
	zfmaddpd zmm0, zmm0, zmm16, zmm30	; 5-8	;; carry1_hi+RNDVAL = carry1 / base + RNDVAL
	vaddpd	zmm8, zmm4, [rsi]		; 6-9	;; x1 = carry5 + values1
	vmovapd	zmm13, zmm1				;; Copy input carry2
	zfmaddpd zmm1, zmm1, zmm17, zmm30	; 6-9	;; carry2_hi+RNDVAL = carry2 / base + RNDVAL
	vaddpd	zmm9, zmm5, [rsi+r13]		; 7-10	;; x2 = carry6 + values2
	vmovapd	zmm14, zmm2				;; Copy input carry3
	zfmaddpd zmm2, zmm2, zmm18, zmm30	; 7-10	;; carry3_hi+RNDVAL = carry3 / base + RNDVAL
	vaddpd	zmm10, zmm6, [rsi+2*r13]	; 8-11	;; x3 = carry7 + values3
	vmovapd	zmm15, zmm3				;; Copy input carry4
	zfmaddpd zmm3, zmm3, zmm19, zmm30	; 8-11	;; carry4_hi+RNDVAL = carry4 / base + RNDVAL
	vaddpd	zmm11, zmm7, [rsi+r14]		; 9-12	;; x4 = carry8 + values4
; 0-3 next carry, 4-7 avail, 8-11 x1-4, 12-15 in carry1-4, 16-19 1/base

	vsubpd	zmm0, zmm0, zmm30		; 9-12	;; carry1_hi = carry1_hi+RNDVAL - RNDVAL (next carry1)
	vblendmpd zmm4 {k1}, zmm29, zmm27	; 10	;; Create (base) constant
	vsubpd	zmm1, zmm1, zmm30		; 10-13	;; carry2_hi = carry2_hi+RNDVAL - RNDVAL (next carry2)
	vblendmpd zmm5 {k2}, zmm29, zmm27	; 11	;; Create (base) constant
	vsubpd	zmm2, zmm2, zmm30		; 11-14	;; carry3_hi = carry3_hi+RNDVAL - RNDVAL (next carry3)
	vblendmpd zmm6 {k3}, zmm29, zmm27	; 12	;; Create (base) constant
	vsubpd	zmm3, zmm3, zmm30		; 12-15	;; carry4_hi = carry4_hi+RNDVAL - RNDVAL (next carry4)
	vblendmpd zmm7 {k4}, zmm29, zmm27	; 13	;; Create (base) constant
; 0-3 next carry zero based, 4-7 base, 8-11 x1-4, 12-15 in carry1-4, 16-19 1/base

	zfnmaddpd zmm12, zmm0, zmm4, zmm12	; 13-16	;; carry1_lo = carry1 - carry1_hi * base = carry1 - rnd(carry1/base)*base = carry1%base
	zfnmaddpd zmm13, zmm1, zmm5, zmm13	; 14-17	;; carry2_lo = carry2 - carry2_hi * base = carry2 - rnd(carry2/base)*base = carry2%base
	zfnmaddpd zmm14, zmm2, zmm6, zmm14	; 15-18	;; carry3_lo = carry3 - carry3_hi * base = carry3 - rnd(carry3/base)*base = carry3%base
	zfnmaddpd zmm15, zmm3, zmm7, zmm15	; 16-19	;; carry4_lo = carry4 - carry4_hi * base = carry4 - rnd(carry4/base)*base = carry4%base
; 0-3 next carry zero based, 4-7 base, 8-11 x1-4, 12-15 carry_lo1-4, 16-19 1/base

	zfmaddpd zmm8, zmm12, zmm24, zmm8	; 17-20 ;; x1 += carry1_lo * k_lo*mulconst
	zfmaddpd zmm9, zmm13, zmm24, zmm9	; 18-21 ;; x2 += carry2_lo * k_lo*mulconst
	zfmaddpd zmm10, zmm14, zmm24, zmm10	; 19-22 ;; x3 += carry3_lo * k_lo*mulconst
	zfmaddpd zmm11, zmm15, zmm24, zmm11	; 20-23 ;; x4 += carry4_lo * k_lo*mulconst
; 0-3 next carry zero based, 4-7 base, 8-11 x1-4, 12-15 carry_lo1-4, 16-19 1/base

	zfmaddpd zmm16, zmm8, zmm16, zmm30	; 21-24 ;; x1_hi+RNDVAL = x1 / base + RNDVAL = rnd(x1/base)+RNDVAL
	zfmaddpd zmm17, zmm9, zmm17, zmm30	; 22-25 ;; x2_hi+RNDVAL = x2 / base + RNDVAL = rnd(x2/base)+RNDVAL
	zfmaddpd zmm18, zmm10, zmm18, zmm30	; 23-26 ;; x3_hi+RNDVAL = x3 / base + RNDVAL = rnd(x3/base)+RNDVAL
	zfmaddpd zmm19, zmm11, zmm19, zmm30	; 24-27 ;; x4_hi+RNDVAL = x4 / base + RNDVAL = rnd(x4/base)+RNDVAL
; 0-3 next carry zero based, 4-7 base, 8-11 x1-4, 12-15 carry_lo1-4, 16-19 x1-4_hi

	vsubpd	zmm16, zmm16, zmm30		; 25-28 ;; x1_hi = x1_hi+RNDVAL - RNDVAL = rnd(x1/base) (next carry5)
	vsubpd	zmm17, zmm17, zmm30		; 26-29 ;; x2_hi = x2_hi+RNDVAL - RNDVAL = rnd(x2/base) (next carry6)
	vsubpd	zmm18, zmm18, zmm30		; 27-30 ;; x3_hi = x3_hi+RNDVAL - RNDVAL = rnd(x3/base) (next carry7)
	vsubpd	zmm19, zmm19, zmm30		; 28-31 ;; x4_hi = x4_hi+RNDVAL - RNDVAL = rnd(x4/base) (next carry8)
; 0-3 next carry zero based, 4-7 base, 8-11 x1-4, 12-15 carry_lo1-4, 16-19 x1-4_hi

	zfnmaddpd zmm8, zmm16, zmm4, zmm8	; 29-32 ;; new val1 = x1_lo = x1 - x1_hi * base = x1 - rnd(x1/base)*base = x1%base
	vblendmpd zmm4 {k1}, zmm21, zmm20	; 21	;; Create (k_hi*mulconst/base) constant
	zfnmaddpd zmm9, zmm17, zmm5, zmm9	; 30-33 ;; new val2 = x2_lo = x2 - x2_hi * base = x2 - rnd(x2/base)*base = x2%base
	vblendmpd zmm5 {k2}, zmm21, zmm20	; 22	;; Create (k_hi*mulconst/base) constant
	zfnmaddpd zmm10, zmm18, zmm6, zmm10	; 31-34 ;; new val3 = x3_lo = x3 - x3_hi * base = x3 - rnd(x3/base)*base = x3%base
	vblendmpd zmm6 {k3}, zmm21, zmm20	; 23	;; Create (k_hi*mulconst/base) constant
	zfnmaddpd zmm11, zmm19, zmm7, zmm11	; 32-35 ;; new val4 = x4_lo = x4 - x4_hi * base = x4 - rnd(x4/base)*base = x4%base
	vblendmpd zmm7 {k4}, zmm21, zmm20	; 24	;; Create (k_hi*mulconst/base) constant

	zfmaddpd zmm4, zmm12, zmm4, zmm16	; 29-32 ;; next carry5 = carry1_lo * k_hi*mulconst/base + next carry5
	zfmaddpd zmm5, zmm13, zmm5, zmm17	; 30-33 ;; next carry6 = carry2_lo * k_hi*mulconst/base + next carry6
	zfmaddpd zmm6, zmm14, zmm6, zmm18	; 31-34 ;; next carry7 = carry3_lo * k_hi*mulconst/base + next carry7
	zfmaddpd zmm7, zmm15, zmm7, zmm19	; 32-35 ;; next carry8 = carry4_lo * k_hi*mulconst/base + next carry8

	zstore	[rsi], zmm8			;; Save new val1
	zstore	[rsi+r13], zmm9			;; Save new val2
	zstore	[rsi+2*r13], zmm10		;; Save new val3
	zstore	[rsi+r14], zmm11		;; Save new val4

	vcmpneqpd k1, zmm0, zmm22		;; Are any non-zero carries remaining to propagate?
	vcmpneqpd k2, zmm1, zmm22
	vcmpneqpd k3, zmm2, zmm22
	vcmpneqpd k4, zmm3, zmm22
	korw	k5, k1, k2
	korw	k6, k3, k4
	vcmpneqpd k1, zmm4, zmm22		;; Are any non-zero carries remaining to propagate?
	vcmpneqpd k2, zmm5, zmm22
	vcmpneqpd k3, zmm6, zmm22
	vcmpneqpd k4, zmm7, zmm22
	korw	k1, k1, k2
	korw	k2, k3, k4
	korw	k3, k5, k6
	korw	k4, k1, k2
	kortestw k3, k4
	jz	done				;; If carries not found then we're done

	bump	rsi, 128			;; Next source pointer
	mov	rdx, r12			;; If r12 has been been bumped by 4, we'll need to unbump r12 and bump rdi
	shr	rdx, 3				;; Set carry if r12 is in +4 state
	adc	rdi, 0				;; Increment (or not) the biglit table pointer
	xor	r12, 4				;; Bump or unbump pointer into the compressed biglit table
	dec	ebx				;; Test for end of section
	jnz	cloop				;; Loop to next double cache lines within section
	jmp	rloop				;; Loop to rotate carries and go back to section start

	;; Do eighth and last iteration without propagating carries out
cloop_end:
	vaddpd	zmm8, zmm4, [rsi]		;; carry + val1
	vaddpd	zmm9, zmm5, [rsi+r13]		;; carry + val2
	vaddpd	zmm10, zmm6, [rsi+2*r13]	;; carry + val3
	vaddpd	zmm11, zmm7, [rsi+r14]		;; carry + val4
	zstore	[rsi], zmm8			;; Save val1
	zstore	[rsi+r13], zmm9			;; Save val2
	zstore	[rsi+2*r13], zmm10		;; Save val3
	zstore	[rsi+r14], zmm11			;; Save val4

done:	zrestore_top_carries_1d_zpad exec	;; Load carries for next section

	vaddpd	zmm0, zmm0, zmm30		;; Restore RNDVAL to low carries (high carries stay in +0 format)
	vaddpd	zmm1, zmm1, zmm30
	vaddpd	zmm2, zmm2, zmm30
	vaddpd	zmm3, zmm3, zmm30

	pop	r12				;; Restore next section' compressed biglit table pointer
	pop	rdi				;; Restore next section's biglit pointer
	ENDM


; Zpad add/sub/addsub/small mul carries are in +0 format.  There are only 4 carries as the low zpad carry is aleays zero.
zadd_carry_op_1d_zpad MACRO ttp
no ttp	zadd_carry_op_1d_zpad_nottp
ttp	zadd_carry_op_1d_zpad_ttp
	ENDM

;; Rotate 4 carries.  Save every top carry from in zmm23.  Carries are rotated up, least significant word is set to RNDVAL.  Carries are in +0 format.
zrotate_carries_op_1d_zpad MACRO ttp
	vpmovzxbq zmm9, ZMM_PERMUTE3		;; zmm9 = 0+6 0+5 0+4 0+3 0+2 0+1 0+0 8+7
	vpermt2pd zmm8, zmm9, zmm0		;; Save top carry from zmm0 into zmm8
	vpermt2pd zmm0, zmm9, zmm22		;; Rotate carries in zmm0
	vpermt2pd zmm8, zmm9, zmm1		;; Save top carry from zmm1 into zmm8
	vpermt2pd zmm1, zmm9, zmm22		;; Rotate carries in zmm1
	vpermt2pd zmm8, zmm9, zmm2		;; Save top carry from zmm2 into zmm8
	vpermt2pd zmm2, zmm9, zmm22		;; Rotate carries in zmm2
	vpermt2pd zmm8, zmm9, zmm3		;; Save top carry from zmm3 into zmm8
	vpermt2pd zmm3, zmm9, zmm22		;; Rotate carries in zmm3
	vaddpd	zmm23, zmm23, zmm8		;; Accumulate saved top carries in zmm23
	ENDM

;; Initialize 4 carries for next section using the saved top carries.  Carries are in +0 format.
zrestore_top_carries_op_1d_zpad MACRO ttp
	mov	edx, 11111110b			;; For vblendmpd during swizzle
	kmovw	k7, edx
	vpmovzxbq zmm9, ZMM_PERMUTE4		;; zmm9 = 0+7 0+7 0+6 0+5 0+4 0+3 0+2 0+1
	vblendmpd zmm3{k7}, zmm23, zmm22	;; Restore lowest carry from zmm23 and zero into zmm3
	vpermt2pd zmm23, zmm9, zmm23		;; Rotate saved top carries down
	vblendmpd zmm2{k7}, zmm23, zmm22	;; Restore lowest carry from zmm23 and zero into zmm2
	vpermt2pd zmm23, zmm9, zmm23		;; Rotate saved top carries down
	vblendmpd zmm1{k7}, zmm23, zmm22	;; Restore lowest carry from zmm23 and zero into zmm1
	vpermt2pd zmm23, zmm9, zmm23		;; Rotate saved top carries down
	vblendmpd zmm0{k7}, zmm23, zmm22	;; Restore lowest carry from zmm23 and zero into zmm0
	ENDM

zadd_carry_op_1d_zpad_nottp MACRO
	LOCAL	rloop, cloop, cloop_end, done

	vzero	zmm23					;; Top carries are saved here for adding into next section
	vmovapd	zmm22, zmm23				;; Constant zero

	mov	cl, 8					;; Spread carry over a max of 8 words
rloop:	mov	rsi, r9					;; Restore section start pointers
	zrotate_carries_op_1d_zpad noexec
	mov	ebx, addcount1				;; Count of double cache lines in a section

	;; For add/sub/addsub/smallmul the low carry is zero.
cloop:	dec	cl					;; Decrement propagation count
	jz	cloop_end				;; Exit carry propagation loop
	vaddpd	zmm8, zmm0, [rsi]		; 1-4	;; x1 = carry1 + values1
	vaddpd	zmm9, zmm1, [rsi+r13]		; 1-4	;; x2 = carry2 + values2
	vaddpd	zmm10, zmm2, [rsi+2*r13]	; 2-5	;; x3 = carry3 + values3
	vaddpd	zmm11, zmm3, [rsi+r14]		; 2-5	;; x4 = carry4 + values4

	zfmaddpd zmm0, zmm8, zmm28, zmm30	; 9-12	;; next carry+RNDVAL = (x/base + RNDVAL) = rnd(x/base)+RNDVAL
	zfmaddpd zmm1, zmm9, zmm28, zmm30	; 9-12	;; next carry+RNDVAL = (x/base + RNDVAL) = rnd(x/base)+RNDVAL
	zfmaddpd zmm2, zmm10, zmm28, zmm30	; 10-13	;; next carry+RNDVAL = (x/base + RNDVAL) = rnd(x/base)+RNDVAL
	zfmaddpd zmm3, zmm11, zmm28, zmm30	; 10-13	;; next carry+RNDVAL = (x/base + RNDVAL) = rnd(x/base)+RNDVAL

	vsubpd	zmm0, zmm0, zmm30		; 13-16	;; next carry = (next carry+RNDVAL) - RNDVAL = rnd(x/base)
	vsubpd	zmm1, zmm1, zmm30		; 13-16	;; next carry = (next carry+RNDVAL) - RNDVAL = rnd(x/base)
	vsubpd	zmm2, zmm2, zmm30		; 14-17	;; next carry = (next carry+RNDVAL) - RNDVAL = rnd(x/base)
	vsubpd	zmm3, zmm3, zmm30		; 14-17	;; next carry = (next carry+RNDVAL) - RNDVAL = rnd(x/base)

	zfnmaddpd zmm8, zmm0, zmm29, zmm8	; 17-20	;; new value = x - rnd(x/base)*base
	zfnmaddpd zmm9, zmm1, zmm29, zmm9	; 17-20	;; new value = x - rnd(x/base)*base
	zfnmaddpd zmm10, zmm2, zmm29, zmm10	; 18-21	;; new value = x - rnd(x/base)*base
	zfnmaddpd zmm11, zmm3, zmm29, zmm11	; 18-21	;; new value = x - rnd(x/base)*base

	zstore	[rsi], zmm8			;; Save new val1
	zstore	[rsi+r13], zmm9			;; Save new val2
	zstore	[rsi+2*r13], zmm10		;; Save new val3
	zstore	[rsi+r14], zmm11		;; Save new val4

	vcmpneqpd k1, zmm0, zmm22		;; Are any non-zero carries remaining to propagate?
	vcmpneqpd k2, zmm1, zmm22
	vcmpneqpd k3, zmm2, zmm22
	vcmpneqpd k4, zmm3, zmm22
	korw	k1, k1, k2
	korw	k3, k3, k4
	kortestw k1, k3
	jz	done				;; If carries not found then we're done

	bump	rsi, 128			;; Next source pointer
	dec	ebx				;; Test for end of section
	jnz	cloop				;; Loop to next double cache lines within section
	jmp	rloop				;; Loop to rotate carries and go back to section start

	;; Do eighth and last iteration without propagating carries out
cloop_end:
	vaddpd	zmm8, zmm0, [rsi]		;; Values1 += carry
	vaddpd	zmm9, zmm1, [rsi+r13]		;; Values2 += carry
	vaddpd	zmm10, zmm2, [rsi+2*r13]	;; Values3 += carry
	vaddpd	zmm11, zmm3, [rsi+r14]		;; Values4 += carry
	zstore	[rsi], zmm8			;; Save value1
	zstore	[rsi+r13], zmm9			;; Save value2
	zstore	[rsi+2*r13], zmm10		;; Save value3
	zstore	[rsi+r14], zmm11		;; Save value4

done:	zrestore_top_carries_op_1d_zpad noexec	;; Load carries for next section
	ENDM

zadd_carry_op_1d_zpad_ttp MACRO
	LOCAL	rloop, cloop, cloop_end, done

	push	rdi					;; Save next section's biglit pointer
	push	r12					;; Save next section' compressed biglit table pointer

	vzero	zmm23					;; Top carries are saved here for adding into next section
	vmovapd	zmm22, zmm23				;; Constant zero

	mov	cl, 8					;; Spread carry over a max of 8 words
rloop:	mov	rsi, r9					;; Restore section start pointers
	mov	rdi, r8
	mov	r12, rbp
	zrotate_carries_op_1d_zpad exec
	mov	ebx, addcount1				;; Count of double cache lines in a section

	;; For add/sub/addsub/smallmul the low carry is zero.
cloop:	dec	cl					;; Decrement propagation count
	jz	cloop_end				;; Exit carry propagation loop
	movzx	rdx, BYTE PTR [rdi]			;; Load index into compressed biglit table
	mov	edx, [r12+rdx*4]			;; Load 4 big vs. little flags

	kmovw	k1, edx				; 1	;; Load 2 big vs. little flags (16-bit load - we use 8-bits)
	vblendmpd zmm16 {k1}, zmm28, zmm26	; 2	;; Create (1/base) constant
	kshiftrw k2, k1, 8			; 2
	vblendmpd zmm17 {k2}, zmm28, zmm26	; 3	;; Create (1/base) constant
	shr	edx, 16
	kmovw	k3, edx				; 3	;; Load 2 big vs. little flags (16-bit load - we use 8-bits)
	vblendmpd zmm18 {k3}, zmm28, zmm26	; 4	;; Create (1/base) constant
	kshiftrw k4, k3, 8			; 4
	vblendmpd zmm19 {k4}, zmm28, zmm26	; 5	;; Create (1/base) constant

	vaddpd	zmm8, zmm0, [rsi]		; 6-9	;; x1 = carry5 + values1
	vaddpd	zmm9, zmm1, [rsi+r13]		; 7-10	;; x2 = carry6 + values2
	vaddpd	zmm10, zmm2, [rsi+2*r13]	; 8-11	;; x3 = carry7 + values3
	vaddpd	zmm11, zmm3, [rsi+r14]		; 9-12	;; x4 = carry8 + values4

	vblendmpd zmm12 {k1}, zmm29, zmm27	; 10	;; Create (base) constant
	vblendmpd zmm13 {k2}, zmm29, zmm27	; 11	;; Create (base) constant
	vblendmpd zmm14 {k3}, zmm29, zmm27	; 12	;; Create (base) constant
	vblendmpd zmm15 {k4}, zmm29, zmm27	; 13	;; Create (base) constant

	zfmaddpd zmm16, zmm8, zmm16, zmm30	; 21-24 ;; x1_hi+RNDVAL = x1 / base + RNDVAL = rnd(x1/base)+RNDVAL
	zfmaddpd zmm17, zmm9, zmm17, zmm30	; 22-25 ;; x2_hi+RNDVAL = x2 / base + RNDVAL = rnd(x2/base)+RNDVAL
	zfmaddpd zmm18, zmm10, zmm18, zmm30	; 23-26 ;; x3_hi+RNDVAL = x3 / base + RNDVAL = rnd(x3/base)+RNDVAL
	zfmaddpd zmm19, zmm11, zmm19, zmm30	; 24-27 ;; x4_hi+RNDVAL = x4 / base + RNDVAL = rnd(x4/base)+RNDVAL

	vsubpd	zmm0, zmm16, zmm30		; 25-28 ;; x1_hi = x1_hi+RNDVAL - RNDVAL = rnd(x1/base) (next carry1)
	vsubpd	zmm1, zmm17, zmm30		; 26-29 ;; x2_hi = x2_hi+RNDVAL - RNDVAL = rnd(x2/base) (next carry2)
	vsubpd	zmm2, zmm18, zmm30		; 27-30 ;; x3_hi = x3_hi+RNDVAL - RNDVAL = rnd(x3/base) (next carry3)
	vsubpd	zmm3, zmm19, zmm30		; 28-31 ;; x4_hi = x4_hi+RNDVAL - RNDVAL = rnd(x4/base) (next carry4)

	zfnmaddpd zmm8, zmm0, zmm12, zmm8	; 29-32 ;; new val1 = x1_lo = x1 - x1_hi * base = x1 - rnd(x1/base)*base = x1%base
	zfnmaddpd zmm9, zmm1, zmm13, zmm9	; 30-33 ;; new val2 = x2_lo = x2 - x2_hi * base = x2 - rnd(x2/base)*base = x2%base
	zfnmaddpd zmm10, zmm2, zmm14, zmm10	; 31-34 ;; new val3 = x3_lo = x3 - x3_hi * base = x3 - rnd(x3/base)*base = x3%base
	zfnmaddpd zmm11, zmm3, zmm15, zmm11	; 32-35 ;; new val4 = x4_lo = x4 - x4_hi * base = x4 - rnd(x4/base)*base = x4%base

	zstore	[rsi], zmm8			;; Save new val1
	zstore	[rsi+r13], zmm9			;; Save new val2
	zstore	[rsi+2*r13], zmm10		;; Save new val3
	zstore	[rsi+r14], zmm11		;; Save new val4

	vcmpneqpd k1, zmm0, zmm22		;; Are any non-zero carries remaining to propagate?
	vcmpneqpd k2, zmm1, zmm22
	vcmpneqpd k3, zmm2, zmm22
	vcmpneqpd k4, zmm3, zmm22
	korw	k1, k1, k2
	korw	k3, k3, k4
	kortestw k1, k3
	jz	done				;; If carries not found then we're done

	bump	rsi, 128			;; Next source pointer
	mov	rdx, r12			;; If r12 has been been bumped by 4, we'll need to unbump r12 and bump rdi
	shr	rdx, 3				;; Set carry if r12 is in +4 state
	adc	rdi, 0				;; Increment (or not) the biglit table pointer
	xor	r12, 4				;; Bump or unbump pointer into the compressed biglit table
	dec	ebx				;; Test for end of section
	jnz	cloop				;; Loop to next double cache lines within section
	jmp	rloop				;; Loop to rotate carries and go back to section start

	;; Do eighth and last iteration without propagating carries out
cloop_end:
	vaddpd	zmm8, zmm0, [rsi]		;; carry + val1
	vaddpd	zmm9, zmm1, [rsi+r13]		;; carry + val2
	vaddpd	zmm10, zmm2, [rsi+2*r13]	;; carry + val3
	vaddpd	zmm11, zmm3, [rsi+r14]		;; carry + val4
	zstore	[rsi], zmm8			;; Save val1
	zstore	[rsi+r13], zmm9			;; Save val2
	zstore	[rsi+2*r13], zmm10		;; Save val3
	zstore	[rsi+r14], zmm11		;; Save val4

done:	zrestore_top_carries_op_1d_zpad exec	;; Load carries for next section

	pop	r12				;; Restore next section' compressed biglit table pointer
	pop	rdi				;; Restore next section's biglit pointer
	ENDM

; Process the top carries from 3 of the last 4 sections processed in a one-pass FFT normalization.
; zmm3 and zmm7 wraps around to the first double-cache line.  This is handled by the zpad_final macros.
; zmm0-2 and zmm4-6 carries are added to the second, third, and fourth sections (val2, val3, val4 in this macro)
; On input:
; zmm0-2 and zmm4-6 input carries
; zmm3 and zmm7 carries preserved for later processing
; zmm20-31 initialized by zadd_carry_1d_zpad

zadd_carry_1d_zpad_cleanup MACRO ttp
no ttp	zadd_carry_1d_zpad_cleanup_nottp
ttp	zadd_carry_1d_zpad_cleanup_ttp
	ENDM

;; Rotate carries.  Carries are rotated up, least significant word is set to RNDVAL.
zrotate_carries_1d_zpad_cleanup MACRO ttp
	vpmovzxbq zmm9, ZMM_PERMUTE3		;; zmm9 = 0+6 0+5 0+4 0+3 0+2 0+1 0+0 8+7
no ttp	vpermt2pd zmm0, zmm9, zmm30		;; Rotate carries in zmm0
no ttp	vpermt2pd zmm1, zmm9, zmm30		;; Rotate carries in zmm1
no ttp	vpermt2pd zmm2, zmm9, zmm30		;; Rotate carries in zmm2
no ttp	vpermt2pd zmm4, zmm9, zmm30		;; Rotate carries in zmm4
no ttp	vpermt2pd zmm5, zmm9, zmm30		;; Rotate carries in zmm5
no ttp	vpermt2pd zmm6, zmm9, zmm30		;; Rotate carries in zmm6
ttp	vpermt2pd zmm0, zmm9, zmm22		;; Rotate carries in zmm0
ttp	vpermt2pd zmm1, zmm9, zmm22		;; Rotate carries in zmm1
ttp	vpermt2pd zmm2, zmm9, zmm22		;; Rotate carries in zmm2
ttp	vpermt2pd zmm4, zmm9, zmm22		;; Rotate carries in zmm4
ttp	vpermt2pd zmm5, zmm9, zmm22		;; Rotate carries in zmm5
ttp	vpermt2pd zmm6, zmm9, zmm22		;; Rotate carries in zmm6
	ENDM

zadd_carry_1d_zpad_cleanup_nottp MACRO
	LOCAL	noncon, rloop, cloop, cloop_end, done

	mov	cl, 8				;; Spread carry over a max of 8 words
	mov	ebx, 1				;; Only do one AVX double at a time (an oscillating value of 1/2 maxval in eighth word is deadly)
	kmovw	k7, ebx
rloop:	mov	rsi, DESTARG			;; Start of first section
	mov	ebx, addcount1			;; Count of double cache lines in a section

	;; Proper carry propagation requires splitting the lower carry that has not been mul'ed-by-k or mul-by-const
cloop:	dec	cl					;; Decrement propagation count
	jz	cloop_end				;; Exit carry propagation loop
	vmovapd	zmm13, zmm0				;; Copy carry2+RNDVAL
	zfmsubpd zmm0, zmm0, zmm28, zmm26	; 2-5	;; carry2_hi+RNDVAL = (carry2+RNDVAL) / base - (RNDVAL/base-RNDVAL) = rnd(carry2/base)+RNDVAL (next carry2)
	vaddpd	zmm9, zmm4, [rsi+r13]		; 2-5	;; x2+RNDVAL = carry6 + values2
	vmovapd	zmm14, zmm1				;; Copy carry3+RNDVAL
	zfmsubpd zmm1, zmm1, zmm28, zmm26	; 3-6	;; carry3_hi+RNDVAL = (carry3+RNDVAL) / base - (RNDVAL/base-RNDVAL) = rnd(carry3/base)+RNDVAL (next carry3)
	vaddpd	zmm10, zmm5, [rsi+2*r13]	; 3-6	;; x3+RNDVAL = carry7 + values3
	vmovapd	zmm15, zmm2				;; Copy carry4+RNDVAL
	zfmsubpd zmm2, zmm2, zmm28, zmm26	; 4-7	;; carry4_hi+RNDVAL = (carry4+RNDVAL) / base - (RNDVAL/base-RNDVAL) = rnd(carry4/base)+RNDVAL (next carry4)
	vaddpd	zmm11, zmm6, [rsi+r14]		; 4-7	;; x4+RNDVAL = carry8 + values4

	zfmsubpd zmm17, zmm0, zmm29, zmm27	; 6-9	;; tmp2+RNDVAL = (carry2_hi+RNDVAL) * base - (RNDVAL*base-RNDVAL) = rnd(carry2/base)*base+RNDVAL
	zfmsubpd zmm18, zmm1, zmm29, zmm27	; 7-10	;; tmp3+RNDVAL = (carry3_hi+RNDVAL) * base - (RNDVAL*base-RNDVAL) = rnd(carry3/base)*base+RNDVAL
	zfmsubpd zmm19, zmm2, zmm29, zmm27	; 8-11	;; tmp4+RNDVAL = (carry4_hi+RNDVAL) * base - (RNDVAL*base-RNDVAL) = rnd(carry4/base)*base+RNDVAL

	vsubpd	zmm13, zmm13, zmm17		; 10-13	;; carry2_lo = carry2+RNDVAL - tmp2+RNDVAL = carry2 - rnd(carry2/base)*base = carry2%base
	vsubpd	zmm14, zmm14, zmm18		; 11-14	;; carry3_lo = carry3+RNDVAL - tmp3+RNDVAL = carry3 - rnd(carry3/base)*base = carry3%base
	vsubpd	zmm15, zmm15, zmm19		; 12-15	;; carry4_lo = carry4+RNDVAL - tmp4+RNDVAL = carry4 - rnd(carry4/base)*base = carry4%base

	zfmaddpd zmm9, zmm13, zmm24, zmm9	; 14-17 ;; x2+RNDVAL += carry2_lo * k_lo*mulconst
	zfmaddpd zmm10, zmm14, zmm24, zmm10	; 15-18 ;; x3+RNDVAL += carry3_lo * k_lo*mulconst
	zfmaddpd zmm11, zmm15, zmm24, zmm11	; 16-19 ;; x4+RNDVAL += carry4_lo * k_lo*mulconst

	zfmsubpd zmm4, zmm9, zmm28, zmm26	; 18-21 ;; x2_hi+RNDVAL = (x2+RNDVAL) / base - (RNDVAL/base-RNDVAL) = rnd(x2/base)+RNDVAL (next carry6)
	zfmsubpd zmm5, zmm10, zmm28, zmm26	; 19-22 ;; x3_hi+RNDVAL = (x3+RNDVAL) / base - (RNDVAL/base-RNDVAL) = rnd(x3/base)+RNDVAL (next carry7)
	zfmsubpd zmm6, zmm11, zmm28, zmm26	; 20-23 ;; x4_hi+RNDVAL = (x4+RNDVAL) / base - (RNDVAL/base-RNDVAL) = rnd(x4/base)+RNDVAL (next carry8)

	zfmsubpd zmm17, zmm4, zmm29, zmm27	; 22-25 ;; tmp6+RNDVAL = (x2_hi+RNDVAL) * base - (RNDVAL*base-RNDVAL) = rnd(x2/base)*base+RNDVAL
	zfmaddpd zmm4, zmm13, zmm21, zmm4	; 22-25 ;; next carry6 = carry2_lo * k_hi*mulconst/base + next carry6
	zfmsubpd zmm18, zmm5, zmm29, zmm27	; 23-26 ;; tmp7+RNDVAL = (x3_hi+RNDVAL) * base - (RNDVAL*base-RNDVAL) = rnd(x3/base)*base+RNDVAL
	zfmaddpd zmm5, zmm14, zmm21, zmm5	; 23-26 ;; next carry7 = carry3_lo * k_hi*mulconst/base + next carry7
	zfmsubpd zmm19, zmm6, zmm29, zmm27	; 24-27 ;; tmp8+RNDVAL = (x4_hi+RNDVAL) * base - (RNDVAL*base-RNDVAL) = rnd(x4/base)*base+RNDVAL
	zfmaddpd zmm6, zmm15, zmm21, zmm6	; 24-27 ;; next carry8 = carry4_lo * k_hi*mulconst/base + next carry8

	vsubpd	zmm9, zmm9, zmm17		; 26-29 ;; new val2 = x2_lo = x2+RNDVAL - tmp6+RNDVAL = x2 - rnd(x2/base)*base = x2%base
	vsubpd	zmm10, zmm10, zmm18		; 27-30 ;; new val3 = x3_lo = x3+RNDVAL - tmp7+RNDVAL = x3 - rnd(x3/base)*base = x3%base
	vsubpd	zmm11, zmm11, zmm19		; 28-31 ;; new val4 = x4_lo = x4+RNDVAL - tmp8+RNDVAL = x4 - rnd(x4/base)*base = x4%base

	zstore	[rsi+r13]{k7}, zmm9		;; Save new val2
	zstore	[rsi+2*r13]{k7}, zmm10		;; Save new val3
	zstore	[rsi+r14]{k7}, zmm11		;; Save new val4

	vcmpneqpd k1, zmm0, zmm30		;; Are any non-zero carries remaining to propagate?
	vcmpneqpd k2, zmm1, zmm30
	vcmpneqpd k3, zmm2, zmm30
	vcmpneqpd k4, zmm4, zmm30
	vcmpneqpd k5, zmm5, zmm30
	vcmpneqpd k6, zmm6, zmm30
	korw	k1, k1, k2
	korw	k3, k3, k4
	korw	k5, k5, k6
	korw	k1, k1, k3
	korw	k1, k1, k5
;	ktestw	k1, k7				;; UASM 2.48 bug - this does not compile
kandw k1, k1, k7
kortestw k1, k1
	jz	done				;; If carries not found then we're done

	bump	rsi, 128			;; Next source pointer
	dec	ebx				;; Test for end of section
	jnz	cloop				;; Loop to next double cache lines within section
	zrotate_carries_1d_zpad_cleanup noexec
	kshiftlw k7, k7, 1			;; Next AVX word
	jmp	rloop				;; Loop to go back to section start

	;; Do eighth and last iteration without propagating carries out.  By now, zmm0-2 are zero.
cloop_end:
	vsubpd	zmm9, zmm4, zmm30		;; Remove RNDVAL from carries
	vsubpd	zmm10, zmm5, zmm30
	vsubpd	zmm11, zmm6, zmm30
	vaddpd	zmm9, zmm9, [rsi+r13]		;; Values2 += carry
	vaddpd	zmm10, zmm10, [rsi+2*r13]	;; Values3 += carry
	vaddpd	zmm11, zmm11, [rsi+r14]		;; Values4 += carry
	zstore	[rsi+r13]{k7}, zmm9		;; Save value2
	zstore	[rsi+2*r13]{k7}, zmm10		;; Save value3
	zstore	[rsi+r14]{k7}, zmm11		;; Save value4

	;; Mimic two-pass's zprocess_last_two_carries
done:	mov	rsi, DESTARG			;; Start of first section
	vsubsd	xmm3, xmm3, xmm30		;; Remove RNDVAL from carries
	cmp	const_fft, 0			;; Are we also multiplying by a constant?
	je	short noncon			;; Jump if not const
	vmulsd	xmm3, xmm3, ZMM_MULCONST	;; Mul low carry by optional mul-by-const
noncon:	zfmaddsd xmm4, xmm3, xmm28, xmm30	;; val_hi+RNDVAL = val / base + RNDVAL
	vsubsd	xmm4, xmm4, xmm30		;; val_hi = (val_hi+RNDVAL) - RNDVAL
	zfnmaddsd xmm3, xmm4, xmm29, xmm3	;; val_lo = val - val_hi * base
	vmovsd	Q ZMM_TMP1, xmm3		;; Save split carry for later (in zpad_final_div_by_k)
	vmovsd	Q ZMM_TMP2, xmm4
	vmovsd	[rsi+64], xmm7			;; Store wrap-around high carry (+RNDVAL format) in first upper half FFT word
	ENDM

zadd_carry_1d_zpad_cleanup_ttp MACRO
	LOCAL	noncon, rloop, cloop, cloop_end, done

	vsubpd	zmm0, zmm0, zmm30			;; Remove RNDVAL from low carries (high carries already in +0 format)
	vsubpd	zmm1, zmm1, zmm30
	vsubpd	zmm2, zmm2, zmm30
	vsubpd	zmm3, zmm3, zmm30

	mov	cl, 8					;; Spread carry over a max of 8 words
	mov	ebx, 1					;; Only do one AVX double at a time (an oscillating value of 1/2 maxval in eighth word is deadly)
	kmovw	k7, ebx
rloop:	mov	rsi, DESTARG				;; Start of first section
	mov	rdi, norm_biglit_array			;; pointer to big/lit flags
	mov	r12, compressed_biglits			;; pointer to compressed biglit table
	mov	ebx, addcount1				;; Count of double cache lines in a section

cloop:	dec	cl					;; Decrement propagation count
	jz	cloop_end				;; Exit carry propagation loop
	movzx	rdx, BYTE PTR [rdi]			;; Load index into compressed biglit table
	mov	edx, [r12+rdx*4]			;; Load 4 big vs. little flags

	shr	edx, 8
	kmovw	k2, edx				; 1	;; Load 2 big vs. little flags (16-bit load - we use 8-bits)
	vblendmpd zmm17 {k2}, zmm28, zmm26	; 3	;; Create (1/base) constant
	shr	edx, 8
	kmovw	k3, edx				; 3	;; Load 2 big vs. little flags (16-bit load - we use 8-bits)
	vblendmpd zmm18 {k3}, zmm28, zmm26	; 4	;; Create (1/base) constant
	kshiftrw k4, k3, 8			; 4
	vblendmpd zmm19 {k4}, zmm28, zmm26	; 5	;; Create (1/base) constant

	vmovapd	zmm13, zmm0				;; Copy input carry2
	zfmaddpd zmm0, zmm0, zmm17, zmm30	; 6-9	;; carry2_hi+RNDVAL = carry2 / base + RNDVAL
	vaddpd	zmm9, zmm4, [rsi+r13]		; 7-10	;; x2 = carry6 + values2
	vmovapd	zmm14, zmm1				;; Copy input carry3
	zfmaddpd zmm1, zmm1, zmm18, zmm30	; 7-10	;; carry3_hi+RNDVAL = carry3 / base + RNDVAL
	vaddpd	zmm10, zmm5, [rsi+2*r13]	; 8-11	;; x3 = carry7 + values3
	vmovapd	zmm15, zmm2				;; Copy input carry4
	zfmaddpd zmm2, zmm2, zmm19, zmm30	; 8-11	;; carry4_hi+RNDVAL = carry4 / base + RNDVAL
	vaddpd	zmm11, zmm6, [rsi+r14]		; 9-12	;; x4 = carry8 + values4
; 0-2 next carry, 4-6 avail, 9-11 x1-4, 12-15 in carry1-4, 16-19 1/base

	vsubpd	zmm0, zmm0, zmm30		; 10-13	;; carry2_hi = carry2_hi+RNDVAL - RNDVAL (next carry2)
	vblendmpd zmm4 {k2}, zmm29, zmm27	; 11	;; Create (base) constant
	vsubpd	zmm1, zmm1, zmm30		; 11-14	;; carry3_hi = carry3_hi+RNDVAL - RNDVAL (next carry3)
	vblendmpd zmm5 {k3}, zmm29, zmm27	; 12	;; Create (base) constant
	vsubpd	zmm2, zmm2, zmm30		; 12-15	;; carry4_hi = carry4_hi+RNDVAL - RNDVAL (next carry4)
	vblendmpd zmm6 {k4}, zmm29, zmm27	; 13	;; Create (base) constant
; 0-3 next carry zero based, 4-7 base, 8-11 x1-4, 12-15 in carry1-4, 16-19 1/base

	zfnmaddpd zmm13, zmm0, zmm4, zmm13	; 14-17	;; carry2_lo = carry2 - carry2_hi * base = carry2 - rnd(carry2/base)*base = carry2%base
	zfnmaddpd zmm14, zmm1, zmm5, zmm14	; 15-18	;; carry3_lo = carry3 - carry3_hi * base = carry3 - rnd(carry3/base)*base = carry3%base
	zfnmaddpd zmm15, zmm2, zmm6, zmm15	; 16-19	;; carry4_lo = carry4 - carry4_hi * base = carry4 - rnd(carry4/base)*base = carry4%base
; 0-3 next carry zero based, 4-7 base, 8-11 x1-4, 12-15 carry_lo1-4, 16-19 1/base

	zfmaddpd zmm9, zmm13, zmm24, zmm9	; 18-21 ;; x2 += carry2_lo * k_lo*mulconst
	zfmaddpd zmm10, zmm14, zmm24, zmm10	; 19-22 ;; x3 += carry3_lo * k_lo*mulconst
	zfmaddpd zmm11, zmm15, zmm24, zmm11	; 20-23 ;; x4 += carry4_lo * k_lo*mulconst
; 0-3 next carry zero based, 4-7 base, 8-11 x1-4, 12-15 carry_lo1-4, 16-19 1/base

	zfmaddpd zmm17, zmm9, zmm17, zmm30	; 22-25 ;; x2_hi+RNDVAL = x2 / base + RNDVAL = rnd(x2/base)+RNDVAL
	zfmaddpd zmm18, zmm10, zmm18, zmm30	; 23-26 ;; x3_hi+RNDVAL = x3 / base + RNDVAL = rnd(x3/base)+RNDVAL
	zfmaddpd zmm19, zmm11, zmm19, zmm30	; 24-27 ;; x4_hi+RNDVAL = x4 / base + RNDVAL = rnd(x4/base)+RNDVAL
; 0-3 next carry zero based, 4-7 base, 8-11 x1-4, 12-15 carry_lo1-4, 16-19 x1-4_hi

	vsubpd	zmm17, zmm17, zmm30		; 26-29 ;; x2_hi = x2_hi+RNDVAL - RNDVAL = rnd(x2/base) (next carry6)
	vsubpd	zmm18, zmm18, zmm30		; 27-30 ;; x3_hi = x3_hi+RNDVAL - RNDVAL = rnd(x3/base) (next carry7)
	vsubpd	zmm19, zmm19, zmm30		; 28-31 ;; x4_hi = x4_hi+RNDVAL - RNDVAL = rnd(x4/base) (next carry8)
; 0-3 next carry zero based, 4-7 base, 8-11 x1-4, 12-15 carry_lo1-4, 16-19 x1-4_hi

	zfnmaddpd zmm9, zmm17, zmm4, zmm9	; 30-33 ;; new val2 = x2_lo = x2 - x2_hi * base = x2 - rnd(x2/base)*base = x2%base
	vblendmpd zmm4 {k2}, zmm21, zmm20	; 22	;; Create (k_hi*mulconst/base) constant
	zfnmaddpd zmm10, zmm18, zmm5, zmm10	; 31-34 ;; new val3 = x3_lo = x3 - x3_hi * base = x3 - rnd(x3/base)*base = x3%base
	vblendmpd zmm5 {k3}, zmm21, zmm20	; 23	;; Create (k_hi*mulconst/base) constant
	zfnmaddpd zmm11, zmm19, zmm6, zmm11	; 32-35 ;; new val4 = x4_lo = x4 - x4_hi * base = x4 - rnd(x4/base)*base = x4%base
	vblendmpd zmm6 {k4}, zmm21, zmm20	; 24	;; Create (k_hi*mulconst/base) constant

	zfmaddpd zmm4, zmm13, zmm4, zmm17	; 30-33 ;; next carry6 = carry2_lo * k_hi*mulconst/base + next carry6
	zfmaddpd zmm5, zmm14, zmm5, zmm18	; 31-34 ;; next carry7 = carry3_lo * k_hi*mulconst/base + next carry7
	zfmaddpd zmm6, zmm15, zmm6, zmm19	; 32-35 ;; next carry8 = carry4_lo * k_hi*mulconst/base + next carry8

	zstore	[rsi+r13]{k7}, zmm9		;; Save new val2
	zstore	[rsi+2*r13]{k7}, zmm10		;; Save new val3
	zstore	[rsi+r14]{k7}, zmm11		;; Save new val4

	vcmpneqpd k1, zmm0, zmm22		;; Are any non-zero carries remaining to propagate?
	vcmpneqpd k2, zmm1, zmm22
	vcmpneqpd k3, zmm2, zmm22
	vcmpneqpd k4, zmm4, zmm22
	vcmpneqpd k5, zmm5, zmm22
	vcmpneqpd k6, zmm6, zmm22
	korw	k1, k1, k2
	korw	k3, k3, k4
	korw	k5, k5, k6
	korw	k1, k1, k3
	korw	k1, k1, k5
;	ktestw	k1, k7				;; UASM 2.48 bug - this does not compile
kandw k1, k1, k7
kortestw k1, k1
	jz	done				;; If carries not found then we're done

	bump	rsi, 128			;; Next source pointer
	mov	rdx, r12			;; If r12 has been been bumped by 4, we'll need to unbump r12 and bump rdi
	shr	rdx, 3				;; Set carry if r12 is in +4 state
	adc	rdi, 0				;; Increment (or not) the biglit table pointer
	xor	r12, 4				;; Bump or unbump pointer into the compressed biglit table
	dec	ebx				;; Test for end of section
	jnz	cloop				;; Loop to next double cache lines within section
	zrotate_carries_1d_zpad_cleanup exec
	kshiftlw k7, k7, 1			;; Next AVX word
	jmp	rloop				;; Loop to go back to section start

	;; Do eighth and last iteration without propagating carries out.  By now zmm0-zmm2 are zero.
cloop_end:
	vaddpd	zmm9, zmm4, [rsi+r13]		;; carry + val2
	vaddpd	zmm10, zmm5, [rsi+2*r13]	;; carry + val3
	vaddpd	zmm11, zmm6, [rsi+r14]		;; carry + val4
	zstore	[rsi+r13]{k7}, zmm9		;; Save val2
	zstore	[rsi+2*r13]{k7}, zmm10		;; Save val3
	zstore	[rsi+r14]{k7}, zmm11		;; Save val4

	;; Mimic two-pass's zprocess_last_two_carries
done:	mov	rsi, DESTARG			;; Start of first section
	cmp	const_fft, 0			;; Are we also multiplying by a constant?
	je	short noncon			;; Jump if not const
	vmulsd	xmm3, xmm3, ZMM_MULCONST	;; Mul low carry by optional mul-by-const
noncon:	zfmaddsd xmm4, xmm3, xmm26, xmm30	;; val_hi+RNDVAL = val / base + RNDVAL = rnd(val/base)+RNDVAL
	vsubsd	xmm4, xmm4, xmm30		;; val_hi = (val_hi+RNDVAL) - RNDVAL
	zfnmaddsd xmm3, xmm4, xmm27, xmm3	;; val_lo = val - val_hi * base
	vmovsd	Q ZMM_TMP1, xmm3		;; Save split carry for later (in zpad_final_div_by_k)
	vmovsd	Q ZMM_TMP2, xmm4
	vaddsd	xmm7, xmm7, xmm30
	vmovsd	[rsi+64], xmm7			;; Store wrap-around high carry (+RNDVAL format) in first upper half FFT word
	ENDM


zadd_carry_op_1d_zpad_cleanup MACRO ttp
no ttp	zadd_carry_op_1d_zpad_cleanup_nottp
ttp	zadd_carry_op_1d_zpad_cleanup_ttp
	ENDM

;; Rotate 8 carries.  Carries are rotated up, least significant word is set to RNDVAL.
zrotate_carries_op_1d_zpad_cleanup MACRO ttp
	vpmovzxbq zmm9, ZMM_PERMUTE3		;; zmm9 = 0+6 0+5 0+4 0+3 0+2 0+1 0+0 8+7
	vpermt2pd zmm0, zmm9, zmm22		;; Rotate carries in zmm4
	vpermt2pd zmm1, zmm9, zmm22		;; Rotate carries in zmm5
	vpermt2pd zmm2, zmm9, zmm22		;; Rotate carries in zmm6
	ENDM

zadd_carry_op_1d_zpad_cleanup_nottp MACRO
	LOCAL	rloop, cloop, cloop_end, done

	mov	cl, 8				;; Spread carry over a max of 8 words
	mov	ebx, 1				;; Only do one AVX double at a time (an oscillating value of 1/2 maxval in eighth word is deadly)
	kmovw	k7, ebx
rloop:	mov	rsi, DESTARG			;; Start of first section
	mov	ebx, addcount1			;; Count of double cache lines in a section

	;; The following rounding code is simpler than zadd_carry_1d_zpad_nottp as low carry is zero
cloop:	dec	cl				;; Decrement propagation count
	jz	cloop_end			;; Exit carry propagation loop
	vaddpd	zmm9, zmm0, [rsi+r13]		; 2-5	;; x2 = carry1 + values2
	vaddpd	zmm10, zmm1, [rsi+2*r13]	; 3-6	;; x3 = carry2 + values3
	vaddpd	zmm11, zmm2, [rsi+r14]		; 4-7	;; x4 = carry3 + values4

	zfmaddpd zmm0, zmm9, zmm28, zmm30	; 18-21 ;; x2_hi+RNDVAL = x2 / base + RNDVAL
	zfmaddpd zmm1, zmm10, zmm28, zmm30	; 19-22 ;; x3_hi+RNDVAL = x3 / base + RNDVAL
	zfmaddpd zmm2, zmm11, zmm28, zmm30	; 20-23 ;; x4_hi+RNDVAL = x4 / base + RNDVAL

	vsubpd	zmm0, zmm0, zmm30		; 18-21 ;; x2_hi = x2_hi+RNDVAL - RNDVAL (next carry1)
	vsubpd	zmm1, zmm1, zmm30		; 19-22 ;; x3_hi = x3_hi+RNDVAL - RNDVAL (next carry2)
	vsubpd	zmm2, zmm2, zmm30		; 20-23 ;; x4_hi = x4_hi+RNDVAL - RNDVAL (next carry3)

	zfnmaddpd zmm9, zmm0, zmm29, zmm9	; 22-25 ;; new val2 = x2_lo = x2 - x2_hi * base
	zfnmaddpd zmm10, zmm1, zmm29, zmm10	; 23-26 ;; new val3 = x3_lo = x3 - x3_hi * base
	zfnmaddpd zmm11, zmm2, zmm29, zmm11	; 24-27 ;; new val4 = x4_lo = x4 - x4_hi * base

	zstore	[rsi+r13]{k7}, zmm9		;; Save value2
	zstore	[rsi+2*r13]{k7}, zmm10		;; Save value3
	zstore	[rsi+r14]{k7}, zmm11		;; Save value4

	vcmpneqpd k1, zmm0, zmm22		;; Are any non-zero carries remaining to propagate?
	vcmpneqpd k2, zmm1, zmm22
	vcmpneqpd k3, zmm2, zmm22
	korw	k1, k1, k2
	korw	k1, k1, k3
;	ktestw	k1, k7				;; UASM 2.48 bug - this does not compile
kandw k1, k1, k7
kortestw k1, k1
	jz	done				;; If carries not found then we're done

	bump	rsi, 128			;; Next 1st source pointer
	dec	ebx				;; Test for end of section
	jnz	cloop				;; Loop to next double cache lines within section
	zrotate_carries_op_1d_zpad_cleanup noexec
	kshiftlw k7, k7, 1			;; Next AVX word
	jmp	rloop				;; Loop to go back to section start

	;; Do eighth and last iteration without propagating carries out
cloop_end:
	vaddpd	zmm9, zmm0, [rsi+r13]		;; Values2 += carry
	vaddpd	zmm10, zmm1, [rsi+2*r13]	;; Values3 += carry
	vaddpd	zmm11, zmm2, [rsi+r14]		;; Values4 += carry
	zstore	[rsi+r13]{k7}, zmm9		;; Save value2
	zstore	[rsi+2*r13]{k7}, zmm10		;; Save value3
	zstore	[rsi+r14]{k7}, zmm11		;; Save value4

	;; Mimic two-pass's zprocess_last_two_carries_op_zpad
done:	mov	rsi, DESTARG			;; Start of first section
	vmovsd	Q ZMM_TMP1, xmm22		;; Clear split carry for later (in zpad_final_div_by_k)
	vmovsd	Q ZMM_TMP2, xmm22
	vaddsd	xmm3, xmm3, [rsi+64]		;; Add wrap-around high carry (+0 format) in first upper half FFT word
	vmovsd	[rsi+64], xmm3
	ENDM

zadd_carry_op_1d_zpad_cleanup_ttp MACRO
	LOCAL	rloop, cloop, cloop_end, done

	mov	cl, 8				;; Spread carry over a max of 8 words
	mov	ebx, 1				;; Only do one AVX double at a time (an oscillating value of 1/2 maxval in eighth word is deadly)
	kmovw	k7, ebx
rloop:	mov	rsi, DESTARG			;; Back to start of first section
	mov	rdi, norm_biglit_array		;; pointer to big/lit flags
	mov	r12, compressed_biglits		;; pointer to compressed biglit table
	mov	ebx, addcount1			;; Count of double cache lines in a section

	;; The following rounding code is simpler than zadd_carry_1d_zpad_ttp as low carry is zero
cloop:	dec	cl					;; Decrement propagation count
	jz	cloop_end				;; Exit carry propagation loop
	movzx	rdx, BYTE PTR [rdi]			;; Load index into compressed biglit table
	mov	edx, [r12+rdx*4]			;; Load 4 big vs. little flags
	shr	edx, 8					;; Skip flags for first section
	kmovw	k2, edx				; 1	;; Load flags for second section
	vblendmpd zmm17 {k2}, zmm28, zmm26	; 3	;; Create (1/base) constant
	shr	edx, 8
	kmovw	k3, edx				; 3	;; Load 2 big vs. little flags (16-bit load - we use 8-bits)
	vblendmpd zmm18 {k3}, zmm28, zmm26	; 4	;; Create (1/base) constant
	kshiftrw k4, k3, 8			; 4
	vblendmpd zmm19 {k4}, zmm28, zmm26	; 5	;; Create (1/base) constant

	vaddpd	zmm9, zmm0, [rsi+r13]		; 7-10	;; x2 = carry6 + values2
	vaddpd	zmm10, zmm1, [rsi+2*r13]	; 8-11	;; x3 = carry7 + values3
	vaddpd	zmm11, zmm2, [rsi+r14]		; 9-12	;; x4 = carry8 + values4

	vblendmpd zmm13 {k2}, zmm29, zmm27	; 11	;; Create (base) constant
	vblendmpd zmm14 {k3}, zmm29, zmm27	; 12	;; Create (base) constant
	vblendmpd zmm15 {k4}, zmm29, zmm27	; 13	;; Create (base) constant

	zfmaddpd zmm17, zmm9, zmm17, zmm30	; 22-25 ;; x2_hi+RNDVAL = x2 / base + RNDVAL = rnd(x2/base)+RNDVAL
	zfmaddpd zmm18, zmm10, zmm18, zmm30	; 23-26 ;; x3_hi+RNDVAL = x3 / base + RNDVAL = rnd(x3/base)+RNDVAL
	zfmaddpd zmm19, zmm11, zmm19, zmm30	; 24-27 ;; x4_hi+RNDVAL = x4 / base + RNDVAL = rnd(x4/base)+RNDVAL

	vsubpd	zmm0, zmm17, zmm30		; 26-29 ;; x2_hi = x2_hi+RNDVAL - RNDVAL = rnd(x2/base) (next carry6)
	vsubpd	zmm1, zmm18, zmm30		; 27-30 ;; x3_hi = x3_hi+RNDVAL - RNDVAL = rnd(x3/base) (next carry7)
	vsubpd	zmm2, zmm19, zmm30		; 28-31 ;; x4_hi = x4_hi+RNDVAL - RNDVAL = rnd(x4/base) (next carry8)

	zfnmaddpd zmm9, zmm0, zmm13, zmm9	; 30-33 ;; new val2 = x2_lo = x2 - x2_hi * base = x2 - rnd(x2/base)*base = x2%base
	zfnmaddpd zmm10, zmm1, zmm14, zmm10	; 31-34 ;; new val3 = x3_lo = x3 - x3_hi * base = x3 - rnd(x3/base)*base = x3%base
	zfnmaddpd zmm11, zmm2, zmm15, zmm11	; 32-35 ;; new val4 = x4_lo = x4 - x4_hi * base = x4 - rnd(x4/base)*base = x4%base

	zstore	[rsi+r13]{k7}, zmm9		;; Save new val2
	zstore	[rsi+2*r13]{k7}, zmm10		;; Save new val3
	zstore	[rsi+r14]{k7}, zmm11		;; Save new val4

	vcmpneqpd k1, zmm0, zmm22		;; Are any non-zero carries remaining to propagate?
	vcmpneqpd k2, zmm1, zmm22
	vcmpneqpd k3, zmm2, zmm22
	korw	k1, k1, k2
	korw	k1, k1, k3
;	ktestw	k1, k7				;; UASM 2.48 bug - this does not compile
kandw k1, k1, k7
kortestw k1, k1
	jz	done				;; If carries not found then we're done

	bump	rsi, 128			;; Next source pointer
	mov	rdx, r12			;; If r12 has been been bumped by 4, we'll need to unbump r12 and bump rdi
	shr	rdx, 3				;; Set carry if r12 is in +4 state
	adc	rdi, 0				;; Increment (or not) the biglit table pointer
	xor	r12, 4				;; Bump or unbump pointer into the compressed biglit table
	dec	ebx				;; Test for end of section
	jnz	cloop				;; Loop to next double cache lines within section
	zrotate_carries_op_1d_zpad_cleanup exec
	kshiftlw k7, k7, 1			;; Next AVX word
	jmp	rloop				;; Loop to go back to section start

	;; Do eighth and last iteration without propagating carries out
cloop_end:
	vaddpd	zmm9, zmm0, [rsi+r13]		;; carry + val2
	vaddpd	zmm10, zmm1, [rsi+2*r13]	;; carry + val3
	vaddpd	zmm11, zmm2, [rsi+r14]		;; carry + val4
	zstore	[rsi+r13]{k7}, zmm9		;; Save val2
	zstore	[rsi+2*r13]{k7}, zmm10		;; Save val3
	zstore	[rsi+r14]{k7}, zmm11		;; Save val4

	;; Mimic two-pass's zprocess_last_two_carries_op_zpad
done:	mov	rsi, DESTARG			;; Start of first section
	vmovsd	Q ZMM_TMP1, xmm22		;; Clear split carry for later (in zpad_final_div_by_k)
	vmovsd	Q ZMM_TMP2, xmm22
	vaddsd	xmm3, xmm3, [rsi+64]		;; Add wrap-around high carry (+0 format) in first upper half FFT word
	vmovsd	[rsi+64], xmm3
	ENDM


; *************** 1d normalized zero-padded add/sub macro ******************
; This macro adds or subtracts, then "normalizes" four pairs of loword/hiword FFT data values.  This involves
; making sure integers are smaller than the maximum allowable integer, generating carries when necessary.
; rsi = pointer to the first number
; rcx = pointer to the second number
; rbx = pointer to the destination
; r12 = pointer to compressed biglit table
; rdx = register used to load compressed biglit index
; r13 = distance to source/dest #2
; r14 = distance to source/dest #4
; rdi = pointer to array of big vs. little flags
; zmm0-3 = carries in +0 format

; Same code as two-pass _wpn macros
znorm_op_1d_zpad_preload MACRO ttp
no ttp	vbroadcastsd zmm27, ZMM_RNDVAL_TIMES_SMALL_BASE	;; RNDVAL * small_word_base - RNDVAL (needed for zadd_carry_op_zpad)
no ttp	vbroadcastsd zmm26, ZMM_RNDVAL_OVER_SMALL_BASE	;; RNDVAL / small_word_base - RNDVAL
	znorm_op_wpn_zpad_preload ttp
	ENDM

znorm_op_1d_zpad MACRO fop, ttp
	znorm_op_wpn_zpad fop, ttp
	ENDM

; *************** 1d normalized addsub macro ******************
; This macro adds and subtracts, then "normalizes"  two pairs of loword/hiword FFT data values.
; Differs from the non-zero-pad version in that upper half data need not be normalized.

; Same as two-pass _wpn macros

znorm_addsub_1d_zpad_preload MACRO ttp
no ttp	vbroadcastsd zmm27, ZMM_RNDVAL_TIMES_SMALL_BASE	;; RNDVAL * small_word_base - RNDVAL (needed for zadd_carry_op_zpad)
no ttp	vbroadcastsd zmm26, ZMM_RNDVAL_OVER_SMALL_BASE	;; RNDVAL / small_word_base - RNDVAL
	znorm_addsub_wpn_zpad_preload ttp
	ENDM

znorm_addsub_1d_zpad MACRO ttp
	znorm_addsub_wpn_zpad ttp
	ENDM

; Like the non-zpad version except that only 4 carries need to be swapped
; WARNING: We cannot use ZMM_TMP1 and ZMM_TMP2 as they are cleared by zadd_carry_1d_zpad_cleanup
znorm_addsub_1d_zpad_save MACRO code		; Save/restore state of add and subtract carries
IF code EQ 1
	zstore	ZMM_TMP3, zmm4			; Save subtract carries
	zstore	ZMM_TMP4, zmm5
	zstore	ZMM_TMP5, zmm6
	zstore	ZMM_TMP6, zmm7
ELSEIF code EQ 2
	xchg	r9, r15				; Set/save FFT pointer to apply carries to
	zstore	ZMM_TMP7, zmm0			; Save add carries
	vmovapd	zmm0, ZMM_TMP3			; Restore subtract carries in zmm0-3 for processing
	zstore	ZMM_TMP3, zmm1
	vmovapd	zmm1, ZMM_TMP4
	zstore	ZMM_TMP4, zmm2
	vmovapd	zmm2, ZMM_TMP5
	zstore	ZMM_TMP5, zmm3
	vmovapd	zmm3, ZMM_TMP6
ELSEIF code EQ 3
	xchg	r9, r15				; Restore FFT pointers
	vmovapd	zmm4, zmm0			; Move subtract carries back in zmm4-7
	vmovapd	zmm5, zmm1
	vmovapd	zmm6, zmm2
	vmovapd	zmm7, zmm3
	vmovapd	zmm0, ZMM_TMP7			; Restore add carries
	vmovapd	zmm1, ZMM_TMP3
	vmovapd	zmm2, ZMM_TMP4
	vmovapd	zmm3, ZMM_TMP5
ELSE
	mov	rsi, DEST2ARG			; Force cleanup on second destination
	mov	DESTARG, rsi
	vmovapd	zmm0, ZMM_TMP3			; Restore subtract carries for final claenup
	vmovapd	zmm1, ZMM_TMP4
	vmovapd	zmm2, ZMM_TMP5
	vmovapd	zmm3, ZMM_TMP6
ENDIF
	ENDM

; *************** 1d normalized small mul macro ******************
; This macro multiplies FFT data by a small value, then normalizes 4 loword/hiword pairs.
; rsi = pointer to source
; r13 = distance to source/dest #2
; r14 = distance to source/dest #4
; rbx = distance to destination
; r12 = pointer to compressed biglit table
; rdx = register used to load compressed biglit index
; rdi = pointer to array of big vs. little flags
; zmm0 = carry
; zmm31 = small multiplier value

; Same as two-pass _wpn macros

znorm_smallmul_1d_zpad_preload MACRO ttp
no ttp	vbroadcastsd zmm27, ZMM_RNDVAL_TIMES_SMALL_BASE	;; RNDVAL * small_word_base - RNDVAL (needed for zadd_carry_op_zpad)
no ttp	vbroadcastsd zmm26, ZMM_RNDVAL_OVER_SMALL_BASE	;; RNDVAL / small_word_base - RNDVAL
	znorm_smallmul_wpn_zpad_preload ttp
	ENDM

znorm_smallmul_1d_zpad MACRO ttp
	znorm_smallmul_wpn_zpad ttp
	ENDM


;;*******************************************************************************************
;;				Macros for two pass FFTs
;;*******************************************************************************************


; This is the normalization routine when we are computing modulo k*b^n+c
; with a zero-padded b^2n FFT.  We do this by multiplying the lower FFT
; word by k and adding in the upper word times -c.  Of course, this is made
; very tedious because we have to carefully avoid any loss of precision.
;
;; NOTE: In zero pad FFTs, big/lit and fudge factor flags for the low and high words are identical

; For WPN zpad macros, these registers are set on input:
; rbp = pointer to carries
; rdi = pointer to big/little flags
; rsi = pointer to the FFT data (source #1)
; r13 = distance to second FFT section data
; r14 = distance to fourth FFT section data
; r10 = pointer two-to-phi inverse group multipliers
; rdx = regster for loading biglit index
; r12 = compressed biglit table
; zmm0-zmm3 = carry in +RNDVAL format for the lower FFT word (not yet multiplied by k)
; zmm4-zmm7 = true carry for adding into the next FFT word.  In +RNDVAL format for rational FFTs and +0 format for irrational FFTs.
; zmm31 = maxerr

znorm_wpn_zpad_preload MACRO ttp, echk, const, khi
no echk no ttp	znorm_wpn_zpad_noechk_nottp_preload const, khi
   echk no ttp	znorm_wpn_zpad_echk_nottp_preload const, khi
no echk    ttp	znorm_wpn_zpad_noechk_ttp_preload const, khi
   echk    ttp	znorm_wpn_zpad_echk_ttp_preload const, khi
	ENDM

znorm_wpn_zpad MACRO ttp, echk, const, khi
	L1prefetchw rsi+128, L1PREFETCH_ALL
	L1prefetchw rsi+r13+128, L1PREFETCH_ALL
	L1prefetchw rsi+2*r13+128, L1PREFETCH_ALL
	L1prefetchw rsi+r14+128, L1PREFETCH_ALL
no echk no ttp	znorm_wpn_zpad_noechk_nottp const, khi
   echk no ttp	znorm_wpn_zpad_echk_nottp const, khi
no echk    ttp	znorm_wpn_zpad_noechk_ttp const, khi
   echk    ttp	znorm_wpn_zpad_echk_ttp const, khi
	ENDM

; Rational cases

znorm_wpn_zpad_noechk_nottp_preload MACRO const, khi
		vbroadcastsd zmm30, ZMM_RNDVAL				;; RNDVAL
		vbroadcastsd zmm29, ZMM_SMALL_BASE			;; small_word_base
		vbroadcastsd zmm28, ZMM_SMALL_BASE_INVERSE		;; 1 / small_word_base
		vbroadcastsd zmm27, ZMM_RNDVAL_TIMES_SMALL_BASE		;; RNDVAL * small_word_base - RNDVAL
		vbroadcastsd zmm26, ZMM_RNDVAL_OVER_SMALL_BASE		;; RNDVAL / small_word_base - RNDVAL
no const	vbroadcastsd zmm25, ZMM_MINUS_C
const		vbroadcastsd zmm25, ZMM_MINUS_C_TIMES_MULCONST
no const	vbroadcastsd zmm24, ZMM_K_LO				;; k_lo
const		vbroadcastsd zmm24, ZMM_K_TIMES_MULCONST_LO
khi no const	vbroadcastsd zmm31, ZMM_K_HI_OVER_SMALL_BASE		;; k_hi / small_word_base
khi const	vbroadcastsd zmm31, ZMM_K_TIMES_MULCONST_HI_OVER_SMALL_BASE
	ENDM
znorm_wpn_zpad_noechk_nottp MACRO const, khi
	vaddpd	zmm12, zmm30, [rsi+64]		; 3-6	;; val5+RNDVAL = RNDVAL + value5
	vaddpd	zmm13, zmm30, [rsi+r13+64]	; 3-6	;; val6+RNDVAL = RNDVAL + value6
	vaddpd	zmm14, zmm30, [rsi+2*r13+64]	; 4-7	;; val7+RNDVAL = RNDVAL + value7
	vaddpd	zmm15, zmm30, [rsi+r14+64]	; 4-7	;; val8+RNDVAL = RNDVAL + value8
	vaddpd	zmm8, zmm0, [rsi]		; 1-4	;; val1+RNDVAL = carry1+RNDVAL + value1
	vaddpd	zmm9, zmm1, [rsi+r13]		; 1-4	;; val2+RNDVAL = carry2+RNDVAL + value2
	vaddpd	zmm10, zmm2, [rsi+2*r13]	; 2-5	;; val3+RNDVAL = carry3+RNDVAL + value3
	vaddpd	zmm11, zmm3, [rsi+r14]		; 2-5	;; val4+RNDVAL = carry4+RNDVAL + value4
	zfmsubpd zmm16, zmm12, zmm28, zmm26	; 5-8	;; val5_hi+RNDVAL = (val5+RNDVAL) / base - (RNDVAL/base-RNDVAL) = rnd(val5/base)+RNDVAL
	zfmsubpd zmm17, zmm13, zmm28, zmm26	; 5-8	;; val6_hi+RNDVAL = (val6+RNDVAL) / base - (RNDVAL/base-RNDVAL) = rnd(val6/base)+RNDVAL
	zfmsubpd zmm18, zmm14, zmm28, zmm26	; 6-9	;; val7_hi+RNDVAL = (val7+RNDVAL) / base - (RNDVAL/base-RNDVAL) = rnd(val7/base)+RNDVAL
	zfmsubpd zmm19, zmm15, zmm28, zmm26	; 6-9	;; val8_hi+RNDVAL = (val8+RNDVAL) / base - (RNDVAL/base-RNDVAL) = rnd(val8/base)+RNDVAL
	zfmsubpd zmm0, zmm8, zmm28, zmm26	; 7-10	;; val1_hi+RNDVAL = (val1+RNDVAL) / base - (RNDVAL/base-RNDVAL) = rnd(val1/base)+RNDVAL (next carry1)
	zfmsubpd zmm1, zmm9, zmm28, zmm26	; 7-10	;; val2_hi+RNDVAL = (val2+RNDVAL) / base - (RNDVAL/base-RNDVAL) = rnd(val2/base)+RNDVAL (next carry2)
	zfmsubpd zmm2, zmm10, zmm28, zmm26	; 8-11	;; val3_hi+RNDVAL = (val3+RNDVAL) / base - (RNDVAL/base-RNDVAL) = rnd(val3/base)+RNDVAL (next carry3)
	zfmsubpd zmm3, zmm11, zmm28, zmm26	; 8-11	;; val4_hi+RNDVAL = (val4+RNDVAL) / base - (RNDVAL/base-RNDVAL) = rnd(val4/base)+RNDVAL (next carry4)
	zfmsubpd zmm20, zmm16, zmm29, zmm27	; 9-12	;; tmp5+RNDVAL = (val5_hi+RNDVAL) * base - (RNDVAL*base-RNDVAL) = rnd(val5/base)*base+RNDVAL
	zfmsubpd zmm21, zmm17, zmm29, zmm27	; 9-12	;; tmp6+RNDVAL = (val6_hi+RNDVAL) * base - (RNDVAL*base-RNDVAL) = rnd(val6/base)*base+RNDVAL
	zfmsubpd zmm22, zmm18, zmm29, zmm27	; 10-13	;; tmp7+RNDVAL = (val7_hi+RNDVAL) * base - (RNDVAL*base-RNDVAL) = rnd(val7/base)*base+RNDVAL
	zfmsubpd zmm23, zmm19, zmm29, zmm27	; 10-13	;; tmp8+RNDVAL = (val8_hi+RNDVAL) * base - (RNDVAL*base-RNDVAL) = rnd(val8/base)*base+RNDVAL
	vsubpd	zmm16, zmm16, zmm30		; 11-14	;; val5_hi = val5_hi+RNDVAL - RNDVAL
	vsubpd	zmm17, zmm17, zmm30		; 11-14	;; val6_hi = val6_hi+RNDVAL - RNDVAL
	vsubpd	zmm18, zmm18, zmm30		; 12-15	;; val7_hi = val7_hi+RNDVAL - RNDVAL
	vsubpd	zmm19, zmm19, zmm30		; 12-15	;; val8_hi = val8_hi+RNDVAL - RNDVAL
	vsubpd	zmm12, zmm12, zmm20		; 13-16	;; val5_lo = val5+RNDVAL - tmp5+RNDVAL = val5 - rnd(val5/base)*base = val5%base
	vsubpd	zmm13, zmm13, zmm21		; 13-16	;; val6_lo = val6+RNDVAL - tmp6+RNDVAL = val6 - rnd(val6/base)*base = val6%base
	vsubpd	zmm14, zmm14, zmm22		; 14-17	;; val7_lo = val7+RNDVAL - tmp7+RNDVAL = val7 - rnd(val7/base)*base = val7%base
	vsubpd	zmm15, zmm15, zmm23		; 14-17	;; val8_lo = val8+RNDVAL - tmp8+RNDVAL = val8 - rnd(val8/base)*base = val8%base
	zfmsubpd zmm20, zmm0, zmm29, zmm27	; 15-18	;; tmp1+RNDVAL = (val1_hi+RNDVAL) * base - (RNDVAL*base-RNDVAL) = rnd(val1/base)*base+RNDVAL
	zfmsubpd zmm21, zmm1, zmm29, zmm27	; 15-18	;; tmp2+RNDVAL = (val2_hi+RNDVAL) * base - (RNDVAL*base-RNDVAL) = rnd(val2/base)*base+RNDVAL
	zfmsubpd zmm22, zmm2, zmm29, zmm27	; 16-19	;; tmp3+RNDVAL = (val3_hi+RNDVAL) * base - (RNDVAL*base-RNDVAL) = rnd(val3/base)*base+RNDVAL
	zfmsubpd zmm23, zmm3, zmm29, zmm27	; 16-19	;; tmp4+RNDVAL = (val4_hi+RNDVAL) * base - (RNDVAL*base-RNDVAL) = rnd(val4/base)*base+RNDVAL
	zfmaddpd zmm12, zmm12, zmm25, zmm4	; 17-20	;; x1+RNDVAL = val5_lo * -c*mulconst + carry5+RNDVAL
	zfmaddpd zmm13, zmm13, zmm25, zmm5	; 17-20	;; x2+RNDVAL = val6_lo * -c*mulconst + carry6+RNDVAL
	zfmaddpd zmm14, zmm14, zmm25, zmm6	; 18-21	;; x3+RNDVAL = val7_lo * -c*mulconst + carry7+RNDVAL
	zfmaddpd zmm15, zmm15, zmm25, zmm7	; 18-21	;; x4+RNDVAL = val8_lo * -c*mulconst + carry8+RNDVAL
	vsubpd	zmm8, zmm8, zmm20		; 19-22	;; val1_lo = val1+RNDVAL - tmp1+RNDVAL = val1 - rnd(val1/base)*base = val1%base
	vsubpd	zmm9, zmm9, zmm21		; 19-22	;; val2_lo = val2+RNDVAL - tmp2+RNDVAL = val2 - rnd(val2/base)*base = val2%base
	vsubpd	zmm10, zmm10, zmm22		; 20-23	;; val3_lo = val3+RNDVAL - tmp3+RNDVAL = val3 - rnd(val3/base)*base = val3%base
	vsubpd	zmm11, zmm11, zmm23		; 20-23	;; val4_lo = val4+RNDVAL - tmp4+RNDVAL = val4 - rnd(val4/base)*base = val4%base
	zfmaddpd zmm12, zmm8, zmm24, zmm12	; 23-26 ;; x1+RNDVAL += val1_lo * k_lo*mulconst
	zfmaddpd zmm13, zmm9, zmm24, zmm13	; 23-26 ;; x2+RNDVAL += val2_lo * k_lo*mulconst
	zfmaddpd zmm14, zmm10, zmm24, zmm14	; 24-27 ;; x3+RNDVAL += val3_lo * k_lo*mulconst
	zfmaddpd zmm15, zmm11, zmm24, zmm15	; 24-27 ;; x4+RNDVAL += val4_lo * k_lo*mulconst
	zfmsubpd zmm4, zmm12, zmm28, zmm26	; 27-30 ;; x1_hi+RNDVAL = (x1+RNDVAL) / base - (RNDVAL/base-RNDVAL) = rnd(x1/base)+RNDVAL
	zfmsubpd zmm5, zmm13, zmm28, zmm26	; 27-30 ;; x2_hi+RNDVAL = (x2+RNDVAL) / base - (RNDVAL/base-RNDVAL) = rnd(x2/base)+RNDVAL
	zfmsubpd zmm6, zmm14, zmm28, zmm26	; 28-31 ;; x3_hi+RNDVAL = (x3+RNDVAL) / base - (RNDVAL/base-RNDVAL) = rnd(x3/base)+RNDVAL
	zfmsubpd zmm7, zmm15, zmm28, zmm26	; 28-31 ;; x4_hi+RNDVAL = (x4+RNDVAL) / base - (RNDVAL/base-RNDVAL) = rnd(x4/base)+RNDVAL
	zfmsubpd zmm20, zmm4, zmm29, zmm27	; 31-34 ;; tmp9+RNDVAL = (x1_hi+RNDVAL) * base - (RNDVAL*base-RNDVAL) = rnd(x1/base)*base+RNDVAL
	zfmsubpd zmm21, zmm5, zmm29, zmm27	; 31-34 ;; tmp10+RNDVAL = (x2_hi+RNDVAL) * base - (RNDVAL*base-RNDVAL) = rnd(x2/base)*base+RNDVAL
	zfmsubpd zmm22, zmm6, zmm29, zmm27	; 32-35 ;; tmp11+RNDVAL = (x3_hi+RNDVAL) * base - (RNDVAL*base-RNDVAL) = rnd(x3/base)*base+RNDVAL
	zfmsubpd zmm23, zmm7, zmm29, zmm27	; 32-35 ;; tmp12+RNDVAL = (x4_hi+RNDVAL) * base - (RNDVAL*base-RNDVAL) = rnd(x4/base)*base+RNDVAL
	zfmaddpd zmm4, zmm16, zmm25, zmm4	; 33-36 ;; next carry5 = val5_hi * -c*mulconst + x1_hi+RNDVAL
	zfmaddpd zmm5, zmm17, zmm25, zmm5	; 33-36 ;; next carry6 = val6_hi * -c*mulconst + x2_hi+RNDVAL
	zfmaddpd zmm6, zmm18, zmm25, zmm6	; 34-37 ;; next carry7 = val7_hi * -c*mulconst + x3_hi+RNDVAL
	zfmaddpd zmm7, zmm19, zmm25, zmm7	; 34-37 ;; next carry8 = val8_hi * -c*mulconst + x4_hi+RNDVAL
	vsubpd	zmm12, zmm12, zmm20		; 35-38 ;; new val1 = x1_lo = x1+RNDVAL - tmp9+RNDVAL = x1 - rnd(x1/base)*base = x1%base
	vsubpd	zmm13, zmm13, zmm21		; 35-38 ;; new val2 = x2_lo = x2+RNDVAL - tmp10+RNDVAL = x2 - rnd(x2/base)*base = x2%base
	vsubpd	zmm14, zmm14, zmm22		; 36-39 ;; new val1 = x3_lo = x3+RNDVAL - tmp11+RNDVAL = x1 - rnd(x3/base)*base = x3%base
	vsubpd	zmm15, zmm15, zmm23		; 36-39 ;; new val2 = x4_lo = x4+RNDVAL - tmp12+RNDVAL = x2 - rnd(x4/base)*base = x4%base
khi	zfmaddpd zmm4, zmm8, zmm31, zmm4	; 37-40 ;; if khi next carry5 = val1_lo * k_hi*mulconst/base + next carry5
khi	zfmaddpd zmm5, zmm9, zmm31, zmm5	; 37-40 ;; if khi next carry6 = val2_lo * k_hi*mulconst/base + next carry6
khi	zfmaddpd zmm6, zmm10, zmm31, zmm6	; 38-41 ;; if khi next carry7 = val3_lo * k_hi*mulconst/base + next carry7
khi	zfmaddpd zmm7, zmm11, zmm31, zmm7	; 38-41 ;; if khi next carry8 = val4_lo * k_hi*mulconst/base + next carry8

	zstore	[rsi], zmm12				;; Save value1
	zstore	[rsi+r13], zmm13			;; Save value2
	zstore	[rsi+2*r13], zmm14			;; Save value3
	zstore	[rsi+r14], zmm15			;; Save value4
	vzero	zmm8
	zstore	[rsi+64], zmm8				;; Save value5
	zstore	[rsi+r13+64], zmm8			;; Save value6
	zstore	[rsi+2*r13+64], zmm8			;; Save value7
	zstore	[rsi+r14+64], zmm8			;; Save value8
	ENDM

znorm_wpn_zpad_echk_nottp_preload MACRO const, khi
		vbroadcastsd zmm30, ZMM_RNDVAL				;; RNDVAL
		vbroadcastsd zmm29, ZMM_SMALL_BASE			;; small_word_base
		vbroadcastsd zmm28, ZMM_SMALL_BASE_INVERSE		;; 1 / small_word_base
		vbroadcastsd zmm27, ZMM_RNDVAL_TIMES_SMALL_BASE		;; RNDVAL * small_word_base - RNDVAL
		vbroadcastsd zmm26, ZMM_RNDVAL_OVER_SMALL_BASE		;; RNDVAL / small_word_base - RNDVAL
no const	vbroadcastsd zmm25, ZMM_MINUS_C
const		vbroadcastsd zmm25, ZMM_MINUS_C_TIMES_MULCONST
no const	vbroadcastsd zmm24, ZMM_K_LO
const		vbroadcastsd zmm24, ZMM_K_TIMES_MULCONST_LO
	ENDM
znorm_wpn_zpad_echk_nottp MACRO const, khi
	vaddpd	zmm8, zmm0, [rsi]		; 1-4	;; val1+RNDVAL = carry1+RNDVAL + value1
	vaddpd	zmm9, zmm1, [rsi+r13]		; 1-4	;; val2+RNDVAL = carry2+RNDVAL + value2
	vaddpd	zmm10, zmm2, [rsi+2*r13]	; 2-5	;; val3+RNDVAL = carry3+RNDVAL + value3
	vaddpd	zmm11, zmm3, [rsi+r14]		; 2-5	;; val4+RNDVAL = carry4+RNDVAL + value4
	vaddpd	zmm12, zmm30, [rsi+64]		; 3-6	;; val5+RNDVAL = RNDVAL + value5
	vaddpd	zmm13, zmm30, [rsi+r13+64]	; 3-6	;; val6+RNDVAL = RNDVAL + value6
	vaddpd	zmm14, zmm30, [rsi+2*r13+64]	; 4-7	;; val7+RNDVAL = RNDVAL + value7
	vaddpd	zmm15, zmm30, [rsi+r14+64]	; 4-7	;; val8+RNDVAL = RNDVAL + value8

	vsubpd	zmm16, zmm8, zmm0		; 5-8	;; tmp = val1+RNDVAL - carry+RNDVAL = rnd(value)
	vsubpd	zmm17, zmm9, zmm1		; 5-8	;; tmp = val2+RNDVAL - carry+RNDVAL = rnd(value)
	vsubpd	zmm18, zmm10, zmm2		; 6-9	;; tmp = val3+RNDVAL - carry+RNDVAL = rnd(value)
	vsubpd	zmm19, zmm11, zmm3		; 6-9	;; tmp = val4+RNDVAL - carry+RNDVAL = rnd(value)
	vsubpd	zmm20, zmm12, zmm30		; 7-10	;; tmp = val5+RNDVAL - RNDVAL = rnd(value)
	vsubpd	zmm21, zmm13, zmm30		; 7-10	;; tmp = val6+RNDVAL - RNDVAL = rnd(value)
	vsubpd	zmm22, zmm14, zmm30		; 8-11	;; tmp = val7+RNDVAL - RNDVAL = rnd(value)
	vsubpd	zmm23, zmm15, zmm30		; 8-11	;; tmp = val8+RNDVAL - RNDVAL = rnd(value)

	vsubpd	zmm16, zmm16, [rsi]	 	; 9-12	;; err = rnd(value) - value
	vsubpd	zmm17, zmm17, [rsi+r13]	 	; 9-12	;; err = rnd(value) - value
	vsubpd	zmm18, zmm18, [rsi+2*r13]	; 10-13	;; err = rnd(value) - value
	vsubpd	zmm19, zmm19, [rsi+r14]	 	; 10-13	;; err = rnd(value) - value
	vsubpd	zmm20, zmm20, [rsi+64]	 	; 11-14	;; err = rnd(value) - value
	vsubpd	zmm21, zmm21, [rsi+r13+64] 	; 11-14	;; err = rnd(value) - value
	vsubpd	zmm22, zmm22, [rsi+2*r13+64]	; 12-15	;; err = rnd(value) - value
	vsubpd	zmm23, zmm23, [rsi+r14+64] 	; 12-15	;; err = rnd(value) - value

	vbroadcastsd zmm0, ZMM_ABSVAL
	vpandq	zmm16, zmm16, zmm0		; 13	;; err = abs(err)
	vpandq	zmm17, zmm17, zmm0		; 13	;; err = abs(err)
	vpandq	zmm18, zmm18, zmm0		; 14	;; err = abs(err)
	vpandq	zmm19, zmm19, zmm0		; 14	;; err = abs(err)
	vpandq	zmm20, zmm20, zmm0		; 15	;; err = abs(err)
	vpandq	zmm21, zmm21, zmm0		; 15	;; err = abs(err)
	vpandq	zmm22, zmm22, zmm0		; 16	;; err = abs(err)
	vpandq	zmm23, zmm23, zmm0		; 16	;; err = abs(err)

;; BUG - Is there a better grouping available (use regs zmm16 up and interleave with the / base FMA instructions
;; Use more maxerr accumulators?

	vmaxpd	zmm20, zmm20, zmm16		; 17-20	;; accumulate maxerr
	vmaxpd	zmm21, zmm21, zmm17		; 17-20	;; accumulate maxerr
	vmaxpd	zmm22, zmm22, zmm18		; 18-21	;; accumulate maxerr
	vmaxpd	zmm23, zmm23, zmm19		; 18-21	;; accumulate maxerr

	zfmsubpd zmm16, zmm12, zmm28, zmm26	; 19-22	;; val5_hi+RNDVAL = (val5+RNDVAL) / base - (RNDVAL/base-RNDVAL) = rnd(val5/base)+RNDVAL
	zfmsubpd zmm17, zmm13, zmm28, zmm26	; 19-22	;; val6_hi+RNDVAL = (val6+RNDVAL) / base - (RNDVAL/base-RNDVAL) = rnd(val6/base)+RNDVAL
	zfmsubpd zmm18, zmm14, zmm28, zmm26	; 20-23	;; val7_hi+RNDVAL = (val7+RNDVAL) / base - (RNDVAL/base-RNDVAL) = rnd(val7/base)+RNDVAL
	zfmsubpd zmm19, zmm15, zmm28, zmm26	; 20-23	;; val8_hi+RNDVAL = (val8+RNDVAL) / base - (RNDVAL/base-RNDVAL) = rnd(val8/base)+RNDVAL
	zfmsubpd zmm0, zmm8, zmm28, zmm26	; 21-24	;; val1_hi+RNDVAL = (val1+RNDVAL) / base - (RNDVAL/base-RNDVAL) = rnd(val1/base)+RNDVAL (next carry1)
	zfmsubpd zmm1, zmm9, zmm28, zmm26	; 21-24	;; val2_hi+RNDVAL = (val2+RNDVAL) / base - (RNDVAL/base-RNDVAL) = rnd(val2/base)+RNDVAL (next carry2)
	zfmsubpd zmm2, zmm10, zmm28, zmm26	; 22-25	;; val3_hi+RNDVAL = (val3+RNDVAL) / base - (RNDVAL/base-RNDVAL) = rnd(val3/base)+RNDVAL (next carry3)
	zfmsubpd zmm3, zmm11, zmm28, zmm26	; 22-25	;; val4_hi+RNDVAL = (val4+RNDVAL) / base - (RNDVAL/base-RNDVAL) = rnd(val4/base)+RNDVAL (next carry4)

	vmaxpd	zmm22, zmm22, zmm20		; 23-26	;; accumulate maxerr
	vmaxpd	zmm23, zmm23, zmm21		; 23-26	;; accumulate maxerr
	vmaxpd	zmm31, zmm31, zmm22		; 17-26	;; accumulate maxerr
	vmaxpd	zmm31, zmm31, zmm23		; 18-26	;; accumulate maxerr

	zfmsubpd zmm20, zmm16, zmm29, zmm27	; 24-27	;; tmp5+RNDVAL = (val5_hi+RNDVAL) * base - (RNDVAL*base-RNDVAL) = rnd(val5/base)*base+RNDVAL
	zfmsubpd zmm21, zmm17, zmm29, zmm27	; 24-27	;; tmp6+RNDVAL = (val6_hi+RNDVAL) * base - (RNDVAL*base-RNDVAL) = rnd(val6/base)*base+RNDVAL
	zfmsubpd zmm22, zmm18, zmm29, zmm27	; 25-28	;; tmp7+RNDVAL = (val7_hi+RNDVAL) * base - (RNDVAL*base-RNDVAL) = rnd(val7/base)*base+RNDVAL
	zfmsubpd zmm23, zmm19, zmm29, zmm27	; 25-28	;; tmp8+RNDVAL = (val8_hi+RNDVAL) * base - (RNDVAL*base-RNDVAL) = rnd(val8/base)*base+RNDVAL
	vsubpd	zmm16, zmm16, zmm30		; 26-29	;; val5_hi = val5_hi+RNDVAL - RNDVAL
	vsubpd	zmm17, zmm17, zmm30		; 26-29	;; val6_hi = val6_hi+RNDVAL - RNDVAL
	vsubpd	zmm18, zmm18, zmm30		; 27-30	;; val7_hi = val7_hi+RNDVAL - RNDVAL
	vsubpd	zmm19, zmm19, zmm30		; 28-31	;; val8_hi = val8_hi+RNDVAL - RNDVAL
	vsubpd	zmm12, zmm12, zmm20		; 28-31	;; val5_lo = val5+RNDVAL - tmp5+RNDVAL = val5 - rnd(val5/base)*base = val5%base
	vsubpd	zmm13, zmm13, zmm21		; 29-32	;; val6_lo = val6+RNDVAL - tmp6+RNDVAL = val6 - rnd(val6/base)*base = val6%base
	vsubpd	zmm14, zmm14, zmm22		; 29-32	;; val7_lo = val7+RNDVAL - tmp7+RNDVAL = val7 - rnd(val7/base)*base = val7%base
	vsubpd	zmm15, zmm15, zmm23		; 30-33	;; val8_lo = val8+RNDVAL - tmp8+RNDVAL = val8 - rnd(val8/base)*base = val8%base
	zfmsubpd zmm20, zmm0, zmm29, zmm27	; 30-33	;; tmp1+RNDVAL = (val1_hi+RNDVAL) * base - (RNDVAL*base-RNDVAL) = rnd(val1/base)*base+RNDVAL
	zfmsubpd zmm21, zmm1, zmm29, zmm27	; 31-34	;; tmp2+RNDVAL = (val2_hi+RNDVAL) * base - (RNDVAL*base-RNDVAL) = rnd(val2/base)*base+RNDVAL
	zfmsubpd zmm22, zmm2, zmm29, zmm27	; 32-35	;; tmp3+RNDVAL = (val3_hi+RNDVAL) * base - (RNDVAL*base-RNDVAL) = rnd(val3/base)*base+RNDVAL
	zfmsubpd zmm23, zmm3, zmm29, zmm27	; 32-35	;; tmp4+RNDVAL = (val4_hi+RNDVAL) * base - (RNDVAL*base-RNDVAL) = rnd(val4/base)*base+RNDVAL
	zfmaddpd zmm12, zmm12, zmm25, zmm4	; 33-36	;; x1+RNDVAL = val5_lo * -c*mulconst + carry5+RNDVAL
	zfmaddpd zmm13, zmm13, zmm25, zmm5	; 33-36	;; x2+RNDVAL = val6_lo * -c*mulconst + carry6+RNDVAL
	zfmaddpd zmm14, zmm14, zmm25, zmm6	; 34-37	;; x3+RNDVAL = val7_lo * -c*mulconst + carry7+RNDVAL
	zfmaddpd zmm15, zmm15, zmm25, zmm7	; 34-37	;; x4+RNDVAL = val8_lo * -c*mulconst + carry8+RNDVAL
	vsubpd	zmm8, zmm8, zmm20		; 35-38	;; val1_lo = val1+RNDVAL - tmp1+RNDVAL = val1 - rnd(val1/base)*base = val1%base
	vsubpd	zmm9, zmm9, zmm21		; 35-38	;; val2_lo = val2+RNDVAL - tmp2+RNDVAL = val2 - rnd(val2/base)*base = val2%base
	vsubpd	zmm10, zmm10, zmm22		; 36-39	;; val3_lo = val3+RNDVAL - tmp3+RNDVAL = val3 - rnd(val3/base)*base = val3%base
	vsubpd	zmm11, zmm11, zmm23		; 36-39	;; val4_lo = val4+RNDVAL - tmp4+RNDVAL = val4 - rnd(val4/base)*base = val4%base

	zfmaddpd zmm12, zmm8, zmm24, zmm12	; 39-42 ;; x1+RNDVAL += val1_lo * k_lo*mulconst
	zfmaddpd zmm13, zmm9, zmm24, zmm13	; 39-42 ;; x2+RNDVAL += val2_lo * k_lo*mulconst
	zfmaddpd zmm14, zmm10, zmm24, zmm14	; 40-43 ;; x3+RNDVAL += val3_lo * k_lo*mulconst
	zfmaddpd zmm15, zmm11, zmm24, zmm15	; 40-43 ;; x4+RNDVAL += val4_lo * k_lo*mulconst

	zfmsubpd zmm4, zmm12, zmm28, zmm26	; 43-46 ;; x1_hi+RNDVAL = (x1+RNDVAL) / base - (RNDVAL/base-RNDVAL) = rnd(x1/base)+RNDVAL
	zfmsubpd zmm5, zmm13, zmm28, zmm26	; 43-46 ;; x2_hi+RNDVAL = (x2+RNDVAL) / base - (RNDVAL/base-RNDVAL) = rnd(x2/base)+RNDVAL
	zfmsubpd zmm6, zmm14, zmm28, zmm26	; 44-47 ;; x3_hi+RNDVAL = (x3+RNDVAL) / base - (RNDVAL/base-RNDVAL) = rnd(x3/base)+RNDVAL
	zfmsubpd zmm7, zmm15, zmm28, zmm26	; 44-47 ;; x4_hi+RNDVAL = (x4+RNDVAL) / base - (RNDVAL/base-RNDVAL) = rnd(x4/base)+RNDVAL
	zfmsubpd zmm20, zmm4, zmm29, zmm27	; 47-50 ;; tmp9+RNDVAL = (x1_hi+RNDVAL) * base - (RNDVAL*base-RNDVAL) = rnd(x1/base)*base+RNDVAL
	zfmsubpd zmm21, zmm5, zmm29, zmm27	; 47-50 ;; tmp10+RNDVAL = (x2_hi+RNDVAL) * base - (RNDVAL*base-RNDVAL) = rnd(x2/base)*base+RNDVAL
	zfmsubpd zmm22, zmm6, zmm29, zmm27	; 48-51 ;; tmp11+RNDVAL = (x3_hi+RNDVAL) * base - (RNDVAL*base-RNDVAL) = rnd(x3/base)*base+RNDVAL
	zfmsubpd zmm23, zmm7, zmm29, zmm27	; 48-51 ;; tmp12+RNDVAL = (x4_hi+RNDVAL) * base - (RNDVAL*base-RNDVAL) = rnd(x4/base)*base+RNDVAL
	zfmaddpd zmm4, zmm16, zmm25, zmm4	; 49-52 ;; next carry5 = val5_hi * -c*mulconst + x1_hi+RNDVAL
	zfmaddpd zmm5, zmm17, zmm25, zmm5	; 49-52 ;; next carry6 = val6_hi * -c*mulconst + x2_hi+RNDVAL
	zfmaddpd zmm6, zmm18, zmm25, zmm6	; 50-53 ;; next carry7 = val7_hi * -c*mulconst + x3_hi+RNDVAL
	zfmaddpd zmm7, zmm19, zmm25, zmm7	; 50-53 ;; next carry8 = val8_hi * -c*mulconst + x4_hi+RNDVAL
	vsubpd	zmm12, zmm12, zmm20		; 51-54 ;; new val1 = x1_lo = x1+RNDVAL - tmp9+RNDVAL = x1 - rnd(x1/base)*base = x1%base
	vsubpd	zmm13, zmm13, zmm21		; 51-54 ;; new val2 = x2_lo = x2+RNDVAL - tmp10+RNDVAL = x2 - rnd(x2/base)*base = x2%base
	vsubpd	zmm14, zmm14, zmm22		; 52-55 ;; new val1 = x3_lo = x3+RNDVAL - tmp11+RNDVAL = x1 - rnd(x3/base)*base = x3%base
	vsubpd	zmm15, zmm15, zmm23		; 52-55 ;; new val2 = x4_lo = x4+RNDVAL - tmp12+RNDVAL = x2 - rnd(x4/base)*base = x4%base
khi no const	vbroadcastsd zmm23, ZMM_K_HI_OVER_SMALL_BASE	;; k_hi / small_word_base
khi const	vbroadcastsd zmm23, ZMM_K_TIMES_MULCONST_HI_OVER_SMALL_BASE
khi	zfmaddpd zmm4, zmm8, zmm23, zmm4	; 53-56 ;; if khi next carry5 = val1_lo * k_hi*mulconst/base + next carry5
khi	zfmaddpd zmm5, zmm9, zmm23, zmm5	; 53-56 ;; if khi next carry6 = val2_lo * k_hi*mulconst/base + next carry6
khi	zfmaddpd zmm6, zmm10, zmm23, zmm6	; 54-57 ;; if khi next carry7 = val3_lo * k_hi*mulconst/base + next carry7
khi	zfmaddpd zmm7, zmm11, zmm23, zmm7	; 54-57 ;; if khi next carry8 = val4_lo * k_hi*mulconst/base + next carry8

	zstore	[rsi], zmm12				;; Save value1
	zstore	[rsi+r13], zmm13			;; Save value2
	zstore	[rsi+2*r13], zmm14			;; Save value3
	zstore	[rsi+r14], zmm15			;; Save value4
	vzero	zmm8
	zstore	[rsi+64], zmm8				;; Save value5
	zstore	[rsi+r13+64], zmm8			;; Save value6
	zstore	[rsi+2*r13+64], zmm8			;; Save value7
	zstore	[rsi+r14+64], zmm8			;; Save value8
	ENDM

; Irrational versions

znorm_wpn_zpad_noechk_ttp_preload MACRO const, khi
		vbroadcastsd zmm30, ZMM_RNDVAL				;; RNDVAL
		vbroadcastsd zmm29, ZMM_SMALL_BASE			;; small_word_base
		vbroadcastsd zmm28, ZMM_SMALL_BASE_INVERSE		;; 1 / small_word_base
		vbroadcastsd zmm27, ZMM_LARGE_BASE			;; large_word_base
		vbroadcastsd zmm26, ZMM_LARGE_BASE_INVERSE		;; 1 / large_word_base
no const	vbroadcastsd zmm25, ZMM_MINUS_C
const		vbroadcastsd zmm25, ZMM_MINUS_C_TIMES_MULCONST
no const	vbroadcastsd zmm31, ZMM_K_LO
const		vbroadcastsd zmm31, ZMM_K_TIMES_MULCONST_LO
	ENDM
znorm_wpn_zpad_noechk_ttp MACRO const, khi
	movzx	rdx, BYTE PTR [rdi]				;; Load index into compressed biglit table
	vmovapd zmm8, [r10+0*64]				;; Inverse group multiplier
	zfmaddpd zmm12, [rsi+64], zmm8, zmm30		; 1-4	;; val5+RNDVAL = value5*inv_grp_mult + RNDVAL
	vmovapd zmm9, [r10+1*64]				;; Inverse group multiplier
	zfmaddpd zmm13, [rsi+r13+64], zmm9, zmm30	; 1-4	;; val6+RNDVAL = value6*inv_grp_mult + RNDVAL
	vmovapd zmm10, [r10+2*64]				;; Inverse group multiplier
	zfmaddpd zmm14, [rsi+2*r13+64], zmm10, zmm30	; 2-5	;; val7+RNDVAL = value7*inv_grp_mult + RNDVAL
	vmovapd zmm11, [r10+3*64]				;; Inverse group multiplier
	zfmaddpd zmm15, [rsi+r14+64], zmm11, zmm30	; 2-5	;; val8+RNDVAL = value8*inv_grp_mult + RNDVAL
	zfmaddpd zmm8, [rsi], zmm8, zmm0		; 3-6	;; val1+RNDVAL = value1*inv_grp_mult + carry1+RNDVAL
	zfmaddpd zmm9, [rsi+r13], zmm9, zmm1		; 3-6	;; val2+RNDVAL = value2*inv_grp_mult + carry2+RNDVAL
	zfmaddpd zmm10, [rsi+2*r13], zmm10, zmm2	; 4-7	;; val3+RNDVAL = value3*inv_grp_mult + carry3+RNDVAL
	zfmaddpd zmm11, [rsi+r14], zmm11, zmm3		; 4-7	;; val4+RNDVAL = value4*inv_grp_mult + carry4+RNDVAL

	mov	edx, [r12+rdx*4]				;; Load 4 big vs. little flags

	vsubpd	zmm12, zmm12, zmm30			; 5-8	;; val5 = val5+RNDVAL - RNDVAL
	vsubpd	zmm13, zmm13, zmm30			; 5-8	;; val6 = val6+RNDVAL - RNDVAL
	vsubpd	zmm14, zmm14, zmm30			; 6-9	;; val7 = val7+RNDVAL - RNDVAL
	vsubpd	zmm15, zmm15, zmm30			; 6-9	;; val8 = val8+RNDVAL - RNDVAL
	vsubpd	zmm8, zmm8, zmm30			; 7-10	;; val1 = val1+RNDVAL - RNDVAL
	vsubpd	zmm9, zmm9, zmm30			; 7-10	;; val2 = val2+RNDVAL - RNDVAL
	vsubpd	zmm10, zmm10, zmm30			; 8-11	;; val3 = val3+RNDVAL - RNDVAL
	vsubpd	zmm11, zmm11, zmm30			; 8-11	;; val4 = val4+RNDVAL - RNDVAL

	kmovw	k1, edx					; 9	;; Load 2 big vs. little flags (16-bit load - we use 8-bits)
	kshiftrw k2, k1, 8				; 10
	shr	edx, 16
	kmovw	k3, edx					; 11	;; Load 2 big vs. little flags (16-bit load - we use 8-bits)
	kshiftrw k4, k3, 8				; 12

;; Using 4 more registers (loading zmm26-29 every time) would let us interleave val1-4 calculations with val5-8, perhaps the CPU is able to do this automatically
;; But complicates _1d end-of-section carry propagation code

	vmovapd	zmm0, zmm4					;; Register alloation works better if 1/base is put in zmm4-7
	vblendmpd zmm4 {k1}, zmm28, zmm26		; 13	;; Create (1/base) constant
	vmovapd	zmm1, zmm5
	vblendmpd zmm5 {k2}, zmm28, zmm26		; 14	;; Create (1/base) constant
	vmovapd	zmm2, zmm6
	vblendmpd zmm6 {k3}, zmm28, zmm26		; 15	;; Create (1/base) constant
	vmovapd	zmm3, zmm7
	vblendmpd zmm7 {k4}, zmm28, zmm26		; 16	;; Create (1/base) constant

	zfmaddpd zmm16, zmm12, zmm4, zmm30		; 9-12	;; val5_hi+RNDVAL = val5 / base + RNDVAL = rnd(val5/base)+RNDVAL
	zfmaddpd zmm17, zmm13, zmm5, zmm30		; 10-13	;; val6_hi+RNDVAL = val6 / base + RNDVAL = rnd(val6/base)+RNDVAL
	zfmaddpd zmm18, zmm14, zmm6, zmm30		; 11-14	;; val7_hi+RNDVAL = val7 / base + RNDVAL = rnd(val7/base)+RNDVAL
	zfmaddpd zmm19, zmm15, zmm7, zmm30		; 12-15	;; val8_hi+RNDVAL = val8 / base + RNDVAL = rnd(val8/base)+RNDVAL

	vblendmpd zmm20 {k1}, zmm29, zmm27		; 17	;; Create (base) constant
	vblendmpd zmm21 {k2}, zmm29, zmm27		; 17	;; Create (base) constant
	vblendmpd zmm22 {k3}, zmm29, zmm27		; 18	;; Create (base) constant
	vblendmpd zmm23 {k4}, zmm29, zmm27		; 18	;; Create (base) constant

	vsubpd	zmm16, zmm16, zmm30			; 13-16	;; val5_hi = val5_hi+RNDVAL - RNDVAL
	vsubpd	zmm17, zmm17, zmm30			; 14-17	;; val6_hi = val6_hi+RNDVAL - RNDVAL
	vsubpd	zmm18, zmm18, zmm30			; 15-18	;; val7_hi = val7_hi+RNDVAL - RNDVAL
	vsubpd	zmm19, zmm19, zmm30			; 16-19	;; val8_hi = val8_hi+RNDVAL - RNDVAL

	zfnmaddpd zmm12, zmm16, zmm20, zmm12		; 19-22	;; val5_lo = val5 - val5_hi * base = val5%base
	zfnmaddpd zmm13, zmm17, zmm21, zmm13		; 19-22	;; val6_lo = val6 - val6_hi * base = val6%base
	zfnmaddpd zmm14, zmm18, zmm22, zmm14		; 20-23	;; val7_lo = val7 - val7_hi * base = val7%base
	zfnmaddpd zmm15, zmm19, zmm23, zmm15		; 20-23	;; val8_lo = val8 - val8_hi * base = val8%base

	zfmaddpd zmm12, zmm12, zmm25, zmm0		; 23-26	;; x1 = val5_lo * -c*mulconst + carry5
	zfmaddpd zmm13, zmm13, zmm25, zmm1		; 23-26	;; x2 = val6_lo * -c*mulconst + carry6
	zfmaddpd zmm14, zmm14, zmm25, zmm2		; 24-27	;; x3 = val7_lo * -c*mulconst + carry7
	zfmaddpd zmm15, zmm15, zmm25, zmm3		; 24-27	;; x4 = val8_lo * -c*mulconst + carry8

	zfmaddpd zmm0, zmm8, zmm4, zmm30		; 17-20	;; val1_hi+RNDVAL = val1 / base + RNDVAL = rnd(val1/base)+RNDVAL (next carry1)
	zfmaddpd zmm1, zmm9, zmm5, zmm30		; 17-20	;; val2_hi+RNDVAL = val2 / base + RNDVAL = rnd(val2/base)+RNDVAL (next carry2)
	zfmaddpd zmm2, zmm10, zmm6, zmm30		; 18-21	;; val3_hi+RNDVAL = val3 / base + RNDVAL = rnd(val3/base)+RNDVAL (next carry3)
	zfmaddpd zmm3, zmm11, zmm7, zmm30		; 18-21	;; val4_hi+RNDVAL = val4 / base + RNDVAL = rnd(val4/base)+RNDVAL (next carry4)

	vsubpd	zmm24, zmm0, zmm30			; 25-28	;; val1_hi = val1_hi+RNDVAL - RNDVAL
	zfnmaddpd zmm8, zmm24, zmm20, zmm8		; 29-32	;; val1_lo = val1 - val1_hi * base = val1%base
	vsubpd	zmm24, zmm1, zmm30			; 25-28	;; val2_hi = val2_hi+RNDVAL - RNDVAL
	zfnmaddpd zmm9, zmm24, zmm21, zmm9		; 29-32	;; val2_lo = val2 - val2_hi * base = val2%base
	vsubpd	zmm24, zmm2, zmm30			; 26-29	;; val3_hi = val3_hi+RNDVAL - RNDVAL
	zfnmaddpd zmm10, zmm24, zmm22, zmm10		; 30-33	;; val3_lo = val3 - val3_hi * base = val3%base
	vsubpd	zmm24, zmm3, zmm30			; 26-29	;; val4_hi = val4_hi+RNDVAL - RNDVAL
	zfnmaddpd zmm11, zmm24, zmm23, zmm11		; 30-33	;; val4_lo = val4 - val4_hi * base = val4%base

	zfmaddpd zmm12, zmm8, zmm31, zmm12		; 33-36 ;; x1 += val1_lo * k_lo*mulconst
	zfmaddpd zmm13, zmm9, zmm31, zmm13		; 33-36 ;; x2 += val2_lo * k_lo*mulconst
	zfmaddpd zmm14, zmm10, zmm31, zmm14		; 34-37 ;; x3 += val3_lo * k_lo*mulconst
	zfmaddpd zmm15, zmm11, zmm31, zmm15		; 34-37 ;; x4 += val4_lo * k_lo*mulconst

	zfmaddpd zmm4, zmm12, zmm4, zmm30		; 37-40 ;; x1_hi+RNDVAL = x1 / base + RNDVAL = rnd(x1/base)+RNDVAL
	zfmaddpd zmm5, zmm13, zmm5, zmm30		; 37-40 ;; x2_hi+RNDVAL = x2 / base + RNDVAL = rnd(x2/base)+RNDVAL
	zfmaddpd zmm6, zmm14, zmm6, zmm30		; 38-41 ;; x3_hi+RNDVAL = x3 / base + RNDVAL = rnd(x3/base)+RNDVAL
	zfmaddpd zmm7, zmm15, zmm7, zmm30		; 38-41 ;; x4_hi+RNDVAL = x4 / base + RNDVAL = rnd(x4/base)+RNDVAL

	vsubpd	zmm4, zmm4, zmm30			; 41-44	;; x1_hi = x1_hi+RNDVAL - RNDVAL
	vsubpd	zmm5, zmm5, zmm30			; 41-44	;; x2_hi = x2_hi+RNDVAL - RNDVAL
	vsubpd	zmm6, zmm6, zmm30			; 42-45	;; x3_hi = x3_hi+RNDVAL - RNDVAL
	vsubpd	zmm7, zmm7, zmm30			; 42-45	;; x4_hi = x4_hi+RNDVAL - RNDVAL

	zfnmaddpd zmm12, zmm4, zmm20, zmm12		; 45-48 ;; new val1 = x1_lo = x1 - x1_hi * base = x1%base
	zfnmaddpd zmm13, zmm5, zmm21, zmm13		; 45-48 ;; new val2 = x2_lo = x2 - x2_hi * base = x2%base
	zfnmaddpd zmm14, zmm6, zmm22, zmm14		; 46-49 ;; new val3 = x3_lo = x3 - x3_hi * base = x3%base
	zfnmaddpd zmm15, zmm7, zmm23, zmm15		; 46-49 ;; new val4 = x4_lo = x4 - x4_hi * base = x4%base

	zfmaddpd zmm4, zmm16, zmm25, zmm4		; 47-50 ;; next carry5 = val5_hi * -c*mulconst + x1_hi
	zfmaddpd zmm5, zmm17, zmm25, zmm5		; 47-50 ;; next carry6 = val6_hi * -c*mulconst + x2_hi
	zfmaddpd zmm6, zmm18, zmm25, zmm6		; 48-51 ;; next carry7 = val7_hi * -c*mulconst + x3_hi
	zfmaddpd zmm7, zmm19, zmm25, zmm7		; 48-51 ;; next carry8 = val8_hi * -c*mulconst + x4_hi

khi no const	vbroadcastsd zmm20, ZMM_K_HI_OVER_SMALL_BASE
khi no const	vbroadcastsd zmm21, ZMM_K_HI_OVER_LARGE_BASE
khi const	vbroadcastsd zmm20, ZMM_K_TIMES_MULCONST_HI_OVER_SMALL_BASE
khi const	vbroadcastsd zmm21, ZMM_K_TIMES_MULCONST_HI_OVER_LARGE_BASE
khi	vblendmpd zmm16 {k1}, zmm20, zmm21		; 43	;; Create (k_hi*mulconst/base) constant
khi	vblendmpd zmm17 {k2}, zmm20, zmm21		; 43	;; Create (k_hi*mulconst/base) constant
khi	vblendmpd zmm18 {k3}, zmm20, zmm21		; 44	;; Create (k_hi*mulconst/base) constant
khi	vblendmpd zmm19 {k4}, zmm20, zmm21		; 44	;; Create (k_hi*mulconst/base) constant
khi	zfmaddpd zmm4, zmm8, zmm16, zmm4		; 51-54 ;; if khi next carry5 = val1_lo * k_hi*mulconst/base + next carry5
khi	zfmaddpd zmm5, zmm9, zmm17, zmm5		; 51-54 ;; if khi next carry6 = val2_lo * k_hi*mulconst/base + next carry6
khi	zfmaddpd zmm6, zmm10, zmm18, zmm6		; 52-55 ;; if khi next carry7 = val3_lo * k_hi*mulconst/base + next carry7
khi	zfmaddpd zmm7, zmm11, zmm19, zmm7		; 52-55 ;; if khi next carry8 = val4_lo * k_hi*mulconst/base + next carry8

	zstore	[rsi], zmm12					;; Save value1
	zstore	[rsi+r13], zmm13				;; Save value2
	zstore	[rsi+2*r13], zmm14				;; Save value3
	zstore	[rsi+r14], zmm15				;; Save value4
	vzero	zmm8
	zstore	[rsi+64], zmm8					;; Save value5
	zstore	[rsi+r13+64], zmm8				;; Save value6
	zstore	[rsi+2*r13+64], zmm8				;; Save value7
	zstore	[rsi+r14+64], zmm8				;; Save value8
	ENDM

znorm_wpn_zpad_echk_ttp_preload MACRO const, khi
		vbroadcastsd zmm30, ZMM_RNDVAL				;; RNDVAL
		vbroadcastsd zmm29, ZMM_SMALL_BASE			;; small_word_base
		vbroadcastsd zmm28, ZMM_SMALL_BASE_INVERSE		;; 1 / small_word_base
		vbroadcastsd zmm27, ZMM_LARGE_BASE			;; large_word_base
		vbroadcastsd zmm26, ZMM_LARGE_BASE_INVERSE		;; 1 / large_word_base
no const	vbroadcastsd zmm25, ZMM_MINUS_C
const		vbroadcastsd zmm25, ZMM_MINUS_C_TIMES_MULCONST
	ENDM
znorm_wpn_zpad_echk_ttp MACRO const, khi
	movzx	rdx, BYTE PTR [rdi]			;; Load index into compressed biglit table
	vmovapd zmm20, [r10+0*64]				;; Inverse group multiplier
	zfmaddpd zmm8, [rsi], zmm20, zmm0		; 1-4	;; val1+RNDVAL = value1*inv_grp_mult + carry1+RNDVAL
	vmovapd zmm21, [r10+1*64]				;; Inverse group multiplier
	zfmaddpd zmm9, [rsi+r13], zmm21, zmm1		; 1-4	;; val2+RNDVAL = value2*inv_grp_mult + carry2+RNDVAL
	vmovapd zmm22, [r10+2*64]				;; Inverse group multiplier
	zfmaddpd zmm10, [rsi+2*r13], zmm22, zmm2	; 2-5	;; val3+RNDVAL = value3*inv_grp_mult + carry3+RNDVAL
	vmovapd zmm23, [r10+3*64]				;; Inverse group multiplier
	zfmaddpd zmm11, [rsi+r14], zmm23, zmm3		; 2-5	;; val4+RNDVAL = value4*inv_grp_mult + carry4+RNDVAL
	zfmaddpd zmm12, [rsi+64], zmm20, zmm30		; 3-6	;; val5+RNDVAL = value5*inv_grp_mult + RNDVAL
	zfmaddpd zmm13, [rsi+r13+64], zmm21, zmm30	; 3-6	;; val6+RNDVAL = value6*inv_grp_mult + RNDVAL
	zfmaddpd zmm14, [rsi+2*r13+64], zmm22, zmm30	; 4-7	;; val7+RNDVAL = value7*inv_grp_mult + RNDVAL
	zfmaddpd zmm15, [rsi+r14+64], zmm23, zmm30	; 4-7	;; val8+RNDVAL = value8*inv_grp_mult + RNDVAL

	mov	edx, [r12+rdx*4]		;; Load 4 big vs. little flags

	vsubpd	zmm16, zmm8, zmm0			; 5-8	;; tmp = val1+RNDVAL - carry+RNDVAL = rnd(value*inv_grp_mult)
	vsubpd	zmm17, zmm9, zmm1			; 5-8	;; tmp = val2+RNDVAL - carry+RNDVAL = rnd(value*inv_grp_mult)
	vsubpd	zmm18, zmm10, zmm2			; 6-9	;; tmp = val3+RNDVAL - carry+RNDVAL = rnd(value*inv_grp_mult)
	vsubpd	zmm19, zmm11, zmm3			; 6-9	;; tmp = val4+RNDVAL - carry+RNDVAL = rnd(value*inv_grp_mult)
	vsubpd	zmm12, zmm12, zmm30			; 7-10	;; val5 = val5+RNDVAL - RNDVAL = rnd(value*inv_grp_mult)
	vsubpd	zmm13, zmm13, zmm30			; 7-10	;; val6 = val6+RNDVAL - RNDVAL = rnd(value*inv_grp_mult)
	vsubpd	zmm14, zmm14, zmm30			; 8-11	;; val7 = val7+RNDVAL - RNDVAL = rnd(value*inv_grp_mult)
	vsubpd	zmm15, zmm15, zmm30			; 8-11	;; val8 = val8+RNDVAL - RNDVAL = rnd(value*inv_grp_mult)

	zfnmaddpd zmm16, [rsi], zmm20, zmm16		; 9-12	;; err = rnd(value) - rnd(value*inv_grp_mult)
	zfnmaddpd zmm17, [rsi+r13], zmm21, zmm17	; 9-12	;; err = rnd(value) - rnd(value*inv_grp_mult)
	zfnmaddpd zmm18, [rsi+2*r13], zmm22, zmm18	; 10-13	;; err = rnd(value) - rnd(value*inv_grp_mult)
	zfnmaddpd zmm19, [rsi+r14], zmm23, zmm19	; 10-13	;; err = rnd(value) - rnd(value*inv_grp_mult)
	zfnmaddpd zmm20, [rsi+64], zmm20, zmm12		; 11-14	;; err = rnd(value) - rnd(value*inv_grp_mult)
	zfnmaddpd zmm21, [rsi+r13+64], zmm21, zmm13	; 11-14	;; err = rnd(value) - rnd(value*inv_grp_mult)
	zfnmaddpd zmm22, [rsi+2*r13+64], zmm22, zmm14	; 12-15	;; err = rnd(value) - rnd(value*inv_grp_mult)
	zfnmaddpd zmm23, [rsi+r14+64], zmm23, zmm15	; 12-15	;; err = rnd(value) - rnd(value*inv_grp_mult)

	vbroadcastsd zmm0, ZMM_ABSVAL
	vpandq	zmm16, zmm16, zmm0			; 13	;; err = abs(err)
	vpandq	zmm17, zmm17, zmm0			; 13	;; err = abs(err)
	vpandq	zmm18, zmm18, zmm0			; 14	;; err = abs(err)
	vpandq	zmm19, zmm19, zmm0			; 14	;; err = abs(err)
	vpandq	zmm20, zmm20, zmm0			; 15	;; err = abs(err)
	vpandq	zmm21, zmm21, zmm0			; 15	;; err = abs(err)
	vpandq	zmm22, zmm22, zmm0			; 16	;; err = abs(err)
	vpandq	zmm23, zmm23, zmm0			; 16	;; err = abs(err)

	vmaxpd	zmm20, zmm20, zmm16			; 17-20	;; accumulate maxerr
	vmaxpd	zmm21, zmm21, zmm17			; 17-20	;; accumulate maxerr
	vmaxpd	zmm22, zmm22, zmm18			; 18-21	;; accumulate maxerr
	vmaxpd	zmm23, zmm23, zmm19			; 18-21	;; accumulate maxerr

	kmovw	k1, edx					; 9	;; Load 2 big vs. little flags (16-bit load - we use 8-bits)
	kshiftrw k2, k1, 8				; 10
	shr	edx, 16
	kmovw	k3, edx					; 11	;; Load 2 big vs. little flags (16-bit load - we use 8-bits)
	kshiftrw k4, k3, 8				; 12

;; Using 4 more registers (loading zmm26-29 every time) would let us interleave val1-4 calculations with val5-8, perhaps the CPU is able to do this automatically
;; But complicates end-of-section carry propagation code

	vmovapd	zmm0, zmm4					;; Register alloation works better if 1/base is put in zmm4-7
	vblendmpd zmm4 {k1}, zmm28, zmm26		; 13	;; Create (1/base) constant
	vmovapd	zmm1, zmm5
	vblendmpd zmm5 {k2}, zmm28, zmm26		; 14	;; Create (1/base) constant
	vmovapd	zmm2, zmm6
	vblendmpd zmm6 {k3}, zmm28, zmm26		; 15	;; Create (1/base) constant
	vmovapd	zmm3, zmm7
	vblendmpd zmm7 {k4}, zmm28, zmm26		; 16	;; Create (1/base) constant

	zfmaddpd zmm16, zmm12, zmm4, zmm30		; 9-12	;; val5_hi+RNDVAL = val5 / base + RNDVAL = rnd(val5/base)+RNDVAL
	zfmaddpd zmm17, zmm13, zmm5, zmm30		; 10-13	;; val6_hi+RNDVAL = val6 / base + RNDVAL = rnd(val6/base)+RNDVAL
	zfmaddpd zmm18, zmm14, zmm6, zmm30		; 11-14	;; val7_hi+RNDVAL = val7 / base + RNDVAL = rnd(val7/base)+RNDVAL
	zfmaddpd zmm19, zmm15, zmm7, zmm30		; 12-15	;; val8_hi+RNDVAL = val8 / base + RNDVAL = rnd(val8/base)+RNDVAL

	vmaxpd	zmm20, zmm20, zmm21			; 21-24	;; accumulate maxerr
	vmaxpd	zmm22, zmm22, zmm23			; 22-25	;; accumulate maxerr
	vmaxpd	zmm31, zmm31, zmm20			; 25-28	;; accumulate maxerr
	vmaxpd	zmm31, zmm31, zmm22			; 29-32	;; accumulate maxerr

	vblendmpd zmm20 {k1}, zmm29, zmm27		; 17	;; Create (base) constant
	vblendmpd zmm21 {k2}, zmm29, zmm27		; 17	;; Create (base) constant
	vblendmpd zmm22 {k3}, zmm29, zmm27		; 18	;; Create (base) constant
	vblendmpd zmm23 {k4}, zmm29, zmm27		; 18	;; Create (base) constant

	vsubpd	zmm16, zmm16, zmm30			; 13-16	;; val5_hi = val5_hi+RNDVAL - RNDVAL
	vsubpd	zmm17, zmm17, zmm30			; 14-17	;; val6_hi = val6_hi+RNDVAL - RNDVAL
	vsubpd	zmm18, zmm18, zmm30			; 15-18	;; val7_hi = val7_hi+RNDVAL - RNDVAL
	vsubpd	zmm19, zmm19, zmm30			; 16-19	;; val8_hi = val8_hi+RNDVAL - RNDVAL

	vsubpd	zmm8, zmm8, zmm30			; 19-22	;; val1 = val1+RNDVAL - RNDVAL
	vsubpd	zmm9, zmm9, zmm30			; 19-22	;; val2 = val2+RNDVAL - RNDVAL
	vsubpd	zmm10, zmm10, zmm30			; 20-23	;; val3 = val3+RNDVAL - RNDVAL
	vsubpd	zmm11, zmm11, zmm30			; 20-23	;; val4 = val4+RNDVAL - RNDVAL

	zfnmaddpd zmm12, zmm16, zmm20, zmm12		; 19-22	;; val5_lo = val5 - val5_hi * base = val5%base
	zfnmaddpd zmm13, zmm17, zmm21, zmm13		; 19-22	;; val6_lo = val6 - val6_hi * base = val6%base
	zfnmaddpd zmm14, zmm18, zmm22, zmm14		; 20-23	;; val7_lo = val7 - val7_hi * base = val7%base
	zfnmaddpd zmm15, zmm19, zmm23, zmm15		; 20-23	;; val8_lo = val8 - val8_hi * base = val8%base

	zfmaddpd zmm12, zmm12, zmm25, zmm0		; 23-26	;; x1 = val5_lo * -c*mulconst + carry5
	zfmaddpd zmm13, zmm13, zmm25, zmm1		; 23-26	;; x2 = val6_lo * -c*mulconst + carry6
	zfmaddpd zmm14, zmm14, zmm25, zmm2		; 24-27	;; x3 = val7_lo * -c*mulconst + carry7
	zfmaddpd zmm15, zmm15, zmm25, zmm3		; 24-27	;; x4 = val8_lo * -c*mulconst + carry8

	zfmaddpd zmm0, zmm8, zmm4, zmm30		; 17-20	;; val1_hi+RNDVAL = val1 / base + RNDVAL = rnd(val1/base)+RNDVAL (next carry1)
	zfmaddpd zmm1, zmm9, zmm5, zmm30		; 17-20	;; val2_hi+RNDVAL = val2 / base + RNDVAL = rnd(val2/base)+RNDVAL (next carry2)
	zfmaddpd zmm2, zmm10, zmm6, zmm30		; 18-21	;; val3_hi+RNDVAL = val3 / base + RNDVAL = rnd(val3/base)+RNDVAL (next carry3)
	zfmaddpd zmm3, zmm11, zmm7, zmm30		; 18-21	;; val4_hi+RNDVAL = val4 / base + RNDVAL = rnd(val4/base)+RNDVAL (next carry4)

	vsubpd	zmm24, zmm0, zmm30			; 25-28	;; val1_hi = val1_hi+RNDVAL - RNDVAL
	zfnmaddpd zmm8, zmm24, zmm20, zmm8		; 29-32	;; val1_lo = val1 - val1_hi * base = val1%base
	vsubpd	zmm24, zmm1, zmm30			; 25-28	;; val2_hi = val2_hi+RNDVAL - RNDVAL
	zfnmaddpd zmm9, zmm24, zmm21, zmm9		; 29-32	;; val2_lo = val2 - val2_hi * base = val2%base
	vsubpd	zmm24, zmm2, zmm30			; 26-29	;; val3_hi = val3_hi+RNDVAL - RNDVAL
	zfnmaddpd zmm10, zmm24, zmm22, zmm10		; 30-33	;; val3_lo = val3 - val3_hi * base = val3%base
	vsubpd	zmm24, zmm3, zmm30			; 26-29	;; val4_hi = val4_hi+RNDVAL - RNDVAL
	zfnmaddpd zmm11, zmm24, zmm23, zmm11		; 30-33	;; val4_lo = val4 - val4_hi * base = val4%base

no const vbroadcastsd zmm24, ZMM_K_LO
const	vbroadcastsd zmm24, ZMM_K_TIMES_MULCONST_LO
	zfmaddpd zmm12, zmm8, zmm24, zmm12		; 33-36 ;; x1 += val1_lo * k_lo*mulconst
	zfmaddpd zmm13, zmm9, zmm24, zmm13		; 33-36 ;; x2 += val2_lo * k_lo*mulconst
	zfmaddpd zmm14, zmm10, zmm24, zmm14		; 34-37 ;; x3 += val3_lo * k_lo*mulconst
	zfmaddpd zmm15, zmm11, zmm24, zmm15		; 34-37 ;; x4 += val4_lo * k_lo*mulconst

	zfmaddpd zmm4, zmm12, zmm4, zmm30		; 37-40 ;; x1_hi+RNDVAL = x1 / base + RNDVAL = rnd(x1/base)+RNDVAL
	zfmaddpd zmm5, zmm13, zmm5, zmm30		; 37-40 ;; x2_hi+RNDVAL = x2 / base + RNDVAL = rnd(x2/base)+RNDVAL
	zfmaddpd zmm6, zmm14, zmm6, zmm30		; 38-41 ;; x3_hi+RNDVAL = x3 / base + RNDVAL = rnd(x3/base)+RNDVAL
	zfmaddpd zmm7, zmm15, zmm7, zmm30		; 38-41 ;; x4_hi+RNDVAL = x4 / base + RNDVAL = rnd(x4/base)+RNDVAL

	vsubpd	zmm4, zmm4, zmm30			; 41-44	;; x1_hi = x1_hi+RNDVAL - RNDVAL
	vsubpd	zmm5, zmm5, zmm30			; 41-44	;; x2_hi = x2_hi+RNDVAL - RNDVAL
	vsubpd	zmm6, zmm6, zmm30			; 42-45	;; x3_hi = x3_hi+RNDVAL - RNDVAL
	vsubpd	zmm7, zmm7, zmm30			; 42-45	;; x4_hi = x4_hi+RNDVAL - RNDVAL

	zfnmaddpd zmm12, zmm4, zmm20, zmm12		; 45-48 ;; new val1 = x1_lo = x1 - x1_hi * base = x1%base
	zfnmaddpd zmm13, zmm5, zmm21, zmm13		; 45-48 ;; new val2 = x2_lo = x2 - x2_hi * base = x2%base
	zfnmaddpd zmm14, zmm6, zmm22, zmm14		; 46-49 ;; new val3 = x3_lo = x3 - x3_hi * base = x3%base
	zfnmaddpd zmm15, zmm7, zmm23, zmm15		; 46-49 ;; new val4 = x4_lo = x4 - x4_hi * base = x4%base

	zfmaddpd zmm4, zmm16, zmm25, zmm4		; 47-50 ;; next carry5 = val5_hi * -c*mulconst + x1_hi
	zfmaddpd zmm5, zmm17, zmm25, zmm5		; 47-50 ;; next carry6 = val6_hi * -c*mulconst + x2_hi
	zfmaddpd zmm6, zmm18, zmm25, zmm6		; 48-51 ;; next carry7 = val7_hi * -c*mulconst + x3_hi
	zfmaddpd zmm7, zmm19, zmm25, zmm7		; 48-51 ;; next carry8 = val8_hi * -c*mulconst + x4_hi

khi no const	vbroadcastsd zmm20, ZMM_K_HI_OVER_SMALL_BASE
khi no const	vbroadcastsd zmm21, ZMM_K_HI_OVER_LARGE_BASE
khi const	vbroadcastsd zmm20, ZMM_K_TIMES_MULCONST_HI_OVER_SMALL_BASE
khi const	vbroadcastsd zmm21, ZMM_K_TIMES_MULCONST_HI_OVER_LARGE_BASE
khi	vblendmpd zmm16 {k1}, zmm20, zmm21		; 43	;; Create (k_hi*mulconst/base) constant
khi	vblendmpd zmm17 {k2}, zmm20, zmm21		; 43	;; Create (k_hi*mulconst/base) constant
khi	vblendmpd zmm18 {k3}, zmm20, zmm21		; 44	;; Create (k_hi*mulconst/base) constant
khi	vblendmpd zmm19 {k4}, zmm20, zmm21		; 44	;; Create (k_hi*mulconst/base) constant
khi	zfmaddpd zmm4, zmm8, zmm16, zmm4		; 51-54 ;; if khi next carry5 = val1_lo * k_hi*mulconst/base + next carry5
khi	zfmaddpd zmm5, zmm9, zmm17, zmm5		; 51-54 ;; if khi next carry6 = val2_lo * k_hi*mulconst/base + next carry6
khi	zfmaddpd zmm6, zmm10, zmm18, zmm6		; 52-55 ;; if khi next carry7 = val3_lo * k_hi*mulconst/base + next carry7
khi	zfmaddpd zmm7, zmm11, zmm19, zmm7		; 52-55 ;; if khi next carry8 = val4_lo * k_hi*mulconst/base + next carry8

	zstore	[rsi], zmm12					;; Save value1
	zstore	[rsi+r13], zmm13				;; Save value2
	zstore	[rsi+2*r13], zmm14				;; Save value3
	zstore	[rsi+r14], zmm15				;; Save value4
	vzero	zmm8
	zstore	[rsi+64], zmm8					;; Save value5
	zstore	[rsi+r13+64], zmm8				;; Save value6
	zstore	[rsi+2*r13+64], zmm8				;; Save value7
	zstore	[rsi+r14+64], zmm8				;; Save value8
	ENDM


; *************** WPN normalized zero-padded add/sub macro ******************
; This macro adds or subtracts, then "normalizes" four pairs of loword/hiword FFT data values for zero-padded FFTs.
; This is similar to the non-zero-pad case except that upper half data need not be normalized (div-by-k will normalize it)
; and there is half as much big/lit data.
; rsi = pointer to the first number
; rcx = pointer to the second number
; rbx = pointer to the destination
; r12 = pointer to compressed biglit table
; rdx = register used to load compressed biglit index
; r13 = distance to source/dest #2
; r14 = distance to source/dest #4
; rdi = pointer to array of big vs. little flags
; zmm0-3 = carries in +0 format

znorm_op_wpn_zpad_preload MACRO ttp
no ttp	znorm_op_wpn_zpad_nottp_preload
ttp	znorm_op_wpn_zpad_ttp_preload
	ENDM

znorm_op_wpn_zpad MACRO fop, ttp
no ttp	znorm_op_wpn_zpad_nottp fop
ttp	znorm_op_wpn_zpad_ttp fop
	ENDM

znorm_op_wpn_zpad_nottp_preload MACRO
	vbroadcastsd zmm30, ZMM_RNDVAL			;; Load the rounding value
	vbroadcastsd zmm29, ZMM_SMALL_BASE		;; small_word_base
	vbroadcastsd zmm28, ZMM_SMALL_BASE_INVERSE	;; 1 / small_word_base
	vzero	zmm0					;; Start process with no carry
	vmovapd	zmm1, zmm0
	vmovapd	zmm2, zmm0
	vmovapd	zmm3, zmm0
	ENDM
znorm_op_wpn_zpad_nottp MACRO fop
	vaddpd	zmm8, zmm0, [rcx]		; 1-4	;; x = value2 + carry
	vaddpd	zmm9, zmm1, [rcx+r13]		; 1-4	;; x = value2 + carry
	vaddpd	zmm10, zmm2, [rcx+2*r13]	; 2-5	;; x = value2 + carry
	vaddpd	zmm11, zmm3, [rcx+r14]		; 2-5	;; x = value2 + carry

	vmovapd	zmm12, [rcx+64]				;; Load second number
	fop	zmm12, zmm12, [rsi+64]		; 3-6	;; Add/sub first number
	vmovapd	zmm13, [rcx+r13+64]			;; Load second number
	fop	zmm13, zmm13, [rsi+r13+64]	; 3-6	;; Add/sub first number
	vmovapd	zmm14, [rcx+2*r13+64]			;; Load second number
	fop	zmm14, zmm14, [rsi+2*r13+64]	; 4-7	;; Add/sub first number
	vmovapd	zmm15, [rcx+r14+64]			;; Load second number
	fop	zmm15, zmm15, [rsi+r14+64]	; 4-7	;; Add/sub first number

	fop	zmm8, zmm8, [rsi]		; 5-8	;; Add/sub first number, x = (value2 op value1) + carry
	fop	zmm9, zmm9, [rsi+r13]		; 5-8	;; Add/sub first number, x = (value2 op value1) + carry
	fop	zmm10, zmm10, [rsi+2*r13]	; 6-9	;; Add/sub first number, x = (value2 op value1) + carry
	fop	zmm11, zmm11, [rsi+r14]		; 6-9	;; Add/sub first number, x = (value2 op value1) + carry

	zfmaddpd zmm0, zmm8, zmm28, zmm30	; 9-12	;; next carry+RNDVAL = (x/base + RNDVAL) = rnd(x/base)+RNDVAL
	zfmaddpd zmm1, zmm9, zmm28, zmm30	; 9-12	;; next carry+RNDVAL = (x/base + RNDVAL) = rnd(x/base)+RNDVAL
	zfmaddpd zmm2, zmm10, zmm28, zmm30	; 10-13	;; next carry+RNDVAL = (x/base + RNDVAL) = rnd(x/base)+RNDVAL
	zfmaddpd zmm3, zmm11, zmm28, zmm30	; 10-13	;; next carry+RNDVAL = (x/base + RNDVAL) = rnd(x/base)+RNDVAL

	vsubpd	zmm0, zmm0, zmm30		; 13-16	;; next carry = (next carry+RNDVAL) - RNDVAL = rnd(x/base)
	vsubpd	zmm1, zmm1, zmm30		; 13-16	;; next carry = (next carry+RNDVAL) - RNDVAL = rnd(x/base)
	vsubpd	zmm2, zmm2, zmm30		; 14-17	;; next carry = (next carry+RNDVAL) - RNDVAL = rnd(x/base)
	vsubpd	zmm3, zmm3, zmm30		; 14-17	;; next carry = (next carry+RNDVAL) - RNDVAL = rnd(x/base)

	zfnmaddpd zmm8, zmm0, zmm29, zmm8	; 17-20	;; new value = x - rnd(x/base)*base
	zfnmaddpd zmm9, zmm1, zmm29, zmm9	; 17-20	;; new value = x - rnd(x/base)*base
	zfnmaddpd zmm10, zmm2, zmm29, zmm10	; 18-21	;; new value = x - rnd(x/base)*base
	zfnmaddpd zmm11, zmm3, zmm29, zmm11	; 18-21	;; new value = x - rnd(x/base)*base

	zstore	[rbx], zmm8				;; Save value1
	zstore	[rbx+r13], zmm9				;; Save value2
	zstore	[rbx+2*r13], zmm10			;; Save value3
	zstore	[rbx+r14], zmm11			;; Save value4
	zstore	[rbx+64], zmm12				;; Save value5
	zstore	[rbx+r13+64], zmm13			;; Save value6
	zstore	[rbx+2*r13+64], zmm14			;; Save value7
	zstore	[rbx+r14+64], zmm15			;; Save value8
	ENDM

znorm_op_wpn_zpad_ttp_preload MACRO
	vbroadcastsd zmm30, ZMM_RNDVAL			;; Load the rounding value
	vbroadcastsd zmm29, ZMM_SMALL_BASE		;; small_word_base
	vbroadcastsd zmm28, ZMM_SMALL_BASE_INVERSE	;; 1 / small_word_base
	vbroadcastsd zmm27, ZMM_LARGE_BASE		;; large_word_base
	vbroadcastsd zmm26, ZMM_LARGE_BASE_INVERSE	;; 1 / large_word_base
	vzero	zmm0					;; Start process with no carry
	vmovapd	zmm1, zmm0
	vmovapd	zmm2, zmm0
	vmovapd	zmm3, zmm0
	ENDM
znorm_op_wpn_zpad_ttp MACRO fop
	movzx	rdx, BYTE PTR [rdi]			;; Load index into compressed biglit table

	vaddpd	zmm8, zmm0, [rcx]		; 1-4	;; x = value2 + carry
	vaddpd	zmm9, zmm1, [rcx+r13]		; 1-4	;; x = value2 + carry
	vaddpd	zmm10, zmm2, [rcx+2*r13]	; 2-5	;; x = value2 + carry
	vaddpd	zmm11, zmm3, [rcx+r14]		; 2-5	;; x = value2 + carry

	vmovapd	zmm12, [rcx+64]				;; Load second number
	fop	zmm12, zmm12, [rsi+64]		; 3-6	;; Add/sub first number
	vmovapd	zmm13, [rcx+r13+64]			;; Load second number
	fop	zmm13, zmm13, [rsi+r13+64]	; 3-6	;; Add/sub first number
	vmovapd	zmm14, [rcx+2*r13+64]			;; Load second number
	fop	zmm14, zmm14, [rsi+2*r13+64]	; 4-7	;; Add/sub first number
	vmovapd	zmm15, [rcx+r14+64]			;; Load second number
	fop	zmm15, zmm15, [rsi+r14+64]	; 4-7	;; Add/sub first number

	mov	edx, [r12+rdx*4]			;; Load 4 big vs. little flags

	fop	zmm8, zmm8, [rsi]		; 5-8	;; Add/sub first number, x = (value2 op value1) + carry
	fop	zmm9, zmm9, [rsi+r13]		; 5-8	;; Add/sub first number, x = (value2 op value1) + carry
	fop	zmm10, zmm10, [rsi+2*r13]	; 6-9	;; Add/sub first number, x = (value2 op value1) + carry
	fop	zmm11, zmm11, [rsi+r14]		; 6-9	;; Add/sub first number, x = (value2 op value1) + carry

;; UNOPTIMIZED?? Use clocks 7,8 to do masks and blends

	kmovw	k1, edx				; 9	;; Load 2 big vs. little flags (16-bit load - we use 8-bits)
	vblendmpd zmm0 {k1}, zmm28, zmm26	; 9	;; Create (1 / base) constant used in next carry calculation
	zfmaddpd zmm0, zmm8, zmm0, zmm30	; 9-12	;; y+RNDVAL = x / base + RNDVAL (next carry+RNDVAL)
	kshiftrw k2, k1, 8			; 9
	vblendmpd zmm1 {k2}, zmm28, zmm26	; 9	;; Create (1 / base) constant used in next carry calculation
	zfmaddpd zmm1, zmm9, zmm1, zmm30	; 9-12	;; y+RNDVAL = x / base + RNDVAL (next carry+RNDVAL)
	shr	edx, 16
	kmovw	k3, edx				; 9	;; Load 2 big vs. little flags (16-bit load - we use 8-bits)
	vblendmpd zmm2 {k3}, zmm28, zmm26	; 9	;; Create (1 / base) constant used in next carry calculation
	zfmaddpd zmm2, zmm10, zmm2, zmm30	; 10-13	;; y+RNDVAL = x / base + RNDVAL (next carry+RNDVAL)
	kshiftrw k4, k3, 8			; 10
	vblendmpd zmm3 {k4}, zmm28, zmm26	; 10	;; Create (1 / base) constant used in next carry calculation
	zfmaddpd zmm3, zmm11, zmm3, zmm30	; 10-13	;; y+RNDVAL = x / base + RNDVAL (next carry+RNDVAL)

	vsubpd	zmm0, zmm0, zmm30		; 13-16	;; y = rnd(x/base) = y - RNDVAL
	vsubpd	zmm1, zmm1, zmm30		; 13-16	;; y = rnd(x/base) = y - RNDVAL
	vsubpd	zmm2, zmm2, zmm30		; 14-17	;; y = rnd(x/base) = y - RNDVAL
	vsubpd	zmm3, zmm3, zmm30		; 14-17	;; y = rnd(x/base) = y - RNDVAL

	vblendmpd zmm24 {k1}, zmm29, zmm27	; 17	;; Create (base) constant used in new value calculation
	zfnmaddpd zmm8, zmm0, zmm24, zmm8	; 17-20	;; new value = x - y * base
	vblendmpd zmm24 {k2}, zmm29, zmm27	; 17	;; Create (base) constant used in new value calculation
	zfnmaddpd zmm9, zmm1, zmm24, zmm9	; 17-20	;; new value = x - y * base
	vblendmpd zmm24 {k3}, zmm29, zmm27	; 17	;; Create (base) constant used in new value calculation
	zfnmaddpd zmm10, zmm2, zmm24, zmm10	; 18-21	;; new value = x - y * base
	vblendmpd zmm24 {k4}, zmm29, zmm27	; 18	;; Create (base) constant used in new value calculation
	zfnmaddpd zmm11, zmm3, zmm24, zmm11	; 18-21	;; new value = x - y * base

	zstore	[rbx], zmm8			;; Save value1
	zstore	[rbx+r13], zmm9			;; Save value2
	zstore	[rbx+2*r13], zmm10		;; Save value3
	zstore	[rbx+r14], zmm11		;; Save value4
	zstore	[rbx+64], zmm12			;; Save value5
	zstore	[rbx+r13+64], zmm13		;; Save value6
	zstore	[rbx+2*r13+64], zmm14		;; Save value7
	zstore	[rbx+r14+64], zmm15		;; Save value8
	ENDM

;; Final step in the znorm_op process.  Write carries to the carries array (to be processed later)

znorm_op_wpn_zpad_save_carries MACRO
	mov	rbp, DATA_ADDR			; Address of carries array
	vaddpd	zmm0, zmm0, zmm30		; Store carries in +RNDVAL format (to share share code with post-FFT normalize)
	vaddpd	zmm1, zmm1, zmm30
	vaddpd	zmm2, zmm2, zmm30
	vaddpd	zmm3, zmm3, zmm30
	zstore	[rbp+0*128], zmm0		; Save carry out of low word
	zstore	[rbp+1*128], zmm1		; Save carry out of low word
	zstore	[rbp+2*128], zmm2		; Save carry out of low word
	zstore	[rbp+3*128], zmm3		; Save carry out of low word
	ENDM


; *************** WPN normalized addsub macro ******************
; This macro adds and subtracts, then "normalizes"  two pairs of loword/hiword FFT data values.
; Differs from the non-zero-pad version in that upper half data need not be normalized.

znorm_addsub_wpn_zpad_preload MACRO ttp
no ttp	znorm_addsub_wpn_zpad_nottp_preload
ttp	znorm_addsub_wpn_zpad_ttp_preload
	ENDM

znorm_addsub_wpn_zpad MACRO ttp
no ttp	znorm_addsub_wpn_zpad_nottp
ttp	znorm_addsub_wpn_zpad_ttp
	ENDM

znorm_addsub_wpn_zpad_nottp_preload MACRO
	vbroadcastsd zmm30, ZMM_RNDVAL			;; Load the rounding value
	vbroadcastsd zmm29, ZMM_SMALL_BASE		;; small_word_base
	vbroadcastsd zmm28, ZMM_SMALL_BASE_INVERSE	;; 1 / small_word_base
	vzero	zmm0					;; Start process with no carry
	vmovapd	zmm1, zmm0
	vmovapd	zmm2, zmm0
	vmovapd	zmm3, zmm0
	vmovapd	zmm4, zmm0
	vmovapd	zmm5, zmm0
	vmovapd	zmm6, zmm0
	vmovapd	zmm7, zmm0
	ENDM
znorm_addsub_wpn_zpad_nottp MACRO

;; UNOPTIMIZED??

	vmovapd	zmm24, [rsi]			;; Load first number
	vmovapd	zmm25, [rcx]			;; Load second number
	vaddpd	zmm8, zmm24, zmm25		;; first + second number
	vsubpd	zmm12, zmm24, zmm25		;; first - second number
	vmovapd	zmm24, [rsi+r13]		;; Load first number
	vmovapd	zmm25, [rcx+r13]		;; Load second number
	vaddpd	zmm9, zmm24, zmm25		;; first + second number
	vsubpd	zmm13, zmm24, zmm25		;; first - second number
	vmovapd	zmm24, [rsi+2*r13]		;; Load first number
	vmovapd	zmm25, [rcx+2*r13]		;; Load second number
	vaddpd	zmm10, zmm24, zmm25		;; first + second number
	vsubpd	zmm14, zmm24, zmm25		;; first - second number
	vmovapd	zmm24, [rsi+r14]		;; Load first number
	vmovapd	zmm25, [rcx+r14]		;; Load second number
	vaddpd	zmm11, zmm24, zmm25		;; first + second number
	vsubpd	zmm15, zmm24, zmm25		;; first - second number

	vmovapd	zmm24, [rsi+64]			;; Load first number
	vmovapd	zmm25, [rcx+64]			;; Load second number
	vaddpd	zmm16, zmm24, zmm25		;; first + second number
	vsubpd	zmm20, zmm24, zmm25		;; first - second number
	vmovapd	zmm24, [rsi+r13+64]		;; Load first number
	vmovapd	zmm25, [rcx+r13+64]		;; Load second number
	vaddpd	zmm17, zmm24, zmm25		;; first + second number
	vsubpd	zmm21, zmm24, zmm25		;; first - second number
	vmovapd	zmm24, [rsi+2*r13+64]		;; Load first number
	vmovapd	zmm25, [rcx+2*r13+64]		;; Load second number
	vaddpd	zmm18, zmm24, zmm25		;; first + second number
	vsubpd	zmm22, zmm24, zmm25		;; first - second number
	vmovapd	zmm24, [rsi+r14+64]		;; Load first number
	vmovapd	zmm25, [rcx+r14+64]		;; Load second number
	vaddpd	zmm19, zmm24, zmm25		;; first + second number
	vsubpd	zmm23, zmm24, zmm25		;; first - second number

	vaddpd	zmm8, zmm8, zmm0		; 1-4	;; x = value + carry
	vaddpd	zmm12, zmm12, zmm4		; 1-4	;; x = value + carry
	vaddpd	zmm9, zmm9, zmm1		; 2-5	;; x = value + carry
	vaddpd	zmm13, zmm13, zmm5		; 2-5	;; x = value + carry
	vaddpd	zmm10, zmm10, zmm2		; 3-6	;; x = value + carry
	vaddpd	zmm14, zmm14, zmm6		; 3-6	;; x = value + carry
	vaddpd	zmm11, zmm11, zmm3		; 4-7	;; x = value + carry
	vaddpd	zmm15, zmm15, zmm7		; 4-7	;; x = value + carry

	zstore	[rbx+64], zmm16			;; Save add value
	zstore	[rbp+64], zmm20			;; Save subtract value
	zstore	[rbx+r13+64], zmm17		;; Save add value
	zstore	[rbp+r13+64], zmm21		;; Save subtract value
	zstore	[rbx+2*r13+64], zmm18		;; Save add value
	zstore	[rbp+2*r13+64], zmm22		;; Save subtract value
	zstore	[rbx+r14+64], zmm19		;; Save add value
	zstore	[rbp+r14+64], zmm23		;; Save subtract value

	zfmaddpd zmm0, zmm8, zmm28, zmm30	; 5-8	;; next carry+RNDVAL = x/base + RNDVAL = rnd(x/base)+RNDVAL
	zfmaddpd zmm4, zmm12, zmm28, zmm30	; 5-8	;; next carry+RNDVAL = x/base + RNDVAL = rnd(x/base)+RNDVAL
	zfmaddpd zmm1, zmm9, zmm28, zmm30	; 6-9	;; next carry+RNDVAL = x/base + RNDVAL = rnd(x/base)+RNDVAL
	zfmaddpd zmm5, zmm13, zmm28, zmm30	; 6-9	;; next carry+RNDVAL = x/base + RNDVAL = rnd(x/base)+RNDVAL
	zfmaddpd zmm2, zmm10, zmm28, zmm30	; 7-10	;; next carry+RNDVAL = x/base + RNDVAL = rnd(x/base)+RNDVAL
	zfmaddpd zmm6, zmm14, zmm28, zmm30	; 7-10	;; next carry+RNDVAL = x/base + RNDVAL = rnd(x/base)+RNDVAL
	zfmaddpd zmm3, zmm11, zmm28, zmm30	; 8-11	;; next carry+RNDVAL = x/base + RNDVAL = rnd(x/base)+RNDVAL
	zfmaddpd zmm7, zmm15, zmm28, zmm30	; 8-11	;; next carry+RNDVAL = x/base + RNDVAL = rnd(x/base)+RNDVAL

	vsubpd	zmm0, zmm0, zmm30		; 9-12	;; next carry = (next carry+RNDVAL) - RNDVAL = rnd(x/base)
	vsubpd	zmm4, zmm4, zmm30		; 9-12	;; next carry = (next carry+RNDVAL) - RNDVAL = rnd(x/base)
	vsubpd	zmm1, zmm1, zmm30		; 10-13	;; next carry = (next carry+RNDVAL) - RNDVAL = rnd(x/base)
	vsubpd	zmm5, zmm5, zmm30		; 10-13	;; next carry = (next carry+RNDVAL) - RNDVAL = rnd(x/base)
	vsubpd	zmm2, zmm2, zmm30		; 11-14	;; next carry = (next carry+RNDVAL) - RNDVAL = rnd(x/base)
	vsubpd	zmm6, zmm6, zmm30		; 11-14	;; next carry = (next carry+RNDVAL) - RNDVAL = rnd(x/base)
	vsubpd	zmm3, zmm3, zmm30		; 12-15	;; next carry = (next carry+RNDVAL) - RNDVAL = rnd(x/base)
	vsubpd	zmm7, zmm7, zmm30		; 12-15	;; next carry = (next carry+RNDVAL) - RNDVAL = rnd(x/base)

	zfnmaddpd zmm8, zmm0, zmm29, zmm8	; 13-16	;; new value = x - rnd(x/base)*base
	zfnmaddpd zmm12, zmm4, zmm29, zmm12	; 13-16	;; new value = x - rnd(x/base)*base
	zfnmaddpd zmm9, zmm1, zmm29, zmm9	; 14-17	;; new value = x - rnd(x/base)*base
	zfnmaddpd zmm13, zmm5, zmm29, zmm13	; 14-17	;; new value = x - rnd(x/base)*base
	zfnmaddpd zmm10, zmm2, zmm29, zmm10	; 15-18	;; new value = x - rnd(x/base)*base
	zfnmaddpd zmm14, zmm6, zmm29, zmm14	; 15-18	;; new value = x - rnd(x/base)*base
	zfnmaddpd zmm11, zmm3, zmm29, zmm11	; 16-19	;; new value = x - rnd(x/base)*base
	zfnmaddpd zmm15, zmm7, zmm29, zmm15	; 16-19	;; new value = x - rnd(x/base)*base

	zstore	[rbx], zmm8			;; Save add value
	zstore	[rbp], zmm12			;; Save subtract value
	zstore	[rbx+r13], zmm9			;; Save add value
	zstore	[rbp+r13], zmm13		;; Save subtract value
	zstore	[rbx+2*r13], zmm10		;; Save add value
	zstore	[rbp+2*r13], zmm14		;; Save subtract value
	zstore	[rbx+r14], zmm11		;; Save add value
	zstore	[rbp+r14], zmm15		;; Save subtract value
	ENDM

znorm_addsub_wpn_zpad_ttp_preload MACRO
	vbroadcastsd zmm30, ZMM_RNDVAL				;; Load the rounding value
	vbroadcastsd zmm29, ZMM_SMALL_BASE			;; small_word_base
	vbroadcastsd zmm28, ZMM_SMALL_BASE_INVERSE		;; 1 / small_word_base
	vbroadcastsd zmm27, ZMM_LARGE_BASE			;; large_word_base
	vbroadcastsd zmm26, ZMM_LARGE_BASE_INVERSE		;; 1 / large_word_base
	vzero	zmm0				;; Start process with no carry
	vmovapd	zmm1, zmm0
	vmovapd	zmm2, zmm0
	vmovapd	zmm3, zmm0
	vmovapd	zmm4, zmm0
	vmovapd	zmm5, zmm0
	vmovapd	zmm6, zmm0
	vmovapd	zmm7, zmm0
	ENDM
znorm_addsub_wpn_zpad_ttp MACRO
	movzx	rdx, BYTE PTR [rdi]			;; Load index into compressed biglit table

;; UNOPTIMIZED??

	vmovapd	zmm24, [rsi]			;; Load first number
	vmovapd	zmm25, [rcx]			;; Load second number
	vaddpd	zmm8, zmm24, zmm25		;; first + second number
	vsubpd	zmm12, zmm24, zmm25		;; first - second number
	vmovapd	zmm24, [rsi+r13]		;; Load first number
	vmovapd	zmm25, [rcx+r13]		;; Load second number
	vaddpd	zmm9, zmm24, zmm25		;; first + second number
	vsubpd	zmm13, zmm24, zmm25		;; first - second number
	vmovapd	zmm24, [rsi+2*r13]		;; Load first number
	vmovapd	zmm25, [rcx+2*r13]		;; Load second number
	vaddpd	zmm10, zmm24, zmm25		;; first + second number
	vsubpd	zmm14, zmm24, zmm25		;; first - second number
	vmovapd	zmm24, [rsi+r14]		;; Load first number
	vmovapd	zmm25, [rcx+r14]		;; Load second number
	vaddpd	zmm11, zmm24, zmm25		;; first + second number
	vsubpd	zmm15, zmm24, zmm25		;; first - second number

	vmovapd	zmm24, [rsi+64]			;; Load first number
	vmovapd	zmm25, [rcx+64]			;; Load second number
	vaddpd	zmm16, zmm24, zmm25		;; first + second number
	vsubpd	zmm20, zmm24, zmm25		;; first - second number
	vmovapd	zmm24, [rsi+r13+64]		;; Load first number
	vmovapd	zmm25, [rcx+r13+64]		;; Load second number
	vaddpd	zmm17, zmm24, zmm25		;; first + second number
	vsubpd	zmm21, zmm24, zmm25		;; first - second number
	vmovapd	zmm24, [rsi+2*r13+64]		;; Load first number
	vmovapd	zmm25, [rcx+2*r13+64]		;; Load second number
	vaddpd	zmm18, zmm24, zmm25		;; first + second number
	vsubpd	zmm22, zmm24, zmm25		;; first - second number
	vmovapd	zmm24, [rsi+r14+64]		;; Load first number
	vmovapd	zmm25, [rcx+r14+64]		;; Load second number
	vaddpd	zmm19, zmm24, zmm25		;; first + second number
	vsubpd	zmm23, zmm24, zmm25		;; first - second number

	mov	edx, [r12+rdx*4]		;; Load 4 big vs. little flags

	vaddpd	zmm8, zmm8, zmm0		; 1-4	;; x = value + carry
	vaddpd	zmm12, zmm12, zmm4		; 1-4	;; x = value + carry
	vaddpd	zmm9, zmm9, zmm1		; 2-5	;; x = value + carry
	vaddpd	zmm13, zmm13, zmm5		; 2-5	;; x = value + carry
	vaddpd	zmm10, zmm10, zmm2		; 3-6	;; x = value + carry
	vaddpd	zmm14, zmm14, zmm6		; 3-6	;; x = value + carry
	vaddpd	zmm11, zmm11, zmm3		; 4-7	;; x = value + carry
	vaddpd	zmm15, zmm15, zmm7		; 4-7	;; x = value + carry

	zstore	[rbx+64], zmm16			;; Save add value
	zstore	[rbp+64], zmm20			;; Save subtract value
	zstore	[rbx+r13+64], zmm17		;; Save add value
	zstore	[rbp+r13+64], zmm21		;; Save subtract value
	zstore	[rbx+2*r13+64], zmm18		;; Save add value
	zstore	[rbp+2*r13+64], zmm22		;; Save subtract value
	zstore	[rbx+r14+64], zmm19		;; Save add value
	zstore	[rbp+r14+64], zmm23		;; Save subtract value

	kmovw	k1, edx				; 5	;; Load 2 big vs. little flags (16-bit load - we use 8-bits)
	kshiftrw k2, k1, 8			; 6
	shr	edx, 16
	kmovw	k3, edx				; 7	;; Load 2 big vs. little flags (16-bit load - we use 8-bits)
	kshiftrw k4, k3, 8			; 10

	vblendmpd zmm4 {k1}, zmm28, zmm26	; 5	;; Create (1 / base) constant used in next carry calculation
	zfmaddpd zmm0, zmm8, zmm4, zmm30	; 5-8	;; y+RNDVAL = x / base + RNDVAL (next carry+RNDVAL)
	zfmaddpd zmm4, zmm12, zmm4, zmm30	; 5-8	;; y+RNDVAL = x / base + RNDVAL (next carry+RNDVAL)
	vblendmpd zmm5 {k2}, zmm28, zmm26	; 6	;; Create (1 / base) constant used in next carry calculation
	zfmaddpd zmm1, zmm9, zmm5, zmm30	; 6-9	;; y+RNDVAL = x / base + RNDVAL (next carry+RNDVAL)
	zfmaddpd zmm5, zmm13, zmm5, zmm30	; 6-9	;; y+RNDVAL = x / base + RNDVAL (next carry+RNDVAL)
	vblendmpd zmm6 {k3}, zmm28, zmm26	; 7	;; Create (1 / base) constant used in next carry calculation
	zfmaddpd zmm2, zmm10, zmm6, zmm30	; 7-10	;; y+RNDVAL = x / base + RNDVAL (next carry+RNDVAL)
	zfmaddpd zmm6, zmm14, zmm6, zmm30	; 7-10	;; y+RNDVAL = x / base + RNDVAL (next carry+RNDVAL)
	vblendmpd zmm7 {k4}, zmm28, zmm26	; 8	;; Create (1 / base) constant used in next carry calculation
	zfmaddpd zmm3, zmm11, zmm7, zmm30	; 8-11	;; y+RNDVAL = x / base + RNDVAL (next carry+RNDVAL)
	zfmaddpd zmm7, zmm15, zmm7, zmm30	; 8-11	;; y+RNDVAL = x / base + RNDVAL (next carry+RNDVAL)

	vsubpd	zmm0, zmm0, zmm30		; 9-12	;; y = rnd(x/base) = y - RNDVAL
	vsubpd	zmm4, zmm4, zmm30		; 9-12	;; y = rnd(x/base) = y - RNDVAL
	vsubpd	zmm1, zmm1, zmm30		; 10-13	;; y = rnd(x/base) = y - RNDVAL
	vsubpd	zmm5, zmm5, zmm30		; 10-13	;; y = rnd(x/base) = y - RNDVAL
	vsubpd	zmm2, zmm2, zmm30		; 11-14	;; y = rnd(x/base) = y - RNDVAL
	vsubpd	zmm6, zmm6, zmm30		; 11-14	;; y = rnd(x/base) = y - RNDVAL
	vsubpd	zmm3, zmm3, zmm30		; 12-15	;; y = rnd(x/base) = y - RNDVAL
	vsubpd	zmm7, zmm7, zmm30		; 12-15	;; y = rnd(x/base) = y - RNDVAL

	vblendmpd zmm16 {k1}, zmm29, zmm27	; 13	;; Create (base) constant used in new value calculation
	zfnmaddpd zmm8, zmm0, zmm16, zmm8	; 13-16	;; new value = x - y * base
	zfnmaddpd zmm12, zmm4, zmm16, zmm12	; 13-16	;; new value = x - y * base
	vblendmpd zmm17 {k2}, zmm29, zmm27	; 14	;; Create (base) constant used in new value calculation
	zfnmaddpd zmm9, zmm1, zmm17, zmm9	; 14-17	;; new value = x - y * base
	zfnmaddpd zmm13, zmm5, zmm17, zmm13	; 14-17	;; new value = x - y * base
	vblendmpd zmm18 {k3}, zmm29, zmm27	; 15	;; Create (base) constant used in new value calculation
	zfnmaddpd zmm10, zmm2, zmm18, zmm10	; 15-18	;; new value = x - y * base
	zfnmaddpd zmm14, zmm6, zmm18, zmm14	; 15-18	;; new value = x - y * base
	vblendmpd zmm19 {k4}, zmm29, zmm27	; 16	;; Create (base) constant used in new value calculation
	zfnmaddpd zmm11, zmm3, zmm19, zmm11	; 16-19	;; new value = x - y * base
	zfnmaddpd zmm15, zmm7, zmm19, zmm15	; 16-19	;; new value = x - y * base

	zstore	[rbx], zmm8			;; Save add value
	zstore	[rbp], zmm12			;; Save subtract value
	zstore	[rbx+r13], zmm9			;; Save add value
	zstore	[rbp+r13], zmm13		;; Save subtract value
	zstore	[rbx+2*r13], zmm10		;; Save add value
	zstore	[rbp+2*r13], zmm14		;; Save subtract value
	zstore	[rbx+r14], zmm11		;; Save add value
	zstore	[rbp+r14], zmm15		;; Save subtract value
	ENDM

;; Final step in the znorm_addsub process.  Write carries to the carries array (to be processed later)

znorm_addsub_wpn_zpad_save_carries MACRO
	mov	rbp, DATA_ADDR			; Address of add carries array
	vaddpd	zmm0, zmm0, zmm30		; Store carries in +RNDVAL format (to share share code with post-FFT normalize)
	vaddpd	zmm1, zmm1, zmm30
	vaddpd	zmm2, zmm2, zmm30
	vaddpd	zmm3, zmm3, zmm30
	zstore	[rbp+0*128], zmm0		; Save carry out of low word
	zstore	[rbp+1*128], zmm1		; Save carry out of low word
	zstore	[rbp+2*128], zmm2		; Save carry out of low word
	zstore	[rbp+3*128], zmm3		; Save carry out of low word

	mov	rbp, PREMULT_ADDR		; Address of subtract carries array
	vaddpd	zmm4, zmm4, zmm30		; Store carries in +RNDVAL format (to share share code with post-FFT normalize)
	vaddpd	zmm5, zmm5, zmm30
	vaddpd	zmm6, zmm6, zmm30
	vaddpd	zmm7, zmm7, zmm30
	zstore	[rbp+0*128], zmm4		; Save carry out of low word
	zstore	[rbp+1*128], zmm5		; Save carry out of low word
	zstore	[rbp+2*128], zmm6		; Save carry out of low word
	zstore	[rbp+3*128], zmm7		; Save carry out of low word
	ENDM


; *************** WPN normalized small mul macro ******************
; This macro multiplies FFT data by a small value, then normalizes 4 loword/hiword pairs.
; rsi = pointer to source
; r13 = distance to source/dest #2
; r14 = distance to source/dest #4
; rbx = distance to destination
; r12 = pointer to compressed biglit table
; rdx = register used to load compressed biglit index
; rdi = pointer to array of big vs. little flags
; zmm0-3 = carry
; zmm31 = small multiplier value

znorm_smallmul_wpn_zpad_preload MACRO ttp
no ttp	znorm_smallmul_wpn_zpad_nottp_preload
ttp	znorm_smallmul_wpn_zpad_ttp_preload
	ENDM

znorm_smallmul_wpn_zpad MACRO ttp
no ttp	znorm_smallmul_wpn_zpad_nottp
ttp	znorm_smallmul_wpn_zpad_ttp
	ENDM

znorm_smallmul_wpn_zpad_nottp_preload MACRO
	vbroadcastsd zmm30, ZMM_RNDVAL			;; Load the rounding value
	vbroadcastsd zmm29, ZMM_SMALL_BASE		;; small_word_base
	vbroadcastsd zmm28, ZMM_SMALL_BASE_INVERSE	;; 1 / small_word_base
	vzero	zmm0					;; Start process with no carry
	vmovapd	zmm1, zmm0
	vmovapd	zmm2, zmm0
	vmovapd	zmm3, zmm0
	ENDM
znorm_smallmul_wpn_zpad_nottp MACRO
	zfmaddpd zmm8, zmm31, [rsi], zmm0	; 1-4	;; x = value * mulconst + carry
	zfmaddpd zmm9, zmm31, [rsi+r13], zmm1	; 1-4	;; x = value * mulconst + carry
	zfmaddpd zmm10, zmm31, [rsi+2*r13], zmm2; 2-5	;; x = value * mulconst + carry
	zfmaddpd zmm11, zmm31, [rsi+r14], zmm3	; 2-5	;; x = value * mulconst + carry
	vmulpd	zmm12, zmm31, [rsi+64]		; 3-6	;; x = value * mulconst
	vmulpd	zmm13, zmm31, [rsi+r13+64]	; 3-6	;; x = value * mulconst
	vmulpd	zmm14, zmm31, [rsi+2*r13+64]	; 4-7	;; x = value * mulconst
	vmulpd	zmm15, zmm31, [rsi+r14+64]	; 4-7	;; x = value * mulconst
	zfmaddpd zmm0, zmm8, zmm28, zmm30	; 5-8	;; next carry+RNDVAL = (x/base + RNDVAL) = rnd(x/base)+RNDVAL
	zfmaddpd zmm1, zmm9, zmm28, zmm30	; 5-8	;; next carry+RNDVAL = (x/base + RNDVAL) = rnd(x/base)+RNDVAL
	zfmaddpd zmm2, zmm10, zmm28, zmm30	; 6-9	;; next carry+RNDVAL = (x/base + RNDVAL) = rnd(x/base)+RNDVAL
	zfmaddpd zmm3, zmm11, zmm28, zmm30	; 6-9	;; next carry+RNDVAL = (x/base + RNDVAL) = rnd(x/base)+RNDVAL
	vsubpd	zmm0, zmm0, zmm30		; 9-12	;; next carry = (next carry+RNDVAL) - RNDVAL = rnd(x/base)
	vsubpd	zmm1, zmm1, zmm30		; 9-12	;; next carry = (next carry+RNDVAL) - RNDVAL = rnd(x/base)
	vsubpd	zmm2, zmm2, zmm30		; 10-13	;; next carry = (next carry+RNDVAL) - RNDVAL = rnd(x/base)
	vsubpd	zmm3, zmm3, zmm30		; 10-13	;; next carry = (next carry+RNDVAL) - RNDVAL = rnd(x/base)
	zfnmaddpd zmm8, zmm0, zmm29, zmm8	; 13-16	;; new value = x - rnd(x/base)*base
	zfnmaddpd zmm9, zmm1, zmm29, zmm9	; 13-16	;; new value = x - rnd(x/base)*base
	zfnmaddpd zmm10, zmm2, zmm29, zmm10	; 14-17	;; new value = x - rnd(x/base)*base
	zfnmaddpd zmm11, zmm3, zmm29, zmm11	; 14-17	;; new value = x - rnd(x/base)*base
	zstore	[rbx], zmm8			;; Save value1
	zstore	[rbx+64], zmm12			;; Save value5
	zstore	[rbx+r13], zmm9			;; Save value2
	zstore	[rbx+r13+64], zmm13		;; Save value6
	zstore	[rbx+2*r13], zmm10		;; Save value3
	zstore	[rbx+2*r13+64], zmm14		;; Save value7
	zstore	[rbx+r14], zmm11		;; Save value4
	zstore	[rbx+r14+64], zmm15		;; Save value8
	ENDM

znorm_smallmul_wpn_zpad_ttp_preload MACRO
	vbroadcastsd zmm30, ZMM_RNDVAL			;; Load the rounding value
	vbroadcastsd zmm29, ZMM_SMALL_BASE		;; small_word_base
	vbroadcastsd zmm28, ZMM_SMALL_BASE_INVERSE	;; 1 / small_word_base
	vbroadcastsd zmm27, ZMM_LARGE_BASE		;; large_word_base
	vbroadcastsd zmm26, ZMM_LARGE_BASE_INVERSE	;; 1 / large_word_base
	vzero	zmm0					;; Start process with no carry
	vmovapd	zmm1, zmm0
	vmovapd	zmm2, zmm0
	vmovapd	zmm3, zmm0
	ENDM
znorm_smallmul_wpn_zpad_ttp MACRO
	movzx	rdx, BYTE PTR [rdi]			;; Load index into compressed biglit table

;; UNOPTIMIZED??

	zfmaddpd zmm8, zmm31, [rsi], zmm0	; 1-4	;; x = value * mulconst + carry
	zfmaddpd zmm9, zmm31, [rsi+r13], zmm1	; 1-4	;; x = value * mulconst + carry
	zfmaddpd zmm10, zmm31, [rsi+2*r13], zmm2; 2-5	;; x = value * mulconst + carry
	zfmaddpd zmm11, zmm31, [rsi+r14], zmm3	; 2-5	;; x = value * mulconst + carry

	mov	edx, [r12+rdx*4]		;; Load 4 big vs. little flags

	vmulpd	zmm12, zmm31, [rsi+64]		; 3-6	;; x = value * mulconst
	vmulpd	zmm13, zmm31, [rsi+r13+64]	; 3-6	;; x = value * mulconst
	vmulpd	zmm14, zmm31, [rsi+2*r13+64]	; 4-7	;; x = value * mulconst
	vmulpd	zmm15, zmm31, [rsi+r14+64]	; 4-7	;; x = value * mulconst

	kmovw	k1, edx				; 5	;; Load 2 big vs. little flags (16-bit load - we use 8-bits)
	kshiftrw k2, k1, 8			; 6
	shr	edx, 16
	kmovw	k3, edx				; 7	;; Load 2 big vs. little flags (16-bit load - we use 8-bits)
	kshiftrw k4, k3, 8			; 10

	vblendmpd zmm0 {k1}, zmm28, zmm26	; 9	;; Create (1 / base) constant used in next carry calculation
	zfmaddpd zmm0, zmm8, zmm0, zmm30	; 9-12	;; y+RNDVAL = x / base + RNDVAL (next carry+RNDVAL)
	vblendmpd zmm1 {k2}, zmm28, zmm26	; 9	;; Create (1 / base) constant used in next carry calculation
	zfmaddpd zmm1, zmm9, zmm1, zmm30	; 9-12	;; y+RNDVAL = x / base + RNDVAL (next carry+RNDVAL)
	vblendmpd zmm2 {k3}, zmm28, zmm26	; 9	;; Create (1 / base) constant used in next carry calculation
	zfmaddpd zmm2, zmm10, zmm2, zmm30	; 10-13	;; y+RNDVAL = x / base + RNDVAL (next carry+RNDVAL)
	vblendmpd zmm3 {k4}, zmm28, zmm26	; 10	;; Create (1 / base) constant used in next carry calculation
	zfmaddpd zmm3, zmm11, zmm3, zmm30	; 10-13	;; y+RNDVAL = x / base + RNDVAL (next carry+RNDVAL)

	vsubpd	zmm0, zmm0, zmm30		; 13-16	;; y = rnd(x/base) = y - RNDVAL
	vsubpd	zmm1, zmm1, zmm30		; 13-16	;; y = rnd(x/base) = y - RNDVAL
	vsubpd	zmm2, zmm2, zmm30		; 14-17	;; y = rnd(x/base) = y - RNDVAL
	vsubpd	zmm3, zmm3, zmm30		; 14-17	;; y = rnd(x/base) = y - RNDVAL

	vblendmpd zmm24 {k1}, zmm29, zmm27	; 17	;; Create (base) constant used in new value calculation
	zfnmaddpd zmm8, zmm0, zmm24, zmm8	; 17-20	;; new value = x - y * base
	vblendmpd zmm24 {k2}, zmm29, zmm27	; 17	;; Create (base) constant used in new value calculation
	zfnmaddpd zmm9, zmm1, zmm24, zmm9	; 17-20	;; new value = x - y * base
	vblendmpd zmm24 {k3}, zmm29, zmm27	; 18	;; Create (base) constant used in new value calculation
	zfnmaddpd zmm10, zmm2, zmm24, zmm10	; 18-21	;; new value = x - y * base
	vblendmpd zmm24 {k4}, zmm29, zmm27	; 18	;; Create (base) constant used in new value calculation
	zfnmaddpd zmm11, zmm3, zmm24, zmm11	; 18-21	;; new value = x - y * base

	zstore	[rbx], zmm8				;; Save value1
	zstore	[rbx+64], zmm12				;; Save value5
	zstore	[rbx+r13], zmm9				;; Save value2
	zstore	[rbx+r13+64], zmm13			;; Save value6
	zstore	[rbx+2*r13], zmm10			;; Save value3
	zstore	[rbx+2*r13+64], zmm14			;; Save value7
	zstore	[rbx+r14], zmm11			;; Save value4
	zstore	[rbx+r14+64], zmm15			;; Save value8
	ENDM

;; Final step in the znorm_smallmul process.  Write carries to the carries array (to be processed later)

znorm_smallmul_wpn_zpad_save_carries MACRO
	mov	rbp, DATA_ADDR			; Address of carries array
	vaddpd	zmm0, zmm0, zmm30		; Store carries in +RNDVAL format (to share share code with post-FFT normalize)
	vaddpd	zmm1, zmm1, zmm30
	vaddpd	zmm2, zmm2, zmm30
	vaddpd	zmm3, zmm3, zmm30
	zstore	[rbp+0*128], zmm0		; Save carry out of low word
	zstore	[rbp+1*128], zmm1		; Save carry out of low word
	zstore	[rbp+2*128], zmm2		; Save carry out of low word
	zstore	[rbp+3*128], zmm3		; Save carry out of low word
	ENDM


; *************** WPN followup macros ******************

;; Carry rotation and propagation code for zero-padded FFTs.

;; How does the final low and high carry propagation work in a squaring/multiply?  This roughly explains the operations involved.
;; The low carry contains data that has not yet been multiplied by k and not yet been multiplied by the optional mul-by-const.
;; The high carry contains data that has already been multiplied by optional mul-by-const and by -c in preparation for wrapping
;; high half FFT data to the lower half.  Conveniently, this multiply by -c is just what we need to wrap the high carry from
;; above the FFT to the high half of the FFT.
;; The high half FFT data has been zeroed, but ZPAD0-6 and the carries contain data to produce the correct high half of the FFT:
;;
;; high half of FFT is:			low-carry * k * mul-by-const    +     high-carry
;; words above the FFT are:		ZPAD0-6 * mul-by-const
;;
;; To wrap words above FFT:		mul by -c and add to high half of FFT
;; To wrap high half of FFT:		divide by k  -> leave remainder in high half of FFT
;;							mul quotient by -c and add to lower half of FFT
;;
;; Note: We eliminate multiplying the low carry by k because it is later divided by k in the wrap high half of FFT step.

;; How does the final low and high carry propagation work in add/sub/addsub/smallmul?  This roughly explains the operations involved.
;; The high half of the FFT has 4 words of data.  The high carry is zero.  The low carry is a traditional carry that
;; is added into the 4 words of high FFT data.
;; To wrap high half of FFT:		divide by k  -> leave remainder in high half of FFT
;;							mul quotient by -c and add to lower half of FFT
;;


; Process four of the "rows" from the carries table for a squaring/multiply operation on a two-pass zero-padded FFT
; where a "row" is low word / high word pair.
; rbp = pointer to carries
; rsi = pointer to FFT data
; r13 = distance to second FFT section data
; r14 = distance to fourth FFT section data
; r12 = pointer to compressed biglit table
; rdx = register to load compressed biglit index into
; rdi = pointer to big/little flags
; rax, r8, r9, r10 = scratch

zadd_carry_rows_zpad_preload MACRO ttp
no ttp	zadd_carry_rows_zpad_nottp_preload
ttp	zadd_carry_rows_zpad_ttp_preload
	ENDM

zadd_carry_rows_zpad MACRO ttp
no ttp	zadd_carry_rows_zpad_nottp
ttp	zadd_carry_rows_zpad_ttp
	ENDM


zadd_carry_rows_zpad_nottp_preload MACRO
	LOCAL	noncon
	vbroadcastsd zmm30, ZMM_RNDVAL				;; RNDVAL
	vbroadcastsd zmm29, ZMM_SMALL_BASE			;; small_word_base
	vbroadcastsd zmm28, ZMM_SMALL_BASE_INVERSE		;; 1 / small_word_base
	vbroadcastsd zmm27, ZMM_RNDVAL_TIMES_SMALL_BASE		;; RNDVAL * small_word_base - RNDVAL
	vbroadcastsd zmm26, ZMM_RNDVAL_OVER_SMALL_BASE		;; RNDVAL / small_word_base - RNDVAL
	vbroadcastsd zmm25, ZMM_K_LO				;; k_lo
	vbroadcastsd zmm24, ZMM_K_HI_OVER_SMALL_BASE		;; k_hi / small_word_base
	cmp	const_fft, 0					;; Are we also multiplying by a constant?
	je	short noncon					;; Jump if not const
	vbroadcastsd zmm25, ZMM_K_TIMES_MULCONST_LO		;; k_lo * mulconst
	vbroadcastsd zmm24, ZMM_K_TIMES_MULCONST_HI_OVER_SMALL_BASE ;; k_hi * mulconst / small_word_base
noncon:	
	ENDM

zadd_carry_rows_zpad_nottp MACRO
	LOCAL	cloop, cloop_end, done

	vmovapd	zmm0, [rbp+0*128]		;; Load low carry word
	vmovapd	zmm1, [rbp+1*128]		;; Load low carry word
	vmovapd	zmm2, [rbp+2*128]		;; Load low carry word
	vmovapd	zmm3, [rbp+3*128]		;; Load low carry word
	vmovapd	zmm4, [rbp+0*128+64]		;; Load hi carry word
	vmovapd	zmm5, [rbp+1*128+64]		;; Load hi carry word
	vmovapd	zmm6, [rbp+2*128+64]		;; Load hi carry word
	vmovapd	zmm7, [rbp+3*128+64]		;; Load hi carry word

	mov	r8, rsi				;; Save pointers
	mov	al, 8				;; Spread carry over a max of 8 words

	;; Proper carry propagation requires splitting the lower carry that has not been mul'ed-by-k or mul-by-const
cloop:	dec	al					;; Decrement count of sets of 4 carries we've processed
	jz	cloop_end				;; Exit carry propagation loop
	vmovapd	zmm12, zmm0				;; Copy carry1+RNDVAL
	zfmsubpd zmm0, zmm0, zmm28, zmm26	; 1-4	;; carry1_hi+RNDVAL = (carry1+RNDVAL) / base - (RNDVAL/base-RNDVAL) = rnd(carry1/base)+RNDVAL (next carry1)
	vaddpd	zmm8, zmm4, [rsi]		; 1-4	;; x1+RNDVAL = carry5 + values1
	vmovapd	zmm13, zmm1				;; Copy carry2+RNDVAL
	zfmsubpd zmm1, zmm1, zmm28, zmm26	; 2-5	;; carry2_hi+RNDVAL = (carry2+RNDVAL) / base - (RNDVAL/base-RNDVAL) = rnd(carry2/base)+RNDVAL (next carry2)
	vaddpd	zmm9, zmm5, [rsi+r13]		; 2-5	;; x2+RNDVAL = carry6 + values2
	vmovapd	zmm14, zmm2				;; Copy carry3+RNDVAL
	zfmsubpd zmm2, zmm2, zmm28, zmm26	; 3-6	;; carry3_hi+RNDVAL = (carry3+RNDVAL) / base - (RNDVAL/base-RNDVAL) = rnd(carry3/base)+RNDVAL (next carry3)
	vaddpd	zmm10, zmm6, [rsi+2*r13]	; 3-6	;; x3+RNDVAL = carry7 + values3
	vmovapd	zmm15, zmm3				;; Copy carry4+RNDVAL
	zfmsubpd zmm3, zmm3, zmm28, zmm26	; 4-7	;; carry4_hi+RNDVAL = (carry4+RNDVAL) / base - (RNDVAL/base-RNDVAL) = rnd(carry4/base)+RNDVAL (next carry4)
	vaddpd	zmm11, zmm7, [rsi+r14]		; 4-7	;; x4+RNDVAL = carry8 + values4

	zfmsubpd zmm16, zmm0, zmm29, zmm27	; 5-8	;; tmp1+RNDVAL = (carry1_hi+RNDVAL) * base - (RNDVAL*base-RNDVAL) = rnd(carry1/base)*base+RNDVAL
	zfmsubpd zmm17, zmm1, zmm29, zmm27	; 6-9	;; tmp2+RNDVAL = (carry2_hi+RNDVAL) * base - (RNDVAL*base-RNDVAL) = rnd(carry2/base)*base+RNDVAL
	zfmsubpd zmm18, zmm2, zmm29, zmm27	; 7-10	;; tmp3+RNDVAL = (carry3_hi+RNDVAL) * base - (RNDVAL*base-RNDVAL) = rnd(carry3/base)*base+RNDVAL
	zfmsubpd zmm19, zmm3, zmm29, zmm27	; 8-11	;; tmp4+RNDVAL = (carry4_hi+RNDVAL) * base - (RNDVAL*base-RNDVAL) = rnd(carry4/base)*base+RNDVAL

	vsubpd	zmm12, zmm12, zmm16		; 9-12	;; carry1_lo = carry1+RNDVAL - tmp1+RNDVAL = carry1 - rnd(carry1/base)*base = carry1%base
	vsubpd	zmm13, zmm13, zmm17		; 10-13	;; carry2_lo = carry2+RNDVAL - tmp2+RNDVAL = carry2 - rnd(carry2/base)*base = carry2%base
	vsubpd	zmm14, zmm14, zmm18		; 11-14	;; carry3_lo = carry3+RNDVAL - tmp3+RNDVAL = carry3 - rnd(carry3/base)*base = carry3%base
	vsubpd	zmm15, zmm15, zmm19		; 12-15	;; carry4_lo = carry4+RNDVAL - tmp4+RNDVAL = carry4 - rnd(carry4/base)*base = carry4%base

	zfmaddpd zmm8, zmm12, zmm25, zmm8	; 13-16 ;; x1+RNDVAL += carry1_lo * k_lo*mulconst
	zfmaddpd zmm9, zmm13, zmm25, zmm9	; 14-17 ;; x2+RNDVAL += carry2_lo * k_lo*mulconst
	zfmaddpd zmm10, zmm14, zmm25, zmm10	; 15-18 ;; x3+RNDVAL += carry3_lo * k_lo*mulconst
	zfmaddpd zmm11, zmm15, zmm25, zmm11	; 16-19 ;; x4+RNDVAL += carry4_lo * k_lo*mulconst

	zfmsubpd zmm4, zmm8, zmm28, zmm26	; 17-20 ;; x1_hi+RNDVAL = (x1+RNDVAL) / base - (RNDVAL/base-RNDVAL) = rnd(x1/base)+RNDVAL (next carry5)
	zfmsubpd zmm5, zmm9, zmm28, zmm26	; 18-21 ;; x2_hi+RNDVAL = (x2+RNDVAL) / base - (RNDVAL/base-RNDVAL) = rnd(x2/base)+RNDVAL (next carry6)
	zfmsubpd zmm6, zmm10, zmm28, zmm26	; 19-22 ;; x3_hi+RNDVAL = (x3+RNDVAL) / base - (RNDVAL/base-RNDVAL) = rnd(x3/base)+RNDVAL (next carry7)
	zfmsubpd zmm7, zmm11, zmm28, zmm26	; 20-23 ;; x4_hi+RNDVAL = (x4+RNDVAL) / base - (RNDVAL/base-RNDVAL) = rnd(x4/base)+RNDVAL (next carry8)

	zfmsubpd zmm16, zmm4, zmm29, zmm27	; 21-24 ;; tmp5+RNDVAL = (x1_hi+RNDVAL) * base - (RNDVAL*base-RNDVAL) = rnd(x1/base)*base+RNDVAL
	zfmaddpd zmm4, zmm12, zmm24, zmm4	; 21-24 ;; next carry5 = carry1_lo * k_hi*mulconst/base + next carry5
	zfmsubpd zmm17, zmm5, zmm29, zmm27	; 22-25 ;; tmp6+RNDVAL = (x2_hi+RNDVAL) * base - (RNDVAL*base-RNDVAL) = rnd(x2/base)*base+RNDVAL
	zfmaddpd zmm5, zmm13, zmm24, zmm5	; 22-25 ;; next carry6 = carry2_lo * k_hi*mulconst/base + next carry6
	zfmsubpd zmm18, zmm6, zmm29, zmm27	; 23-26 ;; tmp7+RNDVAL = (x3_hi+RNDVAL) * base - (RNDVAL*base-RNDVAL) = rnd(x3/base)*base+RNDVAL
	zfmaddpd zmm6, zmm14, zmm24, zmm6	; 23-26 ;; next carry7 = carry3_lo * k_hi*mulconst/base + next carry7
	zfmsubpd zmm19, zmm7, zmm29, zmm27	; 24-27 ;; tmp8+RNDVAL = (x4_hi+RNDVAL) * base - (RNDVAL*base-RNDVAL) = rnd(x4/base)*base+RNDVAL
	zfmaddpd zmm7, zmm15, zmm24, zmm7	; 24-27 ;; next carry8 = carry4_lo * k_hi*mulconst/base + next carry8

	vsubpd	zmm8, zmm8, zmm16		; 25-28 ;; new val1 = x1_lo = x1+RNDVAL - tmp5+RNDVAL = x1 - rnd(x1/base)*base = x1%base
	vsubpd	zmm9, zmm9, zmm17		; 26-29 ;; new val2 = x2_lo = x2+RNDVAL - tmp6+RNDVAL = x2 - rnd(x2/base)*base = x2%base
	vsubpd	zmm10, zmm10, zmm18		; 27-30 ;; new val3 = x3_lo = x3+RNDVAL - tmp7+RNDVAL = x3 - rnd(x3/base)*base = x3%base
	vsubpd	zmm11, zmm11, zmm19		; 28-31 ;; new val4 = x4_lo = x4+RNDVAL - tmp8+RNDVAL = x4 - rnd(x4/base)*base = x4%base

	zstore	[rsi], zmm8			;; Save new val1
	zstore	[rsi+r13], zmm9			;; Save new val2
	zstore	[rsi+2*r13], zmm10		;; Save new val3
	zstore	[rsi+r14], zmm11		;; Save new val4

	vcmpneqpd k1, zmm0, zmm30		;; Are any non-zero carries remaining to propagate?
	vcmpneqpd k2, zmm1, zmm30
	vcmpneqpd k3, zmm2, zmm30
	vcmpneqpd k4, zmm3, zmm30
	korw	k5, k1, k2
	korw	k6, k3, k4
	vcmpneqpd k1, zmm4, zmm30		;; Are any non-zero carries remaining to propagate?
	vcmpneqpd k2, zmm5, zmm30
	vcmpneqpd k3, zmm6, zmm30
	vcmpneqpd k4, zmm7, zmm30
	korw	k1, k1, k2
	korw	k2, k3, k4
	korw	k3, k5, k6
	korw	k4, k1, k2
	kortestw k3, k4
	jz	done				;; If carries not found then we're done

	add	rsi, pass2blkdst		;; Next FFT data ptr
	jmp	cloop				;; Repeat carry propagation loop

	;; Do eighth and last iteration without propagating carries out
cloop_end:
	vsubpd	zmm4, zmm4, zmm30		;; Remove RNDVAL from traditional carry
	vsubpd	zmm5, zmm5, zmm30
	vsubpd	zmm6, zmm6, zmm30
	vsubpd	zmm7, zmm7, zmm30
	vaddpd	zmm4, zmm4, [rsi]		;; carry + val1
	vaddpd	zmm5, zmm5, [rsi+r13]		;; carry + val2
	vaddpd	zmm6, zmm6, [rsi+2*r13]		;; carry + val3
	vaddpd	zmm7, zmm7, [rsi+r14]		;; carry + val4
	zstore	[rsi], zmm4			;; Save val1
	zstore	[rsi+r13], zmm5			;; Save val2
	zstore	[rsi+2*r13], zmm6		;; Save val3
	zstore	[rsi+r14], zmm7			;; Save val4

done:	mov	rsi, r8				;; Restore pointers
	zstore	[rbp+0*128], zmm30		;; Reset carries array to RNDVAL
	zstore	[rbp+0*128+64], zmm30
	zstore	[rbp+1*128], zmm30
	zstore	[rbp+1*128+64], zmm30
	zstore	[rbp+2*128], zmm30
	zstore	[rbp+2*128+64], zmm30
	zstore	[rbp+3*128], zmm30
	zstore	[rbp+3*128+64], zmm30
	ENDM


zadd_carry_rows_zpad_ttp_preload MACRO
	LOCAL	noncon
	vbroadcastsd zmm30, ZMM_RNDVAL				;; Load the rounding value
	vbroadcastsd zmm29, ZMM_SMALL_BASE			;; small_word_base
	vbroadcastsd zmm28, ZMM_SMALL_BASE_INVERSE		;; 1 / small_word_base
	vbroadcastsd zmm27, ZMM_LARGE_BASE			;; large_word_base
	vbroadcastsd zmm26, ZMM_LARGE_BASE_INVERSE		;; 1 / large_word_base
	vbroadcastsd zmm25, ZMM_K_LO				;; k_lo
	vbroadcastsd zmm24, ZMM_K_HI_OVER_SMALL_BASE		;; k_hi / small_word_base
	vbroadcastsd zmm31, ZMM_K_HI_OVER_LARGE_BASE		;; k_hi / large_word_base
	cmp	const_fft, 0					;; Are we also multiplying by a constant?
	je	short noncon					;; Jump if not const
	vbroadcastsd zmm25, ZMM_K_TIMES_MULCONST_LO		;; k_lo * mulconst
	vbroadcastsd zmm24, ZMM_K_TIMES_MULCONST_HI_OVER_SMALL_BASE ;; k_hi * mulconst / small_word_base
	vbroadcastsd zmm31, ZMM_K_TIMES_MULCONST_HI_OVER_LARGE_BASE ;; k_hi * mulconst / large_word_base
noncon:
	ENDM

zadd_carry_rows_zpad_ttp MACRO
	LOCAL	cloop, cloop_end, done

	vmovapd	zmm0, [rbp+0*128]		;; Load low carry word
	vmovapd	zmm1, [rbp+1*128]		;; Load low carry word
	vmovapd	zmm2, [rbp+2*128]		;; Load low carry word
	vmovapd	zmm3, [rbp+3*128]		;; Load low carry word
	vmovapd	zmm4, [rbp+0*128+64]		;; Load hi carry word
	vmovapd	zmm5, [rbp+1*128+64]		;; Load hi carry word
	vmovapd	zmm6, [rbp+2*128+64]		;; Load hi carry word
	vmovapd	zmm7, [rbp+3*128+64]		;; Load hi carry word
	vsubpd	zmm0, zmm0, zmm30		;; Remove RNDVAL from carries
	vsubpd	zmm1, zmm1, zmm30
	vsubpd	zmm2, zmm2, zmm30
	vsubpd	zmm3, zmm3, zmm30
	vsubpd	zmm4, zmm4, zmm30
	vsubpd	zmm5, zmm5, zmm30
	vsubpd	zmm6, zmm6, zmm30
	vsubpd	zmm7, zmm7, zmm30

	push	rsi				;; Save pointers
	push	rdi
	push	r12
	mov	al, 8				;; Spread carry over a max of 8 words
	sub	r9, r9				;; Clear register used to alternate the increment of rdi

cloop:	dec	al				;; Decrement count of sets of 4 carries we've processed
	jz	cloop_end			;; Exit carry propagation loop
	movzx	rdx, BYTE PTR [rdi]			;; Load index into compressed biglit table
	mov	edx, [r12+rdx*4]			;; Load 4 big vs. little flags
	kmovw	k1, edx				; 1	;; Load 2 big vs. little flags (16-bit load - we use 8-bits)
	vblendmpd zmm16 {k1}, zmm28, zmm26	; 2	;; Create (1/base) constant
	kshiftrw k2, k1, 8			; 2
	vblendmpd zmm17 {k2}, zmm28, zmm26	; 3	;; Create (1/base) constant
	shr	edx, 16
	kmovw	k3, edx				; 3	;; Load 2 big vs. little flags (16-bit load - we use 8-bits)
	vblendmpd zmm18 {k3}, zmm28, zmm26	; 4	;; Create (1/base) constant
	kshiftrw k4, k3, 8			; 4
	vblendmpd zmm19 {k4}, zmm28, zmm26	; 5	;; Create (1/base) constant

	vmovapd	zmm12, zmm0				;; Copy input carry1
	zfmaddpd zmm0, zmm0, zmm16, zmm30	; 5-8	;; carry1_hi+RNDVAL = carry1 / base + RNDVAL
	vaddpd	zmm8, zmm4, [rsi]		; 6-9	;; x1 = carry5 + values1
	vmovapd	zmm13, zmm1				;; Copy input carry2
	zfmaddpd zmm1, zmm1, zmm17, zmm30	; 6-9	;; carry2_hi+RNDVAL = carry2 / base + RNDVAL
	vaddpd	zmm9, zmm5, [rsi+r13]		; 7-10	;; x2 = carry6 + values2
	vmovapd	zmm14, zmm2				;; Copy input carry3
	zfmaddpd zmm2, zmm2, zmm18, zmm30	; 7-10	;; carry3_hi+RNDVAL = carry3 / base + RNDVAL
	vaddpd	zmm10, zmm6, [rsi+2*r13]	; 8-11	;; x3 = carry7 + values3
	vmovapd	zmm15, zmm3				;; Copy input carry4
	zfmaddpd zmm3, zmm3, zmm19, zmm30	; 8-11	;; carry4_hi+RNDVAL = carry4 / base + RNDVAL
	vaddpd	zmm11, zmm7, [rsi+r14]		; 9-12	;; x4 = carry8 + values4

	vsubpd	zmm0, zmm0, zmm30		; 9-12	;; carry1_hi = carry1_hi+RNDVAL - RNDVAL (next carry1)
	vblendmpd zmm20 {k1}, zmm29, zmm27	; 10	;; Create (base) constant
	vsubpd	zmm1, zmm1, zmm30		; 10-13	;; carry2_hi = carry2_hi+RNDVAL - RNDVAL (next carry2)
	vblendmpd zmm21 {k2}, zmm29, zmm27	; 11	;; Create (base) constant
	vsubpd	zmm2, zmm2, zmm30		; 11-14	;; carry3_hi = carry3_hi+RNDVAL - RNDVAL (next carry3)
	vblendmpd zmm22 {k3}, zmm29, zmm27	; 12	;; Create (base) constant
	vsubpd	zmm3, zmm3, zmm30		; 12-15	;; carry4_hi = carry4_hi+RNDVAL - RNDVAL (next carry4)
	vblendmpd zmm23 {k4}, zmm29, zmm27	; 13	;; Create (base) constant

	zfnmaddpd zmm12, zmm0, zmm20, zmm12	; 13-16	;; carry1_lo = carry1 - carry1_hi * base = carry1 - rnd(carry1/base)*base = carry1%base
	zfnmaddpd zmm13, zmm1, zmm21, zmm13	; 14-17	;; carry2_lo = carry2 - carry2_hi * base = carry2 - rnd(carry2/base)*base = carry2%base
	zfnmaddpd zmm14, zmm2, zmm22, zmm14	; 15-18	;; carry3_lo = carry3 - carry3_hi * base = carry3 - rnd(carry3/base)*base = carry3%base
	zfnmaddpd zmm15, zmm3, zmm23, zmm15	; 16-19	;; carry4_lo = carry4 - carry4_hi * base = carry4 - rnd(carry4/base)*base = carry4%base

	zfmaddpd zmm8, zmm12, zmm25, zmm8	; 17-20 ;; x1 += carry1_lo * k_lo*mulconst
	zfmaddpd zmm9, zmm13, zmm25, zmm9	; 18-21 ;; x2 += carry2_lo * k_lo*mulconst
	zfmaddpd zmm10, zmm14, zmm25, zmm10	; 19-22 ;; x3 += carry3_lo * k_lo*mulconst
	zfmaddpd zmm11, zmm15, zmm25, zmm11	; 20-23 ;; x4 += carry4_lo * k_lo*mulconst

	zfmaddpd zmm4, zmm8, zmm16, zmm30	; 21-24 ;; x1_hi+RNDVAL = x1 / base + RNDVAL = rnd(x1/base)+RNDVAL
	vblendmpd zmm16 {k1}, zmm24, zmm31	; 21	;; Create (k_hi*mulconst/base) constant
	zfmaddpd zmm5, zmm9, zmm17, zmm30	; 22-25 ;; x2_hi+RNDVAL = x2 / base + RNDVAL = rnd(x2/base)+RNDVAL
	vblendmpd zmm17 {k2}, zmm24, zmm31	; 22	;; Create (k_hi*mulconst/base) constant
	zfmaddpd zmm6, zmm10, zmm18, zmm30	; 23-26 ;; x3_hi+RNDVAL = x3 / base + RNDVAL = rnd(x3/base)+RNDVAL
	vblendmpd zmm18 {k3}, zmm24, zmm31	; 23	;; Create (k_hi*mulconst/base) constant
	zfmaddpd zmm7, zmm11, zmm19, zmm30	; 24-27 ;; x4_hi+RNDVAL = x4 / base + RNDVAL = rnd(x4/base)+RNDVAL
	vblendmpd zmm19 {k4}, zmm24, zmm31	; 24	;; Create (k_hi*mulconst/base) constant

	vsubpd	zmm4, zmm4, zmm30		; 25-28 ;; x1_hi = x1_hi+RNDVAL - RNDVAL = rnd(x1/base) (next carry5)
	vsubpd	zmm5, zmm5, zmm30		; 26-29 ;; x2_hi = x2_hi+RNDVAL - RNDVAL = rnd(x2/base) (next carry6)
	vsubpd	zmm6, zmm6, zmm30		; 27-30 ;; x3_hi = x3_hi+RNDVAL - RNDVAL = rnd(x3/base) (next carry7)
	vsubpd	zmm7, zmm7, zmm30		; 28-31 ;; x4_hi = x4_hi+RNDVAL - RNDVAL = rnd(x4/base) (next carry8)

	zfnmaddpd zmm8, zmm4, zmm20, zmm8	; 29-32 ;; new val1 = x1_lo = x1 - x1_hi * base = x1 - rnd(x1/base)*base = x1%base
	zfmaddpd zmm4, zmm12, zmm16, zmm4	; 29-32 ;; next carry5 = carry1_lo * k_hi*mulconst/base + next carry5
	zfnmaddpd zmm9, zmm5, zmm21, zmm9	; 30-33 ;; new val2 = x2_lo = x2 - x2_hi * base = x2 - rnd(x2/base)*base = x2%base
	zfmaddpd zmm5, zmm13, zmm17, zmm5	; 30-33 ;; next carry6 = carry2_lo * k_hi*mulconst/base + next carry6
	zfnmaddpd zmm10, zmm6, zmm22, zmm10	; 31-34 ;; new val3 = x3_lo = x3 - x3_hi * base = x3 - rnd(x3/base)*base = x3%base
	zfmaddpd zmm6, zmm14, zmm18, zmm6	; 31-34 ;; next carry7 = carry3_lo * k_hi*mulconst/base + next carry7
	zfnmaddpd zmm11, zmm7, zmm23, zmm11	; 32-35 ;; new val4 = x4_lo = x4 - x4_hi * base = x4 - rnd(x4/base)*base = x4%base
	zfmaddpd zmm7, zmm15, zmm19, zmm7	; 32-35 ;; next carry8 = carry4_lo * k_hi*mulconst/base + next carry8

	zstore	[rsi], zmm8			;; Save new val1
	zstore	[rsi+r13], zmm9			;; Save new val2
	zstore	[rsi+2*r13], zmm10		;; Save new val3
	zstore	[rsi+r14], zmm11		;; Save new val4

	vzero	zmm8				;; Zero
	vcmpneqpd k1, zmm0, zmm8		;; Are any non-zero carries remaining to propagate?
	vcmpneqpd k2, zmm1, zmm8
	vcmpneqpd k3, zmm2, zmm8
	vcmpneqpd k4, zmm3, zmm8
	korw	k5, k1, k2
	korw	k6, k3, k4
	vcmpneqpd k1, zmm4, zmm8		;; Are any non-zero carries remaining to propagate?
	vcmpneqpd k2, zmm5, zmm8
	vcmpneqpd k3, zmm6, zmm8
	vcmpneqpd k4, zmm7, zmm8
	korw	k1, k1, k2
	korw	k2, k3, k4
	korw	k3, k5, k6
	korw	k4, k1, k2
	kortestw k3, k4
	jz	done				;; If carries not found then we're done

	add	rsi, pass2blkdst		;; Next FFT data ptr
	mov	rdx, r12			;; If r12 has been been bumped by 4, we'll need to unbump r12 and bump rdi
	shr	rdx, 3				;; Set carry if r12 is in +4 state
	adc	rdi, 0				;; Increment (or not) the biglit table pointer
	xor	r12, 4				;; Bump or unbump pointer into the compressed biglit table
	jmp	cloop				;; Repeat carry propagation loop

	;; Do eighth and last iteration without propagating carries out
cloop_end:
	vaddpd	zmm4, zmm4, [rsi]		;; carry + val1
	vaddpd	zmm5, zmm5, [rsi+r13]		;; carry + val2
	vaddpd	zmm6, zmm6, [rsi+2*r13]		;; carry + val3
	vaddpd	zmm7, zmm7, [rsi+r14]		;; carry + val4
	zstore	[rsi], zmm4			;; Save val1
	zstore	[rsi+r13], zmm5			;; Save val2
	zstore	[rsi+2*r13], zmm6		;; Save val3
	zstore	[rsi+r14], zmm7			;; Save val4

done:	pop	r12				;; Restore pointers
	pop	rdi
	pop	rsi

	zstore	[rbp+0*128], zmm30		;; Reset carries array to RNDVAL
	zstore	[rbp+0*128+64], zmm30
	zstore	[rbp+1*128], zmm30
	zstore	[rbp+1*128+64], zmm30
	zstore	[rbp+2*128], zmm30
	zstore	[rbp+2*128+64], zmm30
	zstore	[rbp+3*128], zmm30
	zstore	[rbp+3*128+64], zmm30
	ENDM


; Process four of the "rows" from the carries table for a add/sub/addsub/smallmul operation on a two-pass zero-padded FFT
; where a "row" is low word / high word pair.
; rbp = pointer to carries
; rsi = pointer to FFT data
; r13 = source #3
; r14 = distance between source #1 and source #2 (as well as source #3 and source #4)
; rdi = pointer to big/little flags
; rax, r8, r9, r10 = scratch

zadd_carry_rows_op_zpad_preload MACRO ttp
no ttp	zadd_carry_rows_op_zpad_nottp_preload
ttp	zadd_carry_rows_op_zpad_ttp_preload
	ENDM

zadd_carry_rows_op_zpad MACRO ttp
no ttp	zadd_carry_rows_op_zpad_nottp
ttp	zadd_carry_rows_op_zpad_ttp
	ENDM


zadd_carry_rows_op_zpad_nottp_preload MACRO
	vbroadcastsd zmm30, ZMM_RNDVAL				;; Load the rounding value
	vbroadcastsd zmm29, ZMM_SMALL_BASE			;; small_word_base
	vbroadcastsd zmm28, ZMM_SMALL_BASE_INVERSE		;; 1 / small_word_base
	vbroadcastsd zmm27, ZMM_RNDVAL_TIMES_SMALL_BASE		;; RNDVAL * small_word_base - RNDVAL
	vbroadcastsd zmm26, ZMM_RNDVAL_OVER_SMALL_BASE		;; RNDVAL / small_word_base - RNDVAL
	ENDM

zadd_carry_rows_op_zpad_nottp MACRO
	LOCAL	cloop, cloop_end, done

	vmovapd	zmm0, [rbp+0*128]		;; Load low carry word
	vmovapd	zmm1, [rbp+1*128]
	vmovapd	zmm2, [rbp+2*128]
	vmovapd	zmm3, [rbp+3*128]
;; BUG - assert high carry is zero???

	mov	r8, rsi				;; Save pointers
	mov	al, 8				;; Spread carry over a max of 8 words

cloop:	dec	al				;; Decrement count of sets of 4 carries we've processed
	jz	cloop_end			;; Exit carry propagation loop
;;; BUG -- pipeline code below

	vaddpd	zmm8, zmm0, [rsi]		; 1-4	;; x1+RNDVAL = carry + values1
	zfmsubpd zmm0, zmm8, zmm28, zmm26	; 5-8	;; x1_hi+RNDVAL = (x1+RNDVAL) / base - (RNDVAL/base-RNDVAL) = rnd(x1/base)+RNDVAL (next carry)
	zfmsubpd zmm16, zmm0, zmm29, zmm27	; 9-12	;; tmp+RNDVAL = (x1_hi+RNDVAL) * base - (RNDVAL*base-RNDVAL) = rnd(x1/base)*base+RNDVAL
	vsubpd	zmm8, zmm8, zmm16		; 13-16	;; new val1 = x1_lo = x1+RNDVAL - tmp+RNDVAL = x1 - rnd(x1/base)*base = x1%base

	vaddpd	zmm9, zmm1, [rsi+r13]		; 1-4	;; x1+RNDVAL = carry + values1
	zfmsubpd zmm1, zmm9, zmm28, zmm26	; 5-8	;; x1_hi+RNDVAL = (x1+RNDVAL) / base - (RNDVAL/base-RNDVAL) = rnd(x1/base)+RNDVAL (next carry)
	zfmsubpd zmm17, zmm1, zmm29, zmm27	; 9-12	;; tmp+RNDVAL = (x1_hi+RNDVAL) * base - (RNDVAL*base-RNDVAL) = rnd(x1/base)*base+RNDVAL
	vsubpd	zmm9, zmm9, zmm17		; 13-16	;; new val1 = x1_lo = x1+RNDVAL - tmp+RNDVAL = x1 - rnd(x1/base)*base = x1%base

	vaddpd	zmm10, zmm2, [rsi+2*r13]	; 1-4	;; x1+RNDVAL = carry + values1
	zfmsubpd zmm2, zmm10, zmm28, zmm26	; 5-8	;; x1_hi+RNDVAL = (x1+RNDVAL) / base - (RNDVAL/base-RNDVAL) = rnd(x1/base)+RNDVAL (next carry)
	zfmsubpd zmm18, zmm2, zmm29, zmm27	; 9-12	;; tmp+RNDVAL = (x1_hi+RNDVAL) * base - (RNDVAL*base-RNDVAL) = rnd(x1/base)*base+RNDVAL
	vsubpd	zmm10, zmm10, zmm18		; 13-16	;; new val1 = x1_lo = x1+RNDVAL - tmp+RNDVAL = x1 - rnd(x1/base)*base = x1%base

	vaddpd	zmm11, zmm3, [rsi+r14]		; 1-4	;; x1+RNDVAL = carry + values1
	zfmsubpd zmm3, zmm11, zmm28, zmm26	; 5-8	;; x1_hi+RNDVAL = (x1+RNDVAL) / base - (RNDVAL/base-RNDVAL) = rnd(x1/base)+RNDVAL (next carry)
	zfmsubpd zmm19, zmm3, zmm29, zmm27	; 9-12	;; tmp+RNDVAL = (x1_hi+RNDVAL) * base - (RNDVAL*base-RNDVAL) = rnd(x1/base)*base+RNDVAL
	vsubpd	zmm11, zmm11, zmm19		; 13-16	;; new val1 = x1_lo = x1+RNDVAL - tmp+RNDVAL = x1 - rnd(x1/base)*base = x1%base

	zstore	[rsi], zmm8			;; Save new value1
	zstore	[rsi+r13], zmm9
	zstore	[rsi+2*r13], zmm10
	zstore	[rsi+r14], zmm11

	vcmpneqpd k1, zmm0, zmm30		;; Are any non-zero carries remaining to propagate?
	vcmpneqpd k2, zmm1, zmm30
	vcmpneqpd k3, zmm2, zmm30
	vcmpneqpd k4, zmm3, zmm30
	korw	k5, k1, k2
	korw	k6, k3, k4
	kortestw k5, k6
	jz	done				;; If carries not found then we're done

	add	rsi, pass2blkdst		;; Next FFT data ptr
	jmp	cloop				;; Repeat carry propagation loop

	;; Do eighth and last iteration without propagating carries out
cloop_end:
	vsubpd	zmm0, zmm0, zmm30		;; Remove RNDVAL from carries
	vsubpd	zmm1, zmm1, zmm30
	vsubpd	zmm2, zmm2, zmm30
	vsubpd	zmm3, zmm3, zmm30
	vaddpd	zmm8, zmm0, [rsi]		;; Values1 += carry
	vaddpd	zmm9, zmm1, [rsi+r13]		;; Values2 += carry
	vaddpd	zmm10, zmm2, [rsi+2*r13]	;; Values3 += carry
	vaddpd	zmm11, zmm3, [rsi+r14]		;; Values4 += carry
	zstore	[rsi], zmm8			;; Save new value1
	zstore	[rsi+r13], zmm9
	zstore	[rsi+2*r13], zmm10
	zstore	[rsi+r14], zmm11

done:	mov	rsi, r8				;; Restore pointers
	zstore	[rbp+0*128], zmm30		;; Reset carries array to RNDVAL
	zstore	[rbp+1*128], zmm30
	zstore	[rbp+2*128], zmm30
	zstore	[rbp+3*128], zmm30
	ENDM


zadd_carry_rows_op_zpad_ttp_preload MACRO
	vbroadcastsd zmm30, ZMM_RNDVAL				;; Load the rounding value
	vbroadcastsd zmm29, ZMM_SMALL_BASE			;; small_word_base
	vbroadcastsd zmm28, ZMM_SMALL_BASE_INVERSE		;; 1 / small_word_base
	vbroadcastsd zmm27, ZMM_LARGE_BASE			;; large_word_base
	vbroadcastsd zmm26, ZMM_LARGE_BASE_INVERSE		;; 1 / large_word_base
	vzero	zmm31						;; Zero
	ENDM

zadd_carry_rows_op_zpad_ttp MACRO
	LOCAL	cloop, cloop_end, done

	vmovapd	zmm0, [rbp+0*128]		;; Load low carry word
	vmovapd	zmm1, [rbp+1*128]
	vmovapd	zmm2, [rbp+2*128]
	vmovapd	zmm3, [rbp+3*128]
	vsubpd	zmm0, zmm0, zmm30		;; Remove RNDVAL from carries
	vsubpd	zmm1, zmm1, zmm30
	vsubpd	zmm2, zmm2, zmm30
	vsubpd	zmm3, zmm3, zmm30
;; BUG - assert high carry is zero???

	push	rsi				;; Save pointers
	push	rdi
	push	r12
	mov	al, 8				;; Spread carry over a max of 8 words
	sub	r9, r9				;; Clear register used to alternate the increment of rdi

cloop:	dec	al				;; Decrement count of sets of 4 carries we've processed
	jz	cloop_end			;; Exit carry propagation loop

	movzx	rdx, BYTE PTR [rdi]		;; Load index into compressed biglit table
	mov	edx, [r12+rdx*4]		;; Load 4 big vs. little flags

	;; BUG -- pipeline below
	vaddpd	zmm8, zmm0, [rsi]		; 1-4	;; x1 = carry1 + values1
	kmovw	k1, edx				; 1	;; Load 2 big vs. little flags (16-bit load - we use 8-bits)
	vblendmpd zmm12 {k1}, zmm28, zmm26	; 1	;; Create (1/base) constant
	zfmaddpd zmm12, zmm8, zmm12, zmm30	; 5-8	;; x1_hi+RNDVAL = x1 / base + RNDVAL = rnd(x1/base)+RNDVAL
	vsubpd	zmm0, zmm12, zmm30		; 9-12	;; x1_hi = x1_hi+RNDVAL - RNDVAL = rnd(x1/base) (next carry1)
	vblendmpd zmm12 {k1}, zmm29, zmm27	; 9	;; Create (base) constant
	zfnmaddpd zmm8, zmm0, zmm12, zmm8	; 13-16 ;; new val1 = x1_lo = x1 - x1_hi * base = x1 - rnd(x1/base)*base = x1%base

	vaddpd	zmm9, zmm1, [rsi+r13]		; 1-4	;; x2 = carry2 + values2
	kshiftrw k2, k1, 8			; 1
	vblendmpd zmm13 {k2}, zmm28, zmm26	; 1	;; Create (1/base) constant
	zfmaddpd zmm13, zmm9, zmm13, zmm30	; 5-8	;; x2_hi+RNDVAL = x2 / base + RNDVAL = rnd(x2/base)+RNDVAL
	vsubpd	zmm1, zmm13, zmm30		; 9-12	;; x2_hi = x2_hi+RNDVAL - RNDVAL = rnd(x2/base) (next carry2)
	vblendmpd zmm13 {k2}, zmm29, zmm27	; 9	;; Create (base) constant
	zfnmaddpd zmm9, zmm1, zmm13, zmm9	; 13-16 ;; new val2 = x2_lo = x2 - x2_hi * base = x2 - rnd(x2/base)*base = x2%base

	vaddpd	zmm10, zmm2, [rsi+2*r13]	; 1-4	;; x3 = carry3 + values3
	shr	edx, 16
	kmovw	k3, edx				; 7	;; Load 2 big vs. little flags (16-bit load - we use 8-bits)
	vblendmpd zmm14 {k3}, zmm28, zmm26	; 8	;; Create (1/base) constant
	zfmaddpd zmm14, zmm10, zmm14, zmm30	; 5-8	;; x3_hi+RNDVAL = x3 / base + RNDVAL = rnd(x3/base)+RNDVAL
	vsubpd	zmm2, zmm14, zmm30		; 9-12	;; x3_hi = x3_hi+RNDVAL - RNDVAL = rnd(x3/base) (next carry3)
	vblendmpd zmm14 {k3}, zmm29, zmm27	; 9	;; Create (base) constant
	zfnmaddpd zmm10, zmm2, zmm14, zmm10	; 13-16 ;; new val3 = x3_lo = x3 - x3_hi * base = x3 - rnd(x3/base)*base = x3%base

	vaddpd	zmm11, zmm3, [rsi+r14]		; 1-4	;; x4 = carry4 + values4
	kshiftrw k4, k3, 8			; 1
	vblendmpd zmm15 {k4}, zmm28, zmm26	; 2	;; Create (1/base) constant
	zfmaddpd zmm15, zmm11, zmm15, zmm30	; 5-8	;; x4_hi+RNDVAL = x4 / base + RNDVAL = rnd(x4/base)+RNDVAL
	vsubpd	zmm3, zmm15, zmm30		; 9-12	;; x4_hi = x4_hi+RNDVAL - RNDVAL = rnd(x4/base) (next carry4)
	vblendmpd zmm15 {k4}, zmm29, zmm27	; 9	;; Create (base) constant
	zfnmaddpd zmm11, zmm3, zmm15, zmm11	; 13-16 ;; new val4 = x4_lo = x4 - x4_hi * base = x4 - rnd(x4/base)*base = x4%base

	zstore	[rsi], zmm8			;; Save new value1
	zstore	[rsi+r13], zmm9
	zstore	[rsi+2*r13], zmm10
	zstore	[rsi+r14], zmm11

	vcmpneqpd k1, zmm0, zmm31		;; Are any non-zero carries remaining to propagate?
	vcmpneqpd k2, zmm1, zmm31
	vcmpneqpd k3, zmm2, zmm31
	vcmpneqpd k4, zmm3, zmm31
	korw	k5, k1, k2
	korw	k6, k3, k4
	kortestw k5, k6
	jz	done				;; If carries not found then we're done

	add	rsi, pass2blkdst		;; Next FFT data ptr
	xor	r12, 4				;; Bump or unbump pointer into the compressed biglit table
	add	rdi, r9				;; Increment (or not) the biglit table pointer
	xor	r9, 1				;; Alternate the increment for rdi
	jmp	cloop				;; Repeat carry propagation loop

	;; Do eighth and last iteration without propagating carries out
cloop_end:
	vaddpd	zmm8, zmm0, [rsi]		;; Values1 += carry
	vaddpd	zmm9, zmm1, [rsi+r13]		;; Values2 += carry
	vaddpd	zmm10, zmm2, [rsi+2*r13]	;; Values3 += carry
	vaddpd	zmm11, zmm3, [rsi+r14]		;; Values4 += carry
	zstore	[rsi], zmm8			;; Save new value1
	zstore	[rsi+r13], zmm9
	zstore	[rsi+2*r13], zmm10
	zstore	[rsi+r14], zmm11

done:	pop	r12				;; Restore pointers
	pop	rdi
	pop	rsi

	zstore	[rbp+0*128], zmm30		;; Reset carries array to RNDVAL
	zstore	[rbp+1*128], zmm30
	zstore	[rbp+2*128], zmm30
	zstore	[rbp+3*128], zmm30
	ENDM


;; This is a zero-pad version of the macro that handles the last carry rotated out of the low half of the FFT
;; and the last carry rotated out of the high half of the FFT in a squaring/multiply operation.
;; xmm0 = last low half carry
;; xmm1 = last high half carry
;; zmm26-30 = standard preloaded rounding constants
;; The low carry is multiplied by -c*mulconst and added to the low FFT data.  The result may not fit in a double, so the result is split in two.
;; The high carry is moved to the first word above the half way point of the FFT.

zprocess_last_two_carries_zpad MACRO ttp
	LOCAL	noncon

	mov	rbp, carries			;; Reload carries array pointer
	vmovsd	Q [rbp], xmm30			;; Clear the first low carry
	vmovsd	Q [rbp+64], xmm30		;; Clear the first high carry

	;; Save the high carry, zpad_prep_final_div_by_k_after_multiply will add this to ZPAD0 * -c * mulbyconst
	mov	rsi, DESTARG			;; FFT data pointer
	vmovsd	[rsi+64], xmm1			;; Save

	split_first_zpad7_word ttp, xmm0, xmm1	;; Split the low carry
	vsubsd	xmm1, xmm1, xmm30		;; Remove RNDVAL	
	;; Multiply the low carry by the mulbyconst.  Save it for later adding to first FFT words in zpad_final_div_by_k where it will get multiplied by -c.
	cmp	const_fft, 0			;; Are we also multiplying by a constant?
	je	short noncon			;; Jump if not const
	vmovsd	xmm2, ZMM_MULCONST
	vmulsd	xmm0, xmm0, xmm2		;; Mul by optional small mul const
	vmulsd	xmm1, xmm1, xmm2		;; Mul by optional small mul const
noncon:	vmovsd	Q ZMM_TMP1, xmm0		;; Save carry for later
	vmovsd	Q ZMM_TMP2, xmm1		;; Save carry for later
	ENDM

;; This is a zero-pad version of the macro that handles the last carry rotated out of the low half of the FFT
;; in an add/sub/addsub/smallmul operation.  There are no carries generated processing the high half of the FFT.
;; xmm0 = last low half carry
;; zmm26-30 = standard preloaded rounding constants

zprocess_last_two_carries_op_zpad MACRO ttp
	mov	rbp, carries			;; Reload carries array pointer
	vmovsd	[rbp], xmm30			;; Clear the first low carry
	vmovsd	[rbp+64], xmm30			;; Clear the first high carry

	;; Add the low carry to the first high FFT word, zpad_prep_final_div_by_k_after_op will do the rest
	mov	rsi, DESTARG			;; FFT data pointer
	vsubsd	xmm0, xmm0, xmm30		;; Remove RNDVAL
	vaddsd	xmm0, xmm0, Q [rsi+64]		;; Add in the FFT data
	vmovsd	[rsi+64], xmm0			;; Save

	; Clear the carry that is added to lowest FFT data in zpad_final_div_by_k.
	vzero	xmm0
	vmovsd	Q ZMM_TMP1, xmm0
	vmovsd	Q ZMM_TMP2, xmm0
	ENDM

;;
;; This macro prepares for a call to zpad_final_div_by_k after a squaring/multiply operation
;;

zpad_prep_final_div_by_k_after_multiply MACRO ttp
	LOCAL	noncon

	;; Multiply ZPAD0 through ZPAD6 by -c * optional small mul const.  This, in essense, wraps this data from above the FFT data area to the halfway point.
	;; Later on we'll divide this by K to decide which data needs wrapping all the way down to the bottom of the FFT data.

	;; NOTE: ZPAD6 will not be bigger than a big word.  We must be careful to handle c's up to about 30 bits

	mov	rsi, DESTARG			;; Load FFT data pointer

	vmovsd	xmm19, ZMM_MINUS_C		;; -c
	vmovsd	xmm1, ADDIN_VALUE		;; Load the add in value
	cmp	const_fft, 0			;; Are we multiplying result by a constant?
	je	short noncon			;; Jump if not const
	vmovsd	xmm19, ZMM_MINUS_C_TIMES_MULCONST ;; -c * small mul const
	vmulsd	xmm1, xmm1, ZMM_MULCONST	;; Multiply the add in value by the small mul const
noncon:	vaddsd	xmm1, xmm1, POSTADDIN_VALUE	;; Add the post-mul-by-const addin value

	;; when c = -1, 1 = b^n
	;; when c = 1, -1 = b^n, 1 = -b^n
	;; when c = -3, 3 = b^n, 1 = b^n - 2
	;; when c = 3, -3 = b^n, 3 = -b^n, 1 = -b^n - 2
	;; The "- 2" has been precomputed in ZPAD_LSW_ADJUST.  Add ADDIN_VALUE * mul-by-const * ZPAD_LSW_ADJUST into the least significant FFT word
	vmovsd	xmm0, [rsi]
	zfmaddsd xmm0, xmm1, ZPAD_LSW_ADJUST, xmm0
	vmovsd	[rsi], xmm0
	vaddsd	xmm1, xmm1, [rsi+64]		;; We will apply rest of ADDIN_VALUE * mul-by-const at the half way point (saved high carry)

ttp	kmovw	k1, ZMM_FIRST_BIGLIT_VALUES	;; Load first 8 big vs. little flags
	vaddsd	xmm3, xmm30, ZPAD0		;; Load value
	split_first_zpad7_word ttp, xmm3, xmm0
	zfmaddsd xmm3, xmm3, xmm19, xmm1 	;; Mul by -c * optional small mul const, add saved high carry and ADDIN_VALUE * mul-by-const
	split_first_zpad7_word ttp, xmm3, xmm1

ttp	kshiftrw k1, k1, 1			;; Next big vs. little flag
	vaddsd	xmm4, xmm0, ZPAD1		;; Load value + carry
	split_zpad7_word ttp, xmm4, xmm0
	zfmaddsd xmm4, xmm4, xmm19, xmm1	;; Mul by -c * optional small mul const, add carry
	split_zpad7_again ttp, xmm4, xmm1

ttp	kshiftrw k1, k1, 1			;; Next big vs. little flag
	vaddsd	xmm5, xmm0, ZPAD2		;; Load value + carry
	split_zpad7_word ttp, xmm5, xmm0
	zfmaddsd xmm5, xmm5, xmm19, xmm1	;; Mul by -c * optional small mul const, add carry
	split_zpad7_again ttp, xmm5, xmm1

ttp	kshiftrw k1, k1, 1			;; Next big vs. little flag
	vaddsd	xmm6, xmm0, ZPAD3		;; Load value + carry
	split_zpad7_word ttp, xmm6, xmm0
	zfmaddsd xmm6, xmm6, xmm19, xmm1	;; Mul by -c * optional small mul const, add carry
	split_zpad7_again ttp, xmm6, xmm1

ttp	kshiftrw k1, k1, 1			;; Next big vs. little flag
	vaddsd	xmm7, xmm0, ZPAD4		;; Load value + carry
	split_zpad7_word ttp, xmm7, xmm0
	zfmaddsd xmm7, xmm7, xmm19, xmm1	;; Mul by -c * optional small mul const, add carry
	split_zpad7_again ttp, xmm7, xmm1

ttp	kshiftrw k1, k1, 1			;; Next big vs. little flag
	vaddsd	xmm8, xmm0, ZPAD5		;; Load value + carry
	split_zpad7_word ttp, xmm8, xmm0
	zfmaddsd xmm8, xmm8, xmm19, xmm1	;; Mul by -c * optional small mul const, add carry
	split_zpad7_again ttp, xmm8, xmm1

	vaddsd	xmm9, xmm0, ZPAD6		;; Load value + carry
	vsubsd	xmm9, xmm9, xmm30
	zfmaddsd xmm9, xmm9, xmm19, xmm1	;; Mul by -c * optional small mul const, add carry
	vsubsd	xmm9, xmm9, xmm30
	ENDM

;;
;; This macro prepares for a call to zpad_final_div_by_k after an add/sub/addsub/smallmul operation.
;; The data in the upper half of the FFT is "integerized" and copied to registers for the divide by k.
;; This macro assumes the smallmul code did not propagate any upper half data past the fourth word.
;;

zpad_prep_final_div_by_k_after_op MACRO ttp

	mov	rsi, DESTARG			;; Load FFT data pointer

	;; Copy and integerize data from above halfway point to xmm3-xmm9

ttp	kmovw	k1, ZMM_FIRST_BIGLIT_VALUES	;; Load first 8 big vs. little flags
	vaddsd	xmm3, xmm30, Q [rsi+64]		;; value1 + RNDVAL
	split_first_zpad7_word ttp, xmm3, xmm0

	add	rsi, ZMM_SRC_INCR1		;; Next source pointer
ttp	kshiftrw k1, k1, 1			;; Next big vs. little flag
	vaddsd	xmm4, xmm0, [rsi+64]		;; Load value + carry
	split_zpad7_word ttp, xmm4, xmm0

	add	rsi, ZMM_SRC_INCR2		;; Next source pointer
ttp	kshiftrw k1, k1, 1			;; Next big vs. little flag
	vaddsd	xmm5, xmm0, [rsi+64]		;; Load value + carry
	split_zpad7_word ttp, xmm5, xmm0

	add	rsi, ZMM_SRC_INCR3		;; Next source pointer
ttp	kshiftrw k1, k1, 1			;; Next big vs. little flag
	vaddsd	xmm6, xmm0, [rsi+64]		;; Load value + carry
	split_zpad7_word ttp, xmm6, xmm7

ttp	kshiftrw k1, k1, 1			;; Next big vs. little flag
	split_zpad7_word ttp, xmm7, xmm8

ttp	kshiftrw k1, k1, 1			;; Next big vs. little flag
	split_zpad7_word ttp, xmm8, xmm9

	vsubsd	xmm9, xmm9, xmm30		;; Remove RNDVAL
	ENDM


; This is final step in a zero-padded FFT operation.  This macro divides the upper half of a zpad result by k
; storing the remainder in the upper half and multiplying the quotient by -c and adding it to the lower half.

; On input:
; xmm3,xmm4,xmm5,xmm6,xmm7,xmm8,xmm9 are the 7 values representing the unnormalized upper half result of a zero-padded FFT operation
; rsi, rdi, rbp destroyed

zpad_final_div_by_k MACRO ttp
	LOCAL	smallk, mediumk, div_k_done

	;; Divide the upper zpad data above b^n by k.  Store the integer part in xmm6-xmm11
	;; and the remainder in xmm3.  Later we will wrap the integer part
	;; down to the bottom of the FFT data area (multiply by -c).
	;; And we will store the remainder in the upper half of the FFT data area.

	;; Note there are three cases to handle.  K is smaller than a big word.
	;; K is between one and 2 big words in size.  And K is more than 2 big words in size.

	cmp	ZPAD_TYPE, 2			;; Are we dealing with case 1,2,or 3
	jl	smallk				;; One word case
	je	mediumk				;; Two word case

	;; This case does the divide by k where k is three words

	vzero	zmm11				;; Zero quotient word that other cases set

	vmulsd	xmm10, xmm9, ZPAD_INVERSE_K6	;; Mul by shifted 1/k
	vroundsd xmm10, xmm10, xmm10, 0		;; Round to integer
	zfnmaddsd xmm9, xmm10, ZPAD_K6_HI, xmm9 ;; Calculate high bits of remainder
	zfnmaddsd xmm8, xmm10, ZPAD_K6_MID, xmm8 ;; Calculate middle bits of remainder
	zfnmaddsd xmm7, xmm10, ZPAD_K6_LO, xmm7	;; Calculate low bits of remainder

	zfmaddsd xmm8, xmm9, ZPAD_SHIFT6, xmm8	;; Combine remainders (new high bits)
	zfmaddsd xmm0, xmm8, ZPAD_SHIFT5, xmm7	;; Combine high and medium bits
	vmulsd	xmm9, xmm0, ZPAD_INVERSE_K5	;; Mul by shifted 1/k
	vroundsd xmm9, xmm9, xmm9, 0		;; Round to integer
	zfnmaddsd xmm8, xmm9, ZPAD_K5_HI, xmm8	;; Calculate high bits of remainder
	zfnmaddsd xmm7, xmm9, ZPAD_K5_MID, xmm7 ;; Calculate middle bits of remainder
	zfnmaddsd xmm6, xmm9, ZPAD_K5_LO, xmm6	;; Calculate low bits of remainder

	zfmaddsd xmm7, xmm8, ZPAD_SHIFT5, xmm7	;; Combine remainders (new high bits)
	zfmaddsd xmm0, xmm7, ZPAD_SHIFT4, xmm6	;; Combine high and medium bits
	vmulsd	xmm8, xmm0, ZPAD_INVERSE_K4	;; Mul by shifted 1/k
	vroundsd xmm8, xmm8, xmm8, 0		;; Round to integer
	zfnmaddsd xmm7, xmm8, ZPAD_K4_HI, xmm7	;; Calculate high bits of remainder
	zfnmaddsd xmm6, xmm8, ZPAD_K4_MID, xmm6 ;; Calculate middle bits of remainder
	zfnmaddsd xmm5, xmm8, ZPAD_K4_LO, xmm5	;; Calculate low bits of remainder

	zfmaddsd xmm6, xmm7, ZPAD_SHIFT4, xmm6	;; Combine remainders (new high bits)
	zfmaddsd xmm0, xmm6, ZPAD_SHIFT3, xmm5	;; Combine high and medium bits
	vmulsd	xmm7, xmm0, ZPAD_INVERSE_K3	;; Mul by shifted 1/k
	vroundsd xmm7, xmm7, xmm7, 0		;; Round to integer
	zfnmaddsd xmm6, xmm7, ZPAD_K3_HI, xmm6	;; Calculate high bits of remainder
	zfnmaddsd xmm5, xmm7, ZPAD_K3_MID, xmm5 ;; Calculate middle bits of remainder
	zfnmaddsd xmm4, xmm7, ZPAD_K3_LO, xmm4	;; Calculate low bits of remainder

	zfmaddsd xmm5, xmm6, ZPAD_SHIFT3, xmm5	;; Combine remainders (new high bits)
	zfmaddsd xmm0, xmm5, ZPAD_SHIFT2, xmm4	;; Combine high and medium bits
	vmulsd	xmm6, xmm0, ZPAD_INVERSE_K2	;; Mul by shifted 1/k
	vroundsd xmm6, xmm6, xmm6, 0		;; Round to integer
	zfnmaddsd xmm5, xmm6, ZPAD_K2_HI, xmm5	;; Calculate high bits of remainder
	zfnmaddsd xmm4, xmm6, ZPAD_K2_MID, xmm4 ;; Calculate middle bits of remainder
	zfnmaddsd xmm3, xmm6, ZPAD_K2_LO, xmm3	;; Calculate low bits of remainder

	zfmaddsd xmm4, xmm5, ZPAD_SHIFT2, xmm4	;; Combine remainders
	zfmaddsd xmm3, xmm4, ZPAD_SHIFT1, xmm3	;; Combine remainders

	jmp	div_k_done

	;; This case does the divide by k where k is two words
mediumk:
	vmulsd	xmm11, xmm9, ZPAD_INVERSE_K6	;; Mul by shifted 1/k
	vroundsd xmm11, xmm11, xmm11, 0		;; Round to integer
	zfnmaddsd xmm9, xmm11, ZPAD_K6_HI, xmm9 ;; Calculate high bits of remainder
	zfnmaddsd xmm8, xmm11, ZPAD_K6_LO, xmm8 ;; Calculate low bits of remainder

	zfmaddsd xmm8, xmm9, ZPAD_SHIFT6, xmm8	;; Combine remainders
	vmulsd	xmm10, xmm8, ZPAD_INVERSE_K5	;; Mul by shifted 1/k
	vroundsd xmm10, xmm10, xmm10, 0		;; Round to integer
	zfnmaddsd xmm8, xmm10, ZPAD_K5_HI, xmm8 ;; Calculate high bits of remainder
	zfnmaddsd xmm7, xmm10, ZPAD_K5_LO, xmm7 ;; Calculate low bits of remainder

	zfmaddsd xmm7, xmm8, ZPAD_SHIFT5, xmm7	;; Combine remainders
	vmulsd	xmm9, xmm7, ZPAD_INVERSE_K4	;; Mul by shifted 1/k
	vroundsd xmm9, xmm9, xmm9, 0		;; Round to integer
	zfnmaddsd xmm7, xmm9, ZPAD_K4_HI, xmm7	;; Calculate high bits of remainder
	zfnmaddsd xmm6, xmm9, ZPAD_K4_LO, xmm6	;; Calculate low bits of remainder

	zfmaddsd xmm6, xmm7, ZPAD_SHIFT4, xmm6	;; Combine remainders
	vmulsd	xmm8, xmm6, ZPAD_INVERSE_K3	;; Mul by shifted 1/k
	vroundsd xmm8, xmm8, xmm8, 0		;; Round to integer
	zfnmaddsd xmm6, xmm8, ZPAD_K3_HI, xmm6	;; Calculate high bits of remainder
	zfnmaddsd xmm5, xmm8, ZPAD_K3_LO, xmm5	;; Calculate low bits of remainder

	zfmaddsd xmm5, xmm6, ZPAD_SHIFT3, xmm5	;; Combine remainders
	vmulsd	xmm7, xmm5, ZPAD_INVERSE_K2	;; Mul by shifted 1/k
	vroundsd xmm7, xmm7, xmm7, 0		;; Round to integer
	zfnmaddsd xmm5, xmm7, ZPAD_K2_HI, xmm5	;; Calculate high bits of remainder
	zfnmaddsd xmm4, xmm7, ZPAD_K2_LO, xmm4	;; Calculate low bits of remainder

	zfmaddsd xmm4, xmm5, ZPAD_SHIFT2, xmm4	;; Combine remainders
	vmulsd	xmm6, xmm4, ZPAD_INVERSE_K1	;; Mul by shifted 1/k
	vroundsd xmm6, xmm6, xmm6, 0		;; Round to integer
	zfnmaddsd xmm4, xmm6, ZPAD_K1_HI, xmm4	;; Calculate high bits of remainder
	zfnmaddsd xmm3, xmm6, ZPAD_K1_LO, xmm3	;; Calculate low bits of remainder

	zfmaddsd xmm3, xmm4, ZPAD_SHIFT1, xmm3	;; Combine remainders

	jmp	div_k_done

	;; This case does the divide by k where k is one word.  Assume xmm8 and xmm9 are zero.

smallk:	vzero	zmm11				;; Zero quotient word that other cases set

	vmulsd	xmm10, xmm7, ZPAD_INVERSE_K1	;; Mul data by 1/k
	vroundsd xmm10, xmm10, xmm10, 0		;; Round to integer
	zfnmaddsd xmm7, xmm10, ZPAD_K1_LO, xmm7	;; Compute remainder

	zfmaddsd xmm6, xmm7, ZPAD_SHIFT4, xmm6	;; Shift remainder, add in next word to divide by k
	vmulsd	xmm9, xmm6, ZPAD_INVERSE_K1	;; Mul data by 1/k
	vroundsd xmm9, xmm9, xmm9, 0		;; Round to integer
	zfnmaddsd xmm6, xmm9, ZPAD_K1_LO, xmm6	;; Compute remainder

	zfmaddsd xmm5, xmm6, ZPAD_SHIFT3, xmm5	;; Shift remainder, add in next word to divide by k
	vmulsd	xmm8, xmm5, ZPAD_INVERSE_K1	;; Mul data by 1/k
	vroundsd xmm8, xmm8, xmm8, 0		;; Round to integer
	zfnmaddsd xmm5, xmm8, ZPAD_K1_LO, xmm5	;; Compute remainder

	zfmaddsd xmm4, xmm5, ZPAD_SHIFT2, xmm4	;; Shift remainder, add in next word to divide by k
	vmulsd	xmm7, xmm4, ZPAD_INVERSE_K1	;; Mul data by 1/k
	vroundsd xmm7, xmm7, xmm7, 0		;; Round to integer
	zfnmaddsd xmm4, xmm7, ZPAD_K1_LO, xmm4	;; Compute remainder

	zfmaddsd xmm3, xmm4, ZPAD_SHIFT1, xmm3	;; Shift remainder, add in next word to divide by k
	vmulsd	xmm6, xmm3, ZPAD_INVERSE_K1	;; Mul data by 1/k
	vroundsd xmm6, xmm6, xmm6, 0		;; Round to integer
	zfnmaddsd xmm3, xmm6, ZPAD_K1_LO, xmm3	;; Compute remainder

div_k_done:

	;; At this point remainder is in xmm3, quotient is in xmm6,xmm7,xmm8,xmm9,xmm10,xmm11

	;; Now normalize the data above the halfway point.

	mov	rsi, DESTARG			;; Address of squared number

ttp	kmovw	k1, ZMM_FIRST_BIGLIT_VALUES	;; Load first 8 big vs. little flags
	vaddsd	xmm0, xmm3, xmm30		;; Add RNDVAL to remainder of divide by k (which was left in xmm3)
	split_first_zpad7_word ttp, xmm0, xmm1
	vmovsd	Q [rsi+64], xmm0		;; Save value1

	add	rsi, ZMM_SRC_INCR1		;; Next source pointer
ttp	kshiftrw k1, k1, 1			;; Next big vs. little flag
	split_zpad7_word ttp, xmm1, xmm0
	vmovsd	Q [rsi+64], xmm1		;; Save value2

	add	rsi, ZMM_SRC_INCR2		;; Next source pointer
ttp	kshiftrw k1, k1, 1			;; Next big vs. little flag
	split_zpad7_word ttp, xmm0, xmm1
	vmovsd	Q [rsi+64], xmm0		;; Save value3

	add	rsi, ZMM_SRC_INCR3		;; Next source pointer
	vsubsd	xmm1, xmm1, xmm30		;; Remove RNDVAL
	vmovsd	Q [rsi+64], xmm1		;; Save new value4

	;; Mul the quotient by -c in preparation for adding it into the lower FFT data area.
	;; We multiply the wrapped_low_carry*mulconst generated in zprocess_last_two_carries by -c and add to lower FFT data.
	;; Then normalize the bottom FFT data.  We must go 6 words deep in case k is 48-50 bits and c is 32 bits.

	mov	rsi, DESTARG			;; Address of squared number
	vmovsd	xmm19, ZMM_MINUS_C		;; -c

ttp	kmovw	k1, ZMM_FIRST_BIGLIT_VALUES	;; Load first 8 big vs. little flags
	vaddsd	xmm6, xmm6, ZMM_TMP1		;; Add carry from zprocess_last_two_carries which wasn't yet mul'ed by -c
	zfmaddsd xmm6, xmm6, xmm19, Q [rsi]	;; Mul integer part of divide by k by -c, add in the FFT data
	vaddsd	xmm6, xmm6, xmm30		;; Add in RNDVAL
	split_first_zpad7_word ttp, xmm6, xmm1
	vmovsd	Q [rsi], xmm6			;; Save value1

	add	rsi, ZMM_SRC_INCR1		;; Next source pointer
ttp	kshiftrw k1, k1, 1			;; Next big vs. little flag
	vaddsd	xmm7, xmm7, ZMM_TMP2		;; Add carry from zprocess_last_two_carries which wasn't yet mul'ed by -c
	zfmaddsd xmm7, xmm7, xmm19, xmm1	;; Mul integer part of divide by k by -c, add carry
	vaddsd	xmm7, xmm7, Q [rsi]		;; Add in the FFT data
	split_zpad7_word ttp, xmm7, xmm1
	vmovsd	Q [rsi], xmm7			;; Save value2
 
	add	rsi, ZMM_SRC_INCR2		;; Next source pointer
ttp	kshiftrw k1, k1, 1			;; Next big vs. little flag
	zfmaddsd xmm8, xmm8, xmm19, xmm1	;; Mul integer part of divide by k by -c, add carry
	vaddsd	xmm8, xmm8, Q [rsi]		;; Add in the FFT data
	split_zpad7_word ttp, xmm8, xmm1
	vmovsd	Q [rsi], xmm8			;; Save value3

	add	rsi, ZMM_SRC_INCR3		;; Next source pointer
ttp	kshiftrw k1, k1, 1			;; Next big vs. little flag
	zfmaddsd xmm9, xmm9, xmm19, xmm1	;; Mul integer part of divide by k by -c, add carry
	vaddsd	xmm9, xmm9, Q [rsi]		;; Add in the FFT data
	split_zpad7_word ttp, xmm9, xmm1
	vmovsd	Q [rsi], xmm9			;; Save value4

	add	rsi, ZMM_SRC_INCR4		;; Next source pointer
ttp	kshiftrw k1, k1, 1			;; Next big vs. little flag
	zfmaddsd xmm10, xmm10, xmm19, xmm1	;; Mul integer part of divide by k by -c, add carry
	vaddsd	xmm10, xmm10, Q [rsi]		;; Add in the FFT data
	split_zpad7_word ttp, xmm10, xmm1
	vmovsd	Q [rsi], xmm10			;; Save value5

	add	rsi, ZMM_SRC_INCR5		;; Next source pointer
ttp	kshiftrw k1, k1, 1			;; Next big vs. little flag
	zfmaddsd xmm11, xmm11, xmm19, xmm1	;; Mul integer part of divide by k by -c, add carry
	vaddsd	xmm11, xmm11, Q [rsi]		;; Add in the FFT data
	split_zpad7_word ttp, xmm11, xmm1
	vmovsd	Q [rsi], xmm11			;; Save value6

	add	rsi, ZMM_SRC_INCR6		;; Next source pointer
ttp	kshiftrw k1, k1, 1			;; Next big vs. little flag
	vaddsd	xmm0, xmm1, Q [rsi]		;; Add FFT data and carry
	split_zpad7_word ttp, xmm0, xmm1
	vmovsd	Q [rsi], xmm0			;; Save value7

	add	rsi, ZMM_SRC_INCR7		;; Next source pointer
	vsubsd	xmm1, xmm1, xmm30		;; Remove RNDVAL
	vaddsd	xmm1, xmm1, Q [rsi]		;; Add in FFT data
	vmovsd	Q [rsi], xmm1			;; Save value8
	ENDM

