; Copyright 1995-2016 Mersenne Research, Inc., all rights reserved
; Author:  George Woltman
; Email: woltman@alum.mit.edu
;

;************************************************************************
; This macro uses SSE2 instructions to perform the powering and modulo
; step in factoring.  Since the SSE2 shift instructions only accept an
; immediate shift count, we need to have different code for every bit length.
;
; This code can handle factors from 64 bits up to 88 bits (I think!).
; How far it can actually handle depends on the error in computing the
; quotient, which I think is 2 bits, but I could be wrong.

sse2_fac MACRO fac_size

;; Note that at the beginning of the xmm0, xmm1, xmm2 contains the
;; remainder to square.  This remainder can be one more bit than the
;; fac_size.  This is because we do not perform a modulo after the optional
;; multiply by two at the end of this macro.

;; Also note that in very rare cases the remainder could be two more bits
;; than fac_size.  Since the calculated quotient could be one too small,
;; the maximum remainder is 2 * (factor + epsilon).  This extremely rare
;; extra bit should not be a problem as the code below uses 30 bit integers
;; and it shouldn't be a problem handling one more bit.

;; In the comments of this code, we'll assume fac_size is 89 bits and the
;; input remainder is 90 bits (30 bits in each xmm register).

;; rem = A,B,C (30,30,30 bits unsigned)
;; Square rem producing a 180 bit result stored in temp1, temp3, temp5
;; temp5 = CC				(60)
;; temp4 = 2BC				(61)
;; temp3 = 2AC + BB			(62 = 61 + 60)
;; temp2 = 2AB				(61)
;; temp1 = AA				(60)

;; If fac_size is 75 bits or more (the remainder is 76 bits or more), then
;; the squared remainder is more than 150 bits.  In this case, get the very
;; highest bits and multiply them by 2^120 mod factor.  We'll then add
;; these back into the squared remainder as part of our effort to create a
;; 120 bit result.
;;
;; AA_HIGH = temp1 >> 30
;; temp4 += AA_HIGH * TWO_120_MODF3
;; temp3 += AA_HIGH * TWO_120_MODF2
;; temp2 += AA_HIGH * TWO_120_MODF1
;; temp1 = AA & 0x3FFFFFFF

;; Combine temp1 through temp5 into 3 temporaries
;;
;; temp5 += (temp4 & 0x3FFFFFFF) << 30
;; temp3 += (temp4 >> 30) + (temp2 & 0x3FFFFFFF) << 30
;; temp1 += temp2 >> 30
;;
;; While doing the above we interleave work from the paragraph below.
;;
;; Our value is now at most 150 bits.  Take the upper 30 bits and multiply
;; them by 2^120 mod factor.  Then add these back into the squared remainder
;; to create a 120 bit result.
;;
;; Make sure we add any bits from temp3 above 2^60 into temp1
;; temp5 += temp1 * TWO_120_MODF3
;; temp4 += temp1 * TWO_120_MODF2
;; temp3 += temp1 * TWO_120_MODF1
;;
;; Finally, we do a lot of instruction interleaving in hopes of making it
;; easier for the CPU to schedule the instructions.  The clock counts assume
;; near optimal P4 instruction scheduling for the critical path.

	IF fac_size GE 75
	movdqa	xmm3, xmm0		;;1-6   Copy A
	movdqa	xmm4, xmm0		;;2-7   Copy A
	pmuludq	xmm0, xmm0		;;3-10  AA
	movdqa	xmm5, xmm1		;;3-8   Copy B
	pmuludq	xmm1, xmm1		;;5-12  temp3 = BB
	pmuludq	xmm3, xmm2		;;7-14  AC
	pmuludq	xmm4, xmm5		;;9-16  AB
	pmuludq	xmm5, xmm2		;;11-18 BC
	movdqa	xmm6, xmm0		;;11-16 temp1 = AA
	psrlq	xmm0, 30		;;12-13 AA_HIGH = Upper 30 bits of AA
	movdqa	xmm7, XMMWORD PTR XMM_TWO_120_MODF2
	pmuludq	xmm7, xmm0		;;14-21 AA_HIGH * 2_120_MODF2
	paddq	xmm1, xmm7		;;22-27 temp3 += AA_HIGH * 2_120_MODF2
	movdqa	xmm7, XMMWORD PTR XMM_TWO_120_MODF1
	pmuludq	xmm7, xmm0		;;16-23 AA_HIGH * 2_120_MODF1
	psllq	xmm3, 1			;;17-18 2AC
	pmuludq	xmm0, XMMWORD PTR XMM_TWO_120_MODF3
					;;18-25 AA_HIGH * 2_120_MODF3
	psllq	xmm4, 1			;;19-20 temp2 = 2AB
	pmuludq	xmm2, xmm2		;;20-27 temp5 = CC
	psllq	xmm5, 1			;;21-22 temp4 = 2BC
	pand	xmm6, XMMWORD PTR XMM_BITS30
					;;23-24 temp1 = Low 30 bits of AA
	paddq	xmm4, xmm7		;;24-29 temp2 += AA_HIGH * 2_120_MODF1
	paddq	xmm5, xmm0		;;26-31 temp4 += AA_HIGH * 2_120_MODF3
	paddq	xmm1, xmm3		;;28-33 temp3 = 2AC + BB
					;;	temp1-5 = xmm6,4,1,5,2
	movdqa	xmm0, xmm1		;;34-39 New temp3 = temp3
	psrlq	xmm1, 30		;;35-36 Top bits of temp3
	paddq	xmm4, xmm1		;;37-42 Add temp3 top bits into temp2
	pand	xmm0, XMMWORD PTR XMM_BITS30
					;;40-41 temp3 = temp3 & 0x3FFFFFFF
	movdqa	xmm3, xmm4		;;43-48 New temp2 = temp2
	psrlq	xmm4, 30		;;44-45 Top bits of temp2
	paddd	xmm6, xmm4		;;46-47 Add temp2 top bits into temp1
					;;	temp1-5 = xmm6,3,0,5,2
	movdqa	xmm7, XMMWORD PTR XMM_TWO_120_MODF2
	pmuludq	xmm7, xmm6		;;48-55 temp1 * TWO_120_MODF2
	pand	xmm3, XMMWORD PTR XMM_BITS30
					;;49-50 temp2 & 0x3FFFFFFF
	movdqa	xmm1, XMMWORD PTR XMM_TWO_120_MODF1
	pmuludq	xmm1, xmm6		;;50-57 temp1 * TWO_120_MODF1
	psllq	xmm3, 30		;;51-52 temp2 << 30
	pmuludq	xmm6, XMMWORD PTR XMM_TWO_120_MODF3
					;;52-59 temp1 * TWO_120_MODF3
	por	xmm0, xmm3		;;53-54 temp3 += temp2 << 30
	paddq	xmm5, xmm7		;;56-61 temp4 += temp1 * TWO_120_MODF2
	paddq	xmm0, xmm1		;;58-63 temp3 += temp1 * TWO_120_MODF1
	paddq	xmm2, xmm6		;;60-65 temp5 += temp1 * TWO_120_MODF3
					;;	temp3-5 = xmm0,xmm5,xmm2
	movdqa	xmm7, XMMWORD PTR XMM_BITS30
	pand	xmm7, xmm5		;;62-63 Lower 30 bits of temp4
	psrlq	xmm5, 30		;;64-65 temp4 >> 30
	paddq	xmm0, xmm5		;;66-71 temp3 += temp4 >> 30
	psllq	xmm7, 30		;;67-68 (temp4 & 0x3FFFFFFF) << 30
	paddq	xmm7, xmm2		;;69-74 temp5 += (temp4&0x3FFFFFFF)<<30
					;;	Result in xmm0,xmm7
	ENDIF

	IF fac_size LT 75
	movdqa	xmm3, xmm0		;;1-6	Copy A
	movdqa	xmm4, xmm0		;;2-7	Copy A
	pmuludq	xmm0, xmm2		;;3-10	AC
	movdqa	xmm5, xmm1		;;3-8	Copy B
	pmuludq	xmm1, xmm1		;;5-12	temp3 = BB
	pmuludq	xmm3, xmm3		;;7-14	temp1 = AA
	pmuludq	xmm4, xmm5		;;9-16	AB
	psllq	xmm0, 1			;;11-12 2AC
	pmuludq	xmm5, xmm2		;;12-19 BC
	paddq	xmm1, xmm0		;;13-18 temp3 = BB + 2AC
	pmuludq	xmm2, xmm2		;;14-21 temp5 = CC
	psllq	xmm4, 1			;;17-18 temp2 = 2AB
	psllq	xmm5, 1			;;20-21 temp4 = 2BC
					;;	temp1-5 = xmm3,4,1,5,2
	movdqa	xmm0, xmm1		;;22-27 New temp3 = temp3
	psrlq	xmm1, 30		;;23-24 Top bits of temp3
	paddq	xmm4, xmm1		;;25-30 Add temp3 top bits into temp2
	pand	xmm0, XMMWORD PTR XMM_BITS30
					;;28-29 temp3 = temp3 & 0x3FFFFFFF
	movdqa	xmm6, xmm4		;;31-36 New temp2 = temp2
	psrlq	xmm4, 30		;;32-33 Top bits of temp2
	paddd	xmm3, xmm4		;;34-35 Add temp2 top bits into temp1
					;;	temp1-5 = xmm3,6,0,5,2
	movdqa	xmm7, XMMWORD PTR XMM_TWO_120_MODF2
	pmuludq	xmm7, xmm3		;;36-43 temp1 * TWO_120_MODF2
	pand	xmm6, XMMWORD PTR XMM_BITS30
					;;37-38 temp2 & 0x3FFFFFFF
	movdqa	xmm1, XMMWORD PTR XMM_TWO_120_MODF1
	pmuludq	xmm1, xmm3		;;38-45 temp1 * TWO_120_MODF1
	psllq	xmm6, 30		;;39-40 temp2 << 30
	pmuludq	xmm3, XMMWORD PTR XMM_TWO_120_MODF3
					;;40-47 temp1 * TWO_120_MODF3
	por	xmm0, xmm6		;;41-42 temp3 += temp2 << 30
	paddq	xmm5, xmm7		;;44-49 temp4 += temp1 * TWO_120_MODF2
	paddq	xmm0, xmm1		;;46-51 temp3 += temp1 * TWO_120_MODF1
	paddq	xmm2, xmm3		;;48-53 temp5 += temp1 * TWO_120_MODF3
					;;	temp3-5 = xmm0,xmm5,xmm2
	movdqa	xmm7, XMMWORD PTR XMMWORD PTR XMM_BITS30
	pand	xmm7, xmm5		;;50-51 Lower 30 bits of temp4
	psrlq	xmm5, 30		;;52-53 temp4 >> 30
	paddq	xmm0, xmm5		;;54-59 temp3 += temp4 >> 30
	psllq	xmm7, 30		;;55-56 (temp4 & 0x3FFFFFFF) << 30
	paddq	xmm7, xmm2		;;57-62 temp5 += (temp4&0x3FFFFFFF)<<30
					;;	Result in xmm0,xmm7
	ENDIF

;; Squared remainder reduced to 120 bits is in xmm0, xmm7 (call them R1,R2).
;; We'll subtract  quotient * factor from this later, so save R1 for later use.

	movdqa	xmm6, xmm0		;;0-5	Save R1

;; Break the upper 60 bits (R1) into 30 bit chunks (U1,U2) and multiply
;; it by 60 bits of 1/factor (I1,I2).  This will give us the quotient
;; (shifted left somewhat).  We only need to compute the upper 60 bits of
;; the quotient since 120 - fac_size is less than 60.
;;
;; Error analysis:  Shifted quotient is up to 5.999... too small.  One for
;; low bits of temp2, one for uncomputed I2*U2, one for if uncomputed I3 is
;; large, one for if top 30 bits of temp5 (U3) is large, and two for carries
;; that may have been generated by additions of temp5 and not added into U2.
;; This is why we must shift off 3 bits for an accurate quotient, meaning
;; a quotient accuracy of 57 bits.  Thus, using the formula described in the
;; next section for computing the shift amount, the minimum trial factor size
;; this code can handle is 64 bits.
;;
;; temp2 = I2*U1 + I1*U2	(61)
;; temp1 = I1*U1		(60)
;;
;; temp1 += temp2 >> 30		(61)

	psrlq	xmm0, 30		;;1-2	U1 = upper bits of R1
	movdqa	xmm1, XMMWORD PTR XMM_I2
	pmuludq	xmm1, xmm0		;;3-10	temp2 = I2 * U1
	movdqa	xmm3, XMMWORD PTR XMM_I1
	pmuludq	xmm0, xmm3		;;5-12	temp1 = I1 * U1
	pand	xmm6, XMMWORD PTR XMM_BITS30
					;;6-7	U2 = lower bits of R1
	pmuludq	xmm3, xmm6		;;8-15	I1 * U2
	paddq	xmm1, xmm3		;;16-21 temp2 += I1 * U2
	psrlq	xmm1, 30		;;22-23 temp2 >> 30
	paddq	xmm0, xmm1		;;24-29 temp1 += temp2 >> 30

;; 60 bits of shifted quotient are now in xmm0.  R1,R2 are in xmm6,xmm7.

;; Shift the quotient right to get the actual quotient.
;; Define BS (bit shift amount). This is the amount to shift the quotient
;; right to compute the quotient.  BS = 60 - (120 - fac_size + 1).
;; For example, if fac_size = 75, R1/R2 is 120 bits, quotient will
;; be 120-75+1=46 bits, temp1 is 60 bits, BS is 60-46=14 bits.
;;
;; Now shift right to get the quotient:
;;
;; Q2 = (temp1 >> BS) & 0x3FFFFFFF			(30)
;; Q1 = temp1 >> (30 + BS)

	movd	xmm2, XMM_BS
	psrlq	xmm0, xmm2		;;30-31 temp1 >> BS
	movdqa	xmm1, XMMWORD PTR XMM_BITS30
	pand	xmm1, xmm0		;;32-33 Q2 = (temp1 >> BS) & 0x3FFFFFFF
	psrlq	xmm0, 30		;;34-35 Q1 = temp1 >> (30 + BS)

;; Quotient is now in xmm0, xmm1.  R1,R2 are in xmm6, xmm7.

;; Multiply quotient Q1,Q2 by factor F1,F2,F3
;; We only need to compute bottom 90 bits.
;;
;; temp3 = F3*Q2				(60)
;; temp2 = F3*Q1 + F2*Q2			(61)
;; temp1 = F2*Q1 + F1*Q2			(61)
;;
;; QF2 = temp3 + ((temp2 & 0x3FFFFFFF)<<30)	(61)
;; QF1 = temp1 + (temp2 >> 30)			(61)

	movdqa	xmm3, XMMWORD PTR XMM_F2
	pmuludq	xmm3, xmm1		;;35-42 temp2 = F2 * Q2
	movdqa	xmm4, XMMWORD PTR XMM_F3
	pmuludq	xmm4, xmm0		;;37-44 F3 * Q1
	movdqa	xmm5, XMMWORD PTR XMM_F1
	pmuludq	xmm5, xmm1		;;39-46 temp1 = F1 * Q2
	pmuludq	xmm0, XMMWORD PTR XMM_F2;;41-48 F2 * Q1
	pmuludq	xmm1, XMMWORD PTR XMM_F3;;43-50 temp3 = F3 * Q2
	paddq	xmm3, xmm4		;;45-50 temp2 += F3 * Q1
	paddd	xmm5, xmm0		;;49-50 temp1 += F2 * Q1
	movdqa	xmm2, XMMWORD PTR XMM_BITS30
	pand	xmm2, xmm3		;;51-52 temp2 & 0x3FFFFFFF
	psllq	xmm2, 30		;;53-54 (temp2 & 0x3FFFFFFF) << 30
	paddq	xmm1, xmm2		;;55-60 QF2 = temp3 + (low(temp2)<<30)
	psrlq	xmm3, 30		;;56-57 temp2 >> 30
	paddd	xmm5, xmm3		;;58-59 QF1 = temp1 + (temp2 >> 30)

;; Quotient * factor is now in xmm5, xmm1.  R1,R2 are in xmm6, xmm7.

;; Subtract QF1,QF2 from R1,R2
;;
;; temp2 = R2 - QF2			(signed 62)
;; temp1 = R1 - QF1			(signed 62)

	psubq	xmm7, xmm1		;;61-66 temp2 = R2 - QF2
	psubd	xmm6, xmm5		;;63-64 temp1 = R1 - QF1

;; If mul by 2 is required:  mul temp1, temp2 by 2 here

	movd	xmm1, XMM_SHIFTER[rdi*8]
	psllq	xmm6, xmm1		;;65-66 temp1 *= 1 or 2
	psllq	xmm7, xmm1		;;67-68 temp2 *= 1 or 2

;; Break remainder in xmm6,xmm7 into 30 bit quantities.
;;
;; RES3 = temp2 & 0x3FFFFFFF		(30)
;; temp2 >>= 30				(signed 32)
;; RES2 = temp2 & 0x3FFFFFFF		(30)
;; temp2 >>= 30				(signed 2)
;; temp1 += temp2
;; RES1 = temp1 & 0x0FFFFFFF		(28)

	movdqa	xmm1, XMMWORD PTR XMM_BITS30	;; Mask to get 30 bits
	movdqa	xmm2, xmm7		;;69-74 Copy temp2
	psrlq	xmm7, 30		;;69-70 temp2 >>= 30
	pand	xmm1, xmm7		;;71-72 RES2 = temp2 & 0x3FFFFFFF
	psrlq	xmm7, 2			;;73-74 temp2 >>= 2
	psrad	xmm7, 28		;;75-76 temp2 >>= 28
	paddd	xmm6, xmm7		;;77-78 temp1 += temp2
	pand	xmm2, XMMWORD PTR XMM_BITS30
					;;78-79 RES3 = temp2 & 0x3FFFFFFF
	movdqa	xmm0, XMMWORD PTR XMM_BITS28	;; Mask to get 28 bits
	pand	xmm0, xmm6		;;80-81 RES1 = temp1 & 0x0FFFFFFF
	ENDM


; This macro does some initialization work and then performs a subset of the
; above working on the initial value.  This subset avoids the squaring of the
; remainder.  The initial value can be up to 60 bits larger than the factor
; size.

sse2_fac_initval MACRO
	LOCAL	retry, ok1, ok2

;; Break 60 bits of factor inverse into 30 bit chunks

retry:	movdqa	xmm7, XMMWORD PTR XMM_INVFAC ; Load factor inverse
	psrlq	xmm7, 3			; Convert 63 bit inverse to 60 bits
	movdqa	xmm6, XMMWORD PTR XMM_BITS30
	pand	xmm6, xmm7		; Q2 = bottom bits of quotient
	psrlq	xmm7, 30		; Q1 = top bits of quotient
	movdqa	XMMWORD PTR XMM_I2, xmm6
	movdqa	XMMWORD PTR XMM_I1, xmm7

;; Make sure the quotient isn't one too large.
;; Multiply quotient Q1,Q2 by factor F1,F2,F3
;; We only need to compute bottom 90 bits.
;;
;; temp3 = F3*Q2				(60)
;; temp2 = F3*Q1 + F2*Q2			(61)
;; temp1 = F2*Q1 + F1*Q2			(61)
;;
;; QF2 = temp3 + ((temp2 & 0x3FFFFFFF)<<30)	(61)
;; QF1 = temp1 + (temp2 >> 30)			(61)

	movdqa	xmm3, XMMWORD PTR XMM_F2
	pmuludq	xmm3, xmm6		;; temp2 = F2 * Q2
	movdqa	xmm4, XMMWORD PTR XMM_F3
	pmuludq	xmm4, xmm7		;; F3 * Q1
	movdqa	xmm5, XMMWORD PTR XMM_F1
	pmuludq	xmm5, xmm6		;; temp1 = F1 * Q2
	pmuludq	xmm7, XMMWORD PTR XMM_F2;; F2 * Q1
	pmuludq	xmm6, XMMWORD PTR XMM_F3;; temp3 = F3 * Q2
	paddq	xmm3, xmm4		;; temp2 += F3 * Q1
	paddd	xmm5, xmm7		;; temp1 += F2 * Q1
	movdqa	xmm2, XMMWORD PTR XMM_BITS30
	pand	xmm2, xmm3		;; temp2 & 0x3FFFFFFF
	psllq	xmm2, 30		;; (temp2 & 0x3FFFFFFF) << 30
	paddq	xmm6, xmm2		;; QF2 = temp3 + (low(temp2)<<30)
	psrlq	xmm3, 30		;; temp2 >> 30
	paddd	xmm5, xmm3		;; QF1 = temp1 + (temp2 >> 30)

;; If QF2 is above 2^60, then add these carry bits into QF1

	psrlq	xmm6, 60
	paddd	xmm5, xmm6		;; QF1 = QF1 + (QF2 >> 60)

;; Validate QF1.  The top bits should be ones (zeros indicate Q*F exceeded
;; a power of two by a little bit).  Detect these cases, decrement the
;; inverse factor and try again.

	psllq	xmm5, 4			;; Test top bits of QF
	pmovmskb edx, xmm5
	test	edx, 0008h		;; See if lower overflowed
	jnz	short ok1		;; Jump if remainder is ok
	movdqa	xmm7, XMMWORD PTR XMM_INVFAC ;; Subtract one from an INVFAC
	psubq	xmm7, XMMWORD PTR XMM_LOWONE
	movdqa	XMMWORD PTR XMM_INVFAC, xmm7
	jmp	retry			;; Redo init code
ok1:	test	edx, 0800h		;; See if higher overflowed
	jnz	short ok2		;; Jump if remainder is ok
	movdqa	xmm7, XMMWORD PTR XMM_INVFAC ;; Subtract one from an INVFAC
	psubq	xmm7, XMMWORD PTR XMM_HIGHONE
	movdqa	XMMWORD PTR XMM_INVFAC, xmm7
	jmp	retry			;; Redo init code
ok2:

;; Compute quotient for 2^120 / factor call it Q1,Q2.

	movdqa	xmm7, XMMWORD PTR XMM_INVFAC ; Load factor inverse
	movd	xmm0, XMM_INIT120BS
	psrlq	xmm7, xmm0		; Compute 2^120 / factor
	movdqa	xmm6, XMMWORD PTR XMM_BITS30
	pand	xmm6, xmm7		; Q2 = bottom bits of quotient
	psrlq	xmm7, 30		; Q1 = top bits of quotient

;; Multiply quotient Q1,Q2 by factor F1,F2,F3
;; We only need to compute bottom 90 bits.
;;
;; temp3 = F3*Q2				(60)
;; temp2 = F3*Q1 + F2*Q2			(61)
;; temp1 = F2*Q1 + F1*Q2			(61)
;;
;; QF2 = temp3 + ((temp2 & 0x3FFFFFFF)<<30)	(61)
;; QF1 = temp1 + (temp2 >> 30)			(61)

	movdqa	xmm3, XMMWORD PTR XMM_F2
	pmuludq	xmm3, xmm6		;; temp2 = F2 * Q2
	movdqa	xmm4, XMMWORD PTR XMM_F3
	pmuludq	xmm4, xmm7		;; F3 * Q1
	movdqa	xmm5, XMMWORD PTR XMM_F1
	pmuludq	xmm5, xmm6		;; temp1 = F1 * Q2
	pmuludq	xmm7, XMMWORD PTR XMM_F2;; F2 * Q1
	pmuludq	xmm6, XMMWORD PTR XMM_F3;; temp3 = F3 * Q2
	paddq	xmm3, xmm4		;; temp2 += F3 * Q1
	paddd	xmm5, xmm7		;; temp1 += F2 * Q1
	movdqa	xmm2, XMMWORD PTR XMM_BITS30
	pand	xmm2, xmm3		;; temp2 & 0x3FFFFFFF
	psllq	xmm2, 30		;; (temp2 & 0x3FFFFFFF) << 30
	paddq	xmm6, xmm2		;; QF2 = temp3 + (low(temp2)<<30)
	psrlq	xmm3, 30		;; temp2 >> 30
	paddd	xmm5, xmm3		;; QF1 = temp1 + (temp2 >> 30)

;; Subtract QF1,QF2 from zero to get 90 bits of 2^120 mod factor.

	pxor	xmm1, xmm1
	pxor	xmm0, xmm0
	psubq	xmm1, xmm6		;; temp2 = Low 60 bits of remainder
	psubd	xmm0, xmm5		;; temp1 = Low 30 bits of remainder

;; Break remainder in xmm0,xmm1 into 30 bit quantities and save
;;
;; RES3 = temp2 & 0x3FFFFFFF		(30)
;; temp2 >>= 30				(signed 32)
;; RES2 = temp2 & 0x3FFFFFFF		(30)
;; temp2 >>= 30				(signed 2)
;; temp1 += temp2
;; RES1 = temp1 & 0x0FFFFFFF		(28)

	movdqa	xmm4, XMMWORD PTR XMM_BITS30	;; Mask to get 30 bits
	movdqa	xmm5, xmm1		;; Copy temp2
	psrlq	xmm1, 30		;; temp2 >>= 30
	pand	xmm4, xmm1		;; RES2 = temp2 & 0x3FFFFFFF
	psrlq	xmm1, 2			;; temp2 >>= 2
	psrad	xmm1, 28		;; temp2 >>= 28
	paddd	xmm0, xmm1		;; temp1 += temp2
	pand	xmm5, XMMWORD PTR XMM_BITS30	;; RES3 = temp2 & 0x3FFFFFFF
	pand	xmm0, XMMWORD PTR XMM_BITS28	;; RES1 = temp1 & 0x0FFFFFFF
	movdqa	XMMWORD PTR XMM_TWO_120_MODF2, xmm4
	movdqa	XMMWORD PTR XMM_TWO_120_MODF3, xmm5
	movdqa	XMMWORD PTR XMM_TWO_120_MODF1, xmm0

;; 1/factor is the shifted quotient.  Shift 1/factor right to get the actual
;; quotient.  XMM_INITBS is the shift amount defined by this formula:
;;	INITBS = 120 - bit_length(initval)
;;
;; Q2 = (temp1 >> INITBS) & 0x3FFFFFFF			(30)
;; Q1 = temp1 >> (30 + INITBS)

	movdqa	xmm0, XMMWORD PTR XMM_INVFAC	;; Load 1/factor
	movd	xmm1, XMM_INITBS
	psrlq	xmm0, xmm1		;; temp1 >> INITBS
	movdqa	xmm1, XMMWORD PTR XMM_BITS30
	pand	xmm1, xmm0		;; Q2 = (temp1 >> BS) & 0x3FFFFFFF
	psrlq	xmm0, 30		;; Q1 = temp1 >> (30 + BS)

;; Load initial value

	movdqa	xmm6, XMMWORD PTR XMM_INITVAL	;; Load R1
	pxor	xmm7, xmm7		;; Clear R2

;; Quotient is now in xmm0, xmm1.  R1,R2 are in xmm6, xmm7.

;; Multiply quotient Q1,Q2 by factor F1,F2,F3
;; We only need to compute bottom 90 bits.
;;
;; temp3 = F3*Q2				(60)
;; temp2 = F3*Q1 + F2*Q2			(61)
;; temp1 = F2*Q1 + F1*Q2			(61)
;;
;; QF2 = temp1 + ((temp2 & 0x3FFFFFFF)<<30)	(61)
;; QF1 = temp1 + (temp2 >> 30)			(61)

	movdqa	xmm3, XMMWORD PTR XMM_F2
	pmuludq	xmm3, xmm1		;; temp2 = F2 * Q2
	movdqa	xmm4, XMMWORD PTR XMM_F3
	pmuludq	xmm4, xmm0		;; F3 * Q1
	movdqa	xmm5, XMMWORD PTR XMM_F1
	pmuludq	xmm5, xmm1		;; temp1 = F1 * Q2
	pmuludq	xmm0, XMMWORD PTR XMM_F2;; F2 * Q1
	pmuludq	xmm1, XMMWORD PTR XMM_F3;; temp3 = F3 * Q2
	paddq	xmm3, xmm4		;; temp2 += F3 * Q1
	paddd	xmm5, xmm0		;; temp1 += F2 * Q1
	movdqa	xmm2, XMMWORD PTR XMM_BITS30
	pand	xmm2, xmm3		;; temp2 & 0x3FFFFFFF
	psllq	xmm2, 30		;; (temp2 & 0x3FFFFFFF) << 30
	paddq	xmm1, xmm2		;; QF2 = temp3 + (low(temp2)<<30)
	psrlq	xmm3, 30		;; temp2 >> 30
	paddd	xmm5, xmm3		;; QF1 = temp1 + (temp2 >> 30)

;; Quotient * factor is now in xmm5, xmm1.  R1,R2 are in xmm6, xmm7.

;; Subtract QF1,QF2 from R1,R2
;;
;; temp2 = R2 - QF2			(signed 62)
;; temp1 = R1 - QF1			(signed 62)

	psubq	xmm7, xmm1		;; temp2 = R2 - QF2
	psubd	xmm6, xmm5		;; temp1 = R1 - QF1

;; Break remainder in xmm6,xmm7 into 30 bit quantities.
;;
;; RES3 = temp2 & 0x3FFFFFFF		(30)
;; temp2 >>= 30				(signed 32)
;; RES2 = temp2 & 0x3FFFFFFF		(30)
;; temp2 >>= 30				(signed 2)
;; temp1 += temp2
;; RES1 = temp1 & 0x0FFFFFFF		(28)

	movdqa	xmm1, XMMWORD PTR XMM_BITS30	;; Mask to get 30 bits
	movdqa	xmm2, xmm7		;; Copy temp2
	psrlq	xmm7, 30		;; temp2 >>= 30
	pand	xmm1, xmm7		;; RES2 = temp2 & 0x3FFFFFFF
	psrlq	xmm7, 2			;; temp2 >>= 2
	psrad	xmm7, 28		;; temp2 >>= 28
	paddd	xmm6, xmm7		;; temp1 += temp2
	pand	xmm2, XMMWORD PTR XMM_BITS30	;; RES3 = temp2 & 0x3FFFFFFF
	movdqa	xmm0, XMMWORD PTR XMM_BITS28	;; Mask to get 28 bits
	pand	xmm0, xmm6		;; RES1 = temp1 & 0x0FFFFFFF
	ENDM





;************************************************************************
; These updated macros uses SSE2 instructions to perform the powering and modulo step
; in factoring for four factors at a time.
;

; This macro does some initialization work and then performs a subset of the
; above working on the initial value.  This subset avoids the squaring of the
; remainder.  The initial value can be up to 60 bits larger than the factor
; size.

sse2_fac_initval MACRO
	LOCAL	retry, invfac_ok, invfac_adjust

;; Preload constants

	movdqa	xmm15, XMMWORD PTR XMM_BITS30

;; Break 60 bits of factor inverse into 30 bit chunks

	movdqa	xmm1, XMMWORD PTR XMM_INVFAC ;; Load factor inverse
	movdqa	xmm7, XMMWORD PTR XMM_INVFACa ;; Load factor inverse
retry:	movdqa	xmm3, xmm1
	psrlq	xmm3, 3			;; Convert 63 bit inverse to 60 bits
	movdqa	xmm0, xmm3
	pand	xmm0, xmm15		;; Q2 = bottom bits of quotient
	psrlq	xmm3, 30		;; Q1 = top bits of quotient
	movdqa	XMMWORD PTR XMM_I2, xmm0
	movdqa	XMMWORD PTR XMM_I1, xmm3

	movdqa	xmm9, xmm7
	psrlq	xmm9, 3			;; Convert 63 bit inverse to 60 bits
	movdqa	xmm6, xmm9
	pand	xmm6, xmm15		;; Q2 = bottom bits of quotient
	psrlq	xmm9, 30		;; Q1 = top bits of quotient
	movdqa	XMMWORD PTR XMM_I2a, xmm6
	movdqa	XMMWORD PTR XMM_I1a, xmm9

;; Make sure the quotient isn't one too large.
;; Multiply quotient Q1,Q2 by factor F1,F2,F3
;; We only need to compute bottom 90 bits.
;;
;; temp3 = F3*Q2				(60)
;; temp2 = F3*Q1 + F2*Q2			(61)
;; temp1 = F2*Q1 + F1*Q2			(61)
;;
;; QF2 = temp3 + ((temp2 & 0x3FFFFFFF)<<30)	(61)
;; QF1 = temp1 + (temp2 >> 30)			(61)

	movdqa	xmm4, xmm3
	pmuludq xmm4, XMMWORD PTR XMM_F3 ;; temp2 = Q1 * F3
	movdqa	xmm2, xmm0
	pmuludq xmm2, XMMWORD PTR XMM_F2 ;; Q2 * F2
	pmuludq xmm3, XMMWORD PTR XMM_F2 ;; temp1 = Q1 * F2
	movdqa	xmm5, xmm0
	pmuludq xmm5, XMMWORD PTR XMM_F1 ;; Q2 * F1
	pmuludq xmm0, XMMWORD PTR XMM_F3 ;; temp3 = Q2 * F3
	paddq	xmm4, xmm2		;; temp2 += Q2 * F2
	paddd	xmm3, xmm5		;; temp1 += Q2 * F1
	movdqa	xmm5, xmm4
	pand	xmm5, xmm15		;; temp2 & 0x3FFFFFFF
	psrlq	xmm4, 30		;; temp2 >> 30
	psllq	xmm5, 30		;; (temp2 & 0x3FFFFFFF) << 30
	paddq	xmm0, xmm5		;; QF2 = temp3 + ((temp2 & 0x3FFFFFFF)<<30)
	paddd	xmm3, xmm4		;; QF1 = temp1 + (temp2 >> 30)

	movdqa	xmm10, xmm9
	pmuludq xmm10, XMMWORD PTR XMM_F3a ;; temp2 = Q1 * F3
	movdqa	xmm8, xmm6
	pmuludq xmm8, XMMWORD PTR XMM_F2a ;; Q2 * F2
	pmuludq xmm9, XMMWORD PTR XMM_F2a ;; temp1 = Q1 * F2
	movdqa	xmm11, xmm6
	pmuludq xmm11, XMMWORD PTR XMM_F1a ;; Q2 * F1
	pmuludq xmm6, XMMWORD PTR XMM_F3a ;; temp3 = Q2 * F3
	paddq	xmm10, xmm8		;; temp2 += Q2 * F2
	paddd	xmm9, xmm11		;; temp1 += Q2 * F1
	movdqa	xmm11, xmm10
	pand	xmm11, xmm15		;; temp2 & 0x3FFFFFFF
	psrlq	xmm10, 30		;; temp2 >> 30
	psllq	xmm11, 30		;; (temp2 & 0x3FFFFFFF) << 30
	paddq	xmm6, xmm11		;; QF2 = temp3 + ((temp2 & 0x3FFFFFFF)<<30)
	paddd	xmm9, xmm10		;; QF1 = temp1 + (temp2 >> 30)

;; If QF2 is above 2^60, then add these carry bits into QF1

	psrlq	xmm0, 60
	paddd	xmm3, xmm0		;; QF1 = QF1 + (QF2 >> 60)

	psrlq	xmm6, 60
	paddd	xmm9, xmm6		;; QF1 = QF1 + (QF2 >> 60)

;; Validate QF1.  The top bits should be ones (zeros indicate Q*F exceeded
;; a power of two by a little bit).  Detect these cases, decrement the
;; inverse factor and try again.  We used to use pcmpeqq but that is an SSE4 instruction.

	pand	xmm3, XMMWORD PTR YMM_28TH_BIT ;; Test for positive dword values in xmm3 (test 28th bit)
	pxor	xmm2, xmm2
	pcmpeqd xmm3, xmm2		;; Test low 32-bits for zero and always match high 32-bits
	pand	xmm9, XMMWORD PTR YMM_28TH_BIT ;; Test for positive dword values in xmm9 (test 28th bit)
	pcmpeqd xmm9, xmm2		;; Test low 32-bits for zero and always match high 32-bits
	pmovmskb rdx, xmm3
	and	edx, 0F0Fh		;; See if INVFAC values changed
	jnz	short invfac_adjust	;; Jump if INVFACs needed to be adjusted
	pmovmskb rdx, xmm9
	and	edx, 0F0Fh		;; See if INVFAC values changed
	jz	short invfac_ok		;; Jump if no INVFACs needed to be adjusted
invfac_adjust:
	psllq	xmm3, 32		;; Sign extend compare's low 32-bits into a 64-bit signed value
	movdqa	xmm2, xmm3
	psrlq	xmm3, 32
	por	xmm3, xmm2
	psllq	xmm9, 32		;; Sign extend compare's low 32-bits into a 64-bit signed value
	movdqa	xmm2, xmm9
	psrlq	xmm9, 32
	por	xmm9, xmm2
	movdqa	xmm1, XMMWORD PTR XMM_INVFAC ;; Load factor inverse
	paddq	xmm1, xmm3		;; Add -1 to INVFAC where QF1 dword val was positive
	movdqa	XMMWORD PTR XMM_INVFAC, xmm1 ;; Store factor inverse
	movdqa	xmm7, XMMWORD PTR XMM_INVFACa ;; Load factor inverse
	paddq	xmm7, xmm9		;; Add -1 to INVFAC where QF1 dword val was positive
	movdqa	XMMWORD PTR XMM_INVFACa, xmm7 ;; Store factor inverse
	jmp	retry			;; Reprocess new INVFAC values
invfac_ok:

;; Compute quotient for 2^120 / factor call it Q1,Q2.

	movdqa	xmm3, XMMWORD PTR XMM_INVFAC ;; Load factor inverse
	psrlq	xmm3, XMMWORD PTR XMM_INIT120BS ;; Compute 2^120 / factor
	movdqa	xmm0, xmm3
	pand	xmm0, xmm15		;; Q2 = bottom bits of quotient
	psrlq	xmm3, 30		;; Q1 = top bits of quotient

	movdqa	xmm9, XMMWORD PTR XMM_INVFACa ;; Load factor inverse
	psrlq	xmm9, XMMWORD PTR XMM_INIT120BS ;; Compute 2^120 / factor
	movdqa	xmm6, xmm9
	pand	xmm6, xmm15		;; Q2 = bottom bits of quotient
	psrlq	xmm9, 30		;; Q1 = top bits of quotient

;; Multiply quotient Q1,Q2 by factor F1,F2,F3
;; We only need to compute bottom 90 bits.
;;
;; temp3 = F3*Q2				(60)
;; temp2 = F3*Q1 + F2*Q2			(61)
;; temp1 = F2*Q1 + F1*Q2			(61)
;;
;; QF2 = temp3 + ((temp2 & 0x3FFFFFFF)<<30)	(61)
;; QF1 = temp1 + (temp2 >> 30)			(61)

	movdqa	xmm4, xmm3
	pmuludq xmm4, XMMWORD PTR XMM_F3 ;; temp2 = Q1 * F3
	movdqa	xmm2, xmm0
	pmuludq xmm2, XMMWORD PTR XMM_F2 ;; Q2 * F2
	pmuludq xmm3, XMMWORD PTR XMM_F2 ;; temp1 = Q1 * F2
	movdqa	xmm5, xmm0
	pmuludq xmm5, XMMWORD PTR XMM_F1 ;; Q2 * F1
	pmuludq xmm0, XMMWORD PTR XMM_F3 ;; temp3 = Q2 * F3
	paddq	xmm4, xmm2		;; temp2 += Q2 * F2
	paddd	xmm3, xmm5		;; temp1 += Q2 * F1
	movdqa	xmm5, xmm4
	pand	xmm5, xmm15		;; temp2 & 0x3FFFFFFF
	psrlq	xmm4, 30		;; temp2 >> 30
	psllq	xmm5, 30		;; (temp2 & 0x3FFFFFFF) << 30
	paddq	xmm0, xmm5		;; QF2 = temp3 + ((temp2 & 0x3FFFFFFF)<<30)
	paddd	xmm3, xmm4		;; QF1 = temp1 + (temp2 >> 30)

	movdqa	xmm10, xmm9
	pmuludq xmm10, XMMWORD PTR XMM_F3a ;; temp2 = Q1 * F3
	movdqa	xmm8, xmm6
	pmuludq xmm8, XMMWORD PTR XMM_F2a ;; Q2 * F2
	pmuludq xmm9, XMMWORD PTR XMM_F2a ;; temp1 = Q1 * F2
	movdqa	xmm11, xmm6
	pmuludq xmm11, XMMWORD PTR XMM_F1a ;; Q2 * F1
	pmuludq xmm6, XMMWORD PTR XMM_F3a ;; temp3 = Q2 * F3
	paddq	xmm10, xmm8		;; temp2 += Q2 * F2
	paddd	xmm9, xmm11		;; temp1 += Q2 * F1
	movdqa	xmm11, xmm10
	pand	xmm11, xmm15		;; temp2 & 0x3FFFFFFF
	psrlq	xmm10, 30		;; temp2 >> 30
	psllq	xmm11, 30		;; (temp2 & 0x3FFFFFFF) << 30
	paddq	xmm6, xmm11		;; QF2 = temp3 + ((temp2 & 0x3FFFFFFF)<<30)
	paddd	xmm9, xmm10		;; QF1 = temp1 + (temp2 >> 30)

;; Subtract QF1,QF2 from zero to get 90 bits of 2^120 mod factor.

	pxor	xmm5, xmm5
	pxor	xmm4, xmm4
	psubq	xmm5, xmm0		;; temp2 = Low 60 bits of remainder
	psubd	xmm4, xmm3		;; temp1 = Low 30 bits of remainder

	pxor	xmm11, xmm11
	pxor	xmm10, xmm10
	psubq	xmm11, xmm6		;; temp2 = Low 60 bits of remainder
	psubd	xmm10, xmm9		;; temp1 = Low 30 bits of remainder

;; Break remainder in xmm5,xmm4 into 30 bit quantities and save
;;
;; RES3 = temp2 & 0x3FFFFFFF		(30)
;; temp2 >>= 30				(signed 32)
;; RES2 = temp2 & 0x3FFFFFFF		(30)
;; temp2 >>= 30				(signed 2)
;; temp1 += temp2
;; RES1 = temp1 & 0x0FFFFFFF		(28)

	movdqa	xmm2, xmm5
	pand	xmm2, xmm15		;; RES3 = temp2 & 0x3FFFFFFF
	psrlq	xmm5, 30		;; temp2 >>= 30
	movdqa	xmm1, xmm5
	pand	xmm1, xmm15		;; RES2 = temp2 & 0x3FFFFFFF
	psrlq	xmm5, 2			;; temp2 >>= 2
	psrad	xmm5, 28		;; temp2 >>= 28
	paddd	xmm4, xmm5		;; temp1 += temp2
	movdqa	xmm0, xmm4
	pand	xmm0, XMMWORD PTR XMM_BITS28 ;; RES1 = temp1 & 0x0FFFFFFF
	movdqa XMMWORD PTR XMM_TWO_120_MODF3, xmm2
	movdqa XMMWORD PTR XMM_TWO_120_MODF2, xmm1
	movdqa XMMWORD PTR XMM_TWO_120_MODF1, xmm0

	movdqa	xmm8, xmm11
	pand	xmm8, xmm15		;; RES3 = temp2 & 0x3FFFFFFF
	psrlq	xmm11, 30		;; temp2 >>= 30
	movdqa	xmm7, xmm11
	pand	xmm7, xmm15		;; RES2 = temp2 & 0x3FFFFFFF
	psrlq	xmm11, 2		;; temp2 >>= 2
	psrad	xmm11, 28		;; temp2 >>= 28
	paddd	xmm10, xmm11		;; temp1 += temp2
	movdqa	xmm6, xmm10
	pand	xmm6, XMMWORD PTR XMM_BITS28 ;; RES1 = temp1 & 0x0FFFFFFF
	movdqa XMMWORD PTR XMM_TWO_120_MODF3a, xmm8
	movdqa XMMWORD PTR XMM_TWO_120_MODF2a, xmm7
	movdqa XMMWORD PTR XMM_TWO_120_MODF1a, xmm6

;; 1/factor is the shifted quotient.  Shift 1/factor right to get the actual
;; quotient.  XMM_INITBS is the shift amount defined by this formula:
;;	INITBS = 120 - bit_length(initval)
;;
;; Q2 = (temp1 >> INITBS) & 0x3FFFFFFF			(30)
;; Q1 = temp1 >> (30 + INITBS)

	movdqa	xmm0, XMMWORD PTR XMM_INVFAC ;; Load factor inverse
	movd	xmm1, XMM_INITBS
	psrlq	xmm0, xmm1		;; temp1 = YMM_INVFAC >> INITBS
	movdqa	xmm3, xmm0
	psrlq	xmm3, 30		;; Q1 = temp1 >> (30 + BS)
	pand	xmm0, xmm15		;; Q2 = (temp1 >> BS) & 0x3FFFFFFF

	movdqa	xmm6, XMMWORD PTR XMM_INVFACa ;; Load factor inverse
	psrlq	xmm6, xmm1		;; temp1 = YMM_INVFAC >> INITBS
	movdqa	xmm9, xmm6
	psrlq	xmm9, 30		;; Q1 = temp1 >> (30 + BS)
	pand	xmm6, xmm15		;; Q2 = (temp1 >> BS) & 0x3FFFFFFF

;; Quotient is now in xmm3, xmm0.

;; Multiply quotient Q1,Q2 by factor F1,F2,F3
;; We only need to compute bottom 90 bits.
;;
;; temp3 = F3*Q2				(60)
;; temp2 = F3*Q1 + F2*Q2			(61)
;; temp1 = F2*Q1 + F1*Q2			(61)
;;
;; QF2 = temp1 + ((temp2 & 0x3FFFFFFF)<<30)	(61)
;; QF1 = temp1 + (temp2 >> 30)			(61)

	movdqa	xmm4, xmm3
	pmuludq xmm4, XMMWORD PTR XMM_F3 ;; temp2 = Q1 * F3
	movdqa	xmm2, xmm0
	pmuludq xmm2, XMMWORD PTR XMM_F2 ;; Q2 * F2
	pmuludq xmm3, XMMWORD PTR XMM_F2 ;; temp1 = Q1 * F2
	movdqa	xmm5, xmm0
	pmuludq xmm5, XMMWORD PTR XMM_F1 ;; Q2 * F1
	pmuludq xmm0, XMMWORD PTR XMM_F3 ;; temp3 = Q2 * F3
	paddq	xmm4, xmm2		;; temp2 += Q2 * F2
	paddd	xmm3, xmm5		;; temp1 += Q2 * F1
	movdqa	xmm5, xmm4
	pand	xmm5, xmm15		;; temp2 & 0x3FFFFFFF
	psrlq	xmm4, 30		;; temp2 >> 30
	psllq	xmm5, 30		;; (temp2 & 0x3FFFFFFF) << 30
	paddq	xmm0, xmm5		;; QF2 = temp3 + ((temp2 & 0x3FFFFFFF)<<30)
	paddd	xmm3, xmm4		;; QF1 = temp1 + (temp2 >> 30)

	movdqa	xmm10, xmm9
	pmuludq xmm10, XMMWORD PTR XMM_F3a ;; temp2 = Q1 * F3
	movdqa	xmm8, xmm6
	pmuludq xmm8, XMMWORD PTR XMM_F2a ;; Q2 * F2
	pmuludq xmm9, XMMWORD PTR XMM_F2a ;; temp1 = Q1 * F2
	movdqa	xmm11, xmm6
	pmuludq xmm11, XMMWORD PTR XMM_F1a ;; Q2 * F1
	pmuludq xmm6, XMMWORD PTR XMM_F3a ;; temp3 = Q2 * F3
	paddq	xmm10, xmm8		;; temp2 += Q2 * F2
	paddd	xmm9, xmm11		;; temp1 += Q2 * F1
	movdqa	xmm11, xmm10
	pand	xmm11, xmm15		;; temp2 & 0x3FFFFFFF
	psrlq	xmm10, 30		;; temp2 >> 30
	psllq	xmm11, 30		;; (temp2 & 0x3FFFFFFF) << 30
	paddq	xmm6, xmm11		;; QF2 = temp3 + ((temp2 & 0x3FFFFFFF)<<30)
	paddd	xmm9, xmm10		;; QF1 = temp1 + (temp2 >> 30)

;; Load initial value

	pxor	xmm5, xmm5		;; Clear R2
	movdqa	xmm4, XMMWORD PTR XMM_INITVAL ;; Load R1

	pxor	xmm11, xmm11		;; Clear R2
	movdqa	xmm10, XMMWORD PTR XMM_INITVAL ;; Load R1

;; Quotient * factor is now in xmm3, xmm0.  R1,R2 are in xmm4, xmm5.

;; Subtract QF1,QF2 from R1,R2
;;
;; temp2 = R2 - QF2			(signed 62)
;; temp1 = R1 - QF1			(signed 62)

	psubq	xmm5, xmm0		;; temp2 = R2 - QF2
	psubd	xmm4, xmm3		;; temp1 = R1 - QF1

	psubq	xmm11, xmm6		;; temp2 = R2 - QF2
	psubd	xmm10, xmm9		;; temp1 = R1 - QF1
	ENDM


; This code can handle factors from 64 bits up to 88 bits (I think!).
; How far it can actually handle depends on the error in computing the
; quotient, which I think is 2 bits, but I could be wrong.

sse2_fac MACRO fac_size

;; Note that at the beginning of the xmm0, xmm1, xmm2 contains the
;; remainder to square.  This remainder can be one more bit than the
;; fac_size.  This is because we do not perform a modulo after the optional
;; multiply by two at the end of this macro.

;; Also note that in very rare cases the remainder could be two more bits
;; than fac_size.  Since the calculated quotient could be one too small,
;; the maximum remainder is 2 * (factor + epsilon).  This extremely rare
;; extra bit should not be a problem as the code below uses 30 bit integers
;; and it shouldn't be a problem handling one more bit.

;; In the comments of this code, we'll assume fac_size is 89 bits and the
;; input remainder is 90 bits (30 bits in each xmm register).

;; rem = A,B,C (30,30,30 bits unsigned)
;; Square rem producing a 180 bit result stored in temp1, temp3, temp5
;; temp5 = CC				(60)
;; temp4 = 2BC				(61)
;; temp3 = 2AC + BB			(62 = 61 + 60)
;; temp2 = 2AB				(61)
;; temp1 = AA				(60)

;; If fac_size is 75 bits or more (the remainder is 76 bits or more), then
;; the squared remainder is more than 150 bits.  In this case, get the very
;; highest bits and multiply them by 2^120 mod factor.  We'll then add
;; these back into the squared remainder as part of our effort to create a
;; 120 bit result.
;;
;; AA_HIGH = temp1 >> 30
;; temp4 += AA_HIGH * TWO_120_MODF3
;; temp3 += AA_HIGH * TWO_120_MODF2
;; temp2 += AA_HIGH * TWO_120_MODF1
;; temp1 = AA & 0x3FFFFFFF

;; Combine temp1 through temp5 into 3 temporaries
;;
;; temp5 += (temp4 & 0x3FFFFFFF) << 30
;; temp3 += (temp4 >> 30) + (temp2 & 0x3FFFFFFF) << 30
;; temp1 += temp2 >> 30
;;
;; While doing the above we interleave work from the paragraph below.
;;
;; Our value is now at most 150 bits.  Take the upper 30 bits and multiply
;; them by 2^120 mod factor.  Then add these back into the squared remainder
;; to create a 120 bit result.
;;
;; Make sure we add any bits from temp3 above 2^60 into temp1
;; temp5 += temp1 * TWO_120_MODF3
;; temp4 += temp1 * TWO_120_MODF2
;; temp3 += temp1 * TWO_120_MODF1
;;
;; Finally, we do a lot of instruction interleaving in hopes of making it
;; easier for the CPU to schedule the instructions.  The clock counts assume
;; near optimal P4 instruction scheduling for the critical path.

;; Break remainder in xmm4,xmm5 into 30 bit quantities.
;;
;; RES3 = temp2 & 0x3FFFFFFF		(30)
;; temp2 >>= 30				(signed 32)
;; RES2 = temp2 & 0x3FFFFFFF		(30)
;; temp2 >>= 30				(signed 2)
;; temp1 += temp2
;; RES1 = temp1 & 0x0FFFFFFF		(28)

	movdqa	xmm2, xmm5
	pand	xmm2, xmm15		;; RES3 = temp2 & 0x3FFFFFFF
	psrlq	xmm5, 30		;; temp2 >>= 30
	movdqa	xmm1, xmm5
	pand	xmm1, xmm15		;; RES2 = temp2 & 0x3FFFFFFF
	psrlq	xmm5, 2			;; temp2 >>= 2
	psrad	xmm5, 28		;; temp2 >>= 28
	paddd	xmm4, xmm5		;; temp1 += temp2
	movdqa	xmm0, xmm4
	pand	xmm0, XMMWORD PTR XMM_BITS28 ;; RES1 = temp1 & 0x0FFFFFFF

	movdqa	xmm8, xmm11
	pand	xmm8, xmm15		;; RES3 = temp2 & 0x3FFFFFFF
	psrlq	xmm11, 30		;; temp2 >>= 30
	movdqa	xmm7, xmm11
	pand	xmm7, xmm15		;; RES2 = temp2 & 0x3FFFFFFF
	psrlq	xmm11, 2		;; temp2 >>= 2
	psrad	xmm11, 28		;; temp2 >>= 28
	paddd	xmm10, xmm11		;; temp1 += temp2
	movdqa	xmm6, xmm10
	pand	xmm6, XMMWORD PTR XMM_BITS28 ;; RES1 = temp1 & 0x0FFFFFFF

					;; A = xmm0, B = xmm1, C = xmm2

	movdqa	xmm3, xmm0		;; Copy A
	paddq	xmm3, xmm0		;; 2A
	movdqa	xmm5, xmm1		;; Copy B
	paddq	xmm5, xmm1		;; 2B
	pmuludq	xmm1, xmm1		;; temp3 = BB
	movdqa	xmm4, xmm0		;; Copy A
	pmuludq	xmm4, xmm0		;; temp1 = AA
	pmuludq	xmm3, xmm2		;; 2AC = 2A * C
	pmuludq	xmm0, xmm5		;; temp2 = 2AB = A * 2B
	pmuludq	xmm5, xmm2		;; temp4 = 2BC = 2B * C
	pmuludq	xmm2, xmm2		;; temp5 = CC
	paddq	xmm1, xmm3		;; temp3 = BB + 2AC

	movdqa	xmm9, xmm6		;; Copy A
	paddq	xmm9, xmm6		;; 2A
	movdqa	xmm11, xmm7		;; Copy B
	paddq	xmm11, xmm7		;; 2B
	pmuludq	xmm7, xmm7		;; temp3 = BB
	movdqa	xmm10, xmm6		;; Copy A
	pmuludq	xmm10, xmm6		;; temp1 = AA
	pmuludq	xmm9, xmm8		;; 2AC = 2A * C
	pmuludq	xmm6, xmm11		;; temp2 = 2AB = A * 2B
	pmuludq	xmm11, xmm8		;; temp4 = 2BC = 2B * C
	pmuludq	xmm8, xmm8		;; temp5 = CC
	paddq	xmm7, xmm9		;; temp3 = BB + 2AC

	IF fac_size GE 75
	movdqa	xmm3, xmm4		;; Copy AA
	psrlq	xmm3, 30		;; AA_HIGH = Upper 30 bits of AA
	pmuludq xmm3, XMMWORD PTR XMM_TWO_120_MODF2 ;; AA_HIGH * 2_120_MODF2
	paddq	xmm1, xmm3		;; temp3 += AA_HIGH * 2_120_MODF2
	movdqa	xmm3, xmm4		;; Copy AA
	psrlq	xmm3, 30		;; AA_HIGH = Upper 30 bits of AA
	pmuludq xmm3, XMMWORD PTR XMM_TWO_120_MODF1 ;; AA_HIGH * 2_120_MODF1
	paddq	xmm0, xmm3		;; temp2 += AA_HIGH * 2_120_MODF1
	movdqa	xmm3, xmm4		;; Copy AA
	psrlq	xmm3, 30		;; AA_HIGH = Upper 30 bits of AA
	pmuludq xmm3, XMMWORD PTR XMM_TWO_120_MODF3 ;; AA_HIGH * 2_120_MODF3
	paddq	xmm5, xmm3		;; temp4 += AA_HIGH * 2_120_MODF3
	pand	xmm4, xmm15		;; temp1 = Low 30 bits of AA

	movdqa	xmm9, xmm10		;; Copy AA
	psrlq	xmm9, 30		;; AA_HIGH = Upper 30 bits of AA
	pmuludq xmm9, XMMWORD PTR XMM_TWO_120_MODF2a ;; AA_HIGH * 2_120_MODF2
	paddq	xmm7, xmm9		;; temp3 += AA_HIGH * 2_120_MODF2
	movdqa	xmm9, xmm10		;; Copy AA
	psrlq	xmm9, 30		;; AA_HIGH = Upper 30 bits of AA
	pmuludq xmm9, XMMWORD PTR XMM_TWO_120_MODF1a ;; AA_HIGH * 2_120_MODF1
	paddq	xmm6, xmm9		;; temp2 += AA_HIGH * 2_120_MODF1
	movdqa	xmm9, xmm10		;; Copy AA
	psrlq	xmm9, 30		;; AA_HIGH = Upper 30 bits of AA
	pmuludq xmm9, XMMWORD PTR XMM_TWO_120_MODF3a ;; AA_HIGH * 2_120_MODF3
	paddq	xmm11, xmm9		;; temp4 += AA_HIGH * 2_120_MODF3
	pand	xmm10, xmm15		;; temp1 = Low 30 bits of AA
	ENDIF

	movdqa	xmm3, xmm1		;; Copy temp3
	psrlq	xmm3, 30		;; Top bits of temp3
	pand	xmm1, xmm15		;; temp3 = temp3 & 0x3FFFFFFF
	paddq	xmm0, xmm3		;; Add temp3 top bits into temp2
	movdqa	xmm3, xmm0		;; Copy temp2
	psrlq	xmm3, 30		;; Top bits of temp2
	pand	xmm0, xmm15		;; temp2 & 0x3FFFFFFF
	psllq	xmm0, 30		;; temp2 << 30
	paddd	xmm4, xmm3		;; Add temp2 top bits into temp1
	por	xmm1, xmm0		;; temp3 += temp2 << 30
	movdqa	xmm0, xmm4		;; Copy temp1
	pmuludq xmm0, XMMWORD PTR XMM_TWO_120_MODF2 ;; temp1 * TWO_120_MODF2
	movdqa	xmm3, xmm4		;; Copy temp1
	pmuludq xmm3, XMMWORD PTR XMM_TWO_120_MODF1 ;; temp1 * TWO_120_MODF1
	pmuludq xmm4, XMMWORD PTR XMM_TWO_120_MODF3 ;; temp1 * TWO_120_MODF3
	paddq	xmm5, xmm0		;; temp4 += temp1 * TWO_120_MODF2
	paddq	xmm1, xmm3		;; temp3 += temp1 * TWO_120_MODF1
	movdqa	xmm0, xmm5		;; Copy temp4
	psrlq	xmm0, 30		;; temp4 >> 30
	pand	xmm5, xmm15		;; Lower 30 bits of temp4
	psllq	xmm5, 30		;; (temp4 & 0x3FFFFFFF) << 30
	paddq	xmm2, xmm4		;; temp5 += temp1 * TWO_120_MODF3
	paddq	xmm1, xmm0		;; temp3 += temp4 >> 30
	paddq	xmm2, xmm5		;; temp5 += (temp4&0x3FFFFFFF)<<30
					;; Result in xmm1,xmm2

	movdqa	xmm9, xmm7		;; Copy temp3
	psrlq	xmm9, 30		;; Top bits of temp3
	pand	xmm7, xmm15		;; temp3 = temp3 & 0x3FFFFFFF
	paddq	xmm6, xmm9		;; Add temp3 top bits into temp2
	movdqa	xmm9, xmm6		;; Copy temp2
	psrlq	xmm9, 30		;; Top bits of temp2
	pand	xmm6, xmm15		;; temp2 & 0x3FFFFFFF
	psllq	xmm6, 30		;; temp2 << 30
	paddd	xmm10, xmm9		;; Add temp2 top bits into temp1
	por	xmm7, xmm6		;; temp3 += temp2 << 30
	movdqa	xmm6, xmm10		;; Copy temp1
	pmuludq xmm6, XMMWORD PTR XMM_TWO_120_MODF2a ;; temp1 * TWO_120_MODF2
	movdqa	xmm9, xmm10		;; Copy temp1
	pmuludq xmm9, XMMWORD PTR XMM_TWO_120_MODF1a ;; temp1 * TWO_120_MODF1
	pmuludq xmm10, XMMWORD PTR XMM_TWO_120_MODF3a ;; temp1 * TWO_120_MODF3
	paddq	xmm11, xmm6		;; temp4 += temp1 * TWO_120_MODF2
	paddq	xmm7, xmm9		;; temp3 += temp1 * TWO_120_MODF1
	movdqa	xmm6, xmm11		;; Copy temp4
	psrlq	xmm6, 30		;; temp4 >> 30
	pand	xmm11, xmm15		;; Lower 30 bits of temp4
	psllq	xmm11, 30		;; (temp4 & 0x3FFFFFFF) << 30
	paddq	xmm8, xmm10		;; temp5 += temp1 * TWO_120_MODF3
	paddq	xmm7, xmm6		;; temp3 += temp4 >> 30
	paddq	xmm8, xmm11		;; temp5 += (temp4&0x3FFFFFFF)<<30
					;; Result in xmm7,xmm8

;; Squared remainder reduced to 120 bits is in xmm1, xmm2 (call them R1,R2).
;; We'll subtract  quotient * factor from this later, so save R1 for later use.

;; Break the upper 60 bits (R1) into 30 bit chunks (U1,U2) and multiply
;; it by 60 bits of 1/factor (I1,I2).  This will give us the quotient
;; (shifted left somewhat).  We only need to compute the upper 60 bits of
;; the quotient since 120 - fac_size is less than 60.
;;
;; Error analysis:  Shifted quotient is up to 5.999... too small.  One for
;; low bits of temp2, one for uncomputed I2*U2, one for if uncomputed I3 is
;; large, one for if top 30 bits of temp5 (U3) is large, and two for carries
;; that may have been generated by additions of temp5 and not added into U2.
;; This is why we must shift off 3 bits for an accurate quotient, meaning
;; a quotient accuracy of 57 bits.  Thus, using the formula described in the
;; next section for computing the shift amount, the minimum trial factor size
;; this code can handle is 64 bits.
;;
;; temp2 = I2*U1 + I1*U2	(61)
;; temp1 = I1*U1		(60)
;;
;; temp1 += temp2 >> 30		(61)

	movdqa	xmm0, xmm1		;; Copy R1
	psrlq	xmm0, 30		;; U1 = upper bits of R1
	movdqa	xmm3, xmm1		;; Copy R1
	pand	xmm3, xmm15		;; U2 = lower bits of R1
	movdqa	xmm4, xmm0		;; Copy U1
	pmuludq xmm4, XMMWORD PTR XMM_I2 ;; temp2 = I2 * U1
	movdqa	xmm5, xmm3		;; Copy U2
	pmuludq xmm5, XMMWORD PTR XMM_I1 ;; I1 * U2
	pmuludq xmm0, XMMWORD PTR XMM_I1 ;; temp1 = I1 * U1
	paddq	xmm4, xmm5		;; temp2 += I1 * U2
	psrlq	xmm4, 30		;; temp2 >> 30
	paddq	xmm0, xmm4		;; temp1 += temp2 >> 30

	movdqa	xmm6, xmm7		;; Copy R1
	psrlq	xmm6, 30		;; U1 = upper bits of R1
	movdqa	xmm9, xmm7		;; Copy R1
	pand	xmm9, xmm15		;; U2 = lower bits of R1
	movdqa	xmm10, xmm6		;; Copy U1
	pmuludq xmm10, XMMWORD PTR XMM_I2a ;; temp2 = I2 * U1
	movdqa	xmm11, xmm9		;; Copy U2
	pmuludq xmm11, XMMWORD PTR XMM_I1a ;; I1 * U2
	pmuludq xmm6, XMMWORD PTR XMM_I1a ;; temp1 = I1 * U1
	paddq	xmm10, xmm11		;; temp2 += I1 * U2
	psrlq	xmm10, 30		;; temp2 >> 30
	paddq	xmm6, xmm10		;; temp1 += temp2 >> 30

;; 60 bits of shifted quotient are now in xmm0.  R1,R2 are in xmm1,xmm2.

;; Shift the quotient right to get the actual quotient.
;; Define BS (bit shift amount). This is the amount to shift the quotient
;; right to compute the quotient.  BS = 60 - (120 - fac_size + 1).
;; For example, if fac_size = 75, R1/R2 is 120 bits, quotient will
;; be 120-75+1=46 bits, temp1 is 60 bits, BS is 60-46=14 bits.
;;
;; Now shift right to get the quotient:
;;
;; Q2 = (temp1 >> BS) & 0x3FFFFFFF			(30)
;; Q1 = temp1 >> (30 + BS)

	psrlq	xmm0, XMMWORD PTR XMM_BS ;; temp1 >> BS
	movdqa	xmm3, xmm0
	psrlq	xmm3, 30		;; Q1 = temp1 >> (30 + BS)
	pand	xmm0, xmm15		;; Q2 = (temp1 >> BS) & 0x3FFFFFFF

	psrlq	xmm6, XMMWORD PTR XMM_BS ;; temp1 >> BS
	movdqa	xmm9, xmm6
	psrlq	xmm9, 30		;; Q1 = temp1 >> (30 + BS)
	pand	xmm6, xmm15		;; Q2 = (temp1 >> BS) & 0x3FFFFFFF

;; Quotient is now in xmm3, xmm0.  R1,R2 are in xmm1, xmm2.

;; Multiply quotient Q1,Q2 by factor F1,F2,F3
;; We only need to compute bottom 90 bits.
;;
;; temp3 = F3*Q2				(60)
;; temp2 = F3*Q1 + F2*Q2			(61)
;; temp1 = F2*Q1 + F1*Q2			(61)
;;
;; QF2 = temp3 + ((temp2 & 0x3FFFFFFF)<<30)	(61)
;; QF1 = temp1 + (temp2 >> 30)			(61)

	movdqa	xmm4, xmm3
	pmuludq xmm4, XMMWORD PTR XMM_F3 ;; temp2 = Q1 * F3
	movdqa	xmm5, xmm0
	pmuludq xmm5, XMMWORD PTR XMM_F2 ;; Q2 * F2
	pmuludq xmm3, XMMWORD PTR XMM_F2 ;; temp1 = Q1 * F2
	paddq	xmm4, xmm5		;; temp2 += Q2 * F2
	movdqa	xmm5, xmm0
	pmuludq xmm5, XMMWORD PTR XMM_F1 ;; Q2 * F1
	pmuludq xmm0, XMMWORD PTR XMM_F3 ;; temp3 = Q2 * F3
	paddd	xmm3, xmm5		;; temp1 += Q2 * F1
	movdqa	xmm5, xmm4
	pand	xmm5, xmm15		;; temp2 & 0x3FFFFFFF
	psrlq	xmm4, 30		;; temp2 >> 30
	psllq	xmm5, 30		;; (temp2 & 0x3FFFFFFF) << 30
	paddq	xmm0, xmm5		;; QF2 = temp3 + ((temp2 & 0x3FFFFFFF)<<30)
	paddd	xmm3, xmm4		;; QF1 = temp1 + (temp2 >> 30)

	movdqa	xmm10, xmm9
	pmuludq xmm10, XMMWORD PTR XMM_F3a ;; temp2 = Q1 * F3
	movdqa	xmm11, xmm6
	pmuludq xmm11, XMMWORD PTR XMM_F2a ;; Q2 * F2
	pmuludq xmm9, XMMWORD PTR XMM_F2a ;; temp1 = Q1 * F2
	paddq	xmm10, xmm11		;; temp2 += Q2 * F2
	movdqa	xmm11, xmm6
	pmuludq xmm11, XMMWORD PTR XMM_F1a ;; Q2 * F1
	pmuludq xmm6, XMMWORD PTR XMM_F3a ;; temp3 = Q2 * F3
	paddd	xmm9, xmm11		;; temp1 += Q2 * F1
	movdqa	xmm11, xmm10
	pand	xmm11, xmm15		;; temp2 & 0x3FFFFFFF
	psrlq	xmm10, 30		;; temp2 >> 30
	psllq	xmm11, 30		;; (temp2 & 0x3FFFFFFF) << 30
	paddq	xmm6, xmm11		;; QF2 = temp3 + ((temp2 & 0x3FFFFFFF)<<30)
	paddd	xmm9, xmm10		;; QF1 = temp1 + (temp2 >> 30)

;; Quotient * factor is now in xmm3, xmm0.  R1,R2 are in xmm1, xmm2.

;; Subtract QF1,QF2 from R1,R2
;;
;; temp2 = R2 - QF2			(signed 62)
;; temp1 = R1 - QF1			(signed 62)

	movdqa	xmm5, xmm2
	psubq	xmm5, xmm0		;; temp2 = R2 - QF2
	movdqa	xmm4, xmm1
	psubd	xmm4, xmm3		;; temp1 = R1 - QF1

	movdqa	xmm11, xmm8
	psubq	xmm11, xmm6		;; temp2 = R2 - QF2
	movdqa	xmm10, xmm7
	psubd	xmm10, xmm9		;; temp1 = R1 - QF1

;; If mul by 2 is required:  mul temp1, temp2 by 2 here

	movdqu	xmm0, XMMWORD PTR XMM_SHIFTER[rdi*8]
	psllq	xmm5, xmm0		;; temp2 *= 1 or 2
	psllq	xmm4, xmm0		;; temp1 *= 1 or 2

	psllq	xmm11, xmm0		;; temp2 *= 1 or 2
	psllq	xmm10, xmm0		;; temp1 *= 1 or 2
	ENDM

;; Check if remainder in xmm4,xmm5 matches factor + 1 indicating we found a factor!

sse2_compare_part1 MACRO
	psubq	xmm5, XMMWORD PTR YMM_ONE ;; Subtract one to compare result to factor
	movdqa	xmm2, xmm5
	pand	xmm2, xmm15		;; RES3 = temp2 & 0x3FFFFFFF
	psubq	xmm11, XMMWORD PTR YMM_ONE ;; Subtract one to compare result to factor
	movdqa	xmm8, xmm11
	pand	xmm8, xmm15		;; RES3 = temp2 & 0x3FFFFFFF
	ENDM

sse2_compare_part2 MACRO
	psrlq	xmm5, 30		;; temp2 >>= 30
	movdqa	xmm1, xmm5
	pand	xmm1, xmm15		;; RES2 = temp2 & 0x3FFFFFFF
	psrlq	xmm5, 2			;; temp2 >>= 2
	psrad	xmm5, 28		;; temp2 >>= 28
	paddd	xmm4, xmm5		;; temp1 += temp2
	movdqa	xmm0, xmm4
	pand	xmm0, XMMWORD PTR XMM_BITS28 ;; RES1 = temp1 & 0x0FFFFFFF
	psrlq	xmm11, 30		;; temp2 >>= 30
	movdqa	xmm7, xmm11
	pand	xmm7, xmm15		;; RES2 = temp2 & 0x3FFFFFFF
	psrlq	xmm11, 2		;; temp2 >>= 2
	psrad	xmm11, 28		;; temp2 >>= 28
	paddd	xmm10, xmm11		;; temp1 += temp2
	movdqa	xmm6, xmm10
	pand	xmm6, XMMWORD PTR XMM_BITS28 ;; RES1 = temp1 & 0x0FFFFFFF
	ENDM




;;****************************************************************
;; The AVX2 versions of the above macros (four factors at a time)
;;****************************************************************


; This macro does some initialization work and then performs a subset of the
; main loop working on the initial value.  This subset avoids the squaring of the
; remainder.  The initial value can be up to 60 bits larger than the factor
; size.

avx2_fac_initval MACRO
	LOCAL	retry, invfac_ok

;; Preload constants

	vmovdqa	ymm15, YMMWORD PTR YMM_BITS30
	vmovdqa	ymm13, YMMWORD PTR YMM_F2
	vmovdqa	ymm14, YMMWORD PTR YMM_F3

;; Break 60 bits of factor inverse into 30 bit chunks

	vmovdqa	ymm1, YMMWORD PTR YMM_INVFAC			;; Load factor inverse
retry:	vpsrlq	ymm3, ymm1, 3					;; Convert 63 bit inverse to 60 bits
	vpand	ymm0, ymm3, ymm15				;; Q2 = bottom bits of quotient
	vpsrlq	ymm3, ymm3, 30					;; Q1 = top bits of quotient
	vmovdqa	YMMWORD PTR YMM_I2, ymm0			;; The quotient is the factor inverse, save as YMM_I1, YMM_I2
	vmovdqa	YMMWORD PTR YMM_I1, ymm3

;; Make sure the quotient isn't one too large.
;; Multiply quotient Q1,Q2 by factor F1,F2,F3
;; We only need to compute bottom 90 bits.
;;
;; temp3 = F3*Q2				(60)
;; temp2 = F3*Q1 + F2*Q2			(61)
;; temp1 = F2*Q1 + F1*Q2			(61)
;;
;; QF2 = temp3 + ((temp2 & 0x3FFFFFFF)<<30)	(61)
;; QF1 = temp1 + (temp2 >> 30)			(61)

	vpmuludq ymm4, ymm3, ymm14				;; temp2 = Q1 * F3
	vpmuludq ymm2, ymm0, ymm13				;; Q2 * F2
	vpmuludq ymm3, ymm3, ymm13				;; temp1 = Q1 * F2
	vpmuludq ymm5, ymm0, YMMWORD PTR YMM_F1			;; Q2 * F1
	vpmuludq ymm0, ymm0, ymm14				;; temp3 = Q2 * F3
	vpaddq	ymm4, ymm4, ymm2				;; temp2 += Q2 * F2
	vpaddd	ymm3, ymm3, ymm5				;; temp1 += Q2 * F1
	vpand	ymm5, ymm4, ymm15				;; temp2 & 0x3FFFFFFF
	vpsrlq	ymm4, ymm4, 30					;; temp2 >> 30
	vpsllq	ymm5, ymm5, 30					;; (temp2 & 0x3FFFFFFF) << 30
	vpaddq	ymm0, ymm0, ymm5				;; QF2 = temp3 + ((temp2 & 0x3FFFFFFF)<<30)
	vpaddd	ymm3, ymm3, ymm4				;; QF1 = temp1 + (temp2 >> 30)

;; If QF2 is above 2^60, then add these carry bits into QF1

	vpsrlq	ymm0, ymm0, 60
	vpaddd	ymm3, ymm3, ymm0				;; QF1 = QF1 + (QF2 >> 60)

;; Validate QF1.  The top bits should be ones (zeros indicate Q*F exceeded
;; a power of two by a little bit).  Detect these cases, decrement the
;; inverse factor and try again.

	vpand	ymm3, ymm3, YMMWORD PTR YMM_28TH_BIT		;; Test for positive dword values in QF1 (test 28th bit)
	vpxor	ymm2, ymm2, ymm2
	vpcmpeqq ymm3, ymm3, ymm2
	vpmovmskb rdx, ymm3
	and	edx, 0FFFFFFFFh					;; See if INVFAC values changed
	jz	short invfac_ok					;; Jump if no INVFACs needed to be adjusted
	vmovdqa	ymm1, YMMWORD PTR YMM_INVFAC			;; Load factor inverse
	vpaddq	ymm1, ymm1, ymm3				;; Add -1 to INVFAC where QF1 dword val was positive
	vmovdqa	YMMWORD PTR YMM_INVFAC, ymm1			;; Store factor inverse
	jmp	retry						;; Reprocess new INVFAC values
invfac_ok:

;; Compute quotient for 2^120 / factor call it Q1,Q2.

	vmovdqa	ymm1, YMMWORD PTR YMM_INVFAC			;; Load factor inverse
	vpsrlq	ymm3, ymm1, XMMWORD PTR XMM_INIT120BS		;; Compute 2^120 / factor
	vpand	ymm0, ymm3, ymm15				;; Q2 = bottom bits of quotient
	vpsrlq	ymm3, ymm3, 30					;; Q1 = top bits of quotient

;; Multiply quotient Q1,Q2 by factor F1,F2,F3
;; We only need to compute bottom 90 bits.
;;
;; temp3 = F3*Q2				(60)
;; temp2 = F3*Q1 + F2*Q2			(61)
;; temp1 = F2*Q1 + F1*Q2			(61)
;;
;; QF2 = temp3 + ((temp2 & 0x3FFFFFFF)<<30)	(61)
;; QF1 = temp1 + (temp2 >> 30)			(61)

	vpmuludq ymm4, ymm3, ymm14				;; temp2 = Q1 * F3
	vpmuludq ymm2, ymm0, ymm13				;; Q2 * F2
	vpmuludq ymm3, ymm3, ymm13				;; temp1 = Q1 * F2
	vpmuludq ymm5, ymm0, YMMWORD PTR YMM_F1			;; Q2 * F1
	vpmuludq ymm0, ymm0, ymm14				;; temp3 = Q2 * F3
	vpaddq	ymm4, ymm4, ymm2				;; temp2 += Q2 * F2
	vpaddd	ymm3, ymm3, ymm5				;; temp1 += Q2 * F1
	vpand	ymm5, ymm4, ymm15				;; temp2 & 0x3FFFFFFF
	vpsrlq	ymm4, ymm4, 30					;; temp2 >> 30
	vpsllq	ymm5, ymm5, 30					;; (temp2 & 0x3FFFFFFF) << 30
	vpaddq	ymm0, ymm0, ymm5				;; QF2 = temp3 + ((temp2 & 0x3FFFFFFF)<<30)
	vpaddd	ymm3, ymm3, ymm4				;; QF1 = temp1 + (temp2 >> 30)

;; Subtract QF1,QF2 from zero to get 90 bits of 2^120 mod factor.

	vpxor	ymm2, ymm2, ymm2
	vpsubq	ymm5, ymm2, ymm0				;; temp2 = Low 60 bits of remainder
	vpsubd	ymm4, ymm2, ymm3				;; temp1 = Low 30 bits of remainder

;; Break remainder in ymm4,ymm5 into 30 bit quantities and save
;;
;; RES3 = temp2 & 0x3FFFFFFF		(30)
;; temp2 >>= 30				(signed 32)
;; RES2 = temp2 & 0x3FFFFFFF		(30)
;; temp2 >>= 30				(signed 2)
;; temp1 += temp2
;; RES1 = temp1 & 0x0FFFFFFF		(28)

	vpand	ymm2, ymm5, ymm15				;; RES3 = temp2 & 0x3FFFFFFF
	vpsrlq	ymm5, ymm5, 30					;; temp2 >>= 30
	vpand	ymm1, ymm5, ymm15				;; RES2 = temp2 & 0x3FFFFFFF
	vpsrlq	ymm5, ymm5, 2					;; temp2 >>= 2
	vpsrad	ymm5, ymm5, 28					;; temp2 >>= 28
	vpaddd	ymm4, ymm4, ymm5				;; temp1 += temp2
	vpand	ymm0, ymm4, YMMWORD PTR YMM_BITS28		;; RES1 = temp1 & 0x0FFFFFFF
	vmovdqa YMMWORD PTR YMM_TWO_120_MODF3, ymm2
	vmovdqa YMMWORD PTR YMM_TWO_120_MODF2, ymm1
	vmovdqa YMMWORD PTR YMM_TWO_120_MODF1, ymm0

;; 1/factor is the shifted quotient.  Shift 1/factor right to get the actual
;; quotient.  YMM_INITBS is the shift amount defined by this formula:
;;	INITBS = 120 - bit_length(initval)
;;
;; Q2 = (temp1 >> INITBS) & 0x3FFFFFFF			(30)
;; Q1 = temp1 >> (30 + INITBS)

	vmovdqa	ymm1, YMMWORD PTR YMM_INVFAC			;; Load factor inverse
	vpsrlq	ymm0, ymm1, XMMWORD PTR XMM_INITBS		;; temp1 = YMM_INVFAC >> INITBS
	vpsrlq	ymm3, ymm0, 30					;; Q1 = temp1 >> (30 + BS)
	vpand	ymm0, ymm0, ymm15				;; Q2 = (temp1 >> BS) & 0x3FFFFFFF

;; Quotient is now in ymm3, ymm0.

;; Multiply quotient Q1,Q2 by factor F1,F2,F3
;; We only need to compute bottom 90 bits.
;;
;; temp3 = Q2*F3				(60)
;; temp2 = Q1*F3 + Q2*F2			(61)
;; temp1 = Q1*F2 + Q2*F1			(61)
;;
;; QF2 = temp3 + ((temp2 & 0x3FFFFFFF)<<30)	(61)
;; QF1 = temp1 + (temp2 >> 30)			(61)

	vpmuludq ymm4, ymm3, ymm14				;; temp2 = Q1 * F3
	vpmuludq ymm2, ymm0, ymm13				;; Q2 * F2
	vpmuludq ymm3, ymm3, ymm13				;; temp1 = Q1 * F2
	vpmuludq ymm5, ymm0, YMMWORD PTR YMM_F1			;; Q2 * F1
	vpmuludq ymm0, ymm0, ymm14				;; temp3 = Q2 * F3
	vpaddq	ymm4, ymm4, ymm2				;; temp2 += Q2 * F2
	vpaddd	ymm3, ymm3, ymm5				;; temp1 += Q2 * F1
	vpand	ymm5, ymm4, ymm15				;; temp2 & 0x3FFFFFFF
	vpsrlq	ymm4, ymm4, 30					;; temp2 >> 30
	vpsllq	ymm5, ymm5, 30					;; (temp2 & 0x3FFFFFFF) << 30
	vpaddq	ymm0, ymm0, ymm5				;; QF2 = temp3 + ((temp2 & 0x3FFFFFFF)<<30)
	vpaddd	ymm3, ymm3, ymm4				;; QF1 = temp1 + (temp2 >> 30)

;; Load initial value

	vpxor	ymm2, ymm2, ymm2				;; Clear R2
	vmovdqa	ymm1, YMMWORD PTR YMM_INITVAL			;; Load R1

;; Quotient * factor is now in ymm3, ymm0.  R1,R2 are in ymm1, ymm2.

;; Subtract QF1,QF2 from R1,R2
;;
;; temp2 = R2 - QF2			(signed 62)
;; temp1 = R1 - QF1			(signed 62)

	vpsubq	ymm5, ymm2, ymm0				;; temp2 = R2 - QF2
	vpsubd	ymm4, ymm1, ymm3				;; temp1 = R1 - QF1

;; Break remainder in ymm4,ymm5 into 30 bit quantities.
;;
;; RES3 = temp2 & 0x3FFFFFFF		(30)
;; temp2 >>= 30				(signed 32)
;; RES2 = temp2 & 0x3FFFFFFF		(30)
;; temp2 >>= 30				(signed 2)
;; temp1 += temp2
;; RES1 = temp1 & 0x0FFFFFFF		(28)

	vpand	ymm2, ymm5, ymm15				;; RES3 = temp2 & 0x3FFFFFFF
	vpsrlq	ymm5, ymm5, 30					;; temp2 >>= 30
	vpand	ymm1, ymm5, ymm15				;; RES2 = temp2 & 0x3FFFFFFF
	vpsrlq	ymm5, ymm5, 2					;; temp2 >>= 2
	vpsrad	ymm5, ymm5, 28					;; temp2 >>= 28
	vpaddd	ymm4, ymm4, ymm5				;; temp1 += temp2
	vpand	ymm0, ymm4, YMMWORD PTR YMM_BITS28		;; RES1 = temp1 & 0x0FFFFFFF


	vmovdqa	ymm12, YMMWORD PTR YMM_I1
	vmovdqa	ymm9, YMMWORD PTR YMM_TWO_120_MODF1
	vmovdqa	ymm10, YMMWORD PTR YMM_TWO_120_MODF2
	vmovdqa	ymm11, YMMWORD PTR YMM_TWO_120_MODF3
	ENDM


avx2_fac MACRO fac_size

;; Note that at the beginning of the ymm0, ymm1, ymm2 contains the
;; remainder to square.  This remainder can be one more bit than the
;; fac_size.  This is because we do not perform a modulo after the optional
;; multiply by two at the end of this macro.

;; Also note that in very rare cases the remainder could be two more bits
;; than fac_size.  Since the calculated quotient could be one too small,
;; the maximum remainder is 2 * (factor + epsilon).  This extremely rare
;; extra bit should not be a problem as the code below uses 30 bit integers
;; and it shouldn't be a problem handling one more bit.

;; In the comments of this code, we'll assume fac_size is 89 bits and the
;; input remainder is 90 bits (30 bits in each ymm register).

;; rem = A,B,C (30,30,30 bits unsigned)
;; Square rem producing a 180 bit result stored in temp1, temp3, temp5
;; temp5 = CC				(60)
;; temp4 = 2BC				(61)
;; temp3 = 2AC + BB			(62 = 61 + 60)
;; temp2 = 2AB				(61)
;; temp1 = AA				(60)

;; If fac_size is 75 bits or more (the remainder is 76 bits or more), then
;; the squared remainder is more than 150 bits.  In this case, get the very
;; highest bits and multiply them by 2^120 mod factor.  We'll then add
;; these back into the squared remainder as part of our effort to create a
;; 120 bit result.
;;
;; AA_HIGH = temp1 >> 30
;; temp4 += AA_HIGH * TWO_120_MODF3
;; temp3 += AA_HIGH * TWO_120_MODF2
;; temp2 += AA_HIGH * TWO_120_MODF1
;; temp1 = AA & 0x3FFFFFFF

;; Combine temp1 through temp5 into 3 temporaries
;;
;; temp5 += (temp4 & 0x3FFFFFFF) << 30
;; temp3 += (temp4 >> 30) + (temp2 & 0x3FFFFFFF) << 30
;; temp1 += temp2 >> 30
;;
;; While doing the above we interleave work from the paragraph below.
;;
;; Our value is now at most 150 bits.  Take the upper 30 bits and multiply
;; them by 2^120 mod factor.  Then add these back into the squared remainder
;; to create a 120 bit result.
;;
;; Make sure we add any bits from temp3 above 2^60 into temp1
;; temp5 += temp1 * TWO_120_MODF3
;; temp4 += temp1 * TWO_120_MODF2
;; temp3 += temp1 * TWO_120_MODF1
;;
;; Finally, we do a lot of instruction interleaving in hopes of making it
;; easier for the CPU to schedule the instructions.  The clock counts assume
;; my understanding of Haswell instruction scheduling is correct.

					;; A = ymm0, B = ymm1, C = ymm2

	vpaddq	ymm3, ymm0, ymm0				;;	1	2A
	vpaddq	ymm5, ymm1, ymm1				;;	1	2B
	vpmuludq ymm1, ymm1, ymm1				;;1-5		temp3 = BB
	vpmuludq ymm4, ymm0, ymm0				;;1-5		temp1 = AA
	vpmuludq ymm3, ymm3, ymm2				;;2-6		2AC = 2A * C
	vpmuludq ymm0, ymm0, ymm5				;;2-6		temp2 = 2AB = A * 2B
	vpmuludq ymm5, ymm5, ymm2				;;3-7		temp4 = 2BC = 2B * C
	vpmuludq ymm2, ymm2, ymm2				;;3-7		temp5 = CC
	vpaddq	ymm1, ymm1, ymm3				;;	7	temp3 = BB + 2AC
								;;		temp1-5 = ymm4,0,1,5,2

	IF fac_size GE 75

	vpsrlq	ymm3, ymm4, 30					;;6		AA_HIGH = Upper 30 bits of AA
;; or if a reg is free, preserve ymm3
	vpmuludq ymm3, ymm3, ymm10				;;7-11		AA_HIGH * 2_120_MODF2
	vpaddq	ymm1, ymm1, ymm3				;;12		temp3 += AA_HIGH * 2_120_MODF2

	vpsrlq	ymm3, ymm4, 30					;;6		AA_HIGH = Upper 30 bits of AA
;; or if a reg is free, preserve ymm3
	vpmuludq ymm3, ymm3, ymm9				;;7-11		AA_HIGH * 2_120_MODF1
	vpaddq	ymm0, ymm0, ymm3				;;12		temp2 += AA_HIGH * 2_120_MODF1

	vpsrlq	ymm3, ymm4, 30					;;8		AA_HIGH = Upper 30 bits of AA
	vpmuludq ymm3, ymm3, ymm11				;;9-13		AA_HIGH * 2_120_MODF3
	vpaddq	ymm5, ymm5, ymm3				;;14		temp4 += AA_HIGH * 2_120_MODF3

	vpand	ymm4, ymm4, ymm15				;; 8		temp1 = Low 30 bits of AA

	ENDIF

	vpsrlq	ymm3, ymm1, 30					;;8		Top bits of temp3
	vpand	ymm1, ymm1, ymm15				;; 8		temp3 = temp3 & 0x3FFFFFFF
	vpaddq	ymm0, ymm0, ymm3				;;9		Add temp3 top bits into temp2
	vpsrlq	ymm3, ymm0, 30					;;10		Top bits of temp2
	vpand	ymm0, ymm0, ymm15				;; 10		temp2 & 0x3FFFFFFF
	vpsllq	ymm0, ymm0, 30					;;11		temp2 << 30
	vpaddd	ymm4, ymm4, ymm3				;; 11		Add temp2 top bits into temp1
	vpor	ymm1, ymm1, ymm0				;;12		temp3 += temp2 << 30
	vpmuludq ymm0, ymm4, ymm10				;;12-16		temp1 * TWO_120_MODF2
	vpmuludq ymm3, ymm4, ymm9				;;12-16		temp1 * TWO_120_MODF1
	vpmuludq ymm4, ymm4, ymm11				;;13-17		temp1 * TWO_120_MODF3
	vpaddq	ymm5, ymm5, ymm0				;;17		temp4 += temp1 * TWO_120_MODF2
	vpaddq	ymm1, ymm1, ymm3				;;17		temp3 += temp1 * TWO_120_MODF1
	vpsrlq	ymm0, ymm5, 30					;;18		temp4 >> 30
	vpand	ymm5, ymm5, ymm15				;; 18		Lower 30 bits of temp4
	vpsllq	ymm5, ymm5, 30					;;19		(temp4 & 0x3FFFFFFF) << 30
	vpaddq	ymm2, ymm2, ymm4				;; 19		temp5 += temp1 * TWO_120_MODF3
	vpaddq	ymm1, ymm1, ymm0				;;20		temp3 += temp4 >> 30
	vpaddq	ymm2, ymm2, ymm5				;; 20		temp5 += (temp4&0x3FFFFFFF)<<30
								;; Result in ymm1,ymm2

;; Squared remainder reduced to 120 bits is in ymm1, ymm2 (call them R1,R2).
;; We'll subtract quotient * factor from this later.

;; Break the upper 60 bits (R1) into 30 bit chunks (U1,U2) and multiply
;; it by 60 bits of 1/factor (I1,I2).  This will give us the quotient
;; (shifted left somewhat).  We only need to compute the upper 60 bits of
;; the quotient since 120 - fac_size is less than 60.
;;
;; Error analysis:  Shifted quotient is up to 5.999... too small.  One for
;; low bits of temp2, one for uncomputed I2*U2, one for if uncomputed I3 is
;; large, one for if top 30 bits of temp5 (U3) is large, and two for carries
;; that may have been generated by additions of temp5 and not added into U2.
;; This is why we must shift off 3 bits for an accurate quotient, meaning
;; a quotient accuracy of 57 bits.  Thus, using the formula described in the
;; next section for computing the shift amount, the minimum trial factor size
;; this code can handle is 64 bits.
;;
;; temp2 = I2*U1 + I1*U2	(61)
;; temp1 = I1*U1		(60)
;;
;; temp1 += temp2 >> 30		(61)

	vpsrlq	ymm0, ymm1, 30				;;1		U1 = upper bits of R1
	vpand	ymm3, ymm1, ymm15			;; 1		U2 = lower bits of R1
	vpmuludq ymm4, ymm0, YMMWORD PTR YMM_I2		;;2-6		temp2 = I2 * U1
	vpmuludq ymm5, ymm12, ymm3			;;2-6		I1 * U2
	vpmuludq ymm0, ymm12, ymm0			;;3-7		temp1 = I1 * U1
	vpaddq	ymm4, ymm4, ymm5			;;7		temp2 += I1 * U2
	vpsrlq	ymm4, ymm4, 30				;;8		temp2 >> 30
	vpaddq	ymm0, ymm0, ymm4			;;9		temp1 += temp2 >> 30

;; 60 bits of shifted quotient are now in ymm0.  R1,R2 are in ymm1,ymm2.

;; Shift the quotient right to get the actual quotient.
;; Define BS (bit shift amount). This is the amount to shift the quotient
;; right to compute the quotient.  BS = 60 - (120 - fac_size + 1).
;; For example, if fac_size = 75, R1/R2 is 120 bits, quotient will
;; be 120-75+1=46 bits, temp1 is 60 bits, BS is 60-46=14 bits.
;;
;; Now shift right to get the quotient:
;;
;; Q2 = (temp1 >> BS) & 0x3FFFFFFF			(30)
;; Q1 = temp1 >> (30 + BS)

	vpsrlq	ymm0, ymm0, XMMWORD PTR XMM_BS		;;10		temp1 >> BS
	vpsrlq	ymm3, ymm0, 30				;;11		Q1 = temp1 >> (30 + BS)
	vpand	ymm0, ymm0, ymm15			;;11		Q2 = (temp1 >> BS) & 0x3FFFFFFF

;; Quotient is now in ymm3, ymm0.  R1,R2 are in ymm1, ymm2.

;; Multiply quotient Q1,Q2 by factor F1,F2,F3
;; We only need to compute bottom 90 bits.
;;
;; temp3 = Q2*F3				(60)
;; temp2 = Q1*F3 + Q2*F2			(61)
;; temp1 = Q1*F2 + Q2*F1			(61)
;;
;; QF2 = temp3 + ((temp2 & 0x3FFFFFFF)<<30)	(61)
;; QF1 = temp1 + (temp2 >> 30)			(61)

	vpmuludq ymm4, ymm3, ymm14			;;12-16		temp2 = Q1 * F3
	vpmuludq ymm5, ymm0, ymm13			;;12-16		Q2 * F2
	vpmuludq ymm3, ymm3, ymm13			;;13-17		temp1 = Q1 * F2
;; might can use a temp reg to move next inst. to its proper place
	vpaddq	ymm4, ymm4, ymm5			;;17		temp2 += Q2 * F2
	vpmuludq ymm5, ymm0, YMMWORD PTR YMM_F1		;;13-17		Q2 * F1
	vpmuludq ymm0, ymm0, ymm14			;;14-18		temp3 = Q2 * F3
	vpaddd	ymm3, ymm3, ymm5			;;18		temp1 += Q2 * F1

	vpand	ymm5, ymm4, ymm15			;;18		temp2 & 0x3FFFFFFF
	vpsrlq	ymm4, ymm4, 30				;; 18		temp2 >> 30
	vpsllq	ymm5, ymm5, 30				;;19		(temp2 & 0x3FFFFFFF) << 30
	vpaddq	ymm0, ymm0, ymm5			;;20		QF2 = temp3 + ((temp2 & 0x3FFFFFFF)<<30)
	vpaddd	ymm3, ymm3, ymm4			;; 20		QF1 = temp1 + (temp2 >> 30)

;; Quotient * factor is now in ymm3, ymm0.  R1,R2 are in ymm1, ymm2.

;; Subtract QF1,QF2 from R1,R2
;;
;; temp2 = R2 - QF2			(signed 62)
;; temp1 = R1 - QF1			(signed 62)

	vpsubq	ymm5, ymm2, ymm0			;;21		temp2 = R2 - QF2
	vpsubd	ymm4, ymm1, ymm3			;; 21		temp1 = R1 - QF1

;; If mul by 2 is required:  mul temp1, temp2 by 2 here

	vmovdqu	xmm0, XMMWORD PTR XMM_SHIFTER[rdi*8]
	vpsllq	ymm5, ymm5, xmm0			;;22		temp2 *= 1 or 2
	vpsllq	ymm4, ymm4, xmm0			;; 22		temp1 *= 1 or 2

;; Break remainder in ymm4,ymm5 into 30 bit quantities.
;;
;; RES3 = temp2 & 0x3FFFFFFF		(30)
;; temp2 >>= 30				(signed 32)
;; RES2 = temp2 & 0x3FFFFFFF		(30)
;; temp2 >>= 30				(signed 2)
;; temp1 += temp2
;; RES1 = temp1 & 0x0FFFFFFF		(28)

	vpand	ymm2, ymm5, ymm15			;;23		RES3 = temp2 & 0x3FFFFFFF
	vpsrlq	ymm5, ymm5, 30				;; 23		temp2 >>= 30
	vpand	ymm1, ymm5, ymm15			;;24		RES2 = temp2 & 0x3FFFFFFF
	vpsrlq	ymm5, ymm5, 2				;; 24		temp2 >>= 2
	vpsrad	ymm5, ymm5, 28				;;25		temp2 >>= 28
	vpaddd	ymm4, ymm4, ymm5			;;26		temp1 += temp2
	vpand	ymm0, ymm4, YMMWORD PTR YMM_BITS28	;;27		RES1 = temp1 & 0x0FFFFFFF
	ENDM



;;****************************************************************
;; The AVX2 versions of the above macros (eight factors at a time)
;;****************************************************************

; This macro does some initialization work and then performs a subset of the
; main loop working on the initial value.  This subset avoids the squaring of the
; remainder.  The initial value can be up to 60 bits larger than the factor
; size.

avx2_fac_initval MACRO
	LOCAL	retry, invfac_adjust, invfac_ok

;; Preload constants

	vmovdqa	ymm15, YMMWORD PTR YMM_BITS30

;; Break 60 bits of factor inverse into 30 bit chunks

	vmovdqa	ymm1, YMMWORD PTR YMM_INVFAC			;; Load factor inverse
	vmovdqa	ymm7, YMMWORD PTR YMM_INVFACa			;; Load factor inverse
retry:	vpsrlq	ymm3, ymm1, 3					;; Convert 63 bit inverse to 60 bits
	vpand	ymm0, ymm3, ymm15				;; Q2 = bottom bits of quotient
	vpsrlq	ymm3, ymm3, 30					;; Q1 = top bits of quotient
	vmovdqa	YMMWORD PTR YMM_I2, ymm0			;; The quotient is the factor inverse, save as YMM_I1, YMM_I2
	vmovdqa	YMMWORD PTR YMM_I1, ymm3

	vpsrlq	ymm9, ymm7, 3					;; Convert 63 bit inverse to 60 bits
	vpand	ymm6, ymm9, ymm15				;; Q2 = bottom bits of quotient
	vpsrlq	ymm9, ymm9, 30					;; Q1 = top bits of quotient
	vmovdqa	YMMWORD PTR YMM_I2a, ymm6			;; The quotient is the factor inverse, save as YMM_I1, YMM_I2
	vmovdqa	YMMWORD PTR YMM_I1a, ymm9

;; Make sure the quotient isn't one too large.
;; Multiply quotient Q1,Q2 by factor F1,F2,F3
;; We only need to compute bottom 90 bits.
;;
;; temp3 = F3*Q2				(60)
;; temp2 = F3*Q1 + F2*Q2			(61)
;; temp1 = F2*Q1 + F1*Q2			(61)
;;
;; QF2 = temp3 + ((temp2 & 0x3FFFFFFF)<<30)	(61)
;; QF1 = temp1 + (temp2 >> 30)			(61)

	vpmuludq ymm4, ymm3, YMMWORD PTR YMM_F3			;; temp2 = Q1 * F3
	vpmuludq ymm2, ymm0, YMMWORD PTR YMM_F2			;; Q2 * F2
	vpmuludq ymm3, ymm3, YMMWORD PTR YMM_F2			;; temp1 = Q1 * F2
	vpmuludq ymm5, ymm0, YMMWORD PTR YMM_F1			;; Q2 * F1
	vpmuludq ymm0, ymm0, YMMWORD PTR YMM_F3			;; temp3 = Q2 * F3
	vpaddq	ymm4, ymm4, ymm2				;; temp2 += Q2 * F2
	vpaddd	ymm3, ymm3, ymm5				;; temp1 += Q2 * F1
	vpand	ymm5, ymm4, ymm15				;; temp2 & 0x3FFFFFFF
	vpsrlq	ymm4, ymm4, 30					;; temp2 >> 30
	vpsllq	ymm5, ymm5, 30					;; (temp2 & 0x3FFFFFFF) << 30
	vpaddq	ymm0, ymm0, ymm5				;; QF2 = temp3 + ((temp2 & 0x3FFFFFFF)<<30)
	vpaddd	ymm3, ymm3, ymm4				;; QF1 = temp1 + (temp2 >> 30)

	vpmuludq ymm10, ymm9, YMMWORD PTR YMM_F3a		;; temp2 = Q1 * F3
	vpmuludq ymm8, ymm6, YMMWORD PTR YMM_F2a		;; Q2 * F2
	vpmuludq ymm9, ymm9, YMMWORD PTR YMM_F2a		;; temp1 = Q1 * F2
	vpmuludq ymm11, ymm6, YMMWORD PTR YMM_F1a		;; Q2 * F1
	vpmuludq ymm6, ymm6, YMMWORD PTR YMM_F3a		;; temp3 = Q2 * F3
	vpaddq	ymm10, ymm10, ymm8				;; temp2 += Q2 * F2
	vpaddd	ymm9, ymm9, ymm11				;; temp1 += Q2 * F1
	vpand	ymm11, ymm10, ymm15				;; temp2 & 0x3FFFFFFF
	vpsrlq	ymm10, ymm10, 30				;; temp2 >> 30
	vpsllq	ymm11, ymm11, 30				;; (temp2 & 0x3FFFFFFF) << 30
	vpaddq	ymm6, ymm6, ymm11				;; QF2 = temp3 + ((temp2 & 0x3FFFFFFF)<<30)
	vpaddd	ymm9, ymm9, ymm10				;; QF1 = temp1 + (temp2 >> 30)

;; If QF2 is above 2^60, then add these carry bits into QF1

	vpsrlq	ymm0, ymm0, 60
	vpaddd	ymm3, ymm3, ymm0				;; QF1 = QF1 + (QF2 >> 60)

	vpsrlq	ymm6, ymm6, 60
	vpaddd	ymm9, ymm9, ymm6				;; QF1 = QF1 + (QF2 >> 60)

;; Validate QF1.  The top bits should be ones (zeros indicate Q*F exceeded
;; a power of two by a little bit).  Detect these cases, decrement the
;; inverse factor and try again.

	vpand	ymm3, ymm3, YMMWORD PTR YMM_28TH_BIT		;; Test for positive dword values in QF1 (test 28th bit)
	vpxor	ymm2, ymm2, ymm2
	vpcmpeqq ymm3, ymm3, ymm2
	vpand	ymm9, ymm9, YMMWORD PTR YMM_28TH_BIT		;; Test for positive dword values in QF1 (test 28th bit)
	vpcmpeqq ymm9, ymm9, ymm2
	vpmovmskb rdx, ymm3
	and	edx, 0FFFFFFFFh					;; See if INVFAC values changed
	jnz	short invfac_adjust				;; Jump INVFACs need adjustment
	vpmovmskb rdx, ymm9
	and	edx, 0FFFFFFFFh					;; See if INVFAC values changed
	jz	short invfac_ok					;; Jump if no INVFACs needed to be adjusted
invfac_adjust:
	vmovdqa	ymm1, YMMWORD PTR YMM_INVFAC			;; Load factor inverse
	vpaddq	ymm1, ymm1, ymm3				;; Add -1 to INVFAC where QF1 dword val was positive
	vmovdqa	YMMWORD PTR YMM_INVFAC, ymm1			;; Store factor inverse
	vmovdqa	ymm7, YMMWORD PTR YMM_INVFACa			;; Load factor inverse
	vpaddq	ymm7, ymm7, ymm9				;; Add -1 to INVFAC where QF1 dword val was positive
	vmovdqa	YMMWORD PTR YMM_INVFACa, ymm7			;; Store factor inverse
	jmp	retry						;; Reprocess new INVFAC values
invfac_ok:

;; Compute quotient for 2^120 / factor call it Q1,Q2.

	vmovdqa	ymm1, YMMWORD PTR YMM_INVFAC			;; Load factor inverse
	vpsrlq	ymm3, ymm1, XMMWORD PTR XMM_INIT120BS		;; Compute 2^120 / factor
	vpand	ymm0, ymm3, ymm15				;; Q2 = bottom bits of quotient
	vpsrlq	ymm3, ymm3, 30					;; Q1 = top bits of quotient

	vmovdqa	ymm7, YMMWORD PTR YMM_INVFACa			;; Load factor inverse
	vpsrlq	ymm9, ymm7, XMMWORD PTR XMM_INIT120BS		;; Compute 2^120 / factor
	vpand	ymm6, ymm9, ymm15				;; Q2 = bottom bits of quotient
	vpsrlq	ymm9, ymm9, 30					;; Q1 = top bits of quotient

;; Multiply quotient Q1,Q2 by factor F1,F2,F3
;; We only need to compute bottom 90 bits.
;;
;; temp3 = F3*Q2				(60)
;; temp2 = F3*Q1 + F2*Q2			(61)
;; temp1 = F2*Q1 + F1*Q2			(61)
;;
;; QF2 = temp3 + ((temp2 & 0x3FFFFFFF)<<30)	(61)
;; QF1 = temp1 + (temp2 >> 30)			(61)

	vpmuludq ymm4, ymm3, YMMWORD PTR YMM_F3			;; temp2 = Q1 * F3
	vpmuludq ymm2, ymm0, YMMWORD PTR YMM_F2			;; Q2 * F2
	vpmuludq ymm3, ymm3, YMMWORD PTR YMM_F2			;; temp1 = Q1 * F2
	vpmuludq ymm5, ymm0, YMMWORD PTR YMM_F1			;; Q2 * F1
	vpmuludq ymm0, ymm0, YMMWORD PTR YMM_F3			;; temp3 = Q2 * F3
	vpaddq	ymm4, ymm4, ymm2				;; temp2 += Q2 * F2
	vpaddd	ymm3, ymm3, ymm5				;; temp1 += Q2 * F1
	vpand	ymm5, ymm4, ymm15				;; temp2 & 0x3FFFFFFF
	vpsrlq	ymm4, ymm4, 30					;; temp2 >> 30
	vpsllq	ymm5, ymm5, 30					;; (temp2 & 0x3FFFFFFF) << 30
	vpaddq	ymm0, ymm0, ymm5				;; QF2 = temp3 + ((temp2 & 0x3FFFFFFF)<<30)
	vpaddd	ymm3, ymm3, ymm4				;; QF1 = temp1 + (temp2 >> 30)

	vpmuludq ymm10, ymm9, YMMWORD PTR YMM_F3a		;; temp2 = Q1 * F3
	vpmuludq ymm8, ymm6, YMMWORD PTR YMM_F2a		;; Q2 * F2
	vpmuludq ymm9, ymm9, YMMWORD PTR YMM_F2a		;; temp1 = Q1 * F2
	vpmuludq ymm11, ymm6, YMMWORD PTR YMM_F1a		;; Q2 * F1
	vpmuludq ymm6, ymm6, YMMWORD PTR YMM_F3a		;; temp3 = Q2 * F3
	vpaddq	ymm10, ymm10, ymm8				;; temp2 += Q2 * F2
	vpaddd	ymm9, ymm9, ymm11				;; temp1 += Q2 * F1
	vpand	ymm11, ymm10, ymm15				;; temp2 & 0x3FFFFFFF
	vpsrlq	ymm10, ymm10, 30				;; temp2 >> 30
	vpsllq	ymm11, ymm11, 30				;; (temp2 & 0x3FFFFFFF) << 30
	vpaddq	ymm6, ymm6, ymm11				;; QF2 = temp3 + ((temp2 & 0x3FFFFFFF)<<30)
	vpaddd	ymm9, ymm9, ymm10				;; QF1 = temp1 + (temp2 >> 30)

;; Subtract QF1,QF2 from zero to get 90 bits of 2^120 mod factor.

	vpxor	ymm2, ymm2, ymm2
	vpsubq	ymm5, ymm2, ymm0				;; temp2 = Low 60 bits of remainder
	vpsubd	ymm4, ymm2, ymm3				;; temp1 = Low 30 bits of remainder

	vpsubq	ymm11, ymm2, ymm6				;; temp2 = Low 60 bits of remainder
	vpsubd	ymm10, ymm2, ymm9				;; temp1 = Low 30 bits of remainder

;; Break remainder in ymm4,ymm5 into 30 bit quantities and save
;;
;; RES3 = temp2 & 0x3FFFFFFF		(30)
;; temp2 >>= 30				(signed 32)
;; RES2 = temp2 & 0x3FFFFFFF		(30)
;; temp2 >>= 30				(signed 2)
;; temp1 += temp2
;; RES1 = temp1 & 0x0FFFFFFF		(28)

	vpand	ymm2, ymm5, ymm15				;; RES3 = temp2 & 0x3FFFFFFF
	vpsrlq	ymm5, ymm5, 30					;; temp2 >>= 30
	vpand	ymm1, ymm5, ymm15				;; RES2 = temp2 & 0x3FFFFFFF
	vpsrlq	ymm5, ymm5, 2					;; temp2 >>= 2
	vpsrad	ymm5, ymm5, 28					;; temp2 >>= 28
	vpaddd	ymm4, ymm4, ymm5				;; temp1 += temp2
	vpand	ymm0, ymm4, YMMWORD PTR YMM_BITS28		;; RES1 = temp1 & 0x0FFFFFFF
	vmovdqa YMMWORD PTR YMM_TWO_120_MODF3, ymm2
	vmovdqa YMMWORD PTR YMM_TWO_120_MODF2, ymm1
	vmovdqa YMMWORD PTR YMM_TWO_120_MODF1, ymm0

	vpand	ymm8, ymm11, ymm15				;; RES3 = temp2 & 0x3FFFFFFF
	vpsrlq	ymm11, ymm11, 30				;; temp2 >>= 30
	vpand	ymm7, ymm11, ymm15				;; RES2 = temp2 & 0x3FFFFFFF
	vpsrlq	ymm11, ymm11, 2					;; temp2 >>= 2
	vpsrad	ymm11, ymm11, 28				;; temp2 >>= 28
	vpaddd	ymm10, ymm10, ymm11				;; temp1 += temp2
	vpand	ymm6, ymm10, YMMWORD PTR YMM_BITS28		;; RES1 = temp1 & 0x0FFFFFFF
	vmovdqa YMMWORD PTR YMM_TWO_120_MODF3a, ymm8
	vmovdqa YMMWORD PTR YMM_TWO_120_MODF2a, ymm7
	vmovdqa YMMWORD PTR YMM_TWO_120_MODF1a, ymm6

;; 1/factor is the shifted quotient.  Shift 1/factor right to get the actual
;; quotient.  YMM_INITBS is the shift amount defined by this formula:
;;	INITBS = 120 - bit_length(initval)
;;
;; Q2 = (temp1 >> INITBS) & 0x3FFFFFFF			(30)
;; Q1 = temp1 >> (30 + INITBS)

	vmovdqa	ymm1, YMMWORD PTR YMM_INVFAC			;; Load factor inverse
	vpsrlq	ymm0, ymm1, XMMWORD PTR XMM_INITBS		;; temp1 = YMM_INVFAC >> INITBS
	vpsrlq	ymm3, ymm0, 30					;; Q1 = temp1 >> (30 + BS)
	vpand	ymm0, ymm0, ymm15				;; Q2 = (temp1 >> BS) & 0x3FFFFFFF

	vmovdqa	ymm7, YMMWORD PTR YMM_INVFACa			;; Load factor inverse
	vpsrlq	ymm6, ymm7, XMMWORD PTR XMM_INITBS		;; temp1 = YMM_INVFAC >> INITBS
	vpsrlq	ymm9, ymm6, 30					;; Q1 = temp1 >> (30 + BS)
	vpand	ymm6, ymm6, ymm15				;; Q2 = (temp1 >> BS) & 0x3FFFFFFF

;; Quotient is now in ymm3, ymm0.

;; Multiply quotient Q1,Q2 by factor F1,F2,F3
;; We only need to compute bottom 90 bits.
;;
;; temp3 = Q2*F3				(60)
;; temp2 = Q1*F3 + Q2*F2			(61)
;; temp1 = Q1*F2 + Q2*F1			(61)
;;
;; QF2 = temp3 + ((temp2 & 0x3FFFFFFF)<<30)	(61)
;; QF1 = temp1 + (temp2 >> 30)			(61)

	vpmuludq ymm4, ymm3, YMMWORD PTR YMM_F3			;; temp2 = Q1 * F3
	vpmuludq ymm2, ymm0, YMMWORD PTR YMM_F2			;; Q2 * F2
	vpmuludq ymm3, ymm3, YMMWORD PTR YMM_F2			;; temp1 = Q1 * F2
	vpmuludq ymm5, ymm0, YMMWORD PTR YMM_F1			;; Q2 * F1
	vpmuludq ymm0, ymm0, YMMWORD PTR YMM_F3			;; temp3 = Q2 * F3
	vpaddq	ymm4, ymm4, ymm2				;; temp2 += Q2 * F2
	vpaddd	ymm3, ymm3, ymm5				;; temp1 += Q2 * F1
	vpand	ymm5, ymm4, ymm15				;; temp2 & 0x3FFFFFFF
	vpsrlq	ymm4, ymm4, 30					;; temp2 >> 30
	vpsllq	ymm5, ymm5, 30					;; (temp2 & 0x3FFFFFFF) << 30
	vpaddq	ymm0, ymm0, ymm5				;; QF2 = temp3 + ((temp2 & 0x3FFFFFFF)<<30)
	vpaddd	ymm3, ymm3, ymm4				;; QF1 = temp1 + (temp2 >> 30)

	vpmuludq ymm10, ymm9, YMMWORD PTR YMM_F3a		;; temp2 = Q1 * F3
	vpmuludq ymm8, ymm6, YMMWORD PTR YMM_F2a		;; Q2 * F2
	vpmuludq ymm9, ymm9, YMMWORD PTR YMM_F2a		;; temp1 = Q1 * F2
	vpmuludq ymm11, ymm6, YMMWORD PTR YMM_F1a		;; Q2 * F1
	vpmuludq ymm6, ymm6, YMMWORD PTR YMM_F3a		;; temp3 = Q2 * F3
	vpaddq	ymm10, ymm10, ymm8				;; temp2 += Q2 * F2
	vpaddd	ymm9, ymm9, ymm11				;; temp1 += Q2 * F1
	vpand	ymm11, ymm10, ymm15				;; temp2 & 0x3FFFFFFF
	vpsrlq	ymm10, ymm10, 30				;; temp2 >> 30
	vpsllq	ymm11, ymm11, 30				;; (temp2 & 0x3FFFFFFF) << 30
	vpaddq	ymm6, ymm6, ymm11				;; QF2 = temp3 + ((temp2 & 0x3FFFFFFF)<<30)
	vpaddd	ymm9, ymm9, ymm10				;; QF1 = temp1 + (temp2 >> 30)

;; Load initial value

	vpxor	ymm2, ymm2, ymm2				;; Clear R2
	vmovdqa	ymm1, YMMWORD PTR YMM_INITVAL			;; Load R1

;; Quotient * factor is now in ymm3, ymm0.  R1,R2 are in ymm1, ymm2.

;; Subtract QF1,QF2 from R1,R2
;;
;; temp2 = R2 - QF2			(signed 62)
;; temp1 = R1 - QF1			(signed 62)

	vpsubq	ymm5, ymm2, ymm0				;; temp2 = R2 - QF2
	vpsubd	ymm4, ymm1, ymm3				;; temp1 = R1 - QF1

	vpsubq	ymm11, ymm2, ymm6				;; temp2 = R2 - QF2
	vpsubd	ymm10, ymm1, ymm9				;; temp1 = R1 - QF1
	ENDM


; This macro does one iteration of the main loop.  Square a remainder and do a mod.

avx2_fac MACRO fac_size

;; Note that at the beginning of the macro ymm4, ymm5 contains the unnormalized remainder
;; for us to square.  The first order of business is to break the remainder into 30-bit
;; chunks in ymm0, ymm1, ymm2 to square.  This remainder can be one more bit than
;; fac_size.  This is because we do not perform a modulo after the optional
;; multiply by two at the end of this macro.

;; Also note that in very rare cases the remainder could be two more bits
;; than fac_size.  Since the calculated quotient could be one too small,
;; the maximum remainder is 2 * (factor + epsilon).  This extremely rare
;; extra bit should not be a problem as the code below uses 30 bit integers
;; and it shouldn't be a problem handling one more bit.

;; In the comments of this code, we'll assume fac_size is 89 bits and the
;; input remainder is 90 bits (30 bits in each ymm register).

;; rem = A,B,C (30,30,30 bits unsigned)
;; Square rem producing a 180 bit result stored in temp1, temp3, temp5
;; temp5 = CC				(60)
;; temp4 = 2BC				(61)
;; temp3 = 2AC + BB			(62 = 61 + 60)
;; temp2 = 2AB				(61)
;; temp1 = AA				(60)

;; If fac_size is 75 bits or more (the remainder is 76 bits or more), then
;; the squared remainder is more than 150 bits.  In this case, get the very
;; highest bits and multiply them by 2^120 mod factor.  We'll then add
;; these back into the squared remainder as part of our effort to create a
;; 120 bit result.
;;
;; AA_HIGH = temp1 >> 30
;; temp4 += AA_HIGH * TWO_120_MODF3
;; temp3 += AA_HIGH * TWO_120_MODF2
;; temp2 += AA_HIGH * TWO_120_MODF1
;; temp1 = AA & 0x3FFFFFFF

;; Combine temp1 through temp5 into 3 temporaries
;;
;; temp5 += (temp4 & 0x3FFFFFFF) << 30
;; temp3 += (temp4 >> 30) + (temp2 & 0x3FFFFFFF) << 30
;; temp1 += temp2 >> 30
;;
;; While doing the above we interleave work from the paragraph below.
;;
;; Our value is now at most 150 bits.  Take the upper 30 bits and multiply
;; them by 2^120 mod factor.  Then add these back into the squared remainder
;; to create a 120 bit result.
;;
;; Make sure we add any bits from temp3 above 2^60 into temp1
;; temp5 += temp1 * TWO_120_MODF3
;; temp4 += temp1 * TWO_120_MODF2
;; temp3 += temp1 * TWO_120_MODF1
;;
;; Finally, we do a lot of instruction interleaving in hopes of making it
;; easier for the CPU to schedule the instructions.  The clock counts assume
;; my understanding of Haswell instruction scheduling is correct.

;; Break remainder in ymm4,ymm5 into 30 bit quantities.
;;
;; RES3 = temp2 & 0x3FFFFFFF		(30)
;; temp2 >>= 30				(signed 32)
;; RES2 = temp2 & 0x3FFFFFFF		(30)
;; temp2 >>= 30				(signed 2)
;; temp1 += temp2
;; RES1 = temp1 & 0x0FFFFFFF		(28)

	vpand	ymm2, ymm5, ymm15				;; RES3 = temp2 & 0x3FFFFFFF
	vpsrlq	ymm5, ymm5, 30					;; temp2 >>= 30
	vpand	ymm1, ymm5, ymm15				;; RES2 = temp2 & 0x3FFFFFFF
	vpsrlq	ymm5, ymm5, 2					;; temp2 >>= 2
	vpsrad	ymm5, ymm5, 28					;; temp2 >>= 28
	vpaddd	ymm4, ymm4, ymm5				;; temp1 += temp2
	vpand	ymm0, ymm4, YMMWORD PTR YMM_BITS28		;; RES1 = temp1 & 0x0FFFFFFF

	vpand	ymm8, ymm11, ymm15				;; RES3 = temp2 & 0x3FFFFFFF
	vpsrlq	ymm11, ymm11, 30				;; temp2 >>= 30
	vpand	ymm7, ymm11, ymm15				;; RES2 = temp2 & 0x3FFFFFFF
	vpsrlq	ymm11, ymm11, 2					;; temp2 >>= 2
	vpsrad	ymm11, ymm11, 28				;; temp2 >>= 28
	vpaddd	ymm10, ymm10, ymm11				;; temp1 += temp2
	vpand	ymm6, ymm10, YMMWORD PTR YMM_BITS28		;; RES1 = temp1 & 0x0FFFFFFF

								;; A = ymm0, B = ymm1, C = ymm2

	vpaddq	ymm3, ymm0, ymm0				;;0		2A
	vpaddq	ymm5, ymm1, ymm1				;;0		2B
	vpmuludq ymm1, ymm1, ymm1				;;1-5		temp3 = BB
	vpmuludq ymm4, ymm0, ymm0				;;1-5		temp1 = AA
	vpmuludq ymm3, ymm3, ymm2				;;2-6		2AC = 2A * C
	vpmuludq ymm0, ymm0, ymm5				;;2-6		temp2 = 2AB = A * 2B
	vpmuludq ymm5, ymm5, ymm2				;;3-7		temp4 = 2BC = 2B * C
	vpmuludq ymm2, ymm2, ymm2				;;3-7		temp5 = CC
	vpaddq	ymm1, ymm1, ymm3				;;7		temp3 = BB + 2AC
								;;		temp1-5 = ymm4,0,1,5,2

	vpaddq	ymm9, ymm6, ymm6				;;	0	2A
	vpaddq	ymm11, ymm7, ymm7				;;	0	2B
	vpmuludq ymm7, ymm7, ymm7				;;	1-5	temp3 = BB
	vpmuludq ymm10, ymm6, ymm6				;;	1-5	temp1 = AA
	vpmuludq ymm9, ymm9, ymm8				;;	2-6	2AC = 2A * C
	vpmuludq ymm6, ymm6, ymm11				;;	2-6	temp2 = 2AB = A * 2B
	vpmuludq ymm11, ymm11, ymm8				;;	3-7	temp4 = 2BC = 2B * C
	vpmuludq ymm8, ymm8, ymm8				;;	3-7	temp5 = CC
	vpaddq	ymm7, ymm7, ymm9				;;	7	temp3 = BB + 2AC
								;;		temp1-5 = ymm10,6,7,11,8

	IF fac_size GE 75
	vpsrlq	ymm3, ymm4, 30					;;6		AA_HIGH = Upper 30 bits of AA
	vpmuludq ymm12, ymm3, YMMWORD PTR YMM_TWO_120_MODF2	;;7-11		AA_HIGH * 2_120_MODF2
	vpaddq	ymm1, ymm1, ymm12				;;12		temp3 += AA_HIGH * 2_120_MODF2
	vpmuludq ymm12, ymm3, YMMWORD PTR YMM_TWO_120_MODF1	;;7-11		AA_HIGH * 2_120_MODF1
	vpaddq	ymm0, ymm0, ymm12				;;12		temp2 += AA_HIGH * 2_120_MODF1
	vpmuludq ymm3, ymm3, YMMWORD PTR YMM_TWO_120_MODF3	;;9-13		AA_HIGH * 2_120_MODF3
	vpaddq	ymm5, ymm5, ymm3				;;14		temp4 += AA_HIGH * 2_120_MODF3
	vpand	ymm4, ymm4, ymm15				;;8		temp1 = Low 30 bits of AA

	vpsrlq	ymm9, ymm10, 30					;;	6	AA_HIGH = Upper 30 bits of AA
	vpmuludq ymm12, ymm9, YMMWORD PTR YMM_TWO_120_MODF2a	;;	7-11	AA_HIGH * 2_120_MODF2
	vpaddq	ymm7, ymm7, ymm12				;;	12	temp3 += AA_HIGH * 2_120_MODF2
	vpmuludq ymm12, ymm9, YMMWORD PTR YMM_TWO_120_MODF1a	;;	7-11	AA_HIGH * 2_120_MODF1
	vpaddq	ymm6, ymm6, ymm12				;;	12	temp2 += AA_HIGH * 2_120_MODF1
	vpmuludq ymm9, ymm9, YMMWORD PTR YMM_TWO_120_MODF3a	;;	9-13	AA_HIGH * 2_120_MODF3
	vpaddq	ymm11, ymm11, ymm9				;;	14	temp4 += AA_HIGH * 2_120_MODF3
	vpand	ymm10, ymm10, ymm15				;;	8	temp1 = Low 30 bits of AA
	ENDIF

	vpsrlq	ymm3, ymm1, 30					;;8		Top bits of temp3
	vpand	ymm1, ymm1, ymm15				;;8		temp3 = temp3 & 0x3FFFFFFF
	vpaddq	ymm0, ymm0, ymm3				;;9		Add temp3 top bits into temp2
	vpsrlq	ymm3, ymm0, 30					;;10		Top bits of temp2
	vpand	ymm0, ymm0, ymm15				;;10		temp2 & 0x3FFFFFFF
	vpsllq	ymm0, ymm0, 30					;;11		temp2 << 30
	vpaddd	ymm4, ymm4, ymm3				;;11		Add temp2 top bits into temp1
	vpor	ymm1, ymm1, ymm0				;;12		temp3 += temp2 << 30
	vpmuludq ymm0, ymm4, YMMWORD PTR YMM_TWO_120_MODF2	;;12-16		temp1 * TWO_120_MODF2
	vpmuludq ymm3, ymm4, YMMWORD PTR YMM_TWO_120_MODF1	;;12-16		temp1 * TWO_120_MODF1
	vpmuludq ymm4, ymm4, YMMWORD PTR YMM_TWO_120_MODF3	;;13-17		temp1 * TWO_120_MODF3
	vpaddq	ymm5, ymm5, ymm0				;;17		temp4 += temp1 * TWO_120_MODF2
	vpaddq	ymm1, ymm1, ymm3				;;17		temp3 += temp1 * TWO_120_MODF1
	vpsrlq	ymm0, ymm5, 30					;;18		temp4 >> 30
	vpand	ymm5, ymm5, ymm15				;;18		Lower 30 bits of temp4
	vpsllq	ymm5, ymm5, 30					;;19		(temp4 & 0x3FFFFFFF) << 30
	vpaddq	ymm2, ymm2, ymm4				;;19		temp5 += temp1 * TWO_120_MODF3
	vpaddq	ymm1, ymm1, ymm0				;;20		temp3 += temp4 >> 30
	vpaddq	ymm2, ymm2, ymm5				;;20		temp5 += (temp4&0x3FFFFFFF)<<30
								;;		Result in ymm1,ymm2

	vpsrlq	ymm9, ymm7, 30					;;	8	Top bits of temp3
	vpand	ymm7, ymm7, ymm15				;;	8	temp3 = temp3 & 0x3FFFFFFF
	vpaddq	ymm6, ymm6, ymm9				;;	9	Add temp3 top bits into temp2
	vpsrlq	ymm9, ymm6, 30					;;	10	Top bits of temp2
	vpand	ymm6, ymm6, ymm15				;;	10	temp2 & 0x3FFFFFFF
	vpsllq	ymm6, ymm6, 30					;;	11	temp2 << 30
	vpaddd	ymm10, ymm10, ymm9				;;	11	Add temp2 top bits into temp1
	vpor	ymm7, ymm7, ymm6				;;	12	temp3 += temp2 << 30
	vpmuludq ymm6, ymm10, YMMWORD PTR YMM_TWO_120_MODF2a	;;	12-16	temp1 * TWO_120_MODF2
	vpmuludq ymm9, ymm10, YMMWORD PTR YMM_TWO_120_MODF1a	;;	12-16	temp1 * TWO_120_MODF1
	vpmuludq ymm10, ymm10, YMMWORD PTR YMM_TWO_120_MODF3a	;;	13-17	temp1 * TWO_120_MODF3
	vpaddq	ymm11, ymm11, ymm6				;;	17	temp4 += temp1 * TWO_120_MODF2
	vpaddq	ymm7, ymm7, ymm9				;;	17	temp3 += temp1 * TWO_120_MODF1
	vpsrlq	ymm6, ymm11, 30					;;	18	temp4 >> 30
	vpand	ymm11, ymm11, ymm15				;;	18	Lower 30 bits of temp4
	vpsllq	ymm11, ymm11, 30				;;	19	(temp4 & 0x3FFFFFFF) << 30
	vpaddq	ymm8, ymm8, ymm10				;;	19	temp5 += temp1 * TWO_120_MODF3
	vpaddq	ymm7, ymm7, ymm6				;;	20	temp3 += temp4 >> 30
	vpaddq	ymm8, ymm8, ymm11				;;	20	temp5 += (temp4&0x3FFFFFFF)<<30
								;;		Result in ymm7,ymm8

;; Squared remainder reduced to 120 bits is in ymm1, ymm2 (call them R1,R2).
;; We'll subtract quotient * factor from this later.

;; Break the upper 60 bits (R1) into 30 bit chunks (U1,U2) and multiply
;; it by 60 bits of 1/factor (I1,I2).  This will give us the quotient
;; (shifted left somewhat).  We only need to compute the upper 60 bits of
;; the quotient since 120 - fac_size is less than 60.
;;
;; Error analysis:  Shifted quotient is up to 5.999... too small.  One for
;; low bits of temp2, one for uncomputed I2*U2, one for if uncomputed I3 is
;; large, one for if top 30 bits of temp5 (U3) is large, and two for carries
;; that may have been generated by additions of temp5 and not added into U2.
;; This is why we must shift off 3 bits for an accurate quotient, meaning
;; a quotient accuracy of 57 bits.  Thus, using the formula described in the
;; next section for computing the shift amount, the minimum trial factor size
;; this code can handle is 64 bits.
;;
;; temp2 = I2*U1 + I1*U2	(61)
;; temp1 = I1*U1		(60)
;;
;; temp1 += temp2 >> 30		(61)

	vpsrlq	ymm0, ymm1, 30				;;1		U1 = upper bits of R1
	vpand	ymm3, ymm1, ymm15			;;1		U2 = lower bits of R1
	vpmuludq ymm4, ymm0, YMMWORD PTR YMM_I2		;;2-6		temp2 = I2 * U1
	vpmuludq ymm5, ymm3, YMMWORD PTR YMM_I1		;;2-6		I1 * U2
	vpmuludq ymm0, ymm0, YMMWORD PTR YMM_I1		;;3-7		temp1 = I1 * U1
	vpaddq	ymm4, ymm4, ymm5			;;7		temp2 += I1 * U2
	vpsrlq	ymm4, ymm4, 30				;;8		temp2 >> 30
	vpaddq	ymm0, ymm0, ymm4			;;9		temp1 += temp2 >> 30

	vpsrlq	ymm6, ymm7, 30				;;	1	U1 = upper bits of R1
	vpand	ymm9, ymm7, ymm15			;;	1	U2 = lower bits of R1
	vpmuludq ymm10, ymm6, YMMWORD PTR YMM_I2a	;;	2-6	temp2 = I2 * U1
	vpmuludq ymm11, ymm9, YMMWORD PTR YMM_I1a	;;	2-6	I1 * U2
	vpmuludq ymm6, ymm6, YMMWORD PTR YMM_I1a	;;	3-7	temp1 = I1 * U1
	vpaddq	ymm10, ymm10, ymm11			;;	7	temp2 += I1 * U2
	vpsrlq	ymm10, ymm10, 30			;;	8	temp2 >> 30
	vpaddq	ymm6, ymm6, ymm10			;;	9	temp1 += temp2 >> 30

;; 60 bits of shifted quotient are now in ymm0.  R1,R2 are in ymm1,ymm2.

;; Shift the quotient right to get the actual quotient.
;; Define BS (bit shift amount). This is the amount to shift the quotient
;; right to compute the quotient.  BS = 60 - (120 - fac_size + 1).
;; For example, if fac_size = 75, R1/R2 is 120 bits, quotient will
;; be 120-75+1=46 bits, temp1 is 60 bits, BS is 60-46=14 bits.
;;
;; Now shift right to get the quotient:
;;
;; Q2 = (temp1 >> BS) & 0x3FFFFFFF			(30)
;; Q1 = temp1 >> (30 + BS)

	vpsrlq	ymm0, ymm0, XMMWORD PTR XMM_BS		;;10		temp1 >> BS
	vpsrlq	ymm3, ymm0, 30				;;11		Q1 = temp1 >> (30 + BS)
	vpand	ymm0, ymm0, ymm15			;;11		Q2 = (temp1 >> BS) & 0x3FFFFFFF

	vpsrlq	ymm6, ymm6, XMMWORD PTR XMM_BS		;;	10	temp1 >> BS
	vpsrlq	ymm9, ymm6, 30				;;	11	Q1 = temp1 >> (30 + BS)
	vpand	ymm6, ymm6, ymm15			;;	11	Q2 = (temp1 >> BS) & 0x3FFFFFFF

;; Quotient is now in ymm3, ymm0.  R1,R2 are in ymm1, ymm2.

;; Multiply quotient Q1,Q2 by factor F1,F2,F3
;; We only need to compute bottom 90 bits.
;;
;; temp3 = Q2*F3				(60)
;; temp2 = Q1*F3 + Q2*F2			(61)
;; temp1 = Q1*F2 + Q2*F1			(61)
;;
;; QF2 = temp3 + ((temp2 & 0x3FFFFFFF)<<30)	(61)
;; QF1 = temp1 + (temp2 >> 30)			(61)

	vpmuludq ymm4, ymm3, YMMWORD PTR YMM_F3		;;12-16		temp2 = Q1 * F3
	vpmuludq ymm5, ymm0, YMMWORD PTR YMM_F2		;;12-16		Q2 * F2
	vpmuludq ymm3, ymm3, YMMWORD PTR YMM_F2		;;13-17		temp1 = Q1 * F2
;; might can use a temp reg to move next inst. to its proper place
	vpaddq	ymm4, ymm4, ymm5			;;17		temp2 += Q2 * F2
	vpmuludq ymm5, ymm0, YMMWORD PTR YMM_F1		;;13-17		Q2 * F1
	vpmuludq ymm0, ymm0, YMMWORD PTR YMM_F3		;;14-18		temp3 = Q2 * F3
	vpaddd	ymm3, ymm3, ymm5			;;18		temp1 += Q2 * F1

	vpand	ymm5, ymm4, ymm15			;;18		temp2 & 0x3FFFFFFF
	vpsrlq	ymm4, ymm4, 30				;;18		temp2 >> 30
	vpsllq	ymm5, ymm5, 30				;;19		(temp2 & 0x3FFFFFFF) << 30
	vpaddq	ymm0, ymm0, ymm5			;;20		QF2 = temp3 + ((temp2 & 0x3FFFFFFF)<<30)
	vpaddd	ymm3, ymm3, ymm4			;;20		QF1 = temp1 + (temp2 >> 30)


	vpmuludq ymm10, ymm9, YMMWORD PTR YMM_F3a	;;	12-16	temp2 = Q1 * F3
	vpmuludq ymm11, ymm6, YMMWORD PTR YMM_F2a	;;	12-16	Q2 * F2
	vpmuludq ymm9, ymm9, YMMWORD PTR YMM_F2a	;;	13-17	temp1 = Q1 * F2
;; might can use a temp reg to move next inst. to its proper place
	vpaddq	ymm10, ymm10, ymm11			;;	17	temp2 += Q2 * F2
	vpmuludq ymm11, ymm6, YMMWORD PTR YMM_F1a	;;	13-17	Q2 * F1
	vpmuludq ymm6, ymm6, YMMWORD PTR YMM_F3a	;;	14-18	temp3 = Q2 * F3
	vpaddd	ymm9, ymm9, ymm11			;;	18	temp1 += Q2 * F1

	vpand	ymm11, ymm10, ymm15			;;	18	temp2 & 0x3FFFFFFF
	vpsrlq	ymm10, ymm10, 30			;;	18	temp2 >> 30
	vpsllq	ymm11, ymm11, 30			;;	19	(temp2 & 0x3FFFFFFF) << 30
	vpaddq	ymm6, ymm6, ymm11			;;	20	QF2 = temp3 + ((temp2 & 0x3FFFFFFF)<<30)
	vpaddd	ymm9, ymm9, ymm10			;;	20	QF1 = temp1 + (temp2 >> 30)

;; Quotient * factor is now in ymm3, ymm0.  R1,R2 are in ymm1, ymm2.

;; Subtract QF1,QF2 from R1,R2
;;
;; temp2 = R2 - QF2			(signed 62)
;; temp1 = R1 - QF1			(signed 62)

	vpsubq	ymm5, ymm2, ymm0			;;21		temp2 = R2 - QF2
	vpsubd	ymm4, ymm1, ymm3			;;21		temp1 = R1 - QF1

	vpsubq	ymm11, ymm8, ymm6			;;	21	temp2 = R2 - QF2
	vpsubd	ymm10, ymm7, ymm9			;;	21	temp1 = R1 - QF1

;; If mul by 2 is required:  mul temp1, temp2 by 2 here

	vmovdqu	xmm0, XMMWORD PTR XMM_SHIFTER[rdi*8]
	vpsllq	ymm5, ymm5, xmm0			;;22		temp2 *= 1 or 2
	vpsllq	ymm4, ymm4, xmm0			;;22		temp1 *= 1 or 2

	vpsllq	ymm11, ymm11, xmm0			;;	22	temp2 *= 1 or 2
	vpsllq	ymm10, ymm10, xmm0			;;	22	temp1 *= 1 or 2
	ENDM

;; Check if remainder in ymm4,ymm5 matches factor + 1 indicating we found a factor!

avx2_compare_part1 MACRO
	vpsubq	ymm5, ymm5, YMMWORD PTR YMM_ONE		;; Subtract one to compare result to factor
	vpand	ymm2, ymm5, ymm15			;; RES3 = temp2 & 0x3FFFFFFF
	vpsubq	ymm11, ymm11, YMMWORD PTR YMM_ONE	;; Subtract one to compare result to factor
	vpand	ymm8, ymm11, ymm15			;; RES3 = temp2 & 0x3FFFFFFF
	ENDM

avx2_compare_part2 MACRO
	vpsrlq	ymm5, ymm5, 30				;; temp2 >>= 30
	vpand	ymm1, ymm5, ymm15			;; RES2 = temp2 & 0x3FFFFFFF
	vpsrlq	ymm5, ymm5, 2				;; temp2 >>= 2
	vpsrad	ymm5, ymm5, 28				;; temp2 >>= 28
	vpaddd	ymm4, ymm4, ymm5			;; temp1 += temp2
	vpand	ymm0, ymm4, YMMWORD PTR YMM_BITS28	;; RES1 = temp1 & 0x0FFFFFFF
	vpsrlq	ymm11, ymm11, 30			;; temp2 >>= 30
	vpand	ymm7, ymm11, ymm15			;; RES2 = temp2 & 0x3FFFFFFF
	vpsrlq	ymm11, ymm11, 2				;; temp2 >>= 2
	vpsrad	ymm11, ymm11, 28			;; temp2 >>= 28
	vpaddd	ymm10, ymm10, ymm11			;; temp1 += temp2
	vpand	ymm6, ymm10, YMMWORD PTR YMM_BITS28	;; RES1 = temp1 & 0x0FFFFFFF
	ENDM


;;********************************************************
;; The AVX FMA versions of TF macros (4 factors at a time)
;; Timed at 1.85 ms with Options/Benchmark on 3.4 GHz Haswell
;;********************************************************

; This macro does some initialization work and then performs a subset of the
; main loop working on the initial value.  This subset avoids the squaring of the
; remainder.  The initial value can be up to 60 bits larger than the factor
; size.

avx_fma_fac_initval MACRO
	LOCAL	large_initial_value, small_initial_value, equals_split, done

;; Preload constants

	vmovapd	ymm15, YMMWORD PTR YMM_FMA_RND_CONST3		;; Load 3.0 * pow (2.0, (double) (51))
	vmovapd	ymm14, YMMWORD PTR YMM_FMA_RND_CONST2		;; Load 3.0 * pow (2.0, (double) (51 + BITS_LOW))
	lea	r8, YMM_FMA_ONE_OR_TWO				;; q2_dividend = 1-or-2 * q2_dividend + q1f1_hi

;; Convert factor from 2 integers into 2 doubles

	vmovapd	ymm13, YMMWORD PTR YMM_FMA_LOW_MASK		;; Mask for acquiring low bits
	vandpd	ymm0, ymm13, YMMWORD PTR YMM_FMA_FAC_LO_INT	;; Load integer factor low
	vorpd	ymm0, ymm0, ymm15				;; Step 1 to convert factor low to a double
	vorpd	ymm1, ymm15, YMMWORD PTR YMM_FMA_FAC_HI_INT	;; Load integer factor hi
	yfmsubpd ymm1, ymm1, YMMWORD PTR YMM_FMA_TWO_TO_LO, ymm14 ;; Convert integer factor hi to double times 2^lo
	vsubpd	ymm0, ymm0, ymm15				;; Step 2 to convert factor low to a double
	vmovapd	YMMWORD PTR YMM_FMA_FAC_HI, ymm0		;; Save fac_hi
	vmovapd	YMMWORD PTR YMM_FMA_FAC_LO, ymm1		;; Save fac_lo

;; Compute factor inverse

	vaddpd	ymm3, ymm0, ymm1				;; Add fac_hi and fac_lo to form factor accurate to 53 bits
	vmovapd	ymm2, YMMWORD PTR YMM_FMA_ONE
	vdivpd	ymm2, ymm2, ymm3				;; Compute 1 / fac
	vmovapd	YMMWORD PTR YMM_FMA_INVFAC, ymm2		;; Save factor inverse

;; Compute (2^split mod f) * 2^-split.  Used after calculating rem_hi^2.

	vmovapd	ymm13, YMMWORD PTR YMM_FMA_RND_CONST4		;; Load (3.0 * 2^51) * 2^-split
	vaddpd	ymm4, ymm2, ymm13				;; q = 1.0 * facinv + rounding const
	vsubpd	ymm4, ymm4, ymm13				;; q -= rounding const
	yfnmaddpd ymm5, ymm4, ymm0, YMMWORD PTR YMM_FMA_ONE	;; hi = 1.0 - q * fac_hi
	vmovapd	ymm12, YMMWORD PTR YMM_FMA_RND_CONST5		;; Load (3.0 * 2^(51 + LOW_BITS)) * 2^-split
	yfnmaddpd ymm6, ymm4, ymm1, ymm12			;; hi_addin = -q * fac_lo + rounding const
	vsubpd	ymm6, ymm6, ymm12				;; hi_addin -= rounding const
	yfnmsubpd ymm4, ymm4, ymm1, ymm6			;; lo = -q* fac_lo - hi_addin
	vaddpd	ymm5, ymm5, ymm6				;; hi += hi_addin
	vmovapd	YMMWORD PTR YMM_FMA_TWOPOW1_MODF_HI, ymm5	;; Save high bits of modf
	vmovapd	YMMWORD PTR YMM_FMA_TWOPOW1_MODF_LO, ymm4	;; Save low bits of modf

;; Compute (2^(split+1) mod f) * 2^-split.  Used for optional mul-by-2 after computing rem_hi^2.

	vmovapd	ymm5, YMMWORD PTR YMM_FMA_TWO			;; Load 2.0
	yfmaddpd ymm4, ymm5, ymm2, ymm13			;; q = 2.0 * facinv + rounding const
	vsubpd	ymm4, ymm4, ymm13				;; q -= rounding const
	yfnmaddpd ymm3, ymm4, ymm0, ymm5			;; hi = 2.0 - q * fac_hi
	yfnmaddpd ymm6, ymm4, ymm1, ymm12			;; hi_addin = -q * fac_lo + rounding const
	vsubpd	ymm6, ymm6, ymm12				;; hi_addin -= rounding const
	yfnmsubpd ymm4, ymm4, ymm1, ymm6			;; lo = -q* fac_lo - hi_addin
	vaddpd	ymm3, ymm3, ymm6				;; hi += hi_addin
	vmovapd	YMMWORD PTR YMM_FMA_TWOPOW2_MODF_HI, ymm3	;; Save high bits of modf
	vmovapd	YMMWORD PTR YMM_FMA_TWOPOW2_MODF_LO, ymm4	;; Save low bits of modf

;; Now compute the initial remainder, using an optimized version of the main loop code.  We can save
;; several multiplies and adds because the squared value is a power of two. 

;; There are 4 options for the initial value to get a remainder of.  FMA_INITVAL_TYPE is:
;;	-1)  initval < 2^split	(calc using small initval code)
;;	0)   > 2^(split+1)	(calc using large initval code)
;;	1)   == 2^split		(use twopow1_modf)
;;	2)   == 2^(split+1)	(use twopow2_modf)

;; Test which of the 4 initial value types we have.  Branch to the right code.

	cmp	FMA_INITVAL_TYPE, 0
	jl	small_initial_value
	je	short large_initial_value

;; Easiest case of all!  Remainder is either twopow1_modf or twopow2_modf

	vmovapd	ymm13, YMMWORD PTR YMM_FMA_TWOPOW_SPLIT		;; Load 2^split
	cmp	FMA_INITVAL_TYPE, 1
	je	short equals_split
	vmulpd	ymm3, ymm13, ymm3
	vmulpd	ymm4, ymm13, ymm4
	jmp	done

equals_split:
	vmulpd	ymm3, ymm13, YMMWORD PTR YMM_FMA_TWOPOW1_MODF_HI
	vmulpd	ymm4, ymm13, YMMWORD PTR YMM_FMA_TWOPOW1_MODF_LO
	jmp	done

;; Process an initial squared value that is more than 2^(split+1).  To avoid any edge cases with regards
;; to precision of the resulting remainder, we load initial_squared_value / 2 and use the twopow2_modf values
;; ymm0 = fachi, ymm1 = faclo, ymm2 = facinv, ymm3 = twopow2modf_hi, ymm4 = twopow2modf_lo

large_initial_value:

;; calc  AA-hi*TwoPowModFhi hi	* 2^(35+35)		51/50	*2^70
;; calc  AA-hi*TwoPowModFhi lo	* 2^(35)		35	*2^35
;; calc  AA-hi*TwoPowModFlo hi	* 2^(35)		41/40	*2^35
;; calc  AA-hi*TwoPowModFlo lo	* 2^0			35	*2^0

	vmovapd	ymm5, YMMWORD PTR YMM_FMA_INITVAL		;; Load the initial squared value/2 (known as aa_hi in main loop)
	vmovapd	ymm13, YMMWORD PTR YMM_FMA_RND_CONST1		;; Load 3.0 * pow (2.0, (double) (51 + BITS_LOW * 2))
	yfmaddpd ymm6, ymm5, ymm3, ymm13			;; q1f1_hi = aa_hi * twopow2modf_hi + rounding const
	vsubpd	ymm6, ymm6, ymm13				;; q1f1_hi -= rounding const
	yfmsubpd ymm3, ymm5, ymm3, ymm6				;; q1f1_lo = aa_hi * twopow2modf_hi - q1f1_hi
								;; q2_dividend = q1f1_hi
								;; final_hi = q1f1_lo

	yfmaddpd ymm7, ymm5, ymm4, ymm14			;; q1f2_hi = aa_hi * twopow2modf_lo + rounding const
	vsubpd	ymm7, ymm7, ymm14				;; q1f2_hi -= rounding const
	yfmsubpd ymm4, ymm5, ymm4, ymm7				;; q1f2_lo = aa_hi * twopow2modf_lo - q1f2_hi
	vaddpd	ymm3, ymm3, ymm7				;; final_hi += q1f2_hi
								;; final_lo = q1f2_lo

;; calc Q2, remaining bits of quotient
;; calc  Q2*Fhi hi	* 2^(33+35)		53	*2^68
;; calc  Q2*Fhi lo	* 2^35			33	*2^35
;; calc  Q2*Flo hi	* 2^(35+0)		41*	*2^35
;; calc  Q2*Flo lo	* 2^0			35	*2^0

	yfmaddpd ymm2, ymm6, ymm2, ymm15			;; q2 = q2_dividend * facinv + rounding const
	vsubpd	ymm2, ymm2, ymm15				;; q2 -= rounding const

	yfnmaddpd ymm0, ymm2, ymm0, ymm6			;; q2f1_lo = q2_dividend - q2 * fac_hi
	vaddpd	ymm3, ymm3, ymm0				;; final_hi += q2f1_lo

	yfnmaddpd ymm0, ymm2, ymm1, ymm14			;; q2f2_hi = -q2 * fac_lo + rounding const
	vsubpd	ymm0, ymm0, ymm14				;; q2f2_hi -= rounding const
	yfnmsubpd ymm1, ymm2, ymm1, ymm0			;; q2f2_lo = -q2 * fac_lo - q2f2_hi

	vaddpd	ymm3, ymm3, ymm0				;; final_hi += q2f2_hi
	vaddpd	ymm4, ymm4, ymm1				;; final_lo += q2f2_lo

	jmp	done

;; Process an initial squared value that is less than 2^split
;; Calc q, the quotient.  Then calc initial_value - q * fac

small_initial_value:
	vmovapd	ymm3, YMMWORD PTR YMM_FMA_INITVAL		;; Load the initial squared value

	yfmaddpd ymm2, ymm3, ymm2, ymm15			;; q = initial_value * facinv + rounding const
	vsubpd	ymm2, ymm2, ymm15				;; q -= rounding const
	yfnmaddpd ymm3, ymm2, ymm0, ymm3			;; final_hi = initial_value - q * fac_hi

	yfnmaddpd ymm0, ymm2, ymm1, ymm14			;; qf2_hi = -q * fac_lo + rounding const
	vsubpd	ymm0, ymm0, ymm14				;; qf2_hi -= rounding const
	yfnmsubpd ymm4, ymm2, ymm1, ymm0			;; final_lo = -q * fac_lo - qf2_hi

	vaddpd	ymm3, ymm3, ymm0				;; final_hi += q2f2_hi

done:
	ENDM


; This macro does one iteration of the main loop.  Square a remainder and do a mod.
; Note that at the beginning of the macro ymm3, ymm4 contains the semi-normalized remainder
; for us to square.  This input number is between -fac/2*epsilon and +fac/2*epsilon.  The range
; is not perfect because this macro's calculation of Q2 is not exact.

; In the comments of this macro, for illustration purposes we'll assume fac_size is 80 bits
; with 35 bits stored in the low word.  There is no reason to believe this code cannot handle
; at least 96 bits.  In fact, we have simulated this macro in C code with no problems.

avx_fma_fac MACRO fac_size

;; AB^2 will be 160 bits.  From there, we multiply the top 40 bits by the 80-bit value 2^120 mod f.
;; This 120-bit result plus the low 120/121 bits of AB^2 give us a 122-bit dividend in computing Q2.

;; To maximize significant bits in calculating Q2, we want to maximize the precision of AA-lo and 2AB-hi.  Looking at
;; the table below that happens when AA-lo and 2AB-hi use as small an offset as possible without exceeding 53 bits
;; of precision.  Below that would be 34 bits in a low word.

;; SQUARE rem (call rem_hi A and rem_lo B).  A is between 0 and 44.5+epsilon bits.  B is between 0 and 35 normalized bits.
;; Unnormalized low words are the result of 1 doubling and 3 adds.  This adds 2 bits.

;;					normalized		unnormalized
;;				      max bits	offset	      max bits	offset
;;AA-hi		* 2^(50+2*35)		40	*2^120		40	*2^120
;;AA-lo		* 2^(2*35)		50	*2^70		50	*2^70
;;2AB-hi	* 2^(34+36)		46/47	*2^70		48/49	*2^70
;;2AB-lo	* 2^36			34/33	*2^36/37	34/33	*2^36/37
;;BB-hi		* 2^(35+0)		35	*2^35		39/40	*2^35
;;BB-lo		* 2^0			35	*2^0		35	*2^0

	vmovapd	ymm15, YMMWORD PTR YMM_FMA_RND_CONST0		;; Load 3.0 * pow (2.0, (double) (51 + BITS_TOTAL + (BITS_TOTAL + 1) / 2))
	yfmaddpd ymm0, ymm3, ymm3, ymm15			;; aa_hi = rem_hi * rem_hi + rounding const
	vsubpd	ymm0, ymm0, ymm15				;; aa_hi -= rounding const
	yfmsubpd ymm1, ymm3, ymm3, ymm0				;; aa_lo = rem_hi * rem_hi - aa_hi

	vmovapd	ymm15, YMMWORD PTR YMM_FMA_RND_CONST1		;; Load 3.0 * pow (2.0, (double) (51 + BITS_LOW * 2))
	yfmaddpd ymm2, ymm3, ymm4, ymm15			;; ab_hi = rem_hi * rem_lo + rounding const
	vsubpd	ymm2, ymm2, ymm15				;; ab_hi -= rounding const
	yfmsubpd ymm3, ymm3, ymm4, ymm2				;; ab_lo = rem_hi * rem_lo - ab_hi

	yfmaddpd ymm2, YMMWORD PTR YMM_FMA_TWO, ymm2, ymm1	;; q2_dividend = 2AB_hi + AA_lo

	vmovapd	ymm15, YMMWORD PTR YMM_FMA_RND_CONST2		;; Load 3.0 * pow (2.0, (double) (51 + BITS_LOW))
	yfmaddpd ymm1, ymm4, ymm4, ymm15			;; bb_hi = rem_lo * rem_lo + rounding const
	vsubpd	ymm1, ymm1, ymm15				;; bb_hi -= rounding const
	yfmsubpd ymm4, ymm4, ymm4, ymm1				;; bb_lo = rem_lo * rem_lo - bb_hi

	yfmaddpd ymm3, YMMWORD PTR YMM_FMA_TWO, ymm3, ymm1	;; final_hi = 2 * ab_lo + bb_hi

;; ymm0 = aa_hi, ymm2 = q2_dividend, ymm3 = final_hi, ymm4 = bb_lo

;; Set register if a mul by two is required this iteration

	mov	edx, FMA_SHIFTER[rdi*4-4]			;; This is 32 if doubling else 0

;; Since we know the power of two that aa_hi is a multiple of (aa_hi is a multiple of 2^120 in our example),
;; we pre-calculate 2^that_power mod factor, and multiply aa-hi by this pre-calculated value.

;;  Aa-hi is 41/40 bits, TwoPowModFhi is 45 bits, TwoPowModFlo is 35 bits
;; calc  AA-hi*TwoPowModFhi hi	* 2^(35+35)		51/50	*2^70
;; calc  AA-hi*TwoPowModFhi lo	* 2^(35)		35	*2^35
;; calc  AA-hi*TwoPowModFlo hi	* 2^(35)		41/40	*2^35
;; calc  AA-hi*TwoPowModFlo lo	* 2^0			35	*2^0

	vmovapd	ymm15, YMMWORD PTR YMM_FMA_RND_CONST1		;; Load 3.0 * pow (2.0, (double) (51 + BITS_LOW * 2))
	yfmaddpd ymm5, ymm0, YMMWORD PTR YMM_FMA_TWOPOW1_MODF_HI[rdx], ymm15 ;; q1f1_hi = aa_hi * twopowmodf_hi[rdx] + rounding const
	vsubpd	ymm5, ymm5, ymm15				;; q1f1_hi -= rounding const
	yfmsubpd ymm1, ymm0, YMMWORD PTR YMM_FMA_TWOPOW1_MODF_HI[rdx], ymm5 ;; q1f1_lo = aa_hi * twopowmodf_hi[rdx] - q1f1_hi

	yfmaddpd ymm2, YMMWORD PTR [r8][rdx], ymm2, ymm5	;; q2_dividend = 1-or-2 * q2_dividend + q1f1_hi

	yfmaddpd ymm3, YMMWORD PTR [r8][rdx], ymm3, ymm1	;; final_hi = 1-or-2 * final_hi + q1f1_lo

;; ymm0 = aa_hi, ymm2 = q2_dividend, ymm3 = final_hi, ymm4 = bb_lo

	vmovapd	ymm15, YMMWORD PTR YMM_FMA_RND_CONST2		;; Load 3.0 * pow (2.0, (double) (51 + BITS_LOW))
	yfmaddpd ymm1, ymm0, YMMWORD PTR YMM_FMA_TWOPOW1_MODF_LO[rdx], ymm15 ;; q1f2_hi = aa_hi * twopowmodf_lo[rdx] + rounding const
	vsubpd	ymm1, ymm1, ymm15				;; q1f2_hi -= rounding const
	yfmsubpd ymm0, ymm0, YMMWORD PTR YMM_FMA_TWOPOW1_MODF_LO[rdx], ymm1 ;; q1f2_lo = aa_hi * twopowmodf_lo[rdx] - q1f2_hi

	vaddpd	ymm3, ymm3, ymm1				;; final_hi += q1f2_hi

	yfmaddpd ymm4, YMMWORD PTR [r8][rdx], ymm4, ymm0	;; final_lo = one_or_two[rdx] * bb_lo + q1f2_lo;

;; ymm2 = q2_dividend, ymm3 = final_hi, ymm4 = final_lo

;; calc Q2, remaining bits of quotient
;;
;; Q2 is 41 bits, Fhi is 45 bits, Flo is 35 bits
;; calc  Q2*Fhi hi	* 2^(33+35)		53	*2^68
;; calc  Q2*Fhi lo	* 2^35			33	*2^35
;; calc  Q2*Flo hi	* 2^(35+0)		41*	*2^35
;; calc  Q2*Flo lo	* 2^0			35	*2^0

	vmovapd	ymm15, YMMWORD PTR YMM_FMA_RND_CONST3		;; Load 3.0 * pow (2.0, (double) (51))
	yfmaddpd ymm1, ymm2, YMMWORD PTR YMM_FMA_INVFAC, ymm15	;; q2 = q2_dividend * facinv + rounding const
	vsubpd	ymm1, ymm1, ymm15				;; q2 -= rounding const

	yfnmaddpd ymm2, ymm1, YMMWORD PTR YMM_FMA_FAC_HI, ymm2	;; Calc Q2*Fhi lo in one blow:  q2f1_lo = q2_dividend - q2 * fac_hi
	vaddpd	ymm3, ymm3, ymm2				;; final_hi += q2f1_lo

	vmovapd	ymm15, YMMWORD PTR YMM_FMA_RND_CONST2		;; Load 3.0 * pow (2.0, (double) (51 + BITS_LOW))
	yfnmaddpd ymm0, ymm1, YMMWORD PTR YMM_FMA_FAC_LO, ymm15	;; q2f2_hi = -q2 * fac_lo + rounding const
	vsubpd	ymm0, ymm0, ymm15				;; q2f2_hi -= rounding const
	yfnmsubpd ymm1, ymm1, YMMWORD PTR YMM_FMA_FAC_LO, ymm0	;; q2f2_lo = -q2 * fac_lo - q2f2_hi

	vaddpd	ymm3, ymm3, ymm0				;; final_hi += q2f2_hi
	vaddpd	ymm4, ymm4, ymm1				;; final_lo += q2f2_lo
	ENDM


;; Check if remainder in ymm3,ymm4 is one indicating we found a factor!

avx_fma_compare MACRO
	vaddpd	ymm2, ymm3, ymm4				;; Combine fac_hi & fac_lo
	ENDM


;;***************************************************************
;; The AVX FMA versions of the above macros (8 factors at a time)
;; Timed at 1.21 ms with Options/Benchmark on 3.4 GHz Haswell
;;***************************************************************

; This macro does some initialization work and then performs a subset of the
; main loop working on the initial value.  This subset avoids the squaring of the
; remainder.  The initial value can be up to 60 bits larger than the factor
; size.

avx_fma_fac_initval MACRO avx_level
	LOCAL	large_initial_value, small_initial_value, equals_split, done

;; Preload constants

	vmovapd	ymm15, YMMWORD PTR YMM_FMA_RND_CONST3		;; Load 3.0 * pow (2.0, (double) (51))
	vmovapd	ymm14, YMMWORD PTR YMM_FMA_RND_CONST2		;; Load 3.0 * pow (2.0, (double) (51 + BITS_LOW))
	lea	r8, YMM_FMA_ONE_OR_TWO				;; Address of 1.0 or 2.0

;; Convert factor from 2 integers into 2 doubles

	vmovapd	ymm13, YMMWORD PTR YMM_FMA_LOW_MASK		;; Mask for acquiring low bits
	vandpd	ymm0, ymm13, YMMWORD PTR YMM_FMA_FAC_LO_INT	;; Load integer factor low
	vandpd	ymm4, ymm13, YMMWORD PTR YMM_FMA_FAC_LO_INTa	;; Load integer factor low

	vorpd	ymm0, ymm0, ymm15				;; Step 1 to convert factor low to a double
	vorpd	ymm4, ymm4, ymm15				;; Step 1 to convert factor low to a double

	vorpd	ymm10, ymm15, YMMWORD PTR YMM_FMA_FAC_HI_INT	;; Load integer factor hi
	vorpd	ymm11, ymm15, YMMWORD PTR YMM_FMA_FAC_HI_INTa	;; Load integer factor hi

	vmovapd	ymm13, YMMWORD PTR YMM_FMA_TWO_TO_LO		;; Constant for converting integer factor hi
	yfmsubpd ymm10, ymm10, ymm13, ymm14			;; Convert integer factor hi to double times 2^lo
	yfmsubpd ymm11, ymm11, ymm13, ymm14			;; Convert integer factor hi to double times 2^lo

	vsubpd	ymm0, ymm0, ymm15				;; Step 2 to convert factor low to a double
	vsubpd	ymm4, ymm4, ymm15				;; Step 2 to convert factor low to a double

	vmovapd	YMMWORD PTR YMM_FMA_FAC_HI, ymm10		;; Save fac_hi
	vmovapd	YMMWORD PTR YMM_FMA_FAC_HIa, ymm11		;; Save fac_hi

	vmovapd	YMMWORD PTR YMM_FMA_FAC_LO, ymm0		;; Save fac_lo
	vmovapd	YMMWORD PTR YMM_FMA_FAC_LOa, ymm4		;; Save fac_lo

;; Compute factor inverse

	vaddpd	ymm3, ymm10, ymm0				;; Add fac_hi and fac_lo to form factor accurate to 53 bits
	vaddpd	ymm7, ymm11, ymm4				;; Add fac_hi and fac_lo to form factor accurate to 53 bits
	vmovapd	ymm9, YMMWORD PTR YMM_FMA_ONE
	vdivpd	ymm8, ymm9, ymm3				;; Compute 1 / fac
	vdivpd	ymm9, ymm9, ymm7				;; Compute 1 / fac
	vmovapd	YMMWORD PTR YMM_FMA_INVFAC, ymm8		;; Save factor inverse
	vmovapd	YMMWORD PTR YMM_FMA_INVFACa, ymm9		;; Save factor inverse

;; Compute (2^split mod f) * 2^-split.  Used after calculating rem_hi^2.

;; ymm0 = fac_lo, ymm8/9 = facinv, ymm10/11 = fac_hi

	vmovapd	ymm13, YMMWORD PTR YMM_FMA_RND_CONST4		;; Load (3.0 * 2^51) * 2^-split
	vaddpd	ymm1, ymm8, ymm13				;; q = 1.0 * facinv + rounding const
	vaddpd	ymm5, ymm9, ymm13				;; q = 1.0 * facinv + rounding const
	vsubpd	ymm1, ymm1, ymm13				;; q -= rounding const
	vsubpd	ymm5, ymm5, ymm13				;; q -= rounding const
	vmovapd	ymm12, YMMWORD PTR YMM_FMA_ONE			;; Load 1.0
	yfnmaddpd ymm2, ymm1, ymm10, ymm12			;; hi = 1.0 - q * fac_hi
	yfnmaddpd ymm6, ymm5, ymm11, ymm12			;; hi = 1.0 - q * fac_hi
	vmovapd	ymm12, YMMWORD PTR YMM_FMA_RND_CONST5		;; Load (3.0 * 2^(51 + LOW_BITS)) * 2^-split
	yfnmaddpd ymm3, ymm1, ymm0, ymm12			;; hi_addin = -q * fac_lo + rounding const
	yfnmaddpd ymm7, ymm5, ymm4, ymm12			;; hi_addin = -q * fac_lo + rounding const
	vsubpd	ymm3, ymm3, ymm12				;; hi_addin -= rounding const
	vsubpd	ymm7, ymm7, ymm12				;; hi_addin -= rounding const
	vaddpd	ymm2, ymm2, ymm3				;; hi += hi_addin
	vaddpd	ymm6, ymm6, ymm7				;; hi += hi_addin
	yfnmsubpd ymm1, ymm1, ymm0, ymm3			;; lo = -q * fac_lo - hi_addin
	yfnmsubpd ymm5, ymm5, ymm4, ymm7			;; lo = -q * fac_lo - hi_addin
	vmovapd	YMMWORD PTR YMM_FMA_TWOPOW1_MODF_HI, ymm2	;; Save high bits of modf
	vmovapd	YMMWORD PTR YMM_FMA_TWOPOW1_MODF_HIa, ymm6	;; Save high bits of modf
	vmovapd	YMMWORD PTR YMM_FMA_TWOPOW1_MODF_LO, ymm1	;; Save low bits of modf
	vmovapd	YMMWORD PTR YMM_FMA_TWOPOW1_MODF_LOa, ymm5	;; Save low bits of modf

;; Compute (2^(split+1) mod f) * 2^-split.  Used for optional mul-by-2 after computing rem_hi^2.

	vmovapd	ymm12, YMMWORD PTR YMM_FMA_TWO			;; Load 2.0
	yfmaddpd ymm1, ymm12, ymm8, ymm13			;; q = 2.0 * facinv + rounding const
	yfmaddpd ymm5, ymm12, ymm9, ymm13			;; q = 2.0 * facinv + rounding const
	vsubpd	ymm1, ymm1, ymm13				;; q -= rounding const
	vsubpd	ymm5, ymm5, ymm13				;; q -= rounding const
	yfnmaddpd ymm2, ymm1, ymm10, ymm12			;; hi = 2.0 - q * fac_hi
	yfnmaddpd ymm6, ymm5, ymm11, ymm12			;; hi = 2.0 - q * fac_hi
	vmovapd	ymm12, YMMWORD PTR YMM_FMA_RND_CONST5		;; Load (3.0 * 2^(51 + LOW_BITS)) * 2^-split
	yfnmaddpd ymm3, ymm1, ymm0, ymm12			;; hi_addin = -q * fac_lo + rounding const
	yfnmaddpd ymm7, ymm5, ymm4, ymm12			;; hi_addin = -q * fac_lo + rounding const
	vsubpd	ymm3, ymm3, ymm12				;; hi_addin -= rounding const
	vsubpd	ymm7, ymm7, ymm12				;; hi_addin -= rounding const
	vaddpd	ymm2, ymm2, ymm3				;; hi += hi_addin
	vaddpd	ymm6, ymm6, ymm7				;; hi += hi_addin
	yfnmsubpd ymm1, ymm1, ymm0, ymm3			;; lo = -q * fac_lo - hi_addin
	yfnmsubpd ymm5, ymm5, ymm4, ymm7			;; lo = -q * fac_lo - hi_addin
	vmovapd	YMMWORD PTR YMM_FMA_TWOPOW2_MODF_HI, ymm2	;; Save high bits of modf
	vmovapd	YMMWORD PTR YMM_FMA_TWOPOW2_MODF_HIa, ymm6	;; Save high bits of modf
	vmovapd	YMMWORD PTR YMM_FMA_TWOPOW2_MODF_LO, ymm1	;; Save low bits of modf
	vmovapd	YMMWORD PTR YMM_FMA_TWOPOW2_MODF_LOa, ymm5	;; Save low bits of modf

;; ymm0 = fac_lo, ymm1 = twopow2modf_lo, ymm2 = twopow2modf_hi, ymm8/9 = facinv, ymm10/11 = fac_hi


;; Now compute the initial remainder, using an optimized version of the main loop code.  We can save
;; several multiplies and adds because the squared value is a power of two. 

;; There are 4 options for the initial value to get a remainder of.  FMA_INITVAL_TYPE is:
;;	-1)  initval < 2^split	(calc using small initval code)
;;	0)   > 2^(split+1)	(calc using large initval code)
;;	1)   == 2^split		(use twopow1_modf)
;;	2)   == 2^(split+1)	(use twopow2_modf)

;; Test which of the 4 initial value types we have.  Branch to the right code.

	cmp	FMA_INITVAL_TYPE, 0
	jl	small_initial_value
	je	short large_initial_value

;; Easiest case of all!  Remainder is either twopow1_modf or twopow2_modf

	vmovapd	ymm13, YMMWORD PTR YMM_FMA_TWOPOW_SPLIT		;; Load 2^split
	cmp	FMA_INITVAL_TYPE, 1
	je	short equals_split
	vmulpd	ymm2, ymm13, ymm2				;; Return twopow2_modf
	vmulpd	ymm3, ymm13, ymm1
	vmulpd	ymm6, ymm13, ymm6				;; Return twopow2_modf
	vmulpd	ymm7, ymm13, ymm5
	jmp	done

equals_split:
	vmulpd	ymm2, ymm13, YMMWORD PTR YMM_FMA_TWOPOW1_MODF_HI ;; Return twopow1_modf
	vmulpd	ymm3, ymm13, YMMWORD PTR YMM_FMA_TWOPOW1_MODF_LO
	vmulpd	ymm6, ymm13, YMMWORD PTR YMM_FMA_TWOPOW1_MODF_HIa ;; Return twopow1_modf
	vmulpd	ymm7, ymm13, YMMWORD PTR YMM_FMA_TWOPOW1_MODF_LOa
	jmp	done

;; Process an initial squared value that is more than 2^(split+1).  To avoid any edge cases with regards
;; to precision of the resulting remainder, we load initial_squared_value / 2 and use the twopow2_modf values

large_initial_value:

;; calc  AA-hi*TwoPowModFhi hi	* 2^(35+35)		51/50	*2^70
;; calc  AA-hi*TwoPowModFhi lo	* 2^(35)		35	*2^35
;; calc  AA-hi*TwoPowModFlo hi	* 2^(35)		41/40	*2^35
;; calc  AA-hi*TwoPowModFlo lo	* 2^0			35	*2^0

;; calc Q2, remaining bits of quotient
;; calc  Q2*Fhi hi	* 2^(33+35)		53	*2^68
;; calc  Q2*Fhi lo	* 2^35			33	*2^35
;; calc  Q2*Flo hi	* 2^(35+0)		41*	*2^35
;; calc  Q2*Flo lo	* 2^0			35	*2^0

;; ymm0 = fac_lo, ymm1 = twopow2modf_lo, ymm2 = twopow2modf_hi, ymm8/9 = facinv, ymm10/11 = fac_hi

	vmovapd	ymm12, YMMWORD PTR YMM_FMA_INITVAL		;; Load the initial squared value/2 (known as aa_hi in main loop)
	vmovapd	ymm13, YMMWORD PTR YMM_FMA_RND_CONST1		;; Load 3.0 * pow (2.0, (double) (51 + BITS_LOW * 2))

	yfmaddpd ymm0, ymm12, ymm2, ymm13			;; 1-4	; q1f1_hi = aa_hi * twopow2modf_hi + rounding const
	yfmaddpd ymm3, ymm12, ymm1, ymm14			;; 1-4	; q1f2_hi = aa_hi * twopow2modf_lo + rounding const
	yfmaddpd ymm4, ymm12, ymm6, ymm13			;; 2-5	; q1f1_hi = aa_hi * twopow2modf_hi + rounding const
	yfmaddpd ymm7, ymm12, ymm5, ymm14			;; 2-5	; q1f2_hi = aa_hi * twopow2modf_lo + rounding const
	vsubpd	ymm0, ymm0, ymm13				;; 5-8	; q1f1_hi -= rounding const (q2_dividend = q1f1_hi)
	vsubpd	ymm3, ymm3, ymm14				;; 5-8	; q1f2_hi -= rounding const
	vsubpd	ymm4, ymm4, ymm13				;; 6-9	; q1f1_hi -= rounding const (q2_dividend = q1f1_hi)
	vsubpd	ymm7, ymm7, ymm14				;; 6-9	; q1f2_hi -= rounding const
	yfmsubpd ymm2, ymm12, ymm2, ymm0			;; 9-12	; q1f1_lo = aa_hi * twopow2modf_hi - q1f1_hi (final_hi = q1f1_lo)
	yfmsubpd ymm1, ymm12, ymm1, ymm3			;; 9-12	; q1f2_lo = aa_hi * twopow2modf_lo - q1f2_hi (final_lo = q1f2_lo)
	yfmsubpd ymm6, ymm12, ymm6, ymm4			;; 10-13; q1f1_lo = aa_hi * twopow2modf_hi - q1f1_hi (final_hi = q1f1_lo)
	yfmsubpd ymm5, ymm12, ymm5, ymm7			;; 10-13; q1f2_lo = aa_hi * twopow2modf_lo - q1f2_hi (final_lo = q1f2_lo)
	vaddpd	ymm2, ymm2, ymm3				;; 13-16; final_hi += q1f2_hi
	vaddpd	ymm6, ymm6, ymm7				;; 14-17; final_hi += q1f2_hi
	yfmaddpd ymm3, ymm0, ymm8, ymm15			;; 12-15; q2 = q2_dividend * facinv + rounding const
	yfmaddpd ymm7, ymm4, ymm9, ymm15			;; 12-15; q2 = q2_dividend * facinv + rounding const
	vsubpd	ymm3, ymm3, ymm15				;; 16-19; q2 -= rounding const
	vsubpd	ymm7, ymm7, ymm15				;; 16-19; q2 -= rounding const
	yfnmaddpd ymm0, ymm3, ymm10, ymm0			;; 20-23; q2f1_lo = q2_dividend - q2 * fac_hi
	yfnmaddpd ymm4, ymm7, ymm11, ymm4			;; 20-23; q2f1_lo = q2_dividend - q2 * fac_hi
	vaddpd	ymm2, ymm2, ymm0				;; 24-27; final_hi += q2f1_lo
	vaddpd	ymm6, ymm6, ymm4				;; 24-27; final_hi += q2f1_lo
	yfnmaddpd ymm0, ymm3, YMMWORD PTR YMM_FMA_FAC_LO, ymm14	;; 21-24; q2f2_hi = -q2 * fac_lo + rounding const
	yfnmaddpd ymm4, ymm7, YMMWORD PTR YMM_FMA_FAC_LOa, ymm14;; 22-25; q2f2_hi = -q2 * fac_lo + rounding const
	vsubpd	ymm0, ymm0, ymm14				;; 25-28; q2f2_hi -= rounding const
	vsubpd	ymm4, ymm4, ymm14				;; 26-29; q2f2_hi -= rounding const
	vaddpd	ymm2, ymm2, ymm0				;; 29-32; final_hi += q2f2_hi
	vaddpd	ymm6, ymm6, ymm4				;; 30-33; final_hi += q2f2_hi
	yfnmsubpd ymm3, ymm3, YMMWORD PTR YMM_FMA_FAC_LO, ymm0	;; 29-32; q2f2_lo = -q2 * fac_lo - q2f2_hi
	yfnmsubpd ymm7, ymm7, YMMWORD PTR YMM_FMA_FAC_LOa, ymm4	;; 31-34; q2f2_lo = -q2 * fac_lo - q2f2_hi
	vaddpd	ymm3, ymm1, ymm3				;; 33-36; final_lo += q2f2_lo
	vaddpd	ymm7, ymm5, ymm7				;; 35-38; final_lo += q2f2_lo
	jmp	done

;; Process an initial squared value that is less than 2^split
;; Calc q, the quotient.  Then calc initial_value - q * fac

;; ymm0 = fac_lo, ymm1 = twopow2modf_lo, ymm2 = twopow2modf_hi, ymm8/9 = facinv, ymm10/11 = fac_hi

small_initial_value:
	vmovapd	ymm12, YMMWORD PTR YMM_FMA_INITVAL		;; Load the initial squared value
	yfmaddpd ymm1, ymm12, ymm8, ymm15			;; q = initial_value * facinv + rounding const
	yfmaddpd ymm5, ymm12, ymm9, ymm15			;; q = initial_value * facinv + rounding const
	vsubpd	ymm1, ymm1, ymm15				;; q -= rounding const
	vsubpd	ymm5, ymm5, ymm15				;; q -= rounding const
	yfnmaddpd ymm2, ymm1, ymm10, ymm12			;; final_hi = initial_value - q * fac_hi
	yfnmaddpd ymm6, ymm5, ymm11, ymm12			;; final_hi = initial_value - q * fac_hi
	yfnmaddpd ymm3, ymm1, ymm0, ymm14			;; qf2_hi = -q * fac_lo + rounding const
	yfnmaddpd ymm7, ymm5, ymm4, ymm14			;; qf2_hi = -q * fac_lo + rounding const
	vsubpd	ymm3, ymm3, ymm14				;; qf2_hi -= rounding const
	vsubpd	ymm7, ymm7, ymm14				;; qf2_hi -= rounding const
	vaddpd	ymm2, ymm2, ymm3				;; final_hi += q2f2_hi
	vaddpd	ymm6, ymm6, ymm7				;; final_hi += q2f2_hi
	yfnmsubpd ymm3, ymm1, ymm0, ymm3			;; final_lo = -q * fac_lo - qf2_hi
	yfnmsubpd ymm7, ymm5, ymm4, ymm7			;; final_lo = -q * fac_lo - qf2_hi

done:
	ENDM


; This macro does one iteration of the main loop.  Square a remainder and do a mod.
; Note that at the beginning of the macro ymm2, ymm3 contains the semi-normalized remainder
; for us to square.  This input number is between -fac/2*epsilon and +fac/2*epsilon.  The range
; is not perfect because this macro's calculation of Q2 is not exact.

; In the comments of this macro, for illustration purposes we'll assume fac_size is 80 bits
; with 35 bits stored in the low word.  There is no reason to believe this code cannot handle
; at least 96 bits.  In fact, we have simulated this macro in C code with no problems.

avx_fma_fac MACRO fac_size

;; AB^2 will be 160 bits.  From there, we multiply the top 40 bits by the 80-bit value 2^120 mod f.
;; This 120-bit result plus the low 120/121 bits of AB^2 give us a 122-bit dividend in computing Q2.

;; To maximize significant bits in calculating Q2, we want to maximize the precision of AA-lo and 2AB-hi.  Looking at
;; the table below that happens when AA-lo and 2AB-hi use as small an offset as possible without exceeding 53 bits
;; of precision.  Below that would be 34 bits in a low word.

;; SQUARE rem (call rem_hi A and rem_lo B).  A is between 0 and 44.5+epsilon bits.  B is between 0 and 35 normalized bits.
;; Unnormalized low words are the result of 1 doubling and 3 adds.  This adds 2 bits.

;;					normalized		unnormalized
;;				      max bits	offset	      max bits	offset
;;AA-hi		* 2^(50+2*35)		40	*2^120		40	*2^120
;;AA-lo		* 2^(2*35)		50	*2^70		50	*2^70
;;2AB-hi	* 2^(34+36)		46/47	*2^70		48/49	*2^70
;;2AB-lo	* 2^36			34/33	*2^36/37	34/33	*2^36/37
;;BB-hi		* 2^(35+0)		35	*2^35		39/40	*2^35
;;BB-lo		* 2^0			35	*2^0		35	*2^0

;; Since we know the power of two that aa_hi is a multiple of (aa_hi is a multiple of 2^120 in our example),
;; we pre-calculate 2^that_power mod factor, and multiply aa-hi by this pre-calculated value.

;; Aa-hi is 41/40 bits, TwoPowModFhi is 45 bits, TwoPowModFlo is 35 bits
;; calc  AA-hi*TwoPowModFhi hi	* 2^(35+35)		51/50	*2^70
;; calc  AA-hi*TwoPowModFhi lo	* 2^(35)		35	*2^35
;; calc  AA-hi*TwoPowModFlo hi	* 2^(35)		41/40	*2^35
;; calc  AA-hi*TwoPowModFlo lo	* 2^0			35	*2^0

;; calc Q2, remaining bits of quotient
;;
;; Q2 is 41 bits, Fhi is 45 bits, Flo is 35 bits
;; calc  Q2*Fhi hi	* 2^(33+35)		53	*2^68
;; calc  Q2*Fhi lo	* 2^35			33	*2^35
;; calc  Q2*Flo hi	* 2^(35+0)		41*	*2^35
;; calc  Q2*Flo lo	* 2^0			35	*2^0

;; ymm0-3,8,9,10 reserved for set 1, ymm4-7,11,12,13 reserved for set 2
;; ymm14,15 is for common use

	vmovapd	ymm13, YMMWORD PTR YMM_FMA_RND_CONST0		;;	; Load 3.0 * pow (2.0, (double) (51 + BITS_TOTAL + (BITS_TOTAL + 1) / 2))
	yfmaddpd ymm0, ymm2, ymm2, ymm13			;;*1-4	; aa_hi = rem_hi * rem_hi + rounding const
	yfmaddpd ymm4, ymm6, ymm6, ymm13			;;*1-4	; aa_hi = rem_hi * rem_hi + rounding const

	vmovapd	ymm15, YMMWORD PTR YMM_FMA_RND_CONST1		;;	; Load 3.0 * pow (2.0, (double) (51 + BITS_LOW * 2))
	yfmaddpd ymm1, ymm2, ymm3, ymm15			;;*2-5	; ab_hi = rem_hi * rem_lo + rounding const
	yfmaddpd ymm5, ymm6, ymm7, ymm15			;;*2-5	; ab_hi = rem_hi * rem_lo + rounding const

	vmovapd	ymm14, YMMWORD PTR YMM_FMA_RND_CONST2		;;	; Load 3.0 * pow (2.0, (double) (51 + BITS_LOW))
	yfmaddpd ymm8, ymm3, ymm3, ymm14			;; 3-6 ; bb_hi = rem_lo * rem_lo + rounding const
	yfmaddpd ymm11, ymm7, ymm7, ymm14			;; 3-6 ; bb_hi = rem_lo * rem_lo + rounding const

	vsubpd	ymm0, ymm0, ymm13				;;*5-8	; aa_hi -= rounding const
	vsubpd	ymm4, ymm4, ymm13				;;*5-8	; aa_hi -= rounding const

	mov	edx, FMA_SHIFTER[rdi*4-4]			;;	; This is 32 if doubling else 0

	vsubpd	ymm1, ymm1, ymm15				;;*6-9	; ab_hi -= rounding const
	vsubpd	ymm5, ymm5, ymm15				;;*6-9	; ab_hi -= rounding const

	vsubpd	ymm8, ymm8, ymm14				;; 7-10	; bb_hi -= rounding const
	vsubpd	ymm11, ymm11, ymm14				;; 7-10 ; bb_hi -= rounding const

	yfmsubpd ymm9, ymm2, ymm2, ymm0				;;*9-12	; aa_lo = rem_hi * rem_hi - aa_hi
	yfmsubpd ymm12, ymm6, ymm6, ymm4			;;*9-12	; aa_lo = rem_hi * rem_hi - aa_hi

	yfmaddpd ymm10, ymm0, YMMWORD PTR YMM_FMA_TWOPOW1_MODF_HI[rdx], ymm15
								;;*10-13; q1f1_hi = aa_hi * twopowmodf_hi[rdx] + rounding const
	yfmaddpd ymm13, ymm4, YMMWORD PTR YMM_FMA_TWOPOW1_MODF_HIa[rdx], ymm15
								;;*10-13; q1f1_hi = aa_hi * twopowmodf_hi[rdx] + rounding const

	yfmsubpd ymm2, ymm2, ymm3, ymm1				;; 11-14; ab_lo = rem_hi * rem_lo - ab_hi
	yfmsubpd ymm6, ymm6, ymm7, ymm5				;; 11-14; ab_lo = rem_hi * rem_lo - ab_hi

	yfmsubpd ymm3, ymm3, ymm3, ymm8				;; 12-15; bb_lo = rem_lo * rem_lo - bb_hi
	yfmsubpd ymm7, ymm7, ymm7, ymm11			;; 12-15; bb_lo = rem_lo * rem_lo - bb_hi

;; ymm0=aa_hi, ymm1=ab_hi, ymm2=ab_lo, ymm3=bb_lo, ymm8=bb_hi, ymm9=aa_lo, ymm10=q1f1_hi

	yfmaddpd ymm9, YMMWORD PTR YMM_FMA_TWO, ymm1, ymm9	;;*13-16; q2_dividend = 2ab_hi + aa_lo
	yfmaddpd ymm12, YMMWORD PTR YMM_FMA_TWO, ymm5, ymm12	;;*13-16; q2_dividend = 2ab_hi + aa_lo

;; ymm0=aa_hi, ymm2=ab_lo, ymm3=bb_lo, ymm8=bb_hi, ymm9=q2_dividend, ymm10=q1f1_hi

	vsubpd	ymm10, ymm10, ymm15				;;*14-17; q1f1_hi -= rounding const
	vsubpd	ymm13, ymm13, ymm15				;;*14-17; q1f1_hi -= rounding const

	yfmaddpd ymm2, YMMWORD PTR YMM_FMA_TWO, ymm2, ymm8	;; 15-18; final_hi = 2 * ab_lo + bb_hi
	yfmaddpd ymm6, YMMWORD PTR YMM_FMA_TWO, ymm6, ymm11	;; 15-18; final_hi = 2 * ab_lo + bb_hi

;; ymm0=aa_hi, ymm2=final_hi, ymm3=bb_lo, ymm9=q2_dividend, ymm10=q1f1_hi

	yfmaddpd ymm1, ymm0, YMMWORD PTR YMM_FMA_TWOPOW1_MODF_LO[rdx], ymm14 ;; 16-19; q1f2_hi = aa_hi * twopowmodf_lo[rdx] + rounding const
	yfmaddpd ymm5, ymm4, YMMWORD PTR YMM_FMA_TWOPOW1_MODF_LOa[rdx], ymm14;; 16-19; q1f2_hi = aa_hi * twopowmodf_lo[rdx] + rounding const

;; ymm0=aa_hi, ymm1=q1f2_hi, ymm2=final_hi, ymm3=bb_lo, ymm9=q2_dividend, ymm10=q1f1_hi

	vmovapd ymm15, YMMWORD PTR [r8][rdx]			;;	; 1-or-2
	yfmaddpd ymm9, ymm15, ymm9, ymm10			;;*17-20; q2_dividend = 1-or-2 * q2_dividend + q1f1_hi
	yfmaddpd ymm12, ymm15, ymm12, ymm13			;;*17-20; q2_dividend = 1-or-2 * q2_dividend + q1f1_hi

	yfmsubpd ymm10, ymm0, YMMWORD PTR YMM_FMA_TWOPOW1_MODF_HI[rdx], ymm10
								;; 18-21; q1f1_lo = aa_hi * twopowmodf_hi[rdx] - q1f1_hi
	yfmsubpd ymm13, ymm4, YMMWORD PTR YMM_FMA_TWOPOW1_MODF_HIa[rdx], ymm13
								;; 18-21; q1f1_lo = aa_hi * twopowmodf_hi[rdx] - q1f1_hi

	vsubpd	ymm1, ymm1, ymm14				;; 20-23; q1f2_hi -= rounding const
	vsubpd	ymm5, ymm5, ymm14				;; 20-23; q1f2_hi -= rounding const

;; ymm0=aa_hi, ymm1=q1f2_hi, ymm2=final_hi, ymm3=bb_lo, ymm9=q2_dividend, ymm10=q1f1_lo

	vmovapd	ymm15, YMMWORD PTR YMM_FMA_RND_CONST3		;;	; Load 3.0 * pow (2.0, (double) (51))
	yfmaddpd ymm8, ymm9, YMMWORD PTR YMM_FMA_INVFAC, ymm15	;;*21-24; q2 = q2_dividend * facinv + rounding const
	yfmaddpd ymm11, ymm12, YMMWORD PTR YMM_FMA_INVFACa, ymm15;;*21-24; q2 = q2_dividend * facinv + rounding const

;; ymm0=aa_hi, ymm1=q1f2_hi, ymm2=final_hi, ymm3=bb_lo, ymm8=q2, ymm9=q2_dividend, ymm10=q1f1_lo

	vmovapd ymm15, YMMWORD PTR [r8][rdx]			;;	; 1-or-2
	yfmaddpd ymm2, ymm15, ymm2, ymm10			;; 22-25; final_hi = 1-or-2 * final_hi + q1f1_lo
	yfmaddpd ymm6, ymm15, ymm6, ymm13			;; 22-25; final_hi = 1-or-2 * final_hi + q1f1_lo

;; ymm0=aa_hi, ymm1=q1f2_hi, ymm2=final_hi, ymm3=bb_lo, ymm8=q2, ymm9=q2_dividend

	yfmsubpd ymm0, ymm0, YMMWORD PTR YMM_FMA_TWOPOW1_MODF_LO[rdx], ymm1;; 24-27; q1f2_lo = aa_hi * twopowmodf_lo[rdx] - q1f2_hi
	yfmsubpd ymm4, ymm4, YMMWORD PTR YMM_FMA_TWOPOW1_MODF_LOa[rdx], ymm5;; 24-27; q1f2_lo = aa_hi * twopowmodf_lo[rdx] - q1f2_hi

;; ymm0=q1f2_lo, ymm1=q1f2_hi, ymm2=final_hi, ymm3=bb_lo, ymm8=q2, ymm9=q2_dividend

	vmovapd	ymm15, YMMWORD PTR YMM_FMA_RND_CONST3		;;	; Load 3.0 * pow (2.0, (double) (51))
	vsubpd	ymm8, ymm8, ymm15				;;*25-28; q2 -= rounding const
	vsubpd	ymm11, ymm11, ymm15				;;*25-28; q2 -= rounding const

	vaddpd	ymm2, ymm2, ymm1				;; 26-29; final_hi += q1f2_hi
	vaddpd	ymm6, ymm6, ymm5				;; 26-29; final_hi += q1f2_hi

	vmovapd ymm15, YMMWORD PTR [r8][rdx]			;;	; 1-or-2
	yfmaddpd ymm3, ymm15, ymm3, ymm0			;; 28-31; final_lo = one_or_two[rdx] * bb_lo + q1f2_lo;
	yfmaddpd ymm7, ymm15, ymm7, ymm4			;; 28-31; final_lo = one_or_two[rdx] * bb_lo + q1f2_lo;

;; ymm2=final_hi, ymm3=final_lo, ymm8=q2, ymm9=q2_dividend

	vmovapd	ymm15, YMMWORD PTR YMM_FMA_RND_CONST2		;; Load 3.0 * pow (2.0, (double) (51 + BITS_LOW))
	yfnmaddpd ymm0, ymm8, YMMWORD PTR YMM_FMA_FAC_LO, ymm15	;;*29-32; q2f2_hi = -q2 * fac_lo + rounding const
	yfnmaddpd ymm4, ymm11, YMMWORD PTR YMM_FMA_FAC_LOa, ymm15;;*29-32; q2f2_hi = -q2 * fac_lo + rounding const

	yfnmaddpd ymm9, ymm8, YMMWORD PTR YMM_FMA_FAC_HI, ymm9	;;*30-33; Calc Q2*Fhi lo in one blow:  q2f1_lo = q2_dividend - q2 * fac_hi
	yfnmaddpd ymm12, ymm11, YMMWORD PTR YMM_FMA_FAC_HIa, ymm12;;*30-33; Calc Q2*Fhi lo in one blow:  q2f1_lo = q2_dividend - q2 * fac_hi

	vsubpd	ymm0, ymm0, ymm15				;;*33-36; q2f2_hi -= rounding const
	vsubpd	ymm4, ymm4, ymm15				;;*33-36; q2f2_hi -= rounding const

 	vaddpd	ymm2, ymm2, ymm9				;;*34-37; final_hi += q2f1_lo
 	vaddpd	ymm6, ymm6, ymm12				;;*34-37; final_hi += q2f1_lo

	yfnmsubpd ymm8, ymm8, YMMWORD PTR YMM_FMA_FAC_LO, ymm0	;;*37-40; q2f2_lo = -q2 * fac_lo - q2f2_hi
	yfnmsubpd ymm11, ymm11, YMMWORD PTR YMM_FMA_FAC_LOa, ymm4;;*37-40; q2f2_lo = -q2 * fac_lo - q2f2_hi

	vaddpd	ymm2, ymm2, ymm0				;;*38-43; final_hi += q2f2_hi
	vaddpd	ymm6, ymm6, ymm4				;;*38-43; final_hi += q2f2_hi

	vaddpd	ymm3, ymm3, ymm8				;;*41-44; final_lo += q2f2_lo
	vaddpd	ymm7, ymm7, ymm11				;;*41-44; final_lo += q2f2_lo
	ENDM


;; Check if remainder in ymm2,ymm3 is one indicating we found a factor!

avx_fma_compare MACRO
	vaddpd	ymm2, ymm2, ymm3				;; Combine fac_hi & fac_lo
	vaddpd	ymm6, ymm6, ymm7
	ENDM


;;****************************************************************
;; The AVX FMA versions of the above macros (12 factors at a time)
;; Timed at 1.04 ms with Options/Benchmark on 3.4 GHz Haswell
;;****************************************************************

; This macro does some initialization work and then performs a subset of the
; main loop working on the initial value.  This subset avoids the squaring of the
; remainder.  The initial value can be up to 60 bits larger than the factor
; size.

avx_fma_fac_initval MACRO avx_level
	LOCAL	large_initial_value, small_initial_value, equals_split, done

;; Preload constants

	vmovapd	ymm15, YMMWORD PTR YMM_FMA_RND_CONST3		;; Load 3.0 * pow (2.0, (double) (51))
	vmovapd	ymm14, YMMWORD PTR YMM_FMA_RND_CONST2		;; Load 3.0 * pow (2.0, (double) (51 + BITS_LOW))
	lea	r8, YMM_FMA_ONE_OR_TWO				;; Address of 1.0 or 2.0

;; Convert factor from 2 integers into 2 doubles

	vmovapd	ymm13, YMMWORD PTR YMM_FMA_LOW_MASK		;; Mask for acquiring low bits
	vandpd	ymm0, ymm13, YMMWORD PTR YMM_FMA_FAC_LO_INT	;; Load integer factor low
	vandpd	ymm4, ymm13, YMMWORD PTR YMM_FMA_FAC_LO_INTa	;; Load integer factor low
	vandpd	ymm8, ymm13, YMMWORD PTR YMM_FMA_FAC_LO_INTb	;; Load integer factor low

	vorpd	ymm0, ymm0, ymm15				;; Step 1 to convert factor low to a double
	vorpd	ymm4, ymm4, ymm15				;; Step 1 to convert factor low to a double
	vorpd	ymm8, ymm8, ymm15				;; Step 1 to convert factor low to a double

	vorpd	ymm2, ymm15, YMMWORD PTR YMM_FMA_FAC_HI_INT	;; Load integer factor hi
	vorpd	ymm6, ymm15, YMMWORD PTR YMM_FMA_FAC_HI_INTa	;; Load integer factor hi
	vorpd	ymm10, ymm15, YMMWORD PTR YMM_FMA_FAC_HI_INTb	;; Load integer factor hi

	vmovapd	ymm13, YMMWORD PTR YMM_FMA_TWO_TO_LO		;; Constant for converting integer factor hi
	yfmsubpd ymm2, ymm2, ymm13, ymm14			;; Convert integer factor hi to double times 2^lo
	yfmsubpd ymm6, ymm6, ymm13, ymm14			;; Convert integer factor hi to double times 2^lo
	yfmsubpd ymm10, ymm10, ymm13, ymm14			;; Convert integer factor hi to double times 2^lo

	vsubpd	ymm0, ymm0, ymm15				;; Step 2 to convert factor low to a double
	vsubpd	ymm4, ymm4, ymm15				;; Step 2 to convert factor low to a double
	vsubpd	ymm8, ymm8, ymm15				;; Step 2 to convert factor low to a double

	vmovapd	YMMWORD PTR YMM_FMA_FAC_HI, ymm2		;; Save fac_hi
	vmovapd	YMMWORD PTR YMM_FMA_FAC_HIa, ymm6		;; Save fac_hi
	vmovapd	YMMWORD PTR YMM_FMA_FAC_HIb, ymm10		;; Save fac_hi

	vmovapd	YMMWORD PTR YMM_FMA_FAC_LO, ymm0		;; Save fac_lo
	vmovapd	YMMWORD PTR YMM_FMA_FAC_LOa, ymm4		;; Save fac_lo
	vmovapd	YMMWORD PTR YMM_FMA_FAC_LOb, ymm8		;; Save fac_lo

;; Compute factor inverse

	vaddpd	ymm3, ymm2, ymm0				;; Add fac_hi and fac_lo to form factor accurate to 53 bits
	vaddpd	ymm7, ymm6, ymm4				;; Add fac_hi and fac_lo to form factor accurate to 53 bits
	vaddpd	ymm11, ymm10, ymm8				;; Add fac_hi and fac_lo to form factor accurate to 53 bits
	vmovapd	ymm9, YMMWORD PTR YMM_FMA_ONE
	vdivpd	ymm1, ymm9, ymm3				;; Compute 1 / fac
	vdivpd	ymm5, ymm9, ymm7				;; Compute 1 / fac
	vdivpd	ymm9, ymm9, ymm11				;; Compute 1 / fac
	vmovapd	YMMWORD PTR YMM_FMA_INVFAC, ymm1		;; Save factor inverse
	vmovapd	YMMWORD PTR YMM_FMA_INVFACa, ymm5		;; Save factor inverse
	vmovapd	YMMWORD PTR YMM_FMA_INVFACb, ymm9		;; Save factor inverse

;; Compute (2^split mod f) * 2^-split.  Used after calculating rem_hi^2.

;; ymm0 = fac_lo, ymm1 = facinv, ymm2 = fac_hi

	vmovapd	ymm13, YMMWORD PTR YMM_FMA_RND_CONST4		;; Load (3.0 * 2^51) * 2^-split
	vaddpd	ymm1, ymm1, ymm13				;; q = 1.0 * facinv + rounding const
	vaddpd	ymm5, ymm5, ymm13				;; q = 1.0 * facinv + rounding const
	vaddpd	ymm9, ymm9, ymm13				;; q = 1.0 * facinv + rounding const
	vsubpd	ymm1, ymm1, ymm13				;; q -= rounding const
	vsubpd	ymm5, ymm5, ymm13				;; q -= rounding const
	vsubpd	ymm9, ymm9, ymm13				;; q -= rounding const
	vmovapd	ymm12, YMMWORD PTR YMM_FMA_ONE			;; Load 1.0
	yfnmaddpd ymm2, ymm1, ymm2, ymm12			;; hi = 1.0 - q * fac_hi
	yfnmaddpd ymm6, ymm5, ymm6, ymm12			;; hi = 1.0 - q * fac_hi
	yfnmaddpd ymm10, ymm9, ymm10, ymm12			;; hi = 1.0 - q * fac_hi
	vmovapd	ymm12, YMMWORD PTR YMM_FMA_RND_CONST5		;; Load (3.0 * 2^(51 + LOW_BITS)) * 2^-split
	yfnmaddpd ymm3, ymm1, ymm0, ymm12			;; hi_addin = -q * fac_lo + rounding const
	yfnmaddpd ymm7, ymm5, ymm4, ymm12			;; hi_addin = -q * fac_lo + rounding const
	yfnmaddpd ymm11, ymm9, ymm8, ymm12			;; hi_addin = -q * fac_lo + rounding const
	vsubpd	ymm3, ymm3, ymm12				;; hi_addin -= rounding const
	vsubpd	ymm7, ymm7, ymm12				;; hi_addin -= rounding const
	vsubpd	ymm11, ymm11, ymm12				;; hi_addin -= rounding const
	vaddpd	ymm2, ymm2, ymm3				;; hi += hi_addin
	vaddpd	ymm6, ymm6, ymm7				;; hi += hi_addin
	vaddpd	ymm10, ymm10, ymm11				;; hi += hi_addin
	yfnmsubpd ymm1, ymm1, ymm0, ymm3			;; lo = -q * fac_lo - hi_addin
	yfnmsubpd ymm5, ymm5, ymm4, ymm7			;; lo = -q * fac_lo - hi_addin
	yfnmsubpd ymm9, ymm9, ymm8, ymm11			;; lo = -q * fac_lo - hi_addin
	vmovapd	YMMWORD PTR YMM_FMA_TWOPOW1_MODF_HI, ymm2	;; Save high bits of modf
	vmovapd	YMMWORD PTR YMM_FMA_TWOPOW1_MODF_HIa, ymm6	;; Save high bits of modf
	vmovapd	YMMWORD PTR YMM_FMA_TWOPOW1_MODF_HIb, ymm10	;; Save high bits of modf
	vmovapd	YMMWORD PTR YMM_FMA_TWOPOW1_MODF_LO, ymm1	;; Save low bits of modf
	vmovapd	YMMWORD PTR YMM_FMA_TWOPOW1_MODF_LOa, ymm5	;; Save low bits of modf
	vmovapd	YMMWORD PTR YMM_FMA_TWOPOW1_MODF_LOb, ymm9	;; Save low bits of modf

;; Compute (2^(split+1) mod f) * 2^-split.  Used for optional mul-by-2 after computing rem_hi^2.

;; ymm0 = fac_lo

	vmovapd	ymm12, YMMWORD PTR YMM_FMA_TWO			;; Load 2.0
	yfmaddpd ymm1, ymm12, YMMWORD PTR YMM_FMA_INVFAC, ymm13	;; q = 2.0 * facinv + rounding const
	yfmaddpd ymm5, ymm12, YMMWORD PTR YMM_FMA_INVFACa, ymm13;; q = 2.0 * facinv + rounding const
	yfmaddpd ymm9, ymm12, YMMWORD PTR YMM_FMA_INVFACb, ymm13;; q = 2.0 * facinv + rounding const
	vsubpd	ymm1, ymm1, ymm13				;; q -= rounding const
	vsubpd	ymm5, ymm5, ymm13				;; q -= rounding const
	vsubpd	ymm9, ymm9, ymm13				;; q -= rounding const
	yfnmaddpd ymm2, ymm1, YMMWORD PTR YMM_FMA_FAC_HI, ymm12	;; hi = 2.0 - q * fac_hi
	yfnmaddpd ymm6, ymm5, YMMWORD PTR YMM_FMA_FAC_HIa, ymm12;; hi = 2.0 - q * fac_hi
	yfnmaddpd ymm10, ymm9, YMMWORD PTR YMM_FMA_FAC_HIb, ymm12;; hi = 2.0 - q * fac_hi
	vmovapd	ymm12, YMMWORD PTR YMM_FMA_RND_CONST5		;; Load (3.0 * 2^(51 + LOW_BITS)) * 2^-split
	yfnmaddpd ymm3, ymm1, ymm0, ymm12			;; hi_addin = -q * fac_lo + rounding const
	yfnmaddpd ymm7, ymm5, ymm4, ymm12			;; hi_addin = -q * fac_lo + rounding const
	yfnmaddpd ymm11, ymm9, ymm8, ymm12			;; hi_addin = -q * fac_lo + rounding const
	vsubpd	ymm3, ymm3, ymm12				;; hi_addin -= rounding const
	vsubpd	ymm7, ymm7, ymm12				;; hi_addin -= rounding const
	vsubpd	ymm11, ymm11, ymm12				;; hi_addin -= rounding const
	vaddpd	ymm2, ymm2, ymm3				;; hi += hi_addin
	vaddpd	ymm6, ymm6, ymm7				;; hi += hi_addin
	vaddpd	ymm10, ymm10, ymm11				;; hi += hi_addin
	yfnmsubpd ymm1, ymm1, ymm0, ymm3			;; lo = -q * fac_lo - hi_addin
	yfnmsubpd ymm5, ymm5, ymm4, ymm7			;; lo = -q * fac_lo - hi_addin
	yfnmsubpd ymm9, ymm9, ymm8, ymm11			;; lo = -q * fac_lo - hi_addin
	vmovapd	YMMWORD PTR YMM_FMA_TWOPOW2_MODF_HI, ymm2	;; Save high bits of modf
	vmovapd	YMMWORD PTR YMM_FMA_TWOPOW2_MODF_HIa, ymm6	;; Save high bits of modf
	vmovapd	YMMWORD PTR YMM_FMA_TWOPOW2_MODF_HIb, ymm10	;; Save high bits of modf
	vmovapd	YMMWORD PTR YMM_FMA_TWOPOW2_MODF_LO, ymm1	;; Save low bits of modf
	vmovapd	YMMWORD PTR YMM_FMA_TWOPOW2_MODF_LOa, ymm5	;; Save low bits of modf
	vmovapd	YMMWORD PTR YMM_FMA_TWOPOW2_MODF_LOb, ymm9	;; Save low bits of modf

;; Now compute the initial remainder, using an optimized version of the main loop code.  We can save
;; several multiplies and adds because the squared value is a power of two. 

;; There are 4 options for the initial value to get a remainder of.  FMA_INITVAL_TYPE is:
;;	-1)  initval < 2^split	(calc using small initval code)
;;	0)   > 2^(split+1)	(calc using large initval code)
;;	1)   == 2^split		(use twopow1_modf)
;;	2)   == 2^(split+1)	(use twopow2_modf)

;; Test which of the 4 initial value types we have.  Branch to the right code.

	cmp	FMA_INITVAL_TYPE, 0
	jl	small_initial_value
	je	short large_initial_value

;; Easiest case of all!  Remainder is either twopow1_modf or twopow2_modf

	vmovapd	ymm13, YMMWORD PTR YMM_FMA_TWOPOW_SPLIT		;; Load 2^split
	cmp	FMA_INITVAL_TYPE, 1
	je	short equals_split
	vmulpd	ymm2, ymm13, ymm2				;; Return twopow2_modf
	vmulpd	ymm3, ymm13, ymm1
	vmulpd	ymm6, ymm13, ymm6				;; Return twopow2_modf
	vmulpd	ymm7, ymm13, ymm5
	vmulpd	ymm10, ymm13, ymm10				;; Return twopow2_modf
	vmulpd	ymm11, ymm13, ymm9
	jmp	done

equals_split:
	vmulpd	ymm2, ymm13, YMMWORD PTR YMM_FMA_TWOPOW1_MODF_HI ;; Return twopow1_modf
	vmulpd	ymm3, ymm13, YMMWORD PTR YMM_FMA_TWOPOW1_MODF_LO
	vmulpd	ymm6, ymm13, YMMWORD PTR YMM_FMA_TWOPOW1_MODF_HIa ;; Return twopow1_modf
	vmulpd	ymm7, ymm13, YMMWORD PTR YMM_FMA_TWOPOW1_MODF_LOa
	vmulpd	ymm10, ymm13, YMMWORD PTR YMM_FMA_TWOPOW1_MODF_HIb ;; Return twopow1_modf
	vmulpd	ymm11, ymm13, YMMWORD PTR YMM_FMA_TWOPOW1_MODF_LOb
	jmp	done

;; Process an initial squared value that is more than 2^(split+1).  To avoid any edge cases with regards
;; to precision of the resulting remainder, we load initial_squared_value / 2 and use the twopow2_modf values

large_initial_value:

;; calc  AA-hi*TwoPowModFhi hi	* 2^(35+35)		51/50	*2^70
;; calc  AA-hi*TwoPowModFhi lo	* 2^(35)		35	*2^35
;; calc  AA-hi*TwoPowModFlo hi	* 2^(35)		41/40	*2^35
;; calc  AA-hi*TwoPowModFlo lo	* 2^0			35	*2^0

;; calc Q2, remaining bits of quotient
;; calc  Q2*Fhi hi	* 2^(33+35)		53	*2^68
;; calc  Q2*Fhi lo	* 2^35			33	*2^35
;; calc  Q2*Flo hi	* 2^(35+0)		41*	*2^35
;; calc  Q2*Flo lo	* 2^0			35	*2^0

;; ymm0 = fac_lo, ymm1 = twopow2modf_lo, ymm2 = twopow2modf_hi

	vmovapd	ymm12, YMMWORD PTR YMM_FMA_INITVAL		;; Load the initial squared value/2 (known as aa_hi in main loop)
	vmovapd	ymm13, YMMWORD PTR YMM_FMA_RND_CONST1		;; Load 3.0 * pow (2.0, (double) (51 + BITS_LOW * 2))

	yfmaddpd ymm0, ymm12, ymm2, ymm13			;; 1-4	; q1f1_hi = aa_hi * twopow2modf_hi + rounding const
	yfmaddpd ymm3, ymm12, ymm1, ymm14			;; 1-4	; q1f2_hi = aa_hi * twopow2modf_lo + rounding const
	yfmaddpd ymm4, ymm12, ymm6, ymm13			;; 2-5	; q1f1_hi = aa_hi * twopow2modf_hi + rounding const
	yfmaddpd ymm7, ymm12, ymm5, ymm14			;; 2-5	; q1f2_hi = aa_hi * twopow2modf_lo + rounding const
	yfmaddpd ymm8, ymm12, ymm10, ymm13			;; 3-6	; q1f1_hi = aa_hi * twopow2modf_hi + rounding const
	yfmaddpd ymm11, ymm12, ymm9, ymm14			;; 3-6	; q1f2_hi = aa_hi * twopow2modf_lo + rounding const
	vsubpd	ymm0, ymm0, ymm13				;; 5-8	; q1f1_hi -= rounding const (q2_dividend = q1f1_hi)
	vsubpd	ymm3, ymm3, ymm14				;; 5-8	; q1f2_hi -= rounding const
	vsubpd	ymm4, ymm4, ymm13				;; 6-9	; q1f1_hi -= rounding const (q2_dividend = q1f1_hi)
	vsubpd	ymm7, ymm7, ymm14				;; 6-9	; q1f2_hi -= rounding const
	vsubpd	ymm8, ymm8, ymm13				;; 7-10	; q1f1_hi -= rounding const (q2_dividend = q1f1_hi)
	vsubpd	ymm11, ymm11, ymm14				;; 7-10	; q1f2_hi -= rounding const
	yfmsubpd ymm2, ymm12, ymm2, ymm0			;; 9-12	; q1f1_lo = aa_hi * twopow2modf_hi - q1f1_hi (final_hi = q1f1_lo)
	yfmsubpd ymm1, ymm12, ymm1, ymm3			;; 9-12	; q1f2_lo = aa_hi * twopow2modf_lo - q1f2_hi (final_lo = q1f2_lo)
	yfmsubpd ymm6, ymm12, ymm6, ymm4			;; 10-13; q1f1_lo = aa_hi * twopow2modf_hi - q1f1_hi (final_hi = q1f1_lo)
	yfmsubpd ymm5, ymm12, ymm5, ymm7			;; 10-13; q1f2_lo = aa_hi * twopow2modf_lo - q1f2_hi (final_lo = q1f2_lo)
	yfmsubpd ymm10, ymm12, ymm10, ymm8			;; 11-14; q1f1_lo = aa_hi * twopow2modf_hi - q1f1_hi (final_hi = q1f1_lo)
	yfmsubpd ymm9, ymm12, ymm9, ymm11			;; 11-14; q1f2_lo = aa_hi * twopow2modf_lo - q1f2_hi (final_lo = q1f2_lo)
	vaddpd	ymm2, ymm2, ymm3				;; 13-16; final_hi += q1f2_hi
	vaddpd	ymm6, ymm6, ymm7				;; 14-17; final_hi += q1f2_hi
	vaddpd	ymm10, ymm10, ymm11				;; 15-18; final_hi += q1f2_hi
	yfmaddpd ymm3, ymm0, YMMWORD PTR YMM_FMA_INVFAC, ymm15	;; 12-15; q2 = q2_dividend * facinv + rounding const
	yfmaddpd ymm7, ymm4, YMMWORD PTR YMM_FMA_INVFACa, ymm15	;; 12-15; q2 = q2_dividend * facinv + rounding const
	yfmaddpd ymm11, ymm8, YMMWORD PTR YMM_FMA_INVFACb, ymm15;; 13-16; q2 = q2_dividend * facinv + rounding const
	vsubpd	ymm3, ymm3, ymm15				;; 16-19; q2 -= rounding const
	vsubpd	ymm7, ymm7, ymm15				;; 16-19; q2 -= rounding const
	vsubpd	ymm11, ymm11, ymm15				;; 17-20; q2 -= rounding const
	yfnmaddpd ymm0, ymm3, YMMWORD PTR YMM_FMA_FAC_HI, ymm0	;; 20-23; q2f1_lo = q2_dividend - q2 * fac_hi
	yfnmaddpd ymm4, ymm7, YMMWORD PTR YMM_FMA_FAC_HIa, ymm4	;; 20-23; q2f1_lo = q2_dividend - q2 * fac_hi
	yfnmaddpd ymm8, ymm11, YMMWORD PTR YMM_FMA_FAC_HIb, ymm8;; 21-24; q2f1_lo = q2_dividend - q2 * fac_hi
	vaddpd	ymm2, ymm2, ymm0				;; 24-27; final_hi += q2f1_lo
	vaddpd	ymm6, ymm6, ymm4				;; 24-27; final_hi += q2f1_lo
	vaddpd	ymm10, ymm10, ymm8				;; 25-28; final_hi += q2f1_lo
	yfnmaddpd ymm0, ymm3, YMMWORD PTR YMM_FMA_FAC_LO, ymm14	;; 21-24; q2f2_hi = -q2 * fac_lo + rounding const
	yfnmaddpd ymm4, ymm7, YMMWORD PTR YMM_FMA_FAC_LOa, ymm14;; 22-25; q2f2_hi = -q2 * fac_lo + rounding const
	yfnmaddpd ymm8, ymm11, YMMWORD PTR YMM_FMA_FAC_LOb, ymm14;;22-25; q2f2_hi = -q2 * fac_lo + rounding const
	vsubpd	ymm0, ymm0, ymm14				;; 25-28; q2f2_hi -= rounding const
	vsubpd	ymm4, ymm4, ymm14				;; 26-29; q2f2_hi -= rounding const
	vsubpd	ymm8, ymm8, ymm14				;; 26-29; q2f2_hi -= rounding const
	vaddpd	ymm2, ymm2, ymm0				;; 29-32; final_hi += q2f2_hi
	vaddpd	ymm6, ymm6, ymm4				;; 30-33; final_hi += q2f2_hi
	vaddpd	ymm10, ymm10, ymm8				;; 30-33; final_hi += q2f2_hi
	yfnmsubpd ymm3, ymm3, YMMWORD PTR YMM_FMA_FAC_LO, ymm0	;; 29-32; q2f2_lo = -q2 * fac_lo - q2f2_hi
	yfnmsubpd ymm7, ymm7, YMMWORD PTR YMM_FMA_FAC_LOa, ymm4	;; 31-34; q2f2_lo = -q2 * fac_lo - q2f2_hi
	yfnmsubpd ymm11, ymm11, YMMWORD PTR YMM_FMA_FAC_LOb, ymm8;;31-34; q2f2_lo = -q2 * fac_lo - q2f2_hi
	vaddpd	ymm3, ymm1, ymm3				;; 33-36; final_lo += q2f2_lo
	vaddpd	ymm7, ymm5, ymm7				;; 35-38; final_lo += q2f2_lo
	vaddpd	ymm11, ymm9, ymm11				;; 35-38; final_lo += q2f2_lo
	jmp	done

;; Process an initial squared value that is less than 2^split
;; Calc q, the quotient.  Then calc initial_value - q * fac

;; ymm0 = fac_lo, ymm1 = twopow2modf_lo, ymm2 = twopow2modf_hi

small_initial_value:
	vmovapd	ymm12, YMMWORD PTR YMM_FMA_INITVAL		;; Load the initial squared value
	yfmaddpd ymm1, ymm12, YMMWORD PTR YMM_FMA_INVFAC, ymm15	;; q = initial_value * facinv + rounding const
	yfmaddpd ymm5, ymm12, YMMWORD PTR YMM_FMA_INVFACa, ymm15;; q = initial_value * facinv + rounding const
	yfmaddpd ymm9, ymm12, YMMWORD PTR YMM_FMA_INVFACb, ymm15;; q = initial_value * facinv + rounding const
	vsubpd	ymm1, ymm1, ymm15				;; q -= rounding const
	vsubpd	ymm5, ymm5, ymm15				;; q -= rounding const
	vsubpd	ymm9, ymm9, ymm15				;; q -= rounding const
	yfnmaddpd ymm2, ymm1, YMMWORD PTR YMM_FMA_FAC_HI, ymm12	;; final_hi = initial_value - q * fac_hi
	yfnmaddpd ymm6, ymm5, YMMWORD PTR YMM_FMA_FAC_HIa, ymm12;; final_hi = initial_value - q * fac_hi
	yfnmaddpd ymm10, ymm9, YMMWORD PTR YMM_FMA_FAC_HIb, ymm12;; final_hi = initial_value - q * fac_hi
	yfnmaddpd ymm3, ymm1, ymm0, ymm14			;; qf2_hi = -q * fac_lo + rounding const
	yfnmaddpd ymm7, ymm5, ymm4, ymm14			;; qf2_hi = -q * fac_lo + rounding const
	yfnmaddpd ymm11, ymm9, ymm8, ymm14			;; qf2_hi = -q * fac_lo + rounding const
	vsubpd	ymm3, ymm3, ymm14				;; qf2_hi -= rounding const
	vsubpd	ymm7, ymm7, ymm14				;; qf2_hi -= rounding const
	vsubpd	ymm11, ymm11, ymm14				;; qf2_hi -= rounding const
	vaddpd	ymm2, ymm2, ymm3				;; final_hi += q2f2_hi
	vaddpd	ymm6, ymm6, ymm7				;; final_hi += q2f2_hi
	vaddpd	ymm10, ymm10, ymm11				;; final_hi += q2f2_hi
	yfnmsubpd ymm3, ymm1, ymm0, ymm3			;; final_lo = -q * fac_lo - qf2_hi
	yfnmsubpd ymm7, ymm5, ymm4, ymm7			;; final_lo = -q * fac_lo - qf2_hi
	yfnmsubpd ymm11, ymm9, ymm8, ymm11			;; final_lo = -q * fac_lo - qf2_hi

done:
	ENDM


; This macro does one iteration of the main loop.  Square a remainder and do a mod.
; Note that at the beginning of the macro ymm2, ymm3 contains the semi-normalized remainder
; for us to square.  This input number is between -fac/2*epsilon and +fac/2*epsilon.  The range
; is not perfect because this macro's calculation of Q2 is not exact.

; In the comments of this macro, for illustration purposes we'll assume fac_size is 80 bits
; with 35 bits stored in the low word.  There is no reason to believe this code cannot handle
; at least 96 bits.  In fact, we have simulated this macro in C code with no problems.

avx_fma_fac MACRO fac_size

;; AB^2 will be 160 bits.  From there, we multiply the top 40 bits by the 80-bit value 2^120 mod f.
;; This 120-bit result plus the low 120/121 bits of AB^2 give us a 122-bit dividend in computing Q2.

;; To maximize significant bits in calculating Q2, we want to maximize the precision of AA-lo and 2AB-hi.  Looking at
;; the table below that happens when AA-lo and 2AB-hi use as small an offset as possible without exceeding 53 bits
;; of precision.  Below that would be 34 bits in a low word.

;; SQUARE rem (call rem_hi A and rem_lo B).  A is between 0 and 44.5+epsilon bits.  B is between 0 and 35 normalized bits.
;; Unnormalized low words are the result of 1 doubling and 3 adds.  This adds 2 bits.

;;					normalized		unnormalized
;;				      max bits	offset	      max bits	offset
;;AA-hi		* 2^(50+2*35)		40	*2^120		40	*2^120
;;AA-lo		* 2^(2*35)		50	*2^70		50	*2^70
;;2AB-hi	* 2^(34+36)		46/47	*2^70		48/49	*2^70
;;2AB-lo	* 2^36			34/33	*2^36/37	34/33	*2^36/37
;;BB-hi		* 2^(35+0)		35	*2^35		39/40	*2^35
;;BB-lo		* 2^0			35	*2^0		35	*2^0

;; Since we know the power of two that aa_hi is a multiple of (aa_hi is a multiple of 2^120 in our example),
;; we pre-calculate 2^that_power mod factor, and multiply aa-hi by this pre-calculated value.

;; Aa-hi is 41/40 bits, TwoPowModFhi is 45 bits, TwoPowModFlo is 35 bits
;; calc  AA-hi*TwoPowModFhi hi	* 2^(35+35)		51/50	*2^70
;; calc  AA-hi*TwoPowModFhi lo	* 2^(35)		35	*2^35
;; calc  AA-hi*TwoPowModFlo hi	* 2^(35)		41/40	*2^35
;; calc  AA-hi*TwoPowModFlo lo	* 2^0			35	*2^0

;; calc Q2, remaining bits of quotient
;;
;; Q2 is 41 bits, Fhi is 45 bits, Flo is 35 bits
;; calc  Q2*Fhi hi	* 2^(33+35)		53	*2^68
;; calc  Q2*Fhi lo	* 2^35			33	*2^35
;; calc  Q2*Flo hi	* 2^(35+0)		41*	*2^35
;; calc  Q2*Flo lo	* 2^0			35	*2^0

;; ymm0-3 reserved for set 1, ymm4-7 reserved for set 2
;; ymm8-11 reserved for set 3, ymm12-15 are for common use

	vmovapd	ymm15, YMMWORD PTR YMM_FMA_RND_CONST0		;;	; Load 3.0 * pow (2.0, (double) (51 + BITS_TOTAL + (BITS_TOTAL + 1) / 2))
	yfmaddpd ymm0, ymm2, ymm2, ymm15			;;*1-4	; aa_hi = rem_hi * rem_hi + rounding const
	yfmaddpd ymm4, ymm6, ymm6, ymm15			;;*1-4	; aa_hi = rem_hi * rem_hi + rounding const
	yfmaddpd ymm8, ymm10, ymm10, ymm15			;;*2-5	; aa_hi = rem_hi * rem_hi + rounding const
	vsubpd	ymm0, ymm0, ymm15				;;*5-8	; aa_hi -= rounding const
	vsubpd	ymm4, ymm4, ymm15				;;*5-8	; aa_hi -= rounding const
	vsubpd	ymm8, ymm8, ymm15				;;*6-9	; aa_hi -= rounding const
	yfmsubpd ymm1, ymm2, ymm2, ymm0				;;*9-12	; aa_lo = rem_hi * rem_hi - aa_hi
	yfmsubpd ymm5, ymm6, ymm6, ymm4				;;*9-12	; aa_lo = rem_hi * rem_hi - aa_hi
	yfmsubpd ymm9, ymm10, ymm10, ymm8			;;*10-13; aa_lo = rem_hi * rem_hi - aa_hi

	mov	edx, FMA_SHIFTER[rdi*4-4]			;;	; This is 32 if doubling else 0

;; ymm0 = aa_hi, ymm1 = aa_lo, ymm2 = rem_hi, ymm3 = rem_lo

	vmovapd	ymm14, YMMWORD PTR YMM_FMA_RND_CONST1		;;	; Load 3.0 * pow (2.0, (double) (51 + BITS_LOW * 2))
	yfmaddpd ymm15, ymm2, ymm3, ymm14			;;*2-5	; ab_hi = rem_hi * rem_lo + rounding const
	vsubpd	ymm15, ymm15, ymm14				;;*6-9	; ab_hi -= rounding const
	yfmaddpd ymm12, ymm0, YMMWORD PTR YMM_FMA_TWOPOW1_MODF_HI[rdx], ymm14
								;;*10-13; q1f1_hi = aa_hi * twopowmodf_hi[rdx] + rounding const
	yfmsubpd ymm2, ymm2, ymm3, ymm15			;; 11-14; ab_lo = rem_hi * rem_lo - ab_hi
	yfmaddpd ymm1, YMMWORD PTR YMM_FMA_TWO, ymm15, ymm1	;;*13-16; q2_dividend = 2AB_hi + AA_lo

	yfmaddpd ymm15, ymm6, ymm7, ymm14			;;*3-6	; ab_hi = rem_hi * rem_lo + rounding const
	vsubpd	ymm15, ymm15, ymm14				;;*7-10	; ab_hi -= rounding const
	yfmaddpd ymm13, ymm4, YMMWORD PTR YMM_FMA_TWOPOW1_MODF_HIa[rdx], ymm14
								;;*11-14; q1f1_hi = aa_hi * twopowmodf_hi[rdx] + rounding const
	yfmsubpd ymm6, ymm6, ymm7, ymm15			;; 12-15; ab_lo = rem_hi * rem_lo - ab_hi
	yfmaddpd ymm5, YMMWORD PTR YMM_FMA_TWO, ymm15, ymm5	;;*13-16; q2_dividend = 2AB_hi + AA_lo

	yfmaddpd ymm15, ymm10, ymm11, ymm14			;;*3-6	; ab_hi = rem_hi * rem_lo + rounding const
	vsubpd	ymm15, ymm15, ymm14				;;*7-10	; ab_hi -= rounding const
	yfmaddpd ymm14, ymm8, YMMWORD PTR YMM_FMA_TWOPOW1_MODF_HIb[rdx], ymm14
								;;*12-15; q1f1_hi = aa_hi * twopowmodf_hi[rdx] + rounding const
	yfmsubpd ymm10, ymm10, ymm11, ymm15			;; 14-17; ab_lo = rem_hi * rem_lo - ab_hi
	yfmaddpd ymm9, YMMWORD PTR YMM_FMA_TWO, ymm15, ymm9	;;*14-17; q2_dividend = 2AB_hi + AA_lo

	vmovapd	ymm15, YMMWORD PTR YMM_FMA_RND_CONST1		;;	; Reload 3.0 * pow (2.0, (double) (51 + BITS_LOW * 2))
	vsubpd	ymm12, ymm12, ymm15				;;*15-18; q1f1_hi -= rounding const
	vsubpd	ymm13, ymm13, ymm15				;;*15-18; q1f1_hi -= rounding const
	vsubpd	ymm14, ymm14, ymm15				;;*16-19; q1f1_hi -= rounding const

	vmovapd ymm15, YMMWORD PTR [r8][rdx]			;;	; 1-or-2
	yfmaddpd ymm1, ymm15, ymm1, ymm12			;;*19-22; q2_dividend = 1-or-2 * q2_dividend + q1f1_hi
	yfmaddpd ymm5, ymm15, ymm5, ymm13			;;*19-22; q2_dividend = 1-or-2 * q2_dividend + q1f1_hi
	yfmaddpd ymm9, ymm15, ymm9, ymm14			;;*20-23; q2_dividend = 1-or-2 * q2_dividend + q1f1_hi

;; ymm0 = aa_hi, ymm1 = q2_dividend, ymm2 = ab_lo, ymm3 = rem_lo, ymm12/13/14 = q1f1_hi

	yfmaddpd ymm15, ymm3, ymm3, YMMWORD PTR YMM_FMA_RND_CONST2 ;; 4-7 ; bb_hi = rem_lo * rem_lo + rounding const
	vsubpd	ymm15, ymm15, YMMWORD PTR YMM_FMA_RND_CONST2	;; 8-11	; bb_hi -= rounding const
	yfmsubpd ymm3, ymm3, ymm3, ymm15			;; 16-19; bb_lo = rem_lo * rem_lo - bb_hi
	yfmaddpd ymm2, YMMWORD PTR YMM_FMA_TWO, ymm2, ymm15	;; 17-20; final_hi = 2 * ab_lo + bb_hi

	yfmaddpd ymm15, ymm7, ymm7, YMMWORD PTR YMM_FMA_RND_CONST2 ;; 4-7 ; bb_hi = rem_lo * rem_lo + rounding const
	vsubpd	ymm15, ymm15, YMMWORD PTR YMM_FMA_RND_CONST2	;; 8-11	; bb_hi -= rounding const
	yfmsubpd ymm7, ymm7, ymm7, ymm15			;; 17-20; bb_lo = rem_lo * rem_lo - bb_hi
	yfmaddpd ymm6, YMMWORD PTR YMM_FMA_TWO, ymm6, ymm15	;; 18-21; final_hi = 2 * ab_lo + bb_hi

	yfmaddpd ymm15, ymm11, ymm11, YMMWORD PTR YMM_FMA_RND_CONST2 ;; 18-21 ; bb_hi = rem_lo * rem_lo + rounding const
	vsubpd	ymm15, ymm15, YMMWORD PTR YMM_FMA_RND_CONST2	;; 22-25; bb_hi -= rounding const
	yfmsubpd ymm11, ymm11, ymm11, ymm15			;; 26-29; bb_lo = rem_lo * rem_lo - bb_hi
	yfmaddpd ymm10, YMMWORD PTR YMM_FMA_TWO, ymm10, ymm15	;; 26-29; final_hi = 2 * ab_lo + bb_hi

	vmovapd ymm15, YMMWORD PTR [r8][rdx]			;;	; 1-or-2
	yfmsubpd ymm12, ymm0, YMMWORD PTR YMM_FMA_TWOPOW1_MODF_HI[rdx], ymm12
								;; 20-23; q1f1_lo = aa_hi * twopowmodf_hi[rdx] - q1f1_hi
	yfmaddpd ymm2, ymm15, ymm2, ymm12			;; 24-27; final_hi = 1-or-2 * final_hi + q1f1_lo
	yfmsubpd ymm13, ymm4, YMMWORD PTR YMM_FMA_TWOPOW1_MODF_HIa[rdx], ymm13
								;; 21-24; q1f1_lo = aa_hi * twopowmodf_hi[rdx] - q1f1_hi
	yfmaddpd ymm6, ymm15, ymm6, ymm13			;; 25-28; final_hi = 1-or-2 * final_hi + q1f1_lo
	yfmsubpd ymm14, ymm8, YMMWORD PTR YMM_FMA_TWOPOW1_MODF_HIb[rdx], ymm14
								;; 24-27; q1f1_lo = aa_hi * twopowmodf_hi[rdx] - q1f1_hi
	yfmaddpd ymm10, ymm15, ymm10, ymm14			;; 30-33; final_hi = 1-or-2 * final_hi + q1f1_lo

;; ymm0 = aa_hi, ymm1 = q2_dividend, ymm2 = final_hi, ymm3 = bb_lo, ymm15 = 1-or-2

	vmovapd	ymm15, YMMWORD PTR YMM_FMA_RND_CONST2		;;	; Load 3.0 * pow (2.0, (double) (51 + BITS_LOW))
	vmovapd ymm12, YMMWORD PTR YMM_FMA_TWOPOW1_MODF_LO[rdx]	;;	; Load twopowmodf_lo[rdx]
	yfmaddpd ymm13, ymm0, ymm12, ymm15			;; 21-24; q1f2_hi = aa_hi * twopowmodf_lo[rdx] + rounding const
	vsubpd	ymm13, ymm13, ymm15				;; 25-28; q1f2_hi -= rounding const
	yfmsubpd ymm0, ymm0, ymm12, ymm13			;; 29-32; q1f2_lo = aa_hi * twopowmodf_lo[rdx] - q1f2_hi
	vaddpd	ymm2, ymm2, ymm13				;; 29-32; final_hi += q1f2_hi

	vmovapd ymm12, YMMWORD PTR YMM_FMA_TWOPOW1_MODF_LOa[rdx];;	; Load twopowmodf_lo[rdx]
	yfmaddpd ymm13, ymm4, ymm12, ymm15			;; 22-25; q1f2_hi = aa_hi * twopowmodf_lo[rdx] + rounding const
	vsubpd	ymm13, ymm13, ymm15				;; 27-30; q1f2_hi -= rounding const
	yfmsubpd ymm4, ymm4, ymm12, ymm13			;; 31-34; q1f2_lo = aa_hi * twopowmodf_lo[rdx] - q1f2_hi
	vaddpd	ymm6, ymm6, ymm13				;; 31-34; final_hi += q1f2_hi

	vmovapd ymm12, YMMWORD PTR YMM_FMA_TWOPOW1_MODF_LOb[rdx];;	; Load twopowmodf_lo[rdx]
	yfmaddpd ymm13, ymm8, ymm12, ymm15			;; 23-26; q1f2_hi = aa_hi * twopowmodf_lo[rdx] + rounding const
	vsubpd	ymm13, ymm13, ymm15				;; 27-30; q1f2_hi -= rounding const
	yfmsubpd ymm8, ymm8, ymm12, ymm13			;; 32-35; q1f2_lo = aa_hi * twopowmodf_lo[rdx] - q1f2_hi
	vaddpd	ymm10, ymm10, ymm13				;; 32-35; final_hi += q1f2_hi

;; ymm0 = q1f2_lo, ymm1 = q2_dividend, ymm2 = final_hi, ymm3 = bb_lo

	vmovapd	ymm15, YMMWORD PTR YMM_FMA_RND_CONST3		;; Load 3.0 * pow (2.0, (double) (51))
	yfmaddpd ymm12, ymm1, YMMWORD PTR YMM_FMA_INVFAC, ymm15	;;*23-26; q2 = q2_dividend * facinv + rounding const
	yfmaddpd ymm3, YMMWORD PTR [r8][rdx], ymm3, ymm0	;; 33-36; final_lo = one_or_two[rdx] * bb_lo + q1f2_lo;

	yfmaddpd ymm13, ymm5, YMMWORD PTR YMM_FMA_INVFACa, ymm15;;*28-31; q2 = q2_dividend * facinv + rounding const
	yfmaddpd ymm7, YMMWORD PTR [r8][rdx], ymm7, ymm4	;; 35-38; final_lo = one_or_two[rdx] * bb_lo + q1f2_lo;

	yfmaddpd ymm14, ymm9, YMMWORD PTR YMM_FMA_INVFACb, ymm15;;*28-31; q2 = q2_dividend * facinv + rounding const
	yfmaddpd ymm11, YMMWORD PTR [r8][rdx], ymm11, ymm8	;; 36-39; final_lo = one_or_two[rdx] * bb_lo + q1f2_lo;

;; ymm1 = q2_dividend, ymm2 = final_hi, ymm3 = final_lo, ymm12/13/14 = q2

	vsubpd	ymm12, ymm12, ymm15				;;*30-33; q2 -= rounding const
	yfnmaddpd ymm0, ymm12, YMMWORD PTR YMM_FMA_FAC_HI, ymm1	;;*34-37; Calc Q2*Fhi lo in one blow:  q2f1_lo = q2_dividend - q2 * fac_hi
 	vaddpd	ymm2, ymm2, ymm0				;;*38-41; final_hi += q2f1_lo

	vsubpd	ymm13, ymm13, ymm15				;;*33-36; q2 -= rounding const
	yfnmaddpd ymm4, ymm13, YMMWORD PTR YMM_FMA_FAC_HIa, ymm5;;*37-40; Calc Q2*Fhi lo in one blow:  q2f1_lo = q2_dividend - q2 * fac_hi
 	vaddpd	ymm6, ymm6, ymm4				;;*41-44; final_hi += q2f1_lo

	vsubpd	ymm14, ymm14, ymm15				;;*34-37; q2 -= rounding const
	yfnmaddpd ymm8, ymm14, YMMWORD PTR YMM_FMA_FAC_HIb, ymm9;;*38-41; Calc Q2*Fhi lo in one blow:  q2f1_lo = q2_dividend - q2 * fac_hi
 	vaddpd	ymm10, ymm10, ymm8				;;*42-45; final_hi += q2f1_lo

;; ymm2 = final_hi, ymm3 = final_lo, ymm12/13/14 = q2

	vmovapd	ymm15, YMMWORD PTR YMM_FMA_RND_CONST2		;;	; Load 3.0 * pow (2.0, (double) (51 + BITS_LOW))
	yfnmaddpd ymm0, ymm12, YMMWORD PTR YMM_FMA_FAC_LO, ymm15;;*35-38; q2f2_hi = -q2 * fac_lo + rounding const
	vsubpd	ymm0, ymm0, ymm15				;;*39-42; q2f2_hi -= rounding const
	yfnmsubpd ymm12, ymm12, YMMWORD PTR YMM_FMA_FAC_LO, ymm0;;*43-46; q2f2_lo = -q2 * fac_lo - q2f2_hi

	yfnmaddpd ymm4, ymm13, YMMWORD PTR YMM_FMA_FAC_LOa, ymm15;;*37-40; q2f2_hi = -q2 * fac_lo + rounding const
	vsubpd	ymm4, ymm4, ymm15				;;*41-44; q2f2_hi -= rounding const
	yfnmsubpd ymm13, ymm13, YMMWORD PTR YMM_FMA_FAC_LOa, ymm4;;*45-48; q2f2_lo = -q2 * fac_lo - q2f2_hi

	yfnmaddpd ymm8, ymm14, YMMWORD PTR YMM_FMA_FAC_LOb, ymm15;;*39-42; q2f2_hi = -q2 * fac_lo + rounding const
	vsubpd	ymm8, ymm8, ymm15				;;*43-46; q2f2_hi -= rounding const
	yfnmsubpd ymm14, ymm14, YMMWORD PTR YMM_FMA_FAC_LOb, ymm8;;*47-50; q2f2_lo = -q2 * fac_lo - q2f2_hi

	vaddpd	ymm2, ymm2, ymm0				;;*44-47; final_hi += q2f2_hi
	vaddpd	ymm3, ymm3, ymm12				;;*47-50; final_lo += q2f2_lo

	vaddpd	ymm6, ymm6, ymm4				;;*45-48; final_hi += q2f2_hi
	vaddpd	ymm7, ymm7, ymm13				;;*49-52; final_lo += q2f2_lo

	vaddpd	ymm10, ymm10, ymm8				;;*48-51; final_hi += q2f2_hi
	vaddpd	ymm11, ymm11, ymm14				;;*51-54; final_lo += q2f2_lo
	ENDM

;; ALTERNATE IMPLEMENTATION THAT USES SAVES AND RESTORES (even better!)
;; Timed at 1.01 ms with Options/Benchmark on 3.4 GHz Haswell

; This macro does one iteration of the main loop.  Square a remainder and do a mod.
; Note that at the beginning of the macro ymm2, ymm3 contains the semi-normalized remainder
; for us to square.  This input number is between -fac/2*epsilon and +fac/2*epsilon.  The range
; is not perfect because this macro's calculation of Q2 is not exact.

; In the comments of this macro, for illustration purposes we'll assume fac_size is 80 bits
; with 35 bits stored in the low word.  There is no reason to believe this code cannot handle
; at least 96 bits.  In fact, we have simulated this macro in C code with no problems.

avx_fma_fac MACRO fac_size

;; AB^2 will be 160 bits.  From there, we multiply the top 40 bits by the 80-bit value 2^120 mod f.
;; This 120-bit result plus the low 120/121 bits of AB^2 give us a 122-bit dividend in computing Q2.

;; To maximize significant bits in calculating Q2, we want to maximize the precision of AA-lo and 2AB-hi.  Looking at
;; the table below that happens when AA-lo and 2AB-hi use as small an offset as possible without exceeding 53 bits
;; of precision.  Below that would be 34 bits in a low word.

;; SQUARE rem (call rem_hi A and rem_lo B).  A is between 0 and 44.5+epsilon bits.  B is between 0 and 35 normalized bits.
;; Unnormalized low words are the result of 1 doubling and 3 adds.  This adds 2 bits.

;;					normalized		unnormalized
;;				      max bits	offset	      max bits	offset
;;AA-hi		* 2^(50+2*35)		40	*2^120		40	*2^120
;;AA-lo		* 2^(2*35)		50	*2^70		50	*2^70
;;2AB-hi	* 2^(34+36)		46/47	*2^70		48/49	*2^70
;;2AB-lo	* 2^36			34/33	*2^36/37	34/33	*2^36/37
;;BB-hi		* 2^(35+0)		35	*2^35		39/40	*2^35
;;BB-lo		* 2^0			35	*2^0		35	*2^0

;; Since we know the power of two that aa_hi is a multiple of (aa_hi is a multiple of 2^120 in our example),
;; we pre-calculate 2^that_power mod factor, and multiply aa-hi by this pre-calculated value.

;; Aa-hi is 41/40 bits, TwoPowModFhi is 45 bits, TwoPowModFlo is 35 bits
;; calc  AA-hi*TwoPowModFhi hi	* 2^(35+35)		51/50	*2^70
;; calc  AA-hi*TwoPowModFhi lo	* 2^(35)		35	*2^35
;; calc  AA-hi*TwoPowModFlo hi	* 2^(35)		41/40	*2^35
;; calc  AA-hi*TwoPowModFlo lo	* 2^0			35	*2^0

;; calc Q2, remaining bits of quotient
;;
;; Q2 is 41 bits, Fhi is 45 bits, Flo is 35 bits
;; calc  Q2*Fhi hi	* 2^(33+35)		53	*2^68
;; calc  Q2*Fhi lo	* 2^35			33	*2^35
;; calc  Q2*Flo hi	* 2^(35+0)		41*	*2^35
;; calc  Q2*Flo lo	* 2^0			35	*2^0

;; ymm0-3,12 reserved for set 1, ymm4-7,13 reserved for set 2
;; ymm8-11,14 reserved for set 3, ymm15 is for common use

	vmovapd	ymm14, YMMWORD PTR YMM_FMA_RND_CONST0		;;	; Load 3.0 * pow (2.0, (double) (51 + BITS_TOTAL + (BITS_TOTAL + 1) / 2))
	yfmaddpd ymm0, ymm2, ymm2, ymm14			;;*1-4	; aa_hi = rem_hi * rem_hi + rounding const
	yfmaddpd ymm4, ymm6, ymm6, ymm14			;;*1-4	; aa_hi = rem_hi * rem_hi + rounding const
	yfmaddpd ymm8, ymm10, ymm10, ymm14			;;*2-5	; aa_hi = rem_hi * rem_hi + rounding const

	vmovapd	ymm15, YMMWORD PTR YMM_FMA_RND_CONST1		;;	; Load 3.0 * pow (2.0, (double) (51 + BITS_LOW * 2))
	yfmaddpd ymm1, ymm2, ymm3, ymm15			;;*2-5	; ab_hi = rem_hi * rem_lo + rounding const
	yfmaddpd ymm5, ymm6, ymm7, ymm15			;;*3-6	; ab_hi = rem_hi * rem_lo + rounding const
	yfmaddpd ymm9, ymm10, ymm11, ymm15			;;*3-6	; ab_hi = rem_hi * rem_lo + rounding const

	yfmaddpd ymm12, ymm3, ymm3, YMMWORD PTR YMM_FMA_RND_CONST2 ;; 4-7 ; bb_hi = rem_lo * rem_lo + rounding const
	yfmaddpd ymm13, ymm7, ymm7, YMMWORD PTR YMM_FMA_RND_CONST2 ;; 4-7 ; bb_hi = rem_lo * rem_lo + rounding const

	mov	edx, FMA_SHIFTER[rdi*4-4]			;;	; This is 32 if doubling else 0

	vsubpd	ymm0, ymm0, ymm14				;;*5-8	; aa_hi -= rounding const
	vsubpd	ymm4, ymm4, ymm14				;;*5-8	; aa_hi -= rounding const
	vsubpd	ymm8, ymm8, ymm14				;;*6-9	; aa_hi -= rounding const

	vsubpd	ymm1, ymm1, ymm15				;;*6-9	; ab_hi -= rounding const
	vsubpd	ymm5, ymm5, ymm15				;;*7-10	; ab_hi -= rounding const
	vsubpd	ymm9, ymm9, ymm15				;;*7-10	; ab_hi -= rounding const

	yfmaddpd ymm14, ymm11, ymm11, YMMWORD PTR YMM_FMA_RND_CONST2 ;; 8-11 ; bb_hi = rem_lo * rem_lo + rounding const
	vsubpd	ymm12, ymm12, YMMWORD PTR YMM_FMA_RND_CONST2	;; 8-11	; bb_hi -= rounding const

	vmovapd	YMMWORD PTR YMM_FMA_SAVE, ymm2			;;	; Save rem_hi
	vmovapd	YMMWORD PTR YMM_FMA_SAVEa, ymm6			;;	; Save rem_hi
	vmovapd	YMMWORD PTR YMM_FMA_SAVEb, ymm10		;;	; Save rem_hi

	yfmsubpd ymm2, ymm2, ymm2, ymm0				;;*9-12	; aa_lo = rem_hi * rem_hi - aa_hi
	yfmsubpd ymm6, ymm6, ymm6, ymm4				;;*9-12	; aa_lo = rem_hi * rem_hi - aa_hi
	yfmsubpd ymm10, ymm10, ymm10, ymm8			;;*10-13; aa_lo = rem_hi * rem_hi - aa_hi

	vmovapd	YMMWORD PTR YMM_FMA_SAVE2, ymm0			;;	; Save aa_hi
	vmovapd	YMMWORD PTR YMM_FMA_SAVE2a, ymm4		;;	; Save aa_hi
	vmovapd	YMMWORD PTR YMM_FMA_SAVE2b, ymm8		;;	; Save aa_hi

	yfmaddpd ymm0, ymm0, YMMWORD PTR YMM_FMA_TWOPOW1_MODF_HI[rdx], ymm15
								;;*10-13; q1f1_hi = aa_hi * twopowmodf_hi[rdx] + rounding const
	yfmaddpd ymm4, ymm4, YMMWORD PTR YMM_FMA_TWOPOW1_MODF_HIa[rdx], ymm15
								;;*11-14; q1f1_hi = aa_hi * twopowmodf_hi[rdx] + rounding const
	yfmaddpd ymm8, ymm8, YMMWORD PTR YMM_FMA_TWOPOW1_MODF_HIb[rdx], ymm15
								;;*11-14; q1f1_hi = aa_hi * twopowmodf_hi[rdx] + rounding const

	vsubpd	ymm13, ymm13, YMMWORD PTR YMM_FMA_RND_CONST2	;; 12-15; bb_hi -= rounding const
	vsubpd	ymm14, ymm14, YMMWORD PTR YMM_FMA_RND_CONST2	;; 12-15; bb_hi -= rounding const

;; ymm0=q1f1_hi, ymm1=ab_hi, ymm2=aa_lo, ymm3=rem_lo, ymm12=bb_hi, SAVE=rem_hi, SAVE2=aa_hi

	yfmaddpd ymm2, YMMWORD PTR YMM_FMA_TWO, ymm1, ymm2	;;*13-16; q2_dividend = 2AB_hi + AA_lo
	yfmaddpd ymm6, YMMWORD PTR YMM_FMA_TWO, ymm5, ymm6	;;*13-16; q2_dividend = 2AB_hi + AA_lo
	yfmaddpd ymm10, YMMWORD PTR YMM_FMA_TWO, ymm9, ymm10	;;*14-17; q2_dividend = 2AB_hi + AA_lo

;; ymm0=q1f1_hi, ymm1=ab_hi, ymm2=q2_dividend, ymm3=rem_lo, ymm12=bb_hi, SAVE=rem_hi, SAVE2=aa_hi

	vsubpd	ymm0, ymm0, ymm15				;;*14-17; q1f1_hi -= rounding const
	vsubpd	ymm4, ymm4, ymm15				;;*15-18; q1f1_hi -= rounding const
	vsubpd	ymm8, ymm8, ymm15				;;*15-18; q1f1_hi -= rounding const

	yfmsubpd ymm1, YMMWORD PTR YMM_FMA_SAVE, ymm3, ymm1	;; 16-19; ab_lo = rem_hi * rem_lo - ab_hi
	yfmsubpd ymm5, YMMWORD PTR YMM_FMA_SAVEa, ymm7, ymm5	;; 16-19; ab_lo = rem_hi * rem_lo - ab_hi
	yfmsubpd ymm9, YMMWORD PTR YMM_FMA_SAVEb, ymm11, ymm9	;; 17-20; ab_lo = rem_hi * rem_lo - ab_hi

;; ymm0=q1f1_hi, ymm1=ab_lo, ymm2=q2_dividend, ymm3=rem_lo, ymm12=bb_hi, SAVE2=aa_hi

	yfmsubpd ymm3, ymm3, ymm3, ymm12			;; 17-20; bb_lo = rem_lo * rem_lo - bb_hi
	yfmsubpd ymm7, ymm7, ymm7, ymm13			;; 18-21; bb_lo = rem_lo * rem_lo - bb_hi
	yfmsubpd ymm11, ymm11, ymm11, ymm14			;; 18-21; bb_lo = rem_lo * rem_lo - bb_hi

;; ymm0=q1f1_hi, ymm1=ab_lo, ymm2=q2_dividend, ymm3=bb_lo, ymm12=bb_hi, SAVE=bb_lo, SAVE2=aa_hi

	vmovapd ymm15, YMMWORD PTR [r8][rdx]			;;	; 1-or-2
	yfmaddpd ymm2, ymm15, ymm2, ymm0			;;*19-22; q2_dividend = 1-or-2 * q2_dividend + q1f1_hi
	yfmaddpd ymm6, ymm15, ymm6, ymm4			;;*19-22; q2_dividend = 1-or-2 * q2_dividend + q1f1_hi
	yfmaddpd ymm10, ymm15, ymm10, ymm8			;;*20-23; q2_dividend = 1-or-2 * q2_dividend + q1f1_hi

	vmovapd	YMMWORD PTR YMM_FMA_SAVE, ymm3			;;	; Save bb_lo
	vmovapd	YMMWORD PTR YMM_FMA_SAVEa, ymm7			;;	; Save bb_lo
	vmovapd	YMMWORD PTR YMM_FMA_SAVEb, ymm11		;;	; Save bb_lo

	vmovapd ymm3, YMMWORD PTR YMM_FMA_SAVE2			;;	; Reload aa_hi
	vmovapd ymm7, YMMWORD PTR YMM_FMA_SAVE2a		;;	; Reload aa_hi
	vmovapd ymm11, YMMWORD PTR YMM_FMA_SAVE2b		;;	; Reload aa_hi

	yfmsubpd ymm0, ymm3, YMMWORD PTR YMM_FMA_TWOPOW1_MODF_HI[rdx], ymm0
								;; 20-23; q1f1_lo = aa_hi * twopowmodf_hi[rdx] - q1f1_hi
	yfmsubpd ymm4, ymm7, YMMWORD PTR YMM_FMA_TWOPOW1_MODF_HIa[rdx], ymm4
								;; 21-24; q1f1_lo = aa_hi * twopowmodf_hi[rdx] - q1f1_hi
	yfmsubpd ymm8, ymm11, YMMWORD PTR YMM_FMA_TWOPOW1_MODF_HIb[rdx], ymm8
								;; 21-24; q1f1_lo = aa_hi * twopowmodf_hi[rdx] - q1f1_hi

;; ymm0=q1f1_lo, ymm1=ab_lo, ymm2=q2_dividend, ymm3=aa_hi, ymm12=bb_hi, SAVE=bb_lo, SAVE2=aa_hi

	yfmaddpd ymm1, YMMWORD PTR YMM_FMA_TWO, ymm1, ymm12	;; 22-25; final_hi = 2 * ab_lo + bb_hi
	yfmaddpd ymm5, YMMWORD PTR YMM_FMA_TWO, ymm5, ymm13	;; 22-25; final_hi = 2 * ab_lo + bb_hi
	yfmaddpd ymm9, YMMWORD PTR YMM_FMA_TWO, ymm9, ymm14	;; 23-26; final_hi = 2 * ab_lo + bb_hi

;; ymm0=q1f1_lo, ymm1=final_hi, ymm2=q2_dividend, ymm3=aa_hi, SAVE=bb_lo, SAVE2=aa_hi

	vmovapd	ymm15, YMMWORD PTR YMM_FMA_RND_CONST3		;;	; Load 3.0 * pow (2.0, (double) (51))
	yfmaddpd ymm12, ymm2, YMMWORD PTR YMM_FMA_INVFAC, ymm15	;;*23-26; q2 = q2_dividend * facinv + rounding const
	yfmaddpd ymm13, ymm6, YMMWORD PTR YMM_FMA_INVFACa, ymm15;;*24-27; q2 = q2_dividend * facinv + rounding const
	yfmaddpd ymm14, ymm10, YMMWORD PTR YMM_FMA_INVFACb, ymm15;;*24-27; q2 = q2_dividend * facinv + rounding const

;; ymm0=q1f1_lo, ymm1=final_hi, ymm2=q2_dividend, ymm3=aa_hi, ymm12=q2, SAVE=bb_lo, SAVE2=aa_hi

	vmovapd	ymm15, YMMWORD PTR YMM_FMA_RND_CONST2		;;	; Load 3.0 * pow (2.0, (double) (51 + BITS_LOW))
	yfmaddpd ymm3, ymm3, YMMWORD PTR YMM_FMA_TWOPOW1_MODF_LO[rdx], ymm15 ;; 25-28; q1f2_hi = aa_hi * twopowmodf_lo[rdx] + rounding const
	yfmaddpd ymm7, ymm7, YMMWORD PTR YMM_FMA_TWOPOW1_MODF_LOa[rdx], ymm15;; 25-28; q1f2_hi = aa_hi * twopowmodf_lo[rdx] + rounding const
	yfmaddpd ymm11, ymm11, YMMWORD PTR YMM_FMA_TWOPOW1_MODF_LOb[rdx], ymm15;; 26-29; q1f2_hi = aa_hi * twopowmodf_lo[rdx] + rounding const

;; ymm0=q1f1_lo, ymm1=final_hi, ymm2=q2_dividend, ymm3=q1f2_hi, ymm12=q2, SAVE=bb_lo, SAVE2=aa_hi

	vmovapd ymm15, YMMWORD PTR [r8][rdx]			;;	; 1-or-2
	yfmaddpd ymm1, ymm15, ymm1, ymm0			;; 26-29; final_hi = 1-or-2 * final_hi + q1f1_lo
	yfmaddpd ymm5, ymm15, ymm5, ymm4			;; 27-30; final_hi = 1-or-2 * final_hi + q1f1_lo
	yfmaddpd ymm9, ymm15, ymm9, ymm8			;; 27-30; final_hi = 1-or-2 * final_hi + q1f1_lo

;; ymm1=final_hi, ymm2=q2_dividend, ymm3=q1f2_hi, ymm12=q2, SAVE=bb_lo, SAVE2=aa_hi

	vmovapd	ymm15, YMMWORD PTR YMM_FMA_RND_CONST3		;;	; Load 3.0 * pow (2.0, (double) (51))
	vsubpd	ymm12, ymm12, ymm15				;;*28-31; q2 -= rounding const
	vsubpd	ymm13, ymm13, ymm15				;;*28-31; q2 -= rounding const
	vsubpd	ymm14, ymm14, ymm15				;;*29-32; q2 -= rounding const

	vmovapd	ymm15, YMMWORD PTR YMM_FMA_RND_CONST2		;;	; Load 3.0 * pow (2.0, (double) (51 + BITS_LOW))
	vsubpd	ymm3, ymm3, ymm15				;; 29-32; q1f2_hi -= rounding const
	vsubpd	ymm7, ymm7, ymm15				;; 30-33; q1f2_hi -= rounding const
	vsubpd	ymm11, ymm11, ymm15				;; 30-33; q1f2_hi -= rounding const

	yfnmaddpd ymm2, ymm12, YMMWORD PTR YMM_FMA_FAC_HI, ymm2	;;*32-35; Calc Q2*Fhi lo in one blow:  q2f1_lo = q2_dividend - q2 * fac_hi
	yfnmaddpd ymm6, ymm13, YMMWORD PTR YMM_FMA_FAC_HIa, ymm6;;*32-35; Calc Q2*Fhi lo in one blow:  q2f1_lo = q2_dividend - q2 * fac_hi
	yfnmaddpd ymm10, ymm14, YMMWORD PTR YMM_FMA_FAC_HIb, ymm10;;*33-36; Calc Q2*Fhi lo in one blow:  q2f1_lo = q2_dividend - q2 * fac_hi

;; ymm1=final_hi, ymm2=q2f1_lo, ymm3=q1f2_hi, ymm12=q2, SAVE=bb_lo, SAVE2=aa_hi

	yfnmaddpd ymm0, ymm12, YMMWORD PTR YMM_FMA_FAC_LO, ymm15;;*33-36; q2f2_hi = -q2 * fac_lo + rounding const
	yfnmaddpd ymm4, ymm13, YMMWORD PTR YMM_FMA_FAC_LOa, ymm15;;*34-37; q2f2_hi = -q2 * fac_lo + rounding const
	yfnmaddpd ymm8, ymm14, YMMWORD PTR YMM_FMA_FAC_LOb, ymm15;;*34-37; q2f2_hi = -q2 * fac_lo + rounding const

;; ymm0=q2f2_hi, ymm1=final_hi, ymm2=q2f1_lo, ymm3=q1f2_hi, ymm12=q2, SAVE=bb_lo, SAVE2=aa_hi

 	vaddpd	ymm2, ymm1, ymm2				;;*36-39; final_hi += q2f1_lo
 	vaddpd	ymm6, ymm5, ymm6				;;*36-39; final_hi += q2f1_lo
 	vaddpd	ymm10, ymm9, ymm10				;;*37-40; final_hi += q2f1_lo

;; ymm0=q2f2_hi, ymm2=final_hi, ymm3=q1f2_hi, ymm12=q2, SAVE=bb_lo, SAVE2=aa_hi

	vmovapd ymm1, YMMWORD PTR YMM_FMA_SAVE2			;;	; Reload aa_hi
	vmovapd ymm5, YMMWORD PTR YMM_FMA_SAVE2a		;;	; Reload aa_hi
	vmovapd ymm9, YMMWORD PTR YMM_FMA_SAVE2b		;;	; Reload aa_hi

	yfmsubpd ymm1, ymm1, YMMWORD PTR YMM_FMA_TWOPOW1_MODF_LO[rdx], ymm3;; 35-38; q1f2_lo = aa_hi * twopowmodf_lo[rdx] - q1f2_hi
	yfmsubpd ymm5, ymm5, YMMWORD PTR YMM_FMA_TWOPOW1_MODF_LOa[rdx], ymm7;; 35-38; q1f2_lo = aa_hi * twopowmodf_lo[rdx] - q1f2_hi
	yfmsubpd ymm9, ymm9, YMMWORD PTR YMM_FMA_TWOPOW1_MODF_LOb[rdx], ymm11;; 37-40; q1f2_lo = aa_hi * twopowmodf_lo[rdx] - q1f2_hi  (push out)

;; ymm0=q2f2_hi, ymm1=q1f2_lo, ymm2=final_hi, ymm3=q1f2_hi, ymm12=q2, SAVE=bb_lo

	vsubpd	ymm0, ymm0, ymm15				;;*38-41; q2f2_hi -= rounding const
	vsubpd	ymm4, ymm4, ymm15				;;*38-41; q2f2_hi -= rounding const
	vsubpd	ymm8, ymm8, ymm15				;;*39-42; q2f2_hi -= rounding const

	vmovapd ymm15, YMMWORD PTR [r8][rdx]			;;	; 1-or-2
	yfmaddpd ymm1, ymm15, YMMWORD PTR YMM_FMA_SAVE, ymm1	;; 39-42; final_lo = one_or_two[rdx] * bb_lo + q1f2_lo;
	yfmaddpd ymm5, ymm15, YMMWORD PTR YMM_FMA_SAVEa, ymm5	;; 40-43; final_lo = one_or_two[rdx] * bb_lo + q1f2_lo;
	yfmaddpd ymm9, ymm15, YMMWORD PTR YMM_FMA_SAVEb, ymm9	;; 40-43; final_lo = one_or_two[rdx] * bb_lo + q1f2_lo;

;; ymm0=q2f2_hi, ymm1=final_lo, ymm2=final_hi, ymm3=q1f2_hi, ymm12=q2

	vaddpd	ymm2, ymm2, ymm3				;; 41-44; final_hi += q1f2_hi
	vaddpd	ymm6, ymm6, ymm7				;; 41-44; final_hi += q1f2_hi
	vaddpd	ymm10, ymm10, ymm11				;; 42-45; final_hi += q1f2_hi

	yfnmsubpd ymm12, ymm12, YMMWORD PTR YMM_FMA_FAC_LO, ymm0;;*42-45; q2f2_lo = -q2 * fac_lo - q2f2_hi
	yfnmsubpd ymm13, ymm13, YMMWORD PTR YMM_FMA_FAC_LOa, ymm4;;*43-46; q2f2_lo = -q2 * fac_lo - q2f2_hi
	yfnmsubpd ymm14, ymm14, YMMWORD PTR YMM_FMA_FAC_LOb, ymm8;;*43-46; q2f2_lo = -q2 * fac_lo - q2f2_hi

;; ymm0=q2f2_hi, ymm1=final_lo, ymm2=final_hi, ymm12=q2f2_lo

	vaddpd	ymm2, ymm2, ymm0				;;*45-48; final_hi += q2f2_hi
	vaddpd	ymm6, ymm6, ymm4				;;*45-48; final_hi += q2f2_hi
	vaddpd	ymm10, ymm10, ymm8				;;*46-49; final_hi += q2f2_hi

	vaddpd	ymm3, ymm1, ymm12				;;*46-49; final_lo += q2f2_lo
	vaddpd	ymm7, ymm5, ymm13				;;*47-50; final_lo += q2f2_lo
	vaddpd	ymm11, ymm9, ymm14				;;*47-50; final_lo += q2f2_lo
	ENDM

;; Check if remainder in ymm2,ymm3 is one indicating we found a factor!
;; Normalize hi/lo so we can compare to YMM_ONE

avx_fma_compare MACRO
	vaddpd	ymm2, ymm2, ymm3				;; Combine fac_hi & fac_lo
	vaddpd	ymm6, ymm6, ymm7
	vaddpd	ymm10, ymm10, ymm11
	ENDM


;;********************************************************************
;; The AVX-512 FMA versions of the above macros (32 factors at a time)
;;********************************************************************

;; Preload constants

avx512_fma_fac_init_once MACRO
	vbroadcastsd zmm31, QWORD PTR ZMM_FMA_RND_CONST3	;; Load 3.0 * pow (2.0, (double) (51))
	vbroadcastsd zmm30, QWORD PTR ZMM_FMA_RND_CONST2	;; Load 3.0 * pow (2.0, (double) (51 + BITS_LOW))
	lea	r13, ZMM_FMA_ONE_OR_TWO				;; Address of 1.0 or 2.0
	ENDM

; This macro does some initialization work and then performs a subset of the
; main loop working on the initial value.  This subset avoids the squaring of the
; remainder.  The initial value can be up to 60 bits larger than the factor
; size.

avx512_fma_fac_initval MACRO
	LOCAL	large_initial_value, small_initial_value, equals_split, done

;; Convert factor from 2 integers into 2 doubles

	vpbroadcastq zmm29, QWORD PTR ZMM_FMA_LOW_MASK		;; Mask for acquiring low bits
	vpandq	zmm0, zmm29, ZMMWORD PTR ZMM_FMA_FAC_LO_INT	;; Load integer factor low
	vpandq	zmm7, zmm29, ZMMWORD PTR ZMM_FMA_FAC_LO_INTa	;; Load integer factor low
	vpandq	zmm14, zmm29, ZMMWORD PTR ZMM_FMA_FAC_LO_INTb	;; Load integer factor low
	vpandq	zmm21, zmm29, ZMMWORD PTR ZMM_FMA_FAC_LO_INTc	;; Load integer factor low

	vporq	zmm0, zmm0, zmm31				;; Step 1 to convert factor low to a double
	vporq	zmm7, zmm7, zmm31				;; Step 1 to convert factor low to a double
	vporq	zmm14, zmm14, zmm31				;; Step 1 to convert factor low to a double
	vporq	zmm21, zmm21, zmm31				;; Step 1 to convert factor low to a double

	vporq	zmm2, zmm31, ZMMWORD PTR ZMM_FMA_FAC_HI_INT	;; Load integer factor hi
	vporq	zmm9, zmm31, ZMMWORD PTR ZMM_FMA_FAC_HI_INTa	;; Load integer factor hi
	vporq	zmm16, zmm31, ZMMWORD PTR ZMM_FMA_FAC_HI_INTb	;; Load integer factor hi
	vporq	zmm23, zmm31, ZMMWORD PTR ZMM_FMA_FAC_HI_INTc	;; Load integer factor hi

	vbroadcastsd zmm29, QWORD PTR ZMM_FMA_TWO_TO_LO		;; Constant for converting integer factor hi
	yfmsubpd zmm2, zmm2, zmm29, zmm30			;; Convert integer factor hi to double times 2^lo
	yfmsubpd zmm9, zmm9, zmm29, zmm30			;; Convert integer factor hi to double times 2^lo
	yfmsubpd zmm16, zmm16, zmm29, zmm30			;; Convert integer factor hi to double times 2^lo
	yfmsubpd zmm23, zmm23, zmm29, zmm30			;; Convert integer factor hi to double times 2^lo

	vsubpd	zmm0, zmm0, zmm31				;; Step 2 to convert factor low to a double
	vsubpd	zmm7, zmm7, zmm31				;; Step 2 to convert factor low to a double
	vsubpd	zmm14, zmm14, zmm31				;; Step 2 to convert factor low to a double
	vsubpd	zmm21, zmm21, zmm31				;; Step 2 to convert factor low to a double

	vmovapd	ZMMWORD PTR ZMM_FMA_FAC_HI, zmm2		;; Save fac_hi
	vmovapd	ZMMWORD PTR ZMM_FMA_FAC_HIa, zmm9		;; Save fac_hi
	vmovapd	ZMMWORD PTR ZMM_FMA_FAC_HIb, zmm16		;; Save fac_hi
	vmovapd	ZMMWORD PTR ZMM_FMA_FAC_HIc, zmm23		;; Save fac_hi

	vmovapd	ZMMWORD PTR ZMM_FMA_FAC_LO, zmm0		;; Save fac_lo
	vmovapd	ZMMWORD PTR ZMM_FMA_FAC_LOa, zmm7		;; Save fac_lo
	vmovapd	ZMMWORD PTR ZMM_FMA_FAC_LOb, zmm14		;; Save fac_lo
	vmovapd	ZMMWORD PTR ZMM_FMA_FAC_LOc, zmm21		;; Save fac_lo

;; Compute factor inverse

	vaddpd	zmm3, zmm2, zmm0				;; Add fac_hi and fac_lo to form factor accurate to 53 bits
	vaddpd	zmm10, zmm9, zmm7				;; Add fac_hi and fac_lo to form factor accurate to 53 bits
	vaddpd	zmm17, zmm16, zmm14				;; Add fac_hi and fac_lo to form factor accurate to 53 bits
	vaddpd	zmm24, zmm23, zmm21				;; Add fac_hi and fac_lo to form factor accurate to 53 bits
	vmovapd	zmm27, ZMMWORD PTR ZMM_FMA_ONE
	vdivpd	zmm1, zmm27, zmm3				;; Compute 1 / fac
	vdivpd	zmm8, zmm27, zmm10				;; Compute 1 / fac
	vdivpd	zmm15, zmm27, zmm17				;; Compute 1 / fac
	vdivpd	zmm22, zmm27, zmm24				;; Compute 1 / fac
	vmovapd	ZMMWORD PTR ZMM_FMA_INVFAC, zmm1		;; Save factor inverse
	vmovapd	ZMMWORD PTR ZMM_FMA_INVFACa, zmm8		;; Save factor inverse
	vmovapd	ZMMWORD PTR ZMM_FMA_INVFACb, zmm15		;; Save factor inverse
	vmovapd	ZMMWORD PTR ZMM_FMA_INVFACc, zmm22		;; Save factor inverse

;; Compute (2^split mod f) * 2^-split.  Used after calculating rem_hi^2.
;; Compute (2^(split+1) mod f) * 2^-split.  Used for optional mul-by-2 after computing rem_hi^2.

;; zmm0 = fac_lo, zmm1 = facinv, zmm2 = fac_hi, zmm27 = 1.0

	vbroadcastsd zmm29, QWORD PTR ZMM_FMA_RND_CONST4	;; Load (3.0 * 2^51) * 2^-split
	vaddpd	zmm3, zmm1, zmm29				;; q = 1.0 * facinv + rounding const
	vaddpd	zmm10, zmm8, zmm29				;; q = 1.0 * facinv + rounding const
	vaddpd	zmm17, zmm15, zmm29				;; q = 1.0 * facinv + rounding const
	vaddpd	zmm24, zmm22, zmm29				;; q = 1.0 * facinv + rounding const
	vmovapd	zmm28, ZMMWORD PTR ZMM_FMA_TWO			;; Load 2.0
	yfmaddpd zmm4, zmm28, zmm1, zmm29			;; q2 = 2.0 * facinv + rounding const
	yfmaddpd zmm11, zmm28, zmm8, zmm29			;; q2 = 2.0 * facinv + rounding const
	yfmaddpd zmm18, zmm28, zmm15, zmm29			;; q2 = 2.0 * facinv + rounding const
	yfmaddpd zmm25, zmm28, zmm22, zmm29			;; q2 = 2.0 * facinv + rounding const
	vsubpd	zmm3, zmm3, zmm29				;; q -= rounding const
	vsubpd	zmm10, zmm10, zmm29				;; q -= rounding const
	vsubpd	zmm17, zmm17, zmm29				;; q -= rounding const
	vsubpd	zmm24, zmm24, zmm29				;; q -= rounding const
	vsubpd	zmm4, zmm4, zmm29				;; q2 -= rounding const
	vsubpd	zmm11, zmm11, zmm29				;; q2 -= rounding const
	vsubpd	zmm18, zmm18, zmm29				;; q2 -= rounding const
	vsubpd	zmm25, zmm25, zmm29				;; q2 -= rounding const
	vbroadcastsd zmm29, QWORD PTR ZMM_FMA_RND_CONST5	;; Load (3.0 * 2^(51 + LOW_BITS)) * 2^-split
	yfnmaddpd zmm5, zmm3, zmm0, zmm29			;; hi_addin = -q * fac_lo + rounding const
	yfnmaddpd zmm12, zmm10, zmm7, zmm29			;; hi_addin = -q * fac_lo + rounding const
	yfnmaddpd zmm19, zmm17, zmm14, zmm29			;; hi_addin = -q * fac_lo + rounding const
	yfnmaddpd zmm26, zmm24, zmm21, zmm29			;; hi_addin = -q * fac_lo + rounding const
	yfnmaddpd zmm6, zmm3, zmm2, zmm27			;; hi = 1.0 - q * fac_hi
	yfnmaddpd zmm13, zmm10, zmm9, zmm27			;; hi = 1.0 - q * fac_hi
	yfnmaddpd zmm20, zmm17, zmm16, zmm27			;; hi = 1.0 - q * fac_hi
	yfnmaddpd zmm27, zmm24, zmm23, zmm27			;; hi = 1.0 - q * fac_hi
	vsubpd	zmm5, zmm5, zmm29				;; hi_addin -= rounding const
	vsubpd	zmm12, zmm12, zmm29				;; hi_addin -= rounding const
	vsubpd	zmm19, zmm19, zmm29				;; hi_addin -= rounding const
	vsubpd	zmm26, zmm26, zmm29				;; hi_addin -= rounding const
	yfnmaddpd zmm1, zmm4, zmm0, zmm29			;; hi_addin2 = -q2 * fac_lo + rounding const
	yfnmaddpd zmm8, zmm11, zmm7, zmm29			;; hi_addin2 = -q2 * fac_lo + rounding const
	yfnmaddpd zmm15, zmm18, zmm14, zmm29			;; hi_addin2 = -q2 * fac_lo + rounding const
	yfnmaddpd zmm22, zmm25, zmm21, zmm29			;; hi_addin2 = -q2 * fac_lo + rounding const
	vaddpd	zmm6, zmm6, zmm5				;; hi += hi_addin
	vaddpd	zmm13, zmm13, zmm12				;; hi += hi_addin
	vaddpd	zmm20, zmm20, zmm19				;; hi += hi_addin
	vaddpd	zmm27, zmm27, zmm26				;; hi += hi_addin
	yfnmsubpd zmm3, zmm3, zmm0, zmm5			;; lo = -q * fac_lo - hi_addin
	yfnmsubpd zmm10, zmm10, zmm7, zmm12			;; lo = -q * fac_lo - hi_addin
	yfnmsubpd zmm17, zmm17, zmm14, zmm19			;; lo = -q * fac_lo - hi_addin
	yfnmsubpd zmm24, zmm24, zmm21, zmm26			;; lo = -q * fac_lo - hi_addin
	vsubpd	zmm1, zmm1, zmm29				;; hi_addin2 -= rounding const
	vsubpd	zmm8, zmm8, zmm29				;; hi_addin2 -= rounding const
	vsubpd	zmm15, zmm15, zmm29				;; hi_addin2 -= rounding const
	vsubpd	zmm22, zmm22, zmm29				;; hi_addin2 -= rounding const
	yfnmaddpd zmm5, zmm4, zmm2, zmm28			;; hi2 = 2.0 - q2 * fac_hi
	yfnmaddpd zmm12, zmm11, zmm9, zmm28			;; hi2 = 2.0 - q2 * fac_hi
	yfnmaddpd zmm19, zmm18, zmm16, zmm28			;; hi2 = 2.0 - q2 * fac_hi
	yfnmaddpd zmm26, zmm25, zmm23, zmm28			;; hi2 = 2.0 - q2 * fac_hi
	yfnmsubpd zmm4, zmm4, zmm0, zmm1			;; lo2 = -q2 * fac_lo - hi_addin2
	yfnmsubpd zmm11, zmm11, zmm7, zmm8			;; lo2 = -q2 * fac_lo - hi_addin2
	yfnmsubpd zmm18, zmm18, zmm14, zmm15			;; lo2 = -q2 * fac_lo - hi_addin2
	yfnmsubpd zmm25, zmm25, zmm21, zmm22			;; lo2 = -q2 * fac_lo - hi_addin2
	vaddpd	zmm5, zmm5, zmm1				;; hi2 += hi_addin2
	vaddpd	zmm12, zmm12, zmm8				;; hi2 += hi_addin2
	vaddpd	zmm19, zmm19, zmm15				;; hi2 += hi_addin2
	vaddpd	zmm26, zmm26, zmm22				;; hi2 += hi_addin2

	vmovapd	ZMMWORD PTR ZMM_FMA_TWOPOW1_MODF_HI, zmm6	;; Save high bits of modf
	vmovapd	ZMMWORD PTR ZMM_FMA_TWOPOW1_MODF_HIa, zmm13	;; Save high bits of modf
	vmovapd	ZMMWORD PTR ZMM_FMA_TWOPOW1_MODF_HIb, zmm20	;; Save high bits of modf
	vmovapd	ZMMWORD PTR ZMM_FMA_TWOPOW1_MODF_HIc, zmm27	;; Save high bits of modf
	vmovapd	ZMMWORD PTR ZMM_FMA_TWOPOW1_MODF_LO, zmm3	;; Save low bits of modf
	vmovapd	ZMMWORD PTR ZMM_FMA_TWOPOW1_MODF_LOa, zmm10	;; Save low bits of modf
	vmovapd	ZMMWORD PTR ZMM_FMA_TWOPOW1_MODF_LOb, zmm17	;; Save low bits of modf
	vmovapd	ZMMWORD PTR ZMM_FMA_TWOPOW1_MODF_LOc, zmm24	;; Save low bits of modf

	vmovapd	ZMMWORD PTR ZMM_FMA_TWOPOW2_MODF_HI, zmm5	;; Save high bits of modf
	vmovapd	ZMMWORD PTR ZMM_FMA_TWOPOW2_MODF_HIa, zmm12	;; Save high bits of modf
	vmovapd	ZMMWORD PTR ZMM_FMA_TWOPOW2_MODF_HIb, zmm19	;; Save high bits of modf
	vmovapd	ZMMWORD PTR ZMM_FMA_TWOPOW2_MODF_HIc, zmm26	;; Save high bits of modf
	vmovapd	ZMMWORD PTR ZMM_FMA_TWOPOW2_MODF_LO, zmm4	;; Save low bits of modf
	vmovapd	ZMMWORD PTR ZMM_FMA_TWOPOW2_MODF_LOa, zmm11	;; Save low bits of modf
	vmovapd	ZMMWORD PTR ZMM_FMA_TWOPOW2_MODF_LOb, zmm18	;; Save low bits of modf
	vmovapd	ZMMWORD PTR ZMM_FMA_TWOPOW2_MODF_LOc, zmm25	;; Save low bits of modf

;; zmm0 = fac_lo, zmm2 = fac_hi, zmm3 = twopowmodf_lo, zmm4 = twopow2modf_lo, zmm5 = twopow2modf_hi, zmm6 = twopowmodf_hi

;; Now compute the initial remainder, using an optimized version of the main loop code.  We can save
;; several multiplies and adds because the squared value is a power of two. 

;; There are 4 options for the initial value to get a remainder of.  FMA_INITVAL_TYPE is:
;;	-1)  initval < 2^split	(calc using small initval code)
;;	0)   > 2^(split+1)	(calc using large initval code)
;;	1)   == 2^split		(use twopow1_modf)
;;	2)   == 2^(split+1)	(use twopow2_modf)

;; Test which of the 4 initial value types we have.  Branch to the right code.

	cmp	FMA_INITVAL_TYPE, 0
	jl	small_initial_value
	je	short large_initial_value

;; Easiest case of all!  Remainder is either twopow1_modf or twopow2_modf

	vbroadcastsd zmm28, QWORD PTR ZMM_FMA_TWOPOW_SPLIT	;; Load 2^split
	cmp	FMA_INITVAL_TYPE, 1
	je	short equals_split
	vmulpd	zmm2, zmm28, zmm5				;; Return twopow2_modf
	vmulpd	zmm3, zmm28, zmm4
	vmulpd	zmm9, zmm28, zmm12				;; Return twopow2_modf
	vmulpd	zmm10, zmm28, zmm11
	vmulpd	zmm16, zmm28, zmm19				;; Return twopow2_modf
	vmulpd	zmm17, zmm28, zmm18
	vmulpd	zmm23, zmm28, zmm26				;; Return twopow2_modf
	vmulpd	zmm24, zmm28, zmm25
	jmp	done

equals_split:
	vmulpd	zmm2, zmm28, zmm6				;; Return twopow1_modf
	vmulpd	zmm3, zmm28, zmm3
	vmulpd	zmm9, zmm28, zmm13				;; Return twopow1_modf
	vmulpd	zmm10, zmm28, zmm10
	vmulpd	zmm16, zmm28, zmm20				;; Return twopow1_modf
	vmulpd	zmm17, zmm28, zmm17
	vmulpd	zmm23, zmm28, zmm27				;; Return twopow1_modf
	vmulpd	zmm24, zmm28, zmm24
	jmp	done

;; Process an initial squared value that is more than 2^(split+1).  To avoid any edge cases with regards
;; to precision of the resulting remainder, we load initial_squared_value / 2 and use the twopow2_modf values

large_initial_value:

;; calc  AA-hi*TwoPowModFhi hi	* 2^(35+35)		51/50	*2^70
;; calc  AA-hi*TwoPowModFhi lo	* 2^(35)		35	*2^35
;; calc  AA-hi*TwoPowModFlo hi	* 2^(35)		41/40	*2^35
;; calc  AA-hi*TwoPowModFlo lo	* 2^0			35	*2^0

;; calc Q2, remaining bits of quotient
;; calc  Q2*Fhi hi	* 2^(33+35)		53	*2^68
;; calc  Q2*Fhi lo	* 2^35			33	*2^35
;; calc  Q2*Flo hi	* 2^(35+0)		41*	*2^35
;; calc  Q2*Flo lo	* 2^0			35	*2^0

;; zmm0 = fac_lo, zmm2 = fac_hi, zmm4 = twopow2modf_lo, zmm5 = twopow2modf_hi

	vbroadcastsd zmm28, QWORD PTR ZMM_FMA_INITVAL		;; Load the initial squared value/2 (known as aa_hi in main loop)
	vbroadcastsd zmm29, QWORD PTR ZMM_FMA_RND_CONST1	;; Load 3.0 * pow (2.0, (double) (51 + BITS_LOW * 2))
	yfmaddpd zmm3, zmm28, zmm5, zmm29			;; q1f1_hi = aa_hi * twopow2modf_hi + rounding const
	yfmaddpd zmm10, zmm28, zmm12, zmm29			;; q1f1_hi = aa_hi * twopow2modf_hi + rounding const
	yfmaddpd zmm17, zmm28, zmm19, zmm29			;; q1f1_hi = aa_hi * twopow2modf_hi + rounding const
	yfmaddpd zmm24, zmm28, zmm26, zmm29			;; q1f1_hi = aa_hi * twopow2modf_hi + rounding const

	yfmaddpd zmm6, zmm28, zmm4, zmm30			;; q1f2_hi = aa_hi * twopow2modf_lo + rounding const
	yfmaddpd zmm13, zmm28, zmm11, zmm30			;; q1f2_hi = aa_hi * twopow2modf_lo + rounding const
	yfmaddpd zmm20, zmm28, zmm18, zmm30			;; q1f2_hi = aa_hi * twopow2modf_lo + rounding const
	yfmaddpd zmm27, zmm28, zmm25, zmm30			;; q1f2_hi = aa_hi * twopow2modf_lo + rounding const

	vsubpd	zmm3, zmm3, zmm29				;; q1f1_hi -= rounding const (q2_dividend = q1f1_hi)
	vsubpd	zmm10, zmm10, zmm29				;; q1f1_hi -= rounding const (q2_dividend = q1f1_hi)
	vsubpd	zmm17, zmm17, zmm29				;; q1f1_hi -= rounding const (q2_dividend = q1f1_hi)
	vsubpd	zmm24, zmm24, zmm29				;; q1f1_hi -= rounding const (q2_dividend = q1f1_hi)

	vsubpd	zmm6, zmm6, zmm30				;; q1f2_hi -= rounding const
	vsubpd	zmm13, zmm13, zmm30				;; q1f2_hi -= rounding const
	vsubpd	zmm20, zmm20, zmm30				;; q1f2_hi -= rounding const
	vsubpd	zmm27, zmm27, zmm30				;; q1f2_hi -= rounding const

	yfmaddpd zmm1, zmm3, ZMMWORD PTR ZMM_FMA_INVFAC, zmm31	;; q2 = q2_dividend * facinv + rounding const
	yfmaddpd zmm8, zmm10, ZMMWORD PTR ZMM_FMA_INVFACa, zmm31;; q2 = q2_dividend * facinv + rounding const
	yfmaddpd zmm15, zmm17, ZMMWORD PTR ZMM_FMA_INVFACb, zmm31;; q2 = q2_dividend * facinv + rounding const
	yfmaddpd zmm22, zmm24, ZMMWORD PTR ZMM_FMA_INVFACc, zmm31;; q2 = q2_dividend * facinv + rounding const

	yfmsubpd zmm5, zmm28, zmm5, zmm3			;; q1f1_lo = aa_hi * twopow2modf_hi - q1f1_hi (final_hi = q1f1_lo)
	yfmsubpd zmm12, zmm28, zmm12, zmm10			;; q1f1_lo = aa_hi * twopow2modf_hi - q1f1_hi (final_hi = q1f1_lo)
	yfmsubpd zmm19, zmm28, zmm19, zmm17			;; q1f1_lo = aa_hi * twopow2modf_hi - q1f1_hi (final_hi = q1f1_lo)
	yfmsubpd zmm26, zmm28, zmm26, zmm24			;; q1f1_lo = aa_hi * twopow2modf_hi - q1f1_hi (final_hi = q1f1_lo)

	vsubpd	zmm1, zmm1, zmm31				;; q2 -= rounding const
	vsubpd	zmm8, zmm8, zmm31				;; q2 -= rounding const
	vsubpd	zmm15, zmm15, zmm31				;; q2 -= rounding const
	vsubpd	zmm22, zmm22, zmm31				;; q2 -= rounding const

	yfmsubpd zmm4, zmm28, zmm4, zmm6			;; q1f2_lo = aa_hi * twopow2modf_lo - q1f2_hi (final_lo = q1f2_lo)
	yfmsubpd zmm11, zmm28, zmm11, zmm13			;; q1f2_lo = aa_hi * twopow2modf_lo - q1f2_hi (final_lo = q1f2_lo)
	yfmsubpd zmm18, zmm28, zmm18, zmm20			;; q1f2_lo = aa_hi * twopow2modf_lo - q1f2_hi (final_lo = q1f2_lo)
	yfmsubpd zmm25, zmm28, zmm25, zmm27			;; q1f2_lo = aa_hi * twopow2modf_lo - q1f2_hi (final_lo = q1f2_lo)

	yfnmaddpd zmm2, zmm1, zmm2, zmm3			;; q2f1_lo = q2_dividend - q2 * fac_hi
	yfnmaddpd zmm9, zmm8, zmm9, zmm10			;; q2f1_lo = q2_dividend - q2 * fac_hi
	yfnmaddpd zmm16, zmm15, zmm16, zmm17			;; q2f1_lo = q2_dividend - q2 * fac_hi
	yfnmaddpd zmm23, zmm22, zmm23, zmm24			;; q2f1_lo = q2_dividend - q2 * fac_hi

	yfnmaddpd zmm3, zmm1, zmm0, zmm30			;; q2f2_hi = -q2 * fac_lo + rounding const
	yfnmaddpd zmm10, zmm8, zmm7, zmm30			;; q2f2_hi = -q2 * fac_lo + rounding const
	yfnmaddpd zmm17, zmm15, zmm14, zmm30			;; q2f2_hi = -q2 * fac_lo + rounding const
	yfnmaddpd zmm24, zmm22, zmm21, zmm30			;; q2f2_hi = -q2 * fac_lo + rounding const

	vaddpd	zmm5, zmm5, zmm6				;; final_hi += q1f2_hi
	vaddpd	zmm12, zmm12, zmm13				;; final_hi += q1f2_hi
	vaddpd	zmm19, zmm19, zmm20				;; final_hi += q1f2_hi
	vaddpd	zmm26, zmm26, zmm27				;; final_hi += q1f2_hi

	vsubpd	zmm3, zmm3, zmm30				;; q2f2_hi -= rounding const
	vsubpd	zmm10, zmm10, zmm30				;; q2f2_hi -= rounding const
	vsubpd	zmm17, zmm17, zmm30				;; q2f2_hi -= rounding const
	vsubpd	zmm24, zmm24, zmm30				;; q2f2_hi -= rounding const

	vaddpd	zmm2, zmm5, zmm2				;; final_hi += q2f1_lo
	vaddpd	zmm9, zmm12, zmm9				;; final_hi += q2f1_lo
	vaddpd	zmm16, zmm19, zmm16				;; final_hi += q2f1_lo
	vaddpd	zmm23, zmm26, zmm23				;; final_hi += q2f1_lo

	yfnmsubpd zmm0, zmm1, zmm0, zmm3			;; q2f2_lo = -q2 * fac_lo - q2f2_hi
	yfnmsubpd zmm7, zmm8, zmm7, zmm10			;; q2f2_lo = -q2 * fac_lo - q2f2_hi
	yfnmsubpd zmm14, zmm15, zmm14, zmm17			;; q2f2_lo = -q2 * fac_lo - q2f2_hi
	yfnmsubpd zmm21, zmm22, zmm21, zmm24			;; q2f2_lo = -q2 * fac_lo - q2f2_hi

	vaddpd	zmm2, zmm2, zmm3				;; final_hi += q2f2_hi
	vaddpd	zmm9, zmm9, zmm10				;; final_hi += q2f2_hi
	vaddpd	zmm16, zmm16, zmm17				;; final_hi += q2f2_hi
	vaddpd	zmm23, zmm23, zmm24				;; final_hi += q2f2_hi

	vaddpd	zmm3, zmm4, zmm0				;; final_lo += q2f2_lo
	vaddpd	zmm10, zmm11, zmm7				;; final_lo += q2f2_lo
	vaddpd	zmm17, zmm18, zmm14				;; final_lo += q2f2_lo
	vaddpd	zmm24, zmm25, zmm21				;; final_lo += q2f2_lo
	jmp	done

;; Process an initial squared value that is less than 2^split
;; Calc q, the quotient.  Then calc initial_value - q * fac

;; zmm0 = fac_lo, zmm2 = fac_hi

small_initial_value:
	vbroadcastsd zmm28, QWORD PTR ZMM_FMA_INITVAL		;; Load the initial squared value
	yfmaddpd zmm3, zmm28, ZMMWORD PTR ZMM_FMA_INVFAC, zmm31	;; q = initial_value * facinv + rounding const
	yfmaddpd zmm10, zmm28, ZMMWORD PTR ZMM_FMA_INVFACa, zmm31;; q = initial_value * facinv + rounding const
	yfmaddpd zmm17, zmm28, ZMMWORD PTR ZMM_FMA_INVFACb, zmm31;; q = initial_value * facinv + rounding const
	yfmaddpd zmm24, zmm28, ZMMWORD PTR ZMM_FMA_INVFACc, zmm31;; q = initial_value * facinv + rounding const
	vsubpd	zmm3, zmm3, zmm31				;; q -= rounding const
	vsubpd	zmm10, zmm10, zmm31				;; q -= rounding const
	vsubpd	zmm17, zmm17, zmm31				;; q -= rounding const
	vsubpd	zmm24, zmm24, zmm31				;; q -= rounding const
	yfnmaddpd zmm4, zmm3, zmm0, zmm30			;; qf2_hi = -q * fac_lo + rounding const
	yfnmaddpd zmm11, zmm10, zmm7, zmm30			;; qf2_hi = -q * fac_lo + rounding const
	yfnmaddpd zmm18, zmm17, zmm14, zmm30			;; qf2_hi = -q * fac_lo + rounding const
	yfnmaddpd zmm25, zmm24, zmm21, zmm30			;; qf2_hi = -q * fac_lo + rounding const
	yfnmaddpd zmm2, zmm3, zmm2, zmm28			;; final_hi = initial_value - q * fac_hi
	yfnmaddpd zmm9, zmm10, zmm9, zmm28			;; final_hi = initial_value - q * fac_hi
	yfnmaddpd zmm16, zmm17, zmm16, zmm28			;; final_hi = initial_value - q * fac_hi
	yfnmaddpd zmm23, zmm24, zmm23, zmm28			;; final_hi = initial_value - q * fac_hi
	vsubpd	zmm4, zmm4, zmm30				;; qf2_hi -= rounding const
	vsubpd	zmm11, zmm11, zmm30				;; qf2_hi -= rounding const
	vsubpd	zmm18, zmm18, zmm30				;; qf2_hi -= rounding const
	vsubpd	zmm25, zmm25, zmm30				;; qf2_hi -= rounding const
	vaddpd	zmm2, zmm2, zmm4				;; final_hi += qf2_hi
	vaddpd	zmm9, zmm9, zmm11				;; final_hi += qf2_hi
	vaddpd	zmm16, zmm16, zmm18				;; final_hi += qf2_hi
	vaddpd	zmm23, zmm23, zmm25				;; final_hi += qf2_hi
	yfnmsubpd zmm3, zmm3, zmm0, zmm4			;; final_lo = -q * fac_lo - qf2_hi
	yfnmsubpd zmm10, zmm10, zmm7, zmm11			;; final_lo = -q * fac_lo - qf2_hi
	yfnmsubpd zmm17, zmm17, zmm14, zmm18			;; final_lo = -q * fac_lo - qf2_hi
	yfnmsubpd zmm24, zmm24, zmm21, zmm25			;; final_lo = -q * fac_lo - qf2_hi

done:
	ENDM


; This macro does one iteration of the main loop.  Square a remainder and do a mod.
; Note that at the beginning of the macro zmm2, zmm3 contains the semi-normalized remainder
; for us to square.  This input number is between -fac/2*epsilon and +fac/2*epsilon.  The range
; is not perfect because this macro's calculation of Q2 is not exact.

; In the comments of this macro, for illustration purposes we'll assume fac_size is 80 bits
; with 35 bits stored in the low word.  There is no reason to believe this code cannot handle
; at least 96 bits.  In fact, we have simulated this macro in C code with no problems.

avx512_fma_fac MACRO fac_size

;; AB^2 will be 160 bits.  From there, we multiply the top 40 bits by the 80-bit value 2^120 mod f.
;; This 120-bit result plus the low 120/121 bits of AB^2 give us a 122-bit dividend in computing Q2.

;; To maximize significant bits in calculating Q2, we want to maximize the precision of AA-lo and 2AB-hi.  Looking at
;; the table below that happens when AA-lo and 2AB-hi use as small an offset as possible without exceeding 53 bits
;; of precision.  Below that would be 34 bits in a low word.

;; SQUARE rem (call rem_hi A and rem_lo B).  A is between 0 and 44.5+epsilon bits.  B is between 0 and 35 normalized bits.
;; Unnormalized low words are the result of 1 doubling and 3 adds.  This adds 2 bits.

;;					normalized		unnormalized
;;				      max bits	offset	      max bits	offset
;;AA-hi		* 2^(50+2*35)		40	*2^120		40	*2^120
;;AA-lo		* 2^(2*35)		50	*2^70		50	*2^70
;;2AB-hi	* 2^(34+36)		46/47	*2^70		48/49	*2^70
;;2AB-lo	* 2^36			34/33	*2^36/37	34/33	*2^36/37
;;BB-hi		* 2^(35+0)		35	*2^35		39/40	*2^35
;;BB-lo		* 2^0			35	*2^0		35	*2^0

;; Since we know the power of two that aa_hi is a multiple of (aa_hi is a multiple of 2^120 in our example),
;; we pre-calculate 2^that_power mod factor, and multiply aa-hi by this pre-calculated value.

;; Aa-hi is 41/40 bits, TwoPowModFhi is 45 bits, TwoPowModFlo is 35 bits
;; calc  AA-hi*TwoPowModFhi hi	* 2^(35+35)		51/50	*2^70
;; calc  AA-hi*TwoPowModFhi lo	* 2^(35)		35	*2^35
;; calc  AA-hi*TwoPowModFlo hi	* 2^(35)		41/40	*2^35
;; calc  AA-hi*TwoPowModFlo lo	* 2^0			35	*2^0

;; calc Q2, remaining bits of quotient
;;
;; Q2 is 41 bits, Fhi is 45 bits, Flo is 35 bits
;; calc  Q2*Fhi hi	* 2^(33+35)		53	*2^68
;; calc  Q2*Fhi lo	* 2^35			33	*2^35
;; calc  Q2*Flo hi	* 2^(35+0)		41*	*2^35
;; calc  Q2*Flo lo	* 2^0			35	*2^0

;; zmm0-6 reserved for set 1, zmm7-13 reserved for set 2
;; zmm14-20 reserved for set 3, zmm21-27 reserved for set 4, zmm28-31 is for common use

	vbroadcastsd zmm28, QWORD PTR ZMM_FMA_RND_CONST0	;;	; Load 3.0 * pow (2.0, (double) (51 + BITS_TOTAL + (BITS_TOTAL + 1) / 2))
	yfmaddpd zmm0, zmm2, zmm2, zmm28			;;*1-4	; aa_hi = rem_hi * rem_hi + rounding const
	yfmaddpd zmm7, zmm9, zmm9, zmm28			;;*1-4	; aa_hi = rem_hi * rem_hi + rounding const
	yfmaddpd zmm14, zmm16, zmm16, zmm28			;;*2-5	; aa_hi = rem_hi * rem_hi + rounding const
	yfmaddpd zmm21, zmm23, zmm23, zmm28			;;*2-5	; aa_hi = rem_hi * rem_hi + rounding const

	vbroadcastsd zmm29, QWORD PTR ZMM_FMA_RND_CONST1	;;	; Load 3.0 * pow (2.0, (double) (51 + BITS_LOW * 2))
	yfmaddpd zmm1, zmm2, zmm3, zmm29			;;*3-6	; ab_hi = rem_hi * rem_lo + rounding const
	yfmaddpd zmm8, zmm9, zmm10, zmm29			;;*3-6	; ab_hi = rem_hi * rem_lo + rounding const
	yfmaddpd zmm15, zmm16, zmm17, zmm29			;;*4-7	; ab_hi = rem_hi * rem_lo + rounding const
	yfmaddpd zmm22, zmm23, zmm24, zmm29			;;*4-7	; ab_hi = rem_hi * rem_lo + rounding const

	vsubpd	zmm0, zmm0, zmm28				;;*5-8	; aa_hi -= rounding const
	vsubpd	zmm7, zmm7, zmm28				;;*5-8	; aa_hi -= rounding const
	vsubpd	zmm14, zmm14, zmm28				;;*6-9	; aa_hi -= rounding const
	vsubpd	zmm21, zmm21, zmm28				;;*6-9	; aa_hi -= rounding const

	vsubpd	zmm1, zmm1, zmm29				;;*7-10	; ab_hi -= rounding const
	vsubpd	zmm8, zmm8, zmm29				;;*7-10	; ab_hi -= rounding const
	vsubpd	zmm15, zmm15, zmm29				;;*8-11	; ab_hi -= rounding const
	vsubpd	zmm22, zmm22, zmm29				;;*8-11	; ab_hi -= rounding const

;; zmm0=aa_hi, zmm1=ab_hi, zmm2=rem_hi, zmm3=rem_lo

	yfmsubpd zmm4, zmm2, zmm2, zmm0				;;*9-12	; aa_lo = rem_hi * rem_hi - aa_hi
	yfmsubpd zmm11, zmm9, zmm9, zmm7			;;*9-12	; aa_lo = rem_hi * rem_hi - aa_hi
	yfmsubpd zmm18, zmm16, zmm16, zmm14			;;*10-13; aa_lo = rem_hi * rem_hi - aa_hi
	yfmsubpd zmm25, zmm23, zmm23, zmm21			;;*10-13; aa_lo = rem_hi * rem_hi - aa_hi

;; zmm0=aa_hi, zmm1=ab_hi, zmm2=rem_hi, zmm3=rem_lo, zmm4=aa_lo

	mov	edx, FMA_SHIFTER[rdi*4-4]			;;	; This is 64 if doubling else 0

	yfmaddpd zmm5, zmm0, ZMMWORD PTR ZMM_FMA_TWOPOW1_MODF_HI[rdx], zmm29
								;;*11-14; q1f1_hi = aa_hi * twopowmodf_hi[rdx] + rounding const
	yfmaddpd zmm12, zmm7, ZMMWORD PTR ZMM_FMA_TWOPOW1_MODF_HIa[rdx], zmm29
								;;*11-14; q1f1_hi = aa_hi * twopowmodf_hi[rdx] + rounding const
	yfmaddpd zmm19, zmm14, ZMMWORD PTR ZMM_FMA_TWOPOW1_MODF_HIb[rdx], zmm29
								;;*12-15; q1f1_hi = aa_hi * twopowmodf_hi[rdx] + rounding const
	yfmaddpd zmm26, zmm21, ZMMWORD PTR ZMM_FMA_TWOPOW1_MODF_HIc[rdx], zmm29
								;;*12-15; q1f1_hi = aa_hi * twopowmodf_hi[rdx] + rounding const

;; zmm0=aa_hi, zmm1=ab_hi, zmm2=rem_hi, zmm3=rem_lo, zmm4=aa_lo, zmm5=q1f1_hi

	vmovapd	zmm28, ZMMWORD PTR ZMM_FMA_TWO
	yfmaddpd zmm4, zmm28, zmm1, zmm4			;;*13-16; q2_dividend = 2AB_hi + AA_lo
	yfmaddpd zmm11, zmm28, zmm8, zmm11			;;*13-16; q2_dividend = 2AB_hi + AA_lo
	yfmaddpd zmm18, zmm28, zmm15, zmm18			;;*14-17; q2_dividend = 2AB_hi + AA_lo
	yfmaddpd zmm25, zmm28, zmm22, zmm25			;;*14-17; q2_dividend = 2AB_hi + AA_lo

;; zmm0=aa_hi, zmm1=ab_hi, zmm2=rem_hi, zmm3=rem_lo, zmm4=q2_dividend, zmm5=q1f1_hi

	vsubpd	zmm5, zmm5, zmm29				;;*15-18; q1f1_hi -= rounding const
	vsubpd	zmm12, zmm12, zmm29				;;*15-18; q1f1_hi -= rounding const
	vsubpd	zmm19, zmm19, zmm29				;;*16-19; q1f1_hi -= rounding const
	vsubpd	zmm26, zmm26, zmm29				;;*16-19; q1f1_hi -= rounding const

	yfmsubpd zmm2, zmm2, zmm3, zmm1				;; 17-20; ab_lo = rem_hi * rem_lo - ab_hi
	yfmsubpd zmm9, zmm9, zmm10, zmm8			;; 17-20; ab_lo = rem_hi * rem_lo - ab_hi
	yfmsubpd zmm16, zmm16, zmm17, zmm15			;; 18-21; ab_lo = rem_hi * rem_lo - ab_hi
	yfmsubpd zmm23, zmm23, zmm24, zmm22			;; 18-21; ab_lo = rem_hi * rem_lo - ab_hi

;; zmm0=aa_hi, zmm2=ab_lo, zmm3=rem_lo, zmm4=q2_dividend, zmm5=q1f1_hi

	vmovapd zmm29, ZMMWORD PTR [r13][rdx]			;;	; 1-or-2
	yfmaddpd zmm4, zmm29, zmm4, zmm5			;;*19-22; q2_dividend = 1-or-2 * q2_dividend + q1f1_hi
	yfmaddpd zmm11, zmm29, zmm11, zmm12			;;*19-22; q2_dividend = 1-or-2 * q2_dividend + q1f1_hi
	yfmaddpd zmm18, zmm29, zmm18, zmm19			;;*20-23; q2_dividend = 1-or-2 * q2_dividend + q1f1_hi
	yfmaddpd zmm25, zmm29, zmm25, zmm26			;;*20-23; q2_dividend = 1-or-2 * q2_dividend + q1f1_hi

	yfmaddpd zmm1, zmm3, zmm3, zmm30			;; 21-24 ; bb_hi = rem_lo * rem_lo + rounding const
	yfmaddpd zmm8, zmm10, zmm10, zmm30			;; 21-24 ; bb_hi = rem_lo * rem_lo + rounding const
	yfmaddpd zmm15, zmm17, zmm17, zmm30			;; 22-25 ; bb_hi = rem_lo * rem_lo + rounding const
	yfmaddpd zmm22, zmm24, zmm24, zmm30			;; 22-25 ; bb_hi = rem_lo * rem_lo + rounding const

;; zmm0=aa_hi, zmm1=bb_hi, zmm2=ab_lo, zmm3=rem_lo, zmm4=q2_dividend, zmm5=q1f1_hi

	yfmaddpd zmm6, zmm4, ZMMWORD PTR ZMM_FMA_INVFAC, zmm31	;;*23-26; q2 = q2_dividend * facinv + rounding const
	yfmaddpd zmm13, zmm11, ZMMWORD PTR ZMM_FMA_INVFACa, zmm31;;*23-26; q2 = q2_dividend * facinv + rounding const
	yfmaddpd zmm20, zmm18, ZMMWORD PTR ZMM_FMA_INVFACb, zmm31;;*24-27; q2 = q2_dividend * facinv + rounding const
	yfmaddpd zmm27, zmm25, ZMMWORD PTR ZMM_FMA_INVFACc, zmm31;;*24-27; q2 = q2_dividend * facinv + rounding const

;; zmm0=aa_hi, zmm1=bb_hi, zmm2=ab_lo, zmm3=rem_lo, zmm4=q2_dividend, zmm5=q1f1_hi, zmm6=q2

	vsubpd	zmm1, zmm1, zmm30				;; 25-28; bb_hi -= rounding const
	vsubpd	zmm8, zmm8, zmm30				;; 25-28; bb_hi -= rounding const
	vsubpd	zmm15, zmm15, zmm30				;; 26-29; bb_hi -= rounding const
	vsubpd	zmm22, zmm22, zmm30				;; 26-29; bb_hi -= rounding const

	yfmsubpd zmm5, zmm0, ZMMWORD PTR ZMM_FMA_TWOPOW1_MODF_HI[rdx], zmm5
								;; 27-30; q1f1_lo = aa_hi * twopowmodf_hi[rdx] - q1f1_hi
	yfmsubpd zmm12, zmm7, ZMMWORD PTR ZMM_FMA_TWOPOW1_MODF_HIa[rdx], zmm12
								;; 27-30; q1f1_lo = aa_hi * twopowmodf_hi[rdx] - q1f1_hi
	yfmsubpd zmm19, zmm14, ZMMWORD PTR ZMM_FMA_TWOPOW1_MODF_HIb[rdx], zmm19
								;; 28-31; q1f1_lo = aa_hi * twopowmodf_hi[rdx] - q1f1_hi
	yfmsubpd zmm26, zmm21, ZMMWORD PTR ZMM_FMA_TWOPOW1_MODF_HIc[rdx], zmm26
								;; 28-31; q1f1_lo = aa_hi * twopowmodf_hi[rdx] - q1f1_hi

;; zmm0=aa_hi, zmm1=bb_hi, zmm2=ab_lo, zmm3=rem_lo, zmm4=q2_dividend, zmm5=q1f1_lo, zmm6=q2

	vsubpd	zmm6, zmm6, zmm31				;;*29-32; q2 -= rounding const
	vsubpd	zmm13, zmm13, zmm31				;;*29-32; q2 -= rounding const
	vsubpd	zmm20, zmm20, zmm31				;;*30-33; q2 -= rounding const
	vsubpd	zmm27, zmm27, zmm31				;;*30-33; q2 -= rounding const

	yfmsubpd zmm3, zmm3, zmm3, zmm1				;; 31-34; bb_lo = rem_lo * rem_lo - bb_hi
	yfmsubpd zmm10, zmm10, zmm10, zmm8			;; 31-34; bb_lo = rem_lo * rem_lo - bb_hi
	yfmsubpd zmm17, zmm17, zmm17, zmm15			;; 32-35; bb_lo = rem_lo * rem_lo - bb_hi
	yfmsubpd zmm24, zmm24, zmm24, zmm22			;; 32-35; bb_lo = rem_lo * rem_lo - bb_hi

;; zmm0=aa_hi, zmm1=bb_hi, zmm2=ab_lo, zmm3=bb_lo, zmm4=q2_dividend, zmm5=q1f1_lo, zmm6=q2

	yfmaddpd zmm2, zmm28, zmm2, zmm1			;; 33-36; final_hi = 2 * ab_lo + bb_hi
	yfmaddpd zmm9, zmm28, zmm9, zmm8			;; 33-36; final_hi = 2 * ab_lo + bb_hi
	yfmaddpd zmm16, zmm28, zmm16, zmm15			;; 34-37; final_hi = 2 * ab_lo + bb_hi
	yfmaddpd zmm23, zmm28, zmm23, zmm22			;; 34-37; final_hi = 2 * ab_lo + bb_hi

;; zmm0=aa_hi, zmm2=final_hi, zmm3=bb_lo, zmm4=q2_dividend, zmm5=q1f1_lo, zmm6=q2

	yfnmaddpd zmm4, zmm6, ZMMWORD PTR ZMM_FMA_FAC_HI, zmm4	;;*35-38; Calc Q2*Fhi lo in one blow:  q2f1_lo = q2_dividend - q2 * fac_hi
	yfnmaddpd zmm11, zmm13, ZMMWORD PTR ZMM_FMA_FAC_HIa, zmm11;;*35-38; Calc Q2*Fhi lo in one blow:  q2f1_lo = q2_dividend - q2 * fac_hi
	yfnmaddpd zmm18, zmm20, ZMMWORD PTR ZMM_FMA_FAC_HIb, zmm18;;*36-39; Calc Q2*Fhi lo in one blow:  q2f1_lo = q2_dividend - q2 * fac_hi
	yfnmaddpd zmm25, zmm27, ZMMWORD PTR ZMM_FMA_FAC_HIc, zmm25;;*36-39; Calc Q2*Fhi lo in one blow:  q2f1_lo = q2_dividend - q2 * fac_hi

;; zmm0=aa_hi, zmm2=final_hi, zmm3=bb_lo, zmm4=q2f1_lo, zmm5=q1f1_lo, zmm6=q2

	yfmaddpd zmm1, zmm0, ZMMWORD PTR ZMM_FMA_TWOPOW1_MODF_LO[rdx], zmm30 ;; 37-40; q1f2_hi = aa_hi * twopowmodf_lo[rdx] + rounding const
	yfmaddpd zmm8, zmm7, ZMMWORD PTR ZMM_FMA_TWOPOW1_MODF_LOa[rdx], zmm30;; 37-40; q1f2_hi = aa_hi * twopowmodf_lo[rdx] + rounding const
	yfmaddpd zmm15, zmm14, ZMMWORD PTR ZMM_FMA_TWOPOW1_MODF_LOb[rdx], zmm30;; 38-41; q1f2_hi = aa_hi * twopowmodf_lo[rdx] + rounding const
	yfmaddpd zmm22, zmm21, ZMMWORD PTR ZMM_FMA_TWOPOW1_MODF_LOc[rdx], zmm30;; 38-41; q1f2_hi = aa_hi * twopowmodf_lo[rdx] + rounding const

;; zmm0=aa_hi, zmm1=q1f2_hi, zmm2=final_hi, zmm3=bb_lo, zmm4=q2f1_lo, zmm5=q1f1_lo, zmm6=q2

	yfmaddpd zmm2, zmm29, zmm2, zmm5			;; 39-42; final_hi = 1-or-2 * final_hi + q1f1_lo
	yfmaddpd zmm9, zmm29, zmm9, zmm12			;; 39-42; final_hi = 1-or-2 * final_hi + q1f1_lo
	yfmaddpd zmm16, zmm29, zmm16, zmm19			;; 40-43; final_hi = 1-or-2 * final_hi + q1f1_lo
	yfmaddpd zmm23, zmm29, zmm23, zmm26			;; 40-43; final_hi = 1-or-2 * final_hi + q1f1_lo

;; zmm0=aa_hi, zmm1=q1f2_hi, zmm2=final_hi, zmm3=bb_lo, zmm4=q2f1_lo, zmm6=q2

	yfnmaddpd zmm5, zmm6, ZMMWORD PTR ZMM_FMA_FAC_LO, zmm30;;*41-44; q2f2_hi = -q2 * fac_lo + rounding const
	yfnmaddpd zmm12, zmm13, ZMMWORD PTR ZMM_FMA_FAC_LOa, zmm30;;*41-44; q2f2_hi = -q2 * fac_lo + rounding const
	yfnmaddpd zmm19, zmm20, ZMMWORD PTR ZMM_FMA_FAC_LOb, zmm30;;*42-45; q2f2_hi = -q2 * fac_lo + rounding const
	yfnmaddpd zmm26, zmm27, ZMMWORD PTR ZMM_FMA_FAC_LOc, zmm30;;*42-45; q2f2_hi = -q2 * fac_lo + rounding const

;; zmm0=aa_hi, zmm1=q1f2_hi, zmm2=final_hi, zmm3=bb_lo, zmm4=q2f1_lo, zmm5=q2f2_hi, zmm6=q2

	vsubpd	zmm1, zmm1, zmm30				;; 43-46; q1f2_hi -= rounding const
	vsubpd	zmm8, zmm8, zmm30				;; 43-46; q1f2_hi -= rounding const
	vsubpd	zmm15, zmm15, zmm30				;; 44-47; q1f2_hi -= rounding const
	vsubpd	zmm22, zmm22, zmm30				;; 44-47; q1f2_hi -= rounding const

 	vaddpd	zmm2, zmm2, zmm4				;;*45-48; final_hi += q2f1_lo
 	vaddpd	zmm9, zmm9, zmm11				;;*45-48; final_hi += q2f1_lo
 	vaddpd	zmm16, zmm16, zmm18				;;*46-49; final_hi += q2f1_lo
 	vaddpd	zmm23, zmm23, zmm25				;;*46-49; final_hi += q2f1_lo

;; zmm0=aa_hi, zmm1=q1f2_hi, zmm2=final_hi, zmm3=bb_lo, zmm5=q2f2_hi, zmm6=q2

	yfmsubpd zmm0, zmm0, ZMMWORD PTR ZMM_FMA_TWOPOW1_MODF_LO[rdx], zmm1;; 47-50; q1f2_lo = aa_hi * twopowmodf_lo[rdx] - q1f2_hi
	yfmsubpd zmm7, zmm7, ZMMWORD PTR ZMM_FMA_TWOPOW1_MODF_LOa[rdx], zmm8;; 47-50; q1f2_lo = aa_hi * twopowmodf_lo[rdx] - q1f2_hi
	yfmsubpd zmm14, zmm14, ZMMWORD PTR ZMM_FMA_TWOPOW1_MODF_LOb[rdx], zmm15;; 48-51; q1f2_lo = aa_hi * twopowmodf_lo[rdx] - q1f2_hi
	yfmsubpd zmm21, zmm21, ZMMWORD PTR ZMM_FMA_TWOPOW1_MODF_LOc[rdx], zmm22;; 48-51; q1f2_lo = aa_hi * twopowmodf_lo[rdx] - q1f2_hi

;; zmm0=q1f2_lo, zmm1=q1f2_hi, zmm2=final_hi, zmm3=bb_lo, zmm5=q2f2_hi, zmm6=q2

	vsubpd	zmm5, zmm5, zmm30				;;*49-52; q2f2_hi -= rounding const
	vsubpd	zmm12, zmm12, zmm30				;;*49-52; q2f2_hi -= rounding const
	vsubpd	zmm19, zmm19, zmm30				;;*50-53; q2f2_hi -= rounding const
	vsubpd	zmm26, zmm26, zmm30				;;*50-53; q2f2_hi -= rounding const

	vaddpd	zmm2, zmm2, zmm1				;; 51-54; final_hi += q1f2_hi
	vaddpd	zmm9, zmm9, zmm8				;; 51-54; final_hi += q1f2_hi
	vaddpd	zmm16, zmm16, zmm15				;; 52-55; final_hi += q1f2_hi
	vaddpd	zmm23, zmm23, zmm22				;; 52-55; final_hi += q1f2_hi

;; zmm0=q1f2_lo, zmm2=final_hi, zmm3=bb_lo, zmm5=q2f2_hi, zmm6=q2

	yfmaddpd zmm3, zmm29, zmm3, zmm0			;; 53-56; final_lo = one_or_two[rdx] * bb_lo + q1f2_lo
	yfmaddpd zmm10, zmm29, zmm10, zmm7			;; 53-56; final_lo = one_or_two[rdx] * bb_lo + q1f2_lo
	yfmaddpd zmm17, zmm29, zmm17, zmm14			;; 54-57; final_lo = one_or_two[rdx] * bb_lo + q1f2_lo
	yfmaddpd zmm24, zmm29, zmm24, zmm21			;; 54-57; final_lo = one_or_two[rdx] * bb_lo + q1f2_lo

;; zmm2=final_hi, zmm3=final_lo, zmm5=q2f2_hi, zmm6=q2

	yfnmsubpd zmm6, zmm6, ZMMWORD PTR ZMM_FMA_FAC_LO, zmm5	;;*55-58; q2f2_lo = -q2 * fac_lo - q2f2_hi
	yfnmsubpd zmm13, zmm13, ZMMWORD PTR ZMM_FMA_FAC_LOa, zmm12;;*55-58; q2f2_lo = -q2 * fac_lo - q2f2_hi
	yfnmsubpd zmm20, zmm20, ZMMWORD PTR ZMM_FMA_FAC_LOb, zmm19;;*56-59; q2f2_lo = -q2 * fac_lo - q2f2_hi
	yfnmsubpd zmm27, zmm27, ZMMWORD PTR ZMM_FMA_FAC_LOc, zmm26;;*56-59; q2f2_lo = -q2 * fac_lo - q2f2_hi

;; zmm2=final_hi, zmm3=final_lo, zmm5=q2f2_hi, zmm6=q2f2_lo

	vaddpd	zmm2, zmm2, zmm5				;;*57-60; final_hi += q2f2_hi
	vaddpd	zmm9, zmm9, zmm12				;;*57-60; final_hi += q2f2_hi
	vaddpd	zmm16, zmm16, zmm19				;;*58-62; final_hi += q2f2_hi
	vaddpd	zmm23, zmm23, zmm26				;;*58-62; final_hi += q2f2_hi

	vaddpd	zmm3, zmm3, zmm6				;;*59-62; final_lo += q2f2_lo
	vaddpd	zmm10, zmm10, zmm13				;;*59-62; final_lo += q2f2_lo
	vaddpd	zmm17, zmm17, zmm20				;;*60-63; final_lo += q2f2_lo
	vaddpd	zmm24, zmm24, zmm27				;;*60-63; final_lo += q2f2_lo
	ENDM

;; Check if remainder in zmm2,zmm3 is one indicating we found a factor!
;; Normalize hi/lo so we can compare to ZMM_ONE

avx512_fma_compare MACRO
	vaddpd	zmm2, zmm2, zmm3				;; Combine fac_hi & fac_lo
	vaddpd	zmm9, zmm9, zmm10
	vaddpd	zmm16, zmm16, zmm17
	vaddpd	zmm23, zmm23, zmm24
	ENDM

