mpf_sub equal and nearly equal operands

[Kevin Ryde]

A bug has been found in mpf_sub affecting results for subtraction of
equal and nearly equal operands.

For equal operands the result is zero, but stored in an improper form,
one which certain subsequent operations will not recognise.  For
nearly equal operands where many high limbs cancel, the result may
have less precision than requested and can even come out as zero.

The patch below to mpf/sub.c for gmp 4.1.3 addresses both problems.
The two test programs below can be used to check whether you're
affected.  (The zero problem unfortuately is believed to extend back
to the first mpf in gmp 2.0, and the cancellation problem back to
gmp 3.0.)

(All followups to the bugs or discuss list, as appropriate, please.)

-------------- next part --------------
Index: sub.c
===================================================================
RCS file: /home/cvsfiles/gmp/mpf/sub.c,v
retrieving revision 1.11
diff -u -r1.11 sub.c
--- sub.c	14 May 2002 16:59:48 -0000	1.11
+++ sub.c	19 Jul 2004 23:33:54 -0000
@@ -105,26 +105,29 @@
 
 		  if (usize == 0)
 		    {
+                      /* u cancels high limbs of v, result is rest of v */
+		      negate ^= 1;
+                    cancellation:
+                      /* strip high zeros before truncating to prec */
+                      while (vsize != 0 && vp[vsize - 1] == 0)
+                        {
+                          vsize--;
+                          exp--;
+                        }
 		      if (vsize > prec)
 			{
 			  vp += vsize - prec;
 			  vsize = prec;
 			}
-		      rsize = vsize;
-		      tp = (mp_ptr) vp;
-		      negate ^= 1;
-		      goto normalize;
+                      MPN_COPY_INCR (rp, vp, vsize);
+                      rsize = vsize;
+                      goto done;
 		    }
 		  if (vsize == 0)
 		    {
-		      if (usize > prec)
-			{
-			  up += usize - prec;
-			  usize = prec;
-			}
-		      rsize = usize;
-		      tp = (mp_ptr) up;
-		      goto normalize;
+                      vp = up;
+                      vsize = usize;
+                      goto cancellation;
 		    }
 		}
 	      while (up[usize - 1] == vp[vsize - 1]);
@@ -401,6 +404,8 @@
 
  done:
   r->_mp_size = negate ? -rsize : rsize;
+  if (rsize == 0)
+    exp = 0;
   r->_mp_exp = exp;
   TMP_FREE (marker);
 }
-------------- next part --------------
#include "gmp.h"

int
main (void)
{
  mpf_t  f;
  mpf_init (f);
  mpf_set_ui (f, 1L);
  mpf_mul_2exp (f, f, 128L);  /* x=2^128 */
  mpf_sub (f, f, f);
  if (mpf_cmp_ui (f, 1L) < 0)
    printf ("ok\n");
  else
    printf ("bad, don't have x-x < 1\n");
  return 0;
}
-------------- next part --------------
#include "gmp.h"

int
main (void)
{
  mpf_t  w, x, y;
  mpf_init (w);
  mpf_init2 (x, 1024L);
  mpf_init2 (y, 1024L);
  mpf_set_ui (x, 1L);
  mpf_mul_2exp (x, x, 512L);    /* x = 2^512 */
  mpf_add_ui (y, x, 1L);        /* y = x+1 */
  mpf_sub (w, y, x);
  if (mpf_cmp_ui (w, 1L) == 0)
    printf ("ok\n");
  else
    printf ("bad, don't have (x+1)-x == 1\n");
  return 0;
}