[Gmp-commit] /home/hgfiles/gmp: toomH_sqr: avoid overflow when SQR_FFT_THRESH...

[mercurial at gmplib.org]

details:   /home/hgfiles/gmp/rev/ddbd5167c07f
changeset: 13279:ddbd5167c07f
user:      Marco Bodrato <bodrato at mail.dm.unipi.it>
date:      Thu Dec 31 10:12:16 2009 +0100
description:
toomH_sqr: avoid overflow when SQR_FFT_THRESHOLD == MP_SIZE_T_MAX

diffstat:

 ChangeLog               |   3 +++
 mpn/generic/toom6_sqr.c |  22 ++++++++++++----------
 mpn/generic/toom8_sqr.c |  16 +++++++++-------
 3 files changed, 24 insertions(+), 17 deletions(-)

diffs (107 lines):

diff -r 8cf99f9e0e45 -r ddbd5167c07f ChangeLog
--- a/ChangeLog	Thu Dec 31 08:36:05 2009 +0100
+++ b/ChangeLog	Thu Dec 31 10:12:16 2009 +0100
@@ -1,5 +1,8 @@
 2009-12-31  Marco Bodrato <bodrato at mail.dm.unipi.it>
 
+	* mpn/generic/toom8_sqr.c (SQR_TOOM8_MAX): Avoid overflow.
+	* mpn/generic/toom6_sqr.c (SQR_TOOM6_MAX): Likewise.
+
 	* mpn/generic/mulmod_bnm1.c: Don't mention MISUSE any more,
 	simply consider UNLIKELY any unexpected size.
 
diff -r 8cf99f9e0e45 -r ddbd5167c07f mpn/generic/toom6_sqr.c
--- a/mpn/generic/toom6_sqr.c	Thu Dec 31 08:36:05 2009 +0100
+++ b/mpn/generic/toom6_sqr.c	Thu Dec 31 10:12:16 2009 +0100
@@ -33,12 +33,6 @@
 #endif
 
 
-#ifdef  SQR_TOOM8_THRESHOLD
-#define SQR_TOOM6_MAX (SQR_TOOM8_THRESHOLD+6*2-1)
-#else
-#define SQR_TOOM6_MAX (SQR_FFT_THRESHOLD+6*2-1)
-#endif
-
 #if TUNE_PROGRAM_BUILD
 #define MAYBE_sqr_basecase 1
 #define MAYBE_sqr_above_basecase   1
@@ -48,20 +42,28 @@
 #define MAYBE_sqr_above_toom3   1
 #define MAYBE_sqr_above_toom4   1
 #else
+#ifdef  SQR_TOOM8_THRESHOLD
+#define SQR_TOOM6_MAX ((SQR_TOOM8_THRESHOLD+6*2-1+5)/6)
+#else
+#define SQR_TOOM6_MAX					\
+  ((SQR_FFT_THRESHOLD <= MP_SIZE_T_MAX - (6*2-1+5)) ?	\
+   ((SQR_FFT_THRESHOLD+6*2-1+5)/6)			\
+   : MP_SIZE_T_MAX )
+#endif
 #define MAYBE_sqr_basecase					\
   (SQR_TOOM6_THRESHOLD < 6 * SQR_TOOM2_THRESHOLD)
 #define MAYBE_sqr_above_basecase				\
-  (SQR_TOOM6_MAX >= 6 * SQR_TOOM2_THRESHOLD)
+  (SQR_TOOM6_MAX >=  SQR_TOOM2_THRESHOLD)
 #define MAYBE_sqr_toom2						\
   (SQR_TOOM6_THRESHOLD < 6 * SQR_TOOM3_THRESHOLD)
 #define MAYBE_sqr_above_toom2					\
-  (SQR_TOOM6_MAX >= 6 * SQR_TOOM3_THRESHOLD)
+  (SQR_TOOM6_MAX >= SQR_TOOM3_THRESHOLD)
 #define MAYBE_sqr_toom3						\
   (SQR_TOOM6_THRESHOLD < 6 * SQR_TOOM4_THRESHOLD)
 #define MAYBE_sqr_above_toom3					\
-  (SQR_TOOM6_MAX >= 6 * SQR_TOOM4_THRESHOLD)
+  (SQR_TOOM6_MAX >= SQR_TOOM4_THRESHOLD)
 #define MAYBE_sqr_above_toom4					\
-  (SQR_TOOM6_MAX >= 6 * SQR_TOOM6_THRESHOLD)
+  (SQR_TOOM6_MAX >= SQR_TOOM6_THRESHOLD)
 #endif
 
 #define TOOM6_SQR_REC(p, a, n, ws)					\
diff -r 8cf99f9e0e45 -r ddbd5167c07f mpn/generic/toom8_sqr.c
--- a/mpn/generic/toom8_sqr.c	Thu Dec 31 08:36:05 2009 +0100
+++ b/mpn/generic/toom8_sqr.c	Thu Dec 31 10:12:16 2009 +0100
@@ -47,8 +47,6 @@
 #define SQR_TOOM6_THRESHOLD MUL_TOOM6H_THRESHOLD
 #endif
 
-#define SQR_TOOM8_MAX (SQR_FFT_THRESHOLD+8*2-1)
-
 #if TUNE_PROGRAM_BUILD
 #define MAYBE_sqr_basecase 1
 #define MAYBE_sqr_above_basecase   1
@@ -60,24 +58,28 @@
 #define MAYBE_sqr_above_toom4   1
 #define MAYBE_sqr_above_toom6   1
 #else
+#define SQR_TOOM8_MAX					\
+  ((SQR_FFT_THRESHOLD <= MP_SIZE_T_MAX - (8*2-1+7)) ?	\
+   ((SQR_FFT_THRESHOLD+8*2-1+7)/8)			\
+   : MP_SIZE_T_MAX )
 #define MAYBE_sqr_basecase					\
   (SQR_TOOM8_THRESHOLD < 8 * SQR_TOOM2_THRESHOLD)
 #define MAYBE_sqr_above_basecase				\
-  (SQR_TOOM8_MAX >= 8 * SQR_TOOM2_THRESHOLD)
+  (SQR_TOOM8_MAX >= SQR_TOOM2_THRESHOLD)
 #define MAYBE_sqr_toom2						\
   (SQR_TOOM8_THRESHOLD < 8 * SQR_TOOM3_THRESHOLD)
 #define MAYBE_sqr_above_toom2					\
-  (SQR_TOOM8_MAX >= 8 * SQR_TOOM3_THRESHOLD)
+  (SQR_TOOM8_MAX >= SQR_TOOM3_THRESHOLD)
 #define MAYBE_sqr_toom3						\
   (SQR_TOOM8_THRESHOLD < 8 * SQR_TOOM4_THRESHOLD)
 #define MAYBE_sqr_above_toom3					\
-  (SQR_TOOM8_MAX >= 8 * SQR_TOOM4_THRESHOLD)
+  (SQR_TOOM8_MAX >= SQR_TOOM4_THRESHOLD)
 #define MAYBE_sqr_toom4						\
   (SQR_TOOM8_THRESHOLD < 8 * SQR_TOOM6_THRESHOLD)
 #define MAYBE_sqr_above_toom4					\
-  (SQR_TOOM8_MAX >= 8 * SQR_TOOM6_THRESHOLD)
+  (SQR_TOOM8_MAX >= SQR_TOOM6_THRESHOLD)
 #define MAYBE_sqr_above_toom6					\
-  (SQR_TOOM8_MAX >= 8 * SQR_TOOM8_THRESHOLD)
+  (SQR_TOOM8_MAX >= SQR_TOOM8_THRESHOLD)
 #endif
 
 #define TOOM8_SQR_REC(p, a, n, ws)					\