Fixing oob violations in detail::multiply and detail::subtract_nounderflow.

src/biginteger.cpp

@@ -138,17 +138,17 @@ biginteger math::operator%(biginteger rhs, biginteger const & lhs) {
   else {
     auto cmp = detail::compare(rhs.data, lhs.data);
     if (cmp == 0) { return biginteger::ZERO; }
-    else {
+    else if (cmp > 0) {
       detail::divide(rhs.data, lhs.data);
       if (detail::compare(rhs.data, biginteger::ZERO.data) == 0) { return biginteger::ZERO; }
-      else if (rhs.is_negative != lhs.is_negative) {
-        rhs.is_negative = lhs.is_negative;
-        auto tmp = lhs.data;
-        swap(rhs.data, tmp);
-        detail::subtract_nounderflow(rhs.data, tmp);
-      }
-      return rhs;
     }
+    if (rhs.is_negative != lhs.is_negative) {
+      rhs.is_negative = lhs.is_negative;
+      auto tmp = lhs.data;
+      swap(rhs.data, tmp);
+      detail::subtract_nounderflow(rhs.data, tmp);
+    }
+    return rhs;
   }
 }
 

src/bignum_helper.cpp

@@ -63,7 +63,7 @@ namespace math { namespace detail {
       rhs[i+offset] -= lhs[i];
     }
     // Borrow
-    for (size_t i = 0; i < rbnd; ++i) {
+    for (size_t i = 0; i < rbnd-1; ++i) {
       if (rhs[i] < 0) {
         rhs[i] += OVER_SEG;
         rhs[i+1] -= 1;
@@ -79,12 +79,12 @@ namespace math { namespace detail {
     // Multiply
     for (size_t i = rbnd; i > 0; --i) {
       int32_t const value = rhs[i-1];
-      for (size_t j = lbnd; j > offset; --j) {
+      for (size_t j = lbnd; j > 0; --j) {
         // Max input              999,999,999
         // Max output 999,999,998,000,000,001
         int64_t product = int64_t(value) * lhs[j-1];
         int64_t overflow = product / OVER_SEG;
-        size_t const to = i+j-2-offset;
+        size_t const to = i+j-2;
         rhs[to] += int32_t(product - (overflow * OVER_SEG));
         rhs[to+1] += int32_t(overflow);
       }
@@ -99,6 +99,7 @@ namespace math { namespace detail {
       }
     }
     while (rhs.back() == 0) { rhs.pop_back(); }
+    rhs.erase(rhs.begin(), rhs.begin() + ptrdiff_t(offset));
   }
     
   data_type shift10(data_type const & data, int32_t places) {