Bug in fixed-point division.

data61 · Apr 2, 2022 · 565c364 · 565c364
1 parent 07292ec
commit 565c364
Showing 1 changed file with 7 additions and 6 deletions.
diff --git a/Compiler/library.py b/Compiler/library.py
@@ -1876,19 +1876,20 @@ def FPDiv(a, b, k, f, kappa, simplex_flag=False, nearest=False):
     x = alpha - b.extend(2 * k) * w
     base.reset_global_vector_size()
 
-    y = a.extend(2 *k) * w
-    y = y.round(2*k, f, kappa, nearest, signed=True)
+    l_y = k + 3 * f - res_f
+    y = a.extend(l_y) * w
+    y = y.round(l_y, f, kappa, nearest, signed=True)
 
     for i in range(theta - 1):
         x = x.extend(2 * k)
-        y = y.extend(2 * k) * (alpha + x).extend(2 * k)
+        y = y.extend(l_y) * (alpha + x).extend(l_y)
         x = x * x
-        y = y.round(2*k, 2*f, kappa, nearest, signed=True)
+        y = y.round(l_y, 2*f, kappa, nearest, signed=True)
         x = x.round(2*k, 2*f, kappa, nearest, signed=True)
 
     x = x.extend(2 * k)
-    y = y.extend(2 * k) * (alpha + x).extend(2 * k)
-    y = y.round(k + 3 * f - res_f, 3 * f - res_f, kappa, nearest, signed=True)
+    y = y.extend(l_y) * (alpha + x).extend(l_y)
+    y = y.round(l_y, 3 * f - res_f, kappa, nearest, signed=True)
     return y
 
 def AppRcr(b, k, f, kappa=None, simplex_flag=False, nearest=False):