[turbofan] Refactor the CheckedInt32Div/CheckedUint32Div lowering.

Improve the lowering of CheckedInt32Div and CheckedUint32Div for the
case that the right hand side is a known (positive) power of two, as
in that case it's sufficient to just check the relevant bits on the
left hand side and then shift by the appropriate amount of bits.

This is significantly faster than what TurboFan is able to generate
from the general lowering, even with all the MachineOperatorReducer
magic (it even shows as a steady ~1.5% overall improvement on the
Kraken crypto ccm benchmark).

Also turn the general CheckedInt32Div lowering into readable code again,
and make sure that all the bailout cases are properly covered by mjsunit
tests (i.e. the "division by zero" bailout was not covered properly).

Bug: v8:8015
Change-Id: Ibfdd367a6ee5d70dcaa48801858042c5029b7004
Reviewed-on: https://chromium-review.googlesource.com/1236954
Reviewed-by: Sigurd Schneider <sigurds@chromium.org>
Commit-Queue: Benedikt Meurer <bmeurer@chromium.org>
Cr-Commit-Position: refs/heads/master@{#56115}

src/compiler/effect-control-linearizer.cc

@@ -1688,63 +1688,87 @@ Node* EffectControlLinearizer::LowerCheckedInt32Div(Node* node,
                                                     Node* frame_state) {
   Node* lhs = node->InputAt(0);
   Node* rhs = node->InputAt(1);
-
-  auto if_not_positive = __ MakeDeferredLabel();
-  auto if_is_minint = __ MakeDeferredLabel();
-  auto done = __ MakeLabel(MachineRepresentation::kWord32);
-  auto minint_check_done = __ MakeLabel();
-
   Node* zero = __ Int32Constant(0);
 
-  // Check if {rhs} is positive (and not zero).
+  // Check if the {rhs} is a known power of two.
-  Node* check0 = __ Int32LessThan(zero, rhs);
+  Int32Matcher m(rhs);
-  __ GotoIfNot(check0, &if_not_positive);
+  if (m.IsPowerOf2()) {
+    // Since we know that {rhs} is a power of two, we can perform a fast
+    // check to see if the relevant least significant bits of the {lhs}
+    // are all zero, and if so we know that we can perform a division
+    // safely (and fast by doing an arithmetic - aka sign preserving -
+    // right shift on {lhs}).
+    int32_t divisor = m.Value();
+    Node* mask = __ Int32Constant(divisor - 1);
+    Node* shift = __ Int32Constant(WhichPowerOf2(divisor));
+    Node* check = __ Word32Equal(__ Word32And(lhs, mask), zero);
+    __ DeoptimizeIfNot(DeoptimizeReason::kLostPrecision, VectorSlotPair(),
+                       check, frame_state);
+    return __ Word32Sar(lhs, shift);
+  } else {
+    auto if_rhs_positive = __ MakeLabel();
+    auto if_rhs_negative = __ MakeDeferredLabel();
+    auto done = __ MakeLabel(MachineRepresentation::kWord32);
 
-  // Fast case, no additional checking required.
+    // Check if {rhs} is positive (and not zero).
-  __ Goto(&done, __ Int32Div(lhs, rhs));
+    Node* check_rhs_positive = __ Int32LessThan(zero, rhs);
+    __ Branch(check_rhs_positive, &if_rhs_positive, &if_rhs_negative);
 
-  {
+    __ Bind(&if_rhs_positive);
-    __ Bind(&if_not_positive);
+    {
+      // Fast case, no additional checking required.
+      __ Goto(&done, __ Int32Div(lhs, rhs));
+    }
 
-    // Check if {rhs} is zero.
+    __ Bind(&if_rhs_negative);
-    Node* check = __ Word32Equal(rhs, zero);
+    {
-    __ DeoptimizeIf(DeoptimizeReason::kDivisionByZero, VectorSlotPair(), check,
+      auto if_lhs_minint = __ MakeDeferredLabel();
-                    frame_state);
+      auto if_lhs_notminint = __ MakeLabel();
 
-    // Check if {lhs} is zero, as that would produce minus zero.
+      // Check if {rhs} is zero.
-    check = __ Word32Equal(lhs, zero);
+      Node* check_rhs_zero = __ Word32Equal(rhs, zero);
-    __ DeoptimizeIf(DeoptimizeReason::kMinusZero, VectorSlotPair(), check,
+      __ DeoptimizeIf(DeoptimizeReason::kDivisionByZero, VectorSlotPair(),
-                    frame_state);
+                      check_rhs_zero, frame_state);
 
-    // Check if {lhs} is kMinInt and {rhs} is -1, in which case we'd have
+      // Check if {lhs} is zero, as that would produce minus zero.
-    // to return -kMinInt, which is not representable.
+      Node* check_lhs_zero = __ Word32Equal(lhs, zero);
-    Node* minint = __ Int32Constant(std::numeric_limits<int32_t>::min());
+      __ DeoptimizeIf(DeoptimizeReason::kMinusZero, VectorSlotPair(),
-    Node* check1 = graph()->NewNode(machine()->Word32Equal(), lhs, minint);
+                      check_lhs_zero, frame_state);
-    __ GotoIf(check1, &if_is_minint);
-    __ Goto(&minint_check_done);
 
-    __ Bind(&if_is_minint);
+      // Check if {lhs} is kMinInt and {rhs} is -1, in which case we'd have
-    // Check if {rhs} is -1.
+      // to return -kMinInt, which is not representable as Word32.
-    Node* minusone = __ Int32Constant(-1);
+      Node* check_lhs_minint = graph()->NewNode(machine()->Word32Equal(), lhs,
-    Node* is_minus_one = __ Word32Equal(rhs, minusone);
+                                                __ Int32Constant(kMinInt));
-    __ DeoptimizeIf(DeoptimizeReason::kOverflow, VectorSlotPair(), is_minus_one,
+      __ Branch(check_lhs_minint, &if_lhs_minint, &if_lhs_notminint);
-                    frame_state);
-    __ Goto(&minint_check_done);
 
-    __ Bind(&minint_check_done);
+      __ Bind(&if_lhs_minint);
-    // Perform the actual integer division.
+      {
-    __ Goto(&done, __ Int32Div(lhs, rhs));
+        // Check that {rhs} is not -1, otherwise result would be -kMinInt.
+        Node* check_rhs_minusone = __ Word32Equal(rhs, __ Int32Constant(-1));
+        __ DeoptimizeIf(DeoptimizeReason::kOverflow, VectorSlotPair(),
+                        check_rhs_minusone, frame_state);
+
+        // Perform the actual integer division.
+        __ Goto(&done, __ Int32Div(lhs, rhs));
+      }
+
+      __ Bind(&if_lhs_notminint);
+      {
+        // Perform the actual integer division.
+        __ Goto(&done, __ Int32Div(lhs, rhs));
+      }
+    }
+
+    __ Bind(&done);
+    Node* value = done.PhiAt(0);
+
+    // Check if the remainder is non-zero.
+    Node* check = __ Word32Equal(lhs, __ Int32Mul(value, rhs));
+    __ DeoptimizeIfNot(DeoptimizeReason::kLostPrecision, VectorSlotPair(),
+                       check, frame_state);
+
+    return value;
   }
-
-  __ Bind(&done);
-  Node* value = done.PhiAt(0);
-
-  // Check if the remainder is non-zero.
-  Node* check = __ Word32Equal(lhs, __ Int32Mul(rhs, value));
-  __ DeoptimizeIfNot(DeoptimizeReason::kLostPrecision, VectorSlotPair(), check,
-                     frame_state);
-
-  return value;
 }
 
 Node* EffectControlLinearizer::BuildUint32Mod(Node* lhs, Node* rhs) {
@@ -1855,22 +1879,38 @@ Node* EffectControlLinearizer::LowerCheckedUint32Div(Node* node,
                                                      Node* frame_state) {
   Node* lhs = node->InputAt(0);
   Node* rhs = node->InputAt(1);
-
   Node* zero = __ Int32Constant(0);
 
-  // Ensure that {rhs} is not zero, otherwise we'd have to return NaN.
+  // Check if the {rhs} is a known power of two.
-  Node* check = __ Word32Equal(rhs, zero);
+  Uint32Matcher m(rhs);
-  __ DeoptimizeIf(DeoptimizeReason::kDivisionByZero, VectorSlotPair(), check,
+  if (m.IsPowerOf2()) {
-                  frame_state);
+    // Since we know that {rhs} is a power of two, we can perform a fast
+    // check to see if the relevant least significant bits of the {lhs}
+    // are all zero, and if so we know that we can perform a division
+    // safely (and fast by doing a logical - aka zero extending - right
+    // shift on {lhs}).
+    uint32_t divisor = m.Value();
+    Node* mask = __ Uint32Constant(divisor - 1);
+    Node* shift = __ Uint32Constant(WhichPowerOf2(divisor));
+    Node* check = __ Word32Equal(__ Word32And(lhs, mask), zero);
+    __ DeoptimizeIfNot(DeoptimizeReason::kLostPrecision, VectorSlotPair(),
+                       check, frame_state);
+    return __ Word32Shr(lhs, shift);
+  } else {
+    // Ensure that {rhs} is not zero, otherwise we'd have to return NaN.
+    Node* check = __ Word32Equal(rhs, zero);
+    __ DeoptimizeIf(DeoptimizeReason::kDivisionByZero, VectorSlotPair(), check,
+                    frame_state);
 
-  // Perform the actual unsigned integer division.
+    // Perform the actual unsigned integer division.
-  Node* value = __ Uint32Div(lhs, rhs);
+    Node* value = __ Uint32Div(lhs, rhs);
 
-  // Check if the remainder is non-zero.
+    // Check if the remainder is non-zero.
-  check = __ Word32Equal(lhs, __ Int32Mul(rhs, value));
+    check = __ Word32Equal(lhs, __ Int32Mul(rhs, value));
-  __ DeoptimizeIfNot(DeoptimizeReason::kLostPrecision, VectorSlotPair(), check,
+    __ DeoptimizeIfNot(DeoptimizeReason::kLostPrecision, VectorSlotPair(),
-                     frame_state);
+                       check, frame_state);
-  return value;
+    return value;
+  }
 }
 
 Node* EffectControlLinearizer::LowerCheckedUint32Mod(Node* node,

test/mjsunit/compiler/number-divide.js

@@ -2,7 +2,7 @@
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
-// Flags: --allow-natives-syntax --opt
+// Flags: --allow-natives-syntax --opt --noalways-opt
 
 // Test that NumberDivide with Number feedback works if only in the
 // end SimplifiedLowering figures out that the inputs to this operation
@@ -61,3 +61,147 @@
   assertEquals(2, foo(4));
   assertOptimized(foo);
 })();
+
+// Test that SpeculativeNumberDivide turns into CheckedInt32Div, and
+// that the "known power of two divisor" optimization works correctly.
+(function() {
+  function foo(x) { return (x | 0) / 2; }
+
+  // Warmup with proper int32 divisions.
+  assertEquals(1, foo(2));
+  assertEquals(2, foo(4));
+  %OptimizeFunctionOnNextCall(foo);
+  assertEquals(3, foo(6));
+  assertOptimized(foo);
+
+  // Make optimized code fail.
+  assertEquals(0.5, foo(1));
+  assertUnoptimized(foo);
+
+  // Try again with the new feedback, and now it should stay optimized.
+  %OptimizeFunctionOnNextCall(foo);
+  assertEquals(4, foo(8));
+  assertOptimized(foo);
+  assertEquals(0.5, foo(1));
+  assertOptimized(foo);
+})();
+
+// Test that SpeculativeNumberDivide turns into CheckedInt32Div, and
+// that the optimized code properly bails out on "division by zero".
+(function() {
+  function foo(x, y) { return x / y; }
+
+  // Warmup with proper int32 divisions.
+  assertEquals(2, foo(4, 2));
+  assertEquals(2, foo(8, 4));
+  %OptimizeFunctionOnNextCall(foo);
+  assertEquals(1, foo(2, 2));
+  assertOptimized(foo);
+
+  // Make optimized code fail.
+  assertEquals(Infinity, foo(1, 0));
+  assertUnoptimized(foo);
+
+  // Try again with the new feedback, and now it should stay optimized.
+  %OptimizeFunctionOnNextCall(foo);
+  assertEquals(2, foo(2, 1));
+  assertOptimized(foo);
+  assertEquals(Infinity, foo(1, 0));
+  assertOptimized(foo);
+})();
+
+// Test that SpeculativeNumberDivide turns into CheckedInt32Div, and
+// that the optimized code properly bails out on minus zero.
+(function() {
+  function foo(x, y) { return x / y; }
+
+  // Warmup with proper int32 divisions.
+  assertEquals(2, foo(4, 2));
+  assertEquals(2, foo(8, 4));
+  %OptimizeFunctionOnNextCall(foo);
+  assertEquals(1, foo(2, 2));
+  assertOptimized(foo);
+
+  // Make optimized code fail.
+  assertEquals(-0, foo(0, -1));
+  assertUnoptimized(foo);
+
+  // Try again with the new feedback, and now it should stay optimized.
+  %OptimizeFunctionOnNextCall(foo);
+  assertEquals(2, foo(2, 1));
+  assertOptimized(foo);
+  assertEquals(-0, foo(0, -1));
+  assertOptimized(foo);
+})();
+
+// Test that SpeculativeNumberDivide turns into CheckedInt32Div, and
+// that the optimized code properly bails out if result is -kMinInt.
+(function() {
+  function foo(x, y) { return x / y; }
+
+  // Warmup with proper int32 divisions.
+  assertEquals(2, foo(4, 2));
+  assertEquals(2, foo(8, 4));
+  %OptimizeFunctionOnNextCall(foo);
+  assertEquals(1, foo(2, 2));
+  assertOptimized(foo);
+
+  // Make optimized code fail.
+  assertEquals(2147483648, foo(-2147483648, -1));
+  assertUnoptimized(foo);
+
+  // Try again with the new feedback, and now it should stay optimized.
+  %OptimizeFunctionOnNextCall(foo);
+  assertEquals(2, foo(2, 1));
+  assertOptimized(foo);
+  assertEquals(2147483648, foo(-2147483648, -1));
+  assertOptimized(foo);
+})();
+
+// Test that SpeculativeNumberDivide turns into CheckedUint32Div, and
+// that the "known power of two divisor" optimization works correctly.
+(function() {
+  function foo(s) { return s.length / 2; }
+
+  // Warmup with proper uint32 divisions.
+  assertEquals(1, foo("ab".repeat(1)));
+  assertEquals(2, foo("ab".repeat(2)));
+  %OptimizeFunctionOnNextCall(foo);
+  assertEquals(3, foo("ab".repeat(3)));
+  assertOptimized(foo);
+
+  // Make optimized code fail.
+  assertEquals(0.5, foo("a"));
+  assertUnoptimized(foo);
+
+  // Try again with the new feedback, and now it should stay optimized.
+  %OptimizeFunctionOnNextCall(foo);
+  assertEquals(4, foo("ab".repeat(4)));
+  assertOptimized(foo);
+  assertEquals(0.5, foo("a"));
+  assertOptimized(foo);
+})();
+
+// Test that SpeculativeNumberDivide turns into CheckedUint32Div, and
+// that the optimized code properly bails out on "division by zero".
+(function() {
+  function foo(x, y) { return (x >>> 0) / (y >>> 0); }
+
+  // Warmup with proper uint32 divisions.
+  assertEquals(2, foo(4, 2));
+  assertEquals(2, foo(8, 4));
+  %OptimizeFunctionOnNextCall(foo);
+  assertEquals(1, foo(2, 2));
+  assertOptimized(foo);
+
+  // Make optimized code fail.
+  assertEquals(Infinity, foo(1, 0));
+  assertUnoptimized(foo);
+
+  // Try again with the new feedback, and now it should stay optimized.
+  %OptimizeFunctionOnNextCall(foo);
+  assertEquals(2, foo(2, 1));
+  assertOptimized(foo);
+  assertEquals(Infinity, foo(1, 0));
+  assertOptimized(foo);
+})();