diff --git a/compiler/src/jdk.graal.compiler.test/src/jdk/graal/compiler/core/test/CountedLoopMaxTripCountPiTest.java b/compiler/src/jdk.graal.compiler.test/src/jdk/graal/compiler/core/test/CountedLoopMaxTripCountPiTest.java index 8abd83e073f7..bca2735773df 100644 --- a/compiler/src/jdk.graal.compiler.test/src/jdk/graal/compiler/core/test/CountedLoopMaxTripCountPiTest.java +++ b/compiler/src/jdk.graal.compiler.test/src/jdk/graal/compiler/core/test/CountedLoopMaxTripCountPiTest.java @@ -54,6 +54,7 @@ int ascendingSnippet(int start, int limit) { for (int i = start; GraalDirectives.injectIterationCount(101, i < limit); i++) { GraalDirectives.sideEffect(i); sum += i; + GraalDirectives.neverStripMine(); } return sum + maxTripCount; } @@ -64,6 +65,7 @@ int descendingSnippet(int start, int limit) { for (int i = start; GraalDirectives.injectIterationCount(102, i >= limit); i--) { GraalDirectives.sideEffect(i); sum += i; + GraalDirectives.neverStripMine(); } return sum + maxTripCount; } @@ -76,9 +78,12 @@ int descendingSnippet(int start, int limit) { void checkGraph(StructuredGraph graph, LoopsData loops) { loops.detectCountedLoops(); for (Loop loop : loops.loops()) { + if (loop.loopBegin().isStripMinedOuter()) { + continue; + } // max trip count can only be used if a loop does not overflow - loop.counted().createOverFlowGuard(); Assert.assertTrue("expect all loops to be counted", loop.isCounted()); + loop.counted().createOverFlowGuard(); ValueNode maxTripCountNode = loop.counted().maxTripCountNode(); Assert.assertTrue("expect a PiNode for the guarded maxTripCount, got: " + maxTripCountNode, maxTripCountNode instanceof PiNode); } diff --git a/compiler/src/jdk.graal.compiler.test/src/jdk/graal/compiler/core/test/LoopSafepointStateVerificationTest.java b/compiler/src/jdk.graal.compiler.test/src/jdk/graal/compiler/core/test/LoopSafepointStateVerificationTest.java index 92f09a070269..e4daf7b9984d 100644 --- a/compiler/src/jdk.graal.compiler.test/src/jdk/graal/compiler/core/test/LoopSafepointStateVerificationTest.java +++ b/compiler/src/jdk.graal.compiler.test/src/jdk/graal/compiler/core/test/LoopSafepointStateVerificationTest.java @@ -99,8 +99,8 @@ public Optional notApplicableTo(GraphState graphState) { @Override protected void run(StructuredGraph graph, HighTierContext context) { for (LoopBeginNode lb : graph.getNodes(LoopBeginNode.TYPE)) { - lb.disableSafepoint(SafepointState.MUST_NEVER_SAFEPOINT); - lb.disableGuestSafepoint(SafepointState.MUST_NEVER_SAFEPOINT); + lb.setLoopEndSafepoint(SafepointState.MUST_NEVER_SAFEPOINT); + lb.setGuestSafepoint(SafepointState.MUST_NEVER_SAFEPOINT); } } diff --git a/compiler/src/jdk.graal.compiler.test/src/jdk/graal/compiler/core/test/deopt/MonitorDeoptTest.java b/compiler/src/jdk.graal.compiler.test/src/jdk/graal/compiler/core/test/deopt/MonitorDeoptTest.java index 6da74c71b219..31c03bb6e509 100644 --- a/compiler/src/jdk.graal.compiler.test/src/jdk/graal/compiler/core/test/deopt/MonitorDeoptTest.java +++ b/compiler/src/jdk.graal.compiler.test/src/jdk/graal/compiler/core/test/deopt/MonitorDeoptTest.java @@ -181,7 +181,7 @@ private static LoopBeginNode findFirstLoop(StructuredGraph graph) { */ private static void removeLoopSafepoint(StructuredGraph graph) { LoopBeginNode loopBegin = findFirstLoop(graph); - loopBegin.disableSafepoint(SafepointState.MUST_NEVER_SAFEPOINT); + loopBegin.setLoopEndSafepoint(SafepointState.MUST_NEVER_SAFEPOINT); } @Test diff --git a/compiler/src/jdk.graal.compiler.test/src/jdk/graal/compiler/loop/test/LoopFragmentTest.java b/compiler/src/jdk.graal.compiler.test/src/jdk/graal/compiler/loop/test/LoopFragmentTest.java index 4df2607fa73d..535e1e02c194 100644 --- a/compiler/src/jdk.graal.compiler.test/src/jdk/graal/compiler/loop/test/LoopFragmentTest.java +++ b/compiler/src/jdk.graal.compiler.test/src/jdk/graal/compiler/loop/test/LoopFragmentTest.java @@ -59,12 +59,17 @@ protected void checkMidTierGraph(StructuredGraph graph) { } NodeIterable loops = graph.getNodes().filter(LoopBeginNode.class); // Loops might be optimizable after partial unrolling - if (!loops.isEmpty()) { - for (LoopBeginNode loop : loops) { - if (loop.isMainLoop()) { - return; - } + boolean seenLoop = false; + for (LoopBeginNode loop : loops) { + if (loop.isStripMinedOuter()) { + continue; } + seenLoop = true; + if (loop.isMainLoop()) { + return; + } + } + if (seenLoop) { fail("expected a main loop"); } } diff --git a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/java/BytecodeParser.java b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/java/BytecodeParser.java index 5bcfe32a4d77..bdca025d0b15 100644 --- a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/java/BytecodeParser.java +++ b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/java/BytecodeParser.java @@ -3910,9 +3910,9 @@ private LoopBeginNode appendLoopBegin(FixedWithNextNode fixedWithNext, int start EndNode preLoopEnd = graph.add(new EndNode()); LoopBeginNode loopBegin = graph.add(new LoopBeginNode()); if (disableLoopSafepoint()) { - loopBegin.disableSafepoint(SafepointState.MUST_NEVER_SAFEPOINT); - loopBegin.disableGuestSafepoint(SafepointState.MUST_NEVER_SAFEPOINT); - loopBegin.disableLoopExitSafepoint(SafepointState.MUST_NEVER_SAFEPOINT); + loopBegin.setLoopEndSafepoint(SafepointState.MUST_NEVER_SAFEPOINT); + loopBegin.setGuestSafepoint(SafepointState.MUST_NEVER_SAFEPOINT); + loopBegin.setLoopExitSafepoint(SafepointState.MUST_NEVER_SAFEPOINT); } fixedWithNext.setNext(preLoopEnd); // Add the single non-loop predecessor of the loop header. diff --git a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/loop/phases/LoopPartialUnrollPhase.java b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/loop/phases/LoopPartialUnrollPhase.java index 3b6e51fb6179..0db3762b9e9e 100644 --- a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/loop/phases/LoopPartialUnrollPhase.java +++ b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/loop/phases/LoopPartialUnrollPhase.java @@ -31,6 +31,8 @@ import jdk.graal.compiler.debug.DebugContext; import jdk.graal.compiler.graph.Graph; +import jdk.graal.compiler.graph.NodeBitMap; +import jdk.graal.compiler.loop.phases.LoopTransformations.PreMainPostResult; import jdk.graal.compiler.nodes.GraphState; import jdk.graal.compiler.nodes.GraphState.StageFlag; import jdk.graal.compiler.nodes.LoopBeginNode; @@ -56,7 +58,8 @@ private void unroll(StructuredGraph graph, CoreProviders context) { EconomicSetNodeEventListener listener = new EconomicSetNodeEventListener(); boolean changed = true; EconomicMap opaqueUnrolledStrides = null; - boolean prePostInserted = false; + NodeBitMap newMainLoops = null; + while (changed) { changed = false; try (Graph.NodeEventScope nes = graph.trackNodeEvents(listener)) { @@ -72,10 +75,13 @@ private void unroll(StructuredGraph graph, CoreProviders context) { // First perform the pre/post transformation and do the partial // unroll when we come around again. LoopUtility.preserveCounterStampsForDivAfterUnroll(loop); - LoopTransformations.insertPrePostLoops(loop); - prePostInserted = true; + PreMainPostResult res = LoopTransformations.insertPrePostLoops(loop); + if (newMainLoops == null) { + newMainLoops = graph.createNodeBitMap(); + } + newMainLoops.markAndGrow(res.getMainLoop()); changed = true; - } else if (prePostInserted) { + } else if (newMainLoops != null && newMainLoops.isMarkedAndGrow(loop.loopBegin())) { if (opaqueUnrolledStrides == null) { opaqueUnrolledStrides = EconomicMap.create(Equivalence.IDENTITY); } @@ -94,7 +100,7 @@ private void unroll(StructuredGraph graph, CoreProviders context) { listener.getNodes().clear(); } - assert !prePostInserted || checkCounted(graph, context.getLoopsDataProvider(), mark); + assert newMainLoops == null || checkCounted(graph, context.getLoopsDataProvider(), mark); } } if (opaqueUnrolledStrides != null) { diff --git a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/loop/phases/LoopSafepointEliminationPhase.java b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/loop/phases/LoopSafepointEliminationPhase.java index 2011d32d2479..c1b5c01309ec 100644 --- a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/loop/phases/LoopSafepointEliminationPhase.java +++ b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/loop/phases/LoopSafepointEliminationPhase.java @@ -26,12 +26,15 @@ import java.util.Optional; +import org.graalvm.collections.EconomicMap; + import jdk.graal.compiler.core.common.NumUtil; import jdk.graal.compiler.core.common.type.IntegerStamp; import jdk.graal.compiler.core.common.type.Stamp; import jdk.graal.compiler.nodes.FixedNode; import jdk.graal.compiler.nodes.GraphState; import jdk.graal.compiler.nodes.Invoke; +import jdk.graal.compiler.nodes.LoopBeginNode; import jdk.graal.compiler.nodes.LoopBeginNode.SafepointState; import jdk.graal.compiler.nodes.LoopEndNode; import jdk.graal.compiler.nodes.NodeView; @@ -58,12 +61,138 @@ public static class Options { //@formatter:on } - private static final long IntegerRangeDistance = NumUtil.unsafeAbs((long) Integer.MAX_VALUE - (long) Integer.MIN_VALUE); + /** + * Models SafepointState transforms that can be applied to all loops in a graph. + */ + public static class LoopSafepointPlan { + private final EconomicMap loopStates; - public static boolean iterationRangeIsIn32Bit(Loop loop) { - if (loop.counted().getStamp().getBits() <= 32) { - return true; + public LoopSafepointPlan(StructuredGraph g) { + loopStates = EconomicMap.create(); + for (LoopBeginNode lb : g.getNodes(LoopBeginNode.TYPE)) { + loopStates.put(lb, new SafepointStateEffect(lb)); + } + } + + /** + * Apply all safepoint effects on all loops, that is, enable/disable loop safepoint on loop + * ends and exits. + */ + public void apply() { + for (SafepointStateEffect loopPlan : loopStates.getValues()) { + loopPlan.apply(); + } + } + + public void setGuestEndStateAllEnds(Loop loop, SafepointState state) { + loopStates.get(loop.loopBegin()).setGuestEndStateAllEnds(state); + } + + public void setGuestEndState(LoopEndNode loopEnd, SafepointState state) { + loopStates.get(loopEnd.loopBegin()).setGuestEndState(loopEnd, state); + } + + public void setEndState(LoopEndNode loopEnd, SafepointState state) { + LoopBeginNode lb = loopEnd.loopBegin(); + loopStates.get(lb).setEndState(loopEnd, state); + } + + public void setEndStateAllEnds(Loop loop, SafepointState state) { + loopStates.get(loop.loopBegin()).setEndStateAllEnds(state); + } + + public void setExitState(Loop loop, SafepointState state) { + loopStates.get(loop.loopBegin()).setExitState(state); + } + + public boolean canSafepoint(LoopEndNode len) { + return loopStates.get(len.loopBegin()).getEndStates().get(len).canSafepoint(); + } + + } + + /** + * Models SafepointState transforms that can be {@linkplain #apply applied} to a single loop. + */ + public static class SafepointStateEffect { + /** + * The safepoint state that should be assigned to the loop exit nodes of this loop. + */ + private SafepointState exitState; + /** + * The safepoint state assigned to every loop end of this loop. + */ + private final EconomicMap endStates; + /** + * The guest safepoint state assigned to every loop end of this loop. + */ + private final EconomicMap guestEndStates; + /** + * The corresponding loop begin node. + */ + private final LoopBeginNode loopBegin; + + public SafepointStateEffect(LoopBeginNode lb) { + this.loopBegin = lb; + endStates = EconomicMap.create(); + guestEndStates = EconomicMap.create(); + // Start with the states before the elimination + for (LoopEndNode len : lb.loopEnds()) { + endStates.put(len, len.getSafepointState()); + guestEndStates.put(len, len.getGuestSafepointState()); + } + exitState = lb.getLoopExitsSafepointState(); + } + + public SafepointState getExitState() { + return exitState; + } + + public void setExitState(SafepointState exitState) { + this.exitState = exitState; } + + public EconomicMap getEndStates() { + return endStates; + } + + public EconomicMap getGuestEndStates() { + return guestEndStates; + } + + public void setEndState(LoopEndNode len, SafepointState newState) { + endStates.put(len, newState); + } + + public void setEndStateAllEnds(SafepointState newState) { + for (LoopEndNode len : loopBegin.loopEnds()) { + endStates.put(len, newState); + } + } + + public void setGuestEndStateAllEnds(SafepointState newState) { + for (LoopEndNode len : loopBegin.loopEnds()) { + guestEndStates.put(len, newState); + } + } + + public void setGuestEndState(LoopEndNode len, SafepointState newState) { + guestEndStates.put(len, newState); + } + + public void apply() { + for (LoopEndNode len : loopBegin.loopEnds()) { + len.setSafepointState(endStates.get(len)); + len.setGuestSafepointState(guestEndStates.get(len)); + } + loopBegin.setLoopExitSafepoint(exitState); + } + + } + + private static final long IntegerRangeDistance = NumUtil.unsafeAbs((long) Integer.MAX_VALUE - (long) Integer.MIN_VALUE); + + public static boolean loopIsInIterationRange(Loop loop, long distance) { final Stamp limitStamp = loop.counted().getTripCountLimit().stamp(NodeView.DEFAULT); if (limitStamp instanceof IntegerStamp) { final IntegerStamp limitIStamp = (IntegerStamp) limitStamp; @@ -86,7 +215,7 @@ public static boolean iterationRangeIsIn32Bit(Loop loop) { final InductionVariable counter = loop.counted().getLimitCheckedIV(); final long stride = counter.isConstantStride() ? NumUtil.safeAbs(counter.constantStride()) : 1; final long strideRelativeStartToLimitDistance = startToLimitDistance / stride; - return strideRelativeStartToLimitDistance <= IntegerRangeDistance; + return strideRelativeStartToLimitDistance <= distance; } catch (ArithmeticException e) { return false; } @@ -95,6 +224,13 @@ public static boolean iterationRangeIsIn32Bit(Loop loop) { return false; } + public static boolean iterationRangeIsIn32Bit(Loop loop) { + if (loop.counted().getStamp().getBits() <= 32) { + return true; + } + return loopIsInIterationRange(loop, IntegerRangeDistance); + } + @Override public Optional notApplicableTo(GraphState graphState) { return NotApplicable.unlessRunAfter(this, GraphState.StageFlag.LOOP_OVERFLOWS_CHECKED, graphState); @@ -102,14 +238,14 @@ public Optional notApplicableTo(GraphState graphState) { @Override protected void run(StructuredGraph graph, MidTierContext context) { - new Instance(graph, context).optimizeSafepoints(); + new SafepointOptimizer(graph, context).optimizeSafepoints().apply(); } - protected static class Instance { + public static class SafepointOptimizer { private final StructuredGraph graph; private final MidTierContext context; - protected Instance(StructuredGraph graph, MidTierContext context) { + public SafepointOptimizer(StructuredGraph graph, MidTierContext context) { this.graph = graph; this.context = context; } @@ -140,7 +276,7 @@ protected Instance(StructuredGraph graph, MidTierContext context) { * {@code call2} dominate the backedges. We can remove safepoint polls from both of them * because the call will be a guaranteed safepoint. */ - private int disableSafepointsByBodyNodes(Loop loop, ControlFlowGraph cfg) { + private int disableSafepointsByBodyNodes(LoopSafepointPlan safepointPlan, Loop loop, ControlFlowGraph cfg) { int loopEndSafepointsDisabled = 0; for (LoopEndNode loopEnd : loop.loopBegin().loopEnds()) { HIRBlock b = cfg.blockFor(loopEnd); @@ -148,9 +284,9 @@ private int disableSafepointsByBodyNodes(Loop loop, ControlFlowGraph cfg) { assert b != null; for (FixedNode node : b.getNodes()) { boolean canDisableSafepoint = canDisableSafepoint(node, context); - boolean disabledInSubclass = onCallInLoop(loopEnd, node); + boolean disabledInSubclass = onCallInLoop(safepointPlan, loopEnd, node); if (canDisableSafepoint) { - loopEnd.disableSafepoint(); + safepointPlan.setEndState(loopEnd, SafepointState.OPTIMIZER_DISABLED); graph.getOptimizationLog().report(LoopSafepointEliminationPhase.class, "SafepointElimination", loop.loopBegin()); loopEndSafepointsDisabled++; /* @@ -167,7 +303,9 @@ private int disableSafepointsByBodyNodes(Loop loop, ControlFlowGraph cfg) { return loopEndSafepointsDisabled; } - public void optimizeSafepoints() { + public LoopSafepointPlan optimizeSafepoints() { + LoopSafepointPlan graphWidePlan = new LoopSafepointPlan(graph); + final boolean optimisticallyRemoveLoopSafepoints = Options.RemoveLoopSafepoints.getValue(graph.getOptions()); LoopsData loops = context.getLoopsDataProvider().getLoopsData(graph); @@ -175,12 +313,20 @@ public void optimizeSafepoints() { for (Loop loop : loops.loops()) { if (!allowGuestSafepoints()) { - loop.loopBegin().disableGuestSafepoint(SafepointState.MUST_NEVER_SAFEPOINT); + graphWidePlan.setGuestEndStateAllEnds(loop, SafepointState.MUST_NEVER_SAFEPOINT); + } + if (!loop.loopBegin().canEndsSafepoint() && !loop.loopBegin().canEndsGuestSafepoint()) { + /* + * There are no safepoints at the loop ends. Therefore, we want safepoints at + * loop exits. Skip optimization of loop exit safepoints. + */ + continue; } - int loopEndSafepointsDisabled = disableSafepointsByBodyNodes(loop, loops.getCFG()); - final boolean allLoopEndSafepointsDisabled = loopEndSafepointsDisabled == loop.loopBegin().getLoopEndCount(); - if (!allLoopEndSafepointsDisabled && optimisticallyRemoveLoopSafepoints) { - if (optimizeSafepointsForCountedLoop(loop)) { + + int loopEndSafepointsDisabled = disableSafepointsByBodyNodes(graphWidePlan, loop, loops.getCFG()); + final boolean allLoopEndSafepointsDisabledByBodyNodes = loopEndSafepointsDisabled == loop.loopBegin().getLoopEndCount(); + if (!allLoopEndSafepointsDisabledByBodyNodes && optimisticallyRemoveLoopSafepoints) { + if (optimizeSafepointsForCountedLoop(graphWidePlan, loop)) { /* * We removed all loop end safepoints if we do it optimistically for the * entire loop. @@ -188,18 +334,28 @@ public void optimizeSafepoints() { loopEndSafepointsDisabled = loop.loopBegin().getLoopEndCount(); } } + if (!loop.loopBegin().canExitsSafepoint()) { + continue; + } final boolean allLoopEndSafepointsEnabled = loopEndSafepointsDisabled == 0; - if (allLoopEndSafepointsEnabled) { - /* - * Only if ALL paths through the loop are guaranteed to safepoint we can drop - * the exit safepoint. If there is any path left that does not safepoint we - * could be only executing that path and then we need an exit safepoint. - */ - loop.loopBegin().disableLoopExitSafepoint(SafepointState.OPTIMIZER_DISABLED); + // strip mined outer is never counted + final boolean stripMinedOuter = loop.loopBegin().isStripMinedOuter(); + // retain the exit safepoint for all non-counted loops + if (loop.isCounted() || stripMinedOuter) { + if (allLoopEndSafepointsEnabled || allLoopEndSafepointsDisabledByBodyNodes) { + /** + * If all paths inside the loop are guaranteed to trigger a safepoint + * explicitly via SafepointNodes, or implicitly via other nodes (e.g., + * InvokeNode), we can drop the exit safepoint. + */ + graphWidePlan.setExitState(loop, SafepointState.OPTIMIZER_DISABLED); + } } } loops.deleteUnusedNodes(); + + return graphWidePlan; } /** @@ -217,7 +373,7 @@ protected boolean allowGuestSafepoints() { * traversing. */ @SuppressWarnings("unused") - protected boolean onCallInLoop(LoopEndNode loopEnd, FixedNode currentCallNode) { + protected boolean onCallInLoop(LoopSafepointPlan safepointPlan, LoopEndNode loopEnd, FixedNode currentCallNode) { return true; } @@ -225,7 +381,7 @@ protected boolean onCallInLoop(LoopEndNode loopEnd, FixedNode currentCallNode) { * To be implemented by subclasses to compute additional fields. */ @SuppressWarnings("unused") - protected void onSafepointDisabledLoopBegin(Loop loop) { + protected void onSafepointDisabledLoopBegin(LoopSafepointPlan safepointPlan, Loop loop) { } /** @@ -233,14 +389,17 @@ protected void onSafepointDisabledLoopBegin(Loop loop) { * able to remove safepoints from the loop ends of the given loop yet. However, the * optimizer may believe this loop is short running enough to remove safepoints. */ - private boolean optimizeSafepointsForCountedLoop(Loop loop) { + private boolean optimizeSafepointsForCountedLoop(LoopSafepointPlan safepointPlan, Loop loop) { if (loop.isCounted()) { - if (loop.getCFGLoop().getChildren().isEmpty() && - (loop.loopBegin().isPreLoop() || loop.loopBegin().isPostLoop() || loopIsIn32BitRange(loop) || - loop.loopBegin().isStripMinedInner())) { + final boolean leafLoop = loop.getCFGLoop().getChildren().isEmpty(); + final boolean preLoop = loop.loopBegin().isPreLoop(); + final boolean postLoop = loop.loopBegin().isPostLoop(); + final boolean briefLoop = loopIsInBriefRange(loop); + final boolean stripMinedInner = loop.loopBegin().isStripMinedInner(); + if (leafLoop && (preLoop || postLoop || briefLoop || stripMinedInner)) { boolean hasSafepoint = false; for (LoopEndNode loopEnd : loop.loopBegin().loopEnds()) { - hasSafepoint |= loopEnd.canSafepoint(); + hasSafepoint |= loopEnd.getSafepointState().canSafepoint(); } if (hasSafepoint) { if (!loop.counted().counterNeverOverflows()) { @@ -256,18 +415,18 @@ private boolean optimizeSafepointsForCountedLoop(Loop loop) { return false; } } - loop.loopBegin().disableSafepoint(SafepointState.OPTIMIZER_DISABLED); - if (loop.loopBegin().isStripMinedInner()) { + safepointPlan.setEndStateAllEnds(loop, SafepointState.OPTIMIZER_DISABLED); + if (stripMinedInner) { /* * graal strip mined this loop, trust the heuristics and remove the * inner loop safepoint */ - loop.loopBegin().disableGuestSafepoint(SafepointState.OPTIMIZER_DISABLED); + safepointPlan.setGuestEndStateAllEnds(loop, SafepointState.OPTIMIZER_DISABLED); } else { /* * let the shape of the loop decide whether a guest safepoint is needed */ - onSafepointDisabledLoopBegin(loop); + onSafepointDisabledLoopBegin(safepointPlan, loop); } graph.getOptimizationLog().report(LoopSafepointEliminationPhase.class, "SafepointElimination", loop.loopBegin()); return true; @@ -277,7 +436,11 @@ private boolean optimizeSafepointsForCountedLoop(Loop loop) { return false; } - public boolean loopIsIn32BitRange(Loop loop) { + /** + * Determine if this loop is a brief range loop. A brief range loop is one where we assume + * that we can drop the safepoint because the body of the loop executes quickly enough. + */ + public boolean loopIsInBriefRange(Loop loop) { return iterationRangeIsIn32Bit(loop); } diff --git a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/nodes/GraphState.java b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/nodes/GraphState.java index 4510102bf2d1..a8e130b32b44 100644 --- a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/nodes/GraphState.java +++ b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/nodes/GraphState.java @@ -614,6 +614,7 @@ public enum StageFlag { FLOATING_READS, GUARD_MOVEMENT, GUARD_LOWERING, + STRIP_MINING, VALUE_PROXY_REMOVAL, SAFEPOINTS_INSERTION, MID_TIER_LOWERING, diff --git a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/nodes/LoopBeginNode.java b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/nodes/LoopBeginNode.java index 7f6243e996f4..6ac4d0bb730a 100644 --- a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/nodes/LoopBeginNode.java +++ b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/nodes/LoopBeginNode.java @@ -130,6 +130,10 @@ public boolean isProtectedNonOverflowingUnsigned() { /** * State of the safepoint properties of this loop. + * + * For the various points in a loop (e.g. loop ends, loop exits, guest loop ends) for which a + * safepoint may be emitted, a SafepointState value specifies if the emission is (or should be) + * suppressed and if so, why. */ public enum SafepointState { /** @@ -153,19 +157,19 @@ public enum SafepointState { this.canSafepoint = canSafepoint; } - private boolean canSafepoint() { + public boolean canSafepoint() { return canSafepoint; } } - /** See {@link LoopEndNode#canSafepoint} for more information. */ + /** See {@link LoopEndNode#safepointState} for more information. */ SafepointState loopEndsSafepointState; /** Same as {@link #loopEndsSafepointState} but for {@link LoopExitNode}. */ SafepointState loopExitsSafepointState; - /** See {@link LoopEndNode#canGuestSafepoint} for more information. */ + /** See {@link LoopEndNode#guestSafepointState} for more information. */ SafepointState guestLoopEndsSafepointState; /** @@ -262,6 +266,10 @@ public boolean canExitsSafepoint() { return loopExitsSafepointState.canSafepoint(); } + public SafepointState getLoopExitsSafepointState() { + return loopExitsSafepointState; + } + public void setStripMinedInner(boolean stripMinedInner) { this.stripMinedInner = stripMinedInner; } @@ -342,30 +350,40 @@ public void setUnrollFactor(int currentUnrollFactor) { unrollFactor = currentUnrollFactor; } - /** Disables safepoint for the whole loop, i.e., for all {@link LoopEndNode loop ends}. */ - public void disableSafepoint(SafepointState newState) { - GraalError.guarantee(!newState.canSafepoint(), "New state must not safepoint %s", newState); + public void setLoopEndSafepoint(SafepointState newState) { + GraalError.guarantee(loopEndsSafepointState.canSafepoint() || !newState.canSafepoint(), "New state must not allow safepoints if old did not, old=%s, new=%s", loopEndsSafepointState, newState); + if (loopEndsSafepointState == SafepointState.MUST_NEVER_SAFEPOINT) { + GraalError.guarantee(!newState.canSafepoint(), "Safepoints have been disabled for this loop, cannot re-enable them old=%s, new=%s", loopExitsSafepointState, newState); + } /* Store flag locally in case new loop ends are created later on. */ this.loopEndsSafepointState = newState; /* Propagate flag to all existing loop ends. */ for (LoopEndNode loopEnd : loopEnds()) { - loopEnd.disableSafepoint(); + loopEnd.setSafepointState(newState); } } - /** Disables safepoint for the whole loop, i.e., for all {@link LoopEndNode loop ends}. */ - public void disableLoopExitSafepoint(SafepointState newState) { - GraalError.guarantee(!newState.canSafepoint(), "New state must not safepoint %s", newState); + public void setLoopExitSafepoint(SafepointState newState) { + /* + * The safepoint state for the loop exits can change over the course of compilation. It + * depends on what is inside the body of a loop, how many ends, which calls etc. So it can + * be that this becomes weaker and stronger over time as long as we do not force safepoint + * on one that was explicitly disabled. + */ + if (loopExitsSafepointState == SafepointState.MUST_NEVER_SAFEPOINT) { + GraalError.guarantee(!newState.canSafepoint(), "Safepoints have been disabled for this loop, cannot re-enable them old=%s, new=%s", loopExitsSafepointState, newState); + } this.loopExitsSafepointState = newState; } - public void disableGuestSafepoint(SafepointState newState) { - GraalError.guarantee(!newState.canSafepoint(), "New state must not safepoint %s", newState); + public void setGuestSafepoint(SafepointState newState) { + GraalError.guarantee(guestLoopEndsSafepointState.canSafepoint() || !newState.canSafepoint(), "New state must not allow safepoints if old did not, old=%s, new=%s", guestLoopEndsSafepointState, + newState); /* Store flag locally in case new loop ends are created later on. */ this.guestLoopEndsSafepointState = newState; /* Propagate flag to all existing loop ends. */ for (LoopEndNode loopEnd : loopEnds()) { - loopEnd.disableGuestSafepoint(); + loopEnd.setGuestSafepointState(newState); } } diff --git a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/nodes/LoopEndNode.java b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/nodes/LoopEndNode.java index 423a380fb2da..c26ffbab3ccb 100644 --- a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/nodes/LoopEndNode.java +++ b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/nodes/LoopEndNode.java @@ -40,6 +40,7 @@ import jdk.graal.compiler.nodeinfo.NodeCycles; import jdk.graal.compiler.nodeinfo.NodeInfo; import jdk.graal.compiler.nodeinfo.NodeSize; +import jdk.graal.compiler.nodes.LoopBeginNode.SafepointState; import jdk.graal.compiler.nodes.spi.NodeLIRBuilderTool; /** @@ -60,20 +61,20 @@ public final class LoopEndNode extends AbstractEndNode { protected int endIndex; /** - * Most loop ends need a safepoint (flag set to true) so that garbage collection can interrupt a - * long-running (possibly endless) loop. Safepoints may be disabled for two reasons: 1) Some - * code must be safepoint free, i.e., uninterruptible by garbage collection. 2) An optimization - * phase determined that the loop already has another safepoint or cannot be endless, so there - * is no need for a loop-end safepoint. + * Most loop ends need a safepoint ({@link SafepointState#canSafepoint()} yields true) so that + * garbage collection can interrupt a long-running (possibly endless) loop. Safepoints may be + * disabled for two reasons: 1) Some code must be safepoint free, i.e., uninterruptible by + * garbage collection. 2) An optimization phase determined that the loop already has another + * safepoint or cannot be endless, so there is no need for a loop-end safepoint. * * Note that 1) is a hard correctness issue: emitting a safepoint in uninterruptible code is a * bug, i.e., it is not allowed to set the flag back to true once it is false. To ensure that * loop ends that are created late, e.g., during control flow simplifications, have no * safepoints in such cases, the safepoints are actually disabled for the - * {@link LoopBeginNode#canEndsSafepoint loop begin}. New loop ends inherit the flag value from + * {@link LoopBeginNode#canEndsSafepoint loop begin}. New loop ends inherit the state value from * the loop begin. */ - boolean canSafepoint; + SafepointState safepointState; /** * If Graal is used as a compiler for a guest language then in addition to host safepoints there @@ -81,18 +82,18 @@ public final class LoopEndNode extends AbstractEndNode { * to support garbage collectors but to support features like cancellation or reading stack * frames from other threads. *

- * This flag is used to store the information whether safepoints can be disabled for this loop - * end. It depends on the guest language implementation framework whether this flag may ever - * become false. + * The state is used to store the information whether safepoints can be disabled for this loop + * end. It depends on the guest language implementation framework whether this state indicates a + * possible safepoint or not. *

- * If Graal is not used to compile a guest language then this flag will be false - * for all loops seen by the safepoint elimination phase. It will be true before - * safepoint elimination. + * If Graal is not used to compile a guest language then this state will yield + * {@link SafepointState#canSafepoint()} {@code false} for all loops seen by the safepoint + * elimination phase. It will yield true before safepoint elimination. *

* More information on the guest safepoint implementation in Truffle can be found * here. */ - boolean canGuestSafepoint; + SafepointState guestSafepointState; public LoopEndNode(LoopBeginNode begin) { super(TYPE); @@ -100,8 +101,8 @@ public LoopEndNode(LoopBeginNode begin) { assert NumUtil.assertNonNegativeInt(idx); this.endIndex = idx; this.loopBegin = begin; - this.canSafepoint = begin.canEndsSafepoint(); - this.canGuestSafepoint = begin.canEndsGuestSafepoint(); + this.safepointState = begin.loopEndsSafepointState; + this.guestSafepointState = begin.guestLoopEndsSafepointState; } @Override @@ -118,27 +119,12 @@ public void setLoopBegin(LoopBeginNode x) { this.loopBegin = x; } - /** - * Disables safepoints for only this loop end (in contrast to disabling it for - * {@link LoopBeginNode#disableSafepoint(jdk.graal.compiler.nodes.LoopBeginNode.SafepointState) - * the whole loop}. - */ - public void disableSafepoint() { - this.canSafepoint = false; - } - - public void disableGuestSafepoint() { - this.canGuestSafepoint = false; - } - - public boolean canGuestSafepoint() { - assert !canGuestSafepoint || loopBegin().canEndsGuestSafepoint() : "When safepoints are disabled for loop begin, safepoints must be disabled for all loop ends"; - return this.canGuestSafepoint; + public void setSafepointState(SafepointState newState) { + this.safepointState = newState; } - public boolean canSafepoint() { - assert !canSafepoint || loopBegin().canEndsSafepoint() : "When safepoints are disabled for loop begin, safepoints must be disabled for all loop ends"; - return canSafepoint; + public void setGuestSafepointState(SafepointState newState) { + this.guestSafepointState = newState; } @Override @@ -170,6 +156,14 @@ void setEndIndex(int idx) { this.endIndex = idx; } + public SafepointState getSafepointState() { + return safepointState; + } + + public SafepointState getGuestSafepointState() { + return guestSafepointState; + } + @Override public Iterable cfgSuccessors() { return Collections.emptyList(); @@ -180,7 +174,7 @@ public NodeCycles estimatedNodeCycles() { if (!(loopBegin instanceof LoopBeginNode) || loopBegin() == null) { return CYCLES_UNKNOWN; } - if (canSafepoint()) { + if (safepointState.canSafepoint()) { // jmp+read return CYCLES_2; } @@ -192,7 +186,7 @@ protected NodeSize dynamicNodeSizeEstimate() { if (!(loopBegin instanceof LoopBeginNode)) { return SIZE_UNKNOWN; } - if (canSafepoint()) { + if (safepointState.canSafepoint()) { return SIZE_2; } return super.dynamicNodeSizeEstimate(); diff --git a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/nodes/calc/AddNode.java b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/nodes/calc/AddNode.java index ffc5952cdb37..195985caa5a4 100644 --- a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/nodes/calc/AddNode.java +++ b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/nodes/calc/AddNode.java @@ -58,6 +58,10 @@ protected AddNode(NodeClass c, ValueNode x, ValueNode y) { super(c, getArithmeticOpTable(x).getAdd(), x, y); } + protected AddNode(NodeClass c, Stamp betterStartStamp, ValueNode x, ValueNode y) { + super(c, betterStartStamp, x, y); + } + public static ValueNode create(ValueNode x, ValueNode y, NodeView view) { BinaryOp op = ArithmeticOpTable.forStamp(x.stamp(view)).getAdd(); Stamp stamp = op.foldStamp(x.stamp(view), y.stamp(view)); diff --git a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/nodes/calc/BinaryArithmeticNode.java b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/nodes/calc/BinaryArithmeticNode.java index 37551c835047..774e33f6740b 100644 --- a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/nodes/calc/BinaryArithmeticNode.java +++ b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/nodes/calc/BinaryArithmeticNode.java @@ -28,6 +28,7 @@ import static jdk.graal.compiler.nodeinfo.NodeSize.SIZE_1; import java.util.Arrays; + import jdk.graal.compiler.core.common.type.ArithmeticOpTable; import jdk.graal.compiler.core.common.type.ArithmeticOpTable.BinaryOp; import jdk.graal.compiler.core.common.type.ArithmeticStamp; @@ -75,7 +76,7 @@ public static ArithmeticOpTable getArithmeticOpTable(ValueNode forValue) { protected final BinaryOp getOp(ValueNode forX, ValueNode forY) { ArithmeticOpTable table = getArithmeticOpTable(forX); - assert table.equals(getArithmeticOpTable(forY)); + assert table.equals(getArithmeticOpTable(forY)) : Assertions.errorMessage("Invalid table ops", forX, table, forY, getArithmeticOpTable(forY)); return getOp(table); } diff --git a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/nodes/loop/Loop.java b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/nodes/loop/Loop.java index 6e943302d6da..272a0e923118 100644 --- a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/nodes/loop/Loop.java +++ b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/nodes/loop/Loop.java @@ -44,8 +44,10 @@ import jdk.graal.compiler.graph.NodeBitMap; import jdk.graal.compiler.graph.iterators.NodePredicate; import jdk.graal.compiler.nodes.AbstractBeginNode; +import jdk.graal.compiler.nodes.AbstractDeoptimizeNode; import jdk.graal.compiler.nodes.AbstractEndNode; import jdk.graal.compiler.nodes.ConstantNode; +import jdk.graal.compiler.nodes.DeoptimizeNode; import jdk.graal.compiler.nodes.FixedGuardNode; import jdk.graal.compiler.nodes.FixedNode; import jdk.graal.compiler.nodes.FixedWithNextNode; @@ -82,9 +84,34 @@ import jdk.graal.compiler.phases.common.CanonicalizerPhase; /** - * Extra loop data for the given loop. This includes data on which nodes belong to a loop, counted + * Extra loop data for a loop in the IR. This includes data on which nodes belong to a loop, counted * loop information if the compiler detects it as a counted loop. Data about induction variables, * parent loops and much more. + * + * A note on the relation of {@link Loop}, {@link LoopBeginNode} and {@link CFGLoop} in the Graal + * IR. + * + * A {@link Loop} is a data structure used by the optimizer to reason about a loop. It encapsulates + * machinery to compute which (floating) nodes belong to a loop as well as API to duplicate, copy, + * etc a loop. + * + * In contrast a {@link LoopBeginNode} is a marker node in the IR to signal the start of a loop + * structure. A {@link Loop} is a temporary data structure while the {@link LoopBeginNode} is + * permanent. One can compute multiple {@link Loop} for a given {@link LoopBeginNode} loop in the + * IR. + * + * A {@link CFGLoop} is an encapsulation of concepts related to a loop in the context of a + * {@link ControlFlowGraph}. It pulls together the {@link HIRBlock} of a loop and computes extra + * data like depth and child loops etc. necessary to even compute a {@link Loop} for the optimizer. + * + * IN the bigger picture a {@link LoopBeginNode} in the IR signals that a loop data structure + * begins. While computing a {@link ControlFlowGraph} we build the context data structure + * {@link CFGLoop} which is the necessary CFG abstraction to compute a {@link Loop} for the + * optimizer. + * + * Context data for a loop that needs to be preserved over the entire course of compilation is + * attached to a {@link LoopBeginNode} because that is the only permanent data storage associated + * with a loop. */ public class Loop { /** @@ -492,8 +519,29 @@ public void nodesInLoopBranch(NodeBitMap branchNodes, AbstractBeginNode branch) } public EconomicMap getInductionVariables() { - if (ivs == null) { - ivs = findInductionVariables(); + return getInductionVariables(false, false); + } + + /** + * Gets the collection of all {@link InductionVariable} of this loop indexed by their + * {@link InductionVariable#valueNode()}. + * + * If {@code forceReset==true} throws away any previously computed induction variables. The + * collection of the IVs uses the current data in {@link #inside} and thus can change depending + * on when IVs are computed. + * + * If {@code computeDeoptLoopExitIVs==true} computes a potentially broader set of IVs. It + * includes those that have usages outside a loop if such a usage is a framestate used by a + * {@link DeoptimizeNode}. That is necessary because there is concrete discrepancy in loop nodes + * between Java bytecode and the actual liveness. See {@link CFGLoop#getNaturalExits()} for + * details. Every branch leading to a deopt is already outside a loop and thus does not count as + * usages that drive normal IV collection. Certain optimizations still need to see all induction + * variables, also those with usages outside. Note that there are potentially IVs that are still + * not returned by this function if they have usages outside a loop but are not used by a deopt. + */ + public EconomicMap getInductionVariables(boolean forceReset, boolean computeDeoptLoopExitIVs) { + if (ivs == null || forceReset) { + ivs = findInductionVariables(computeDeoptLoopExitIVs); } return ivs; } @@ -528,7 +576,7 @@ public EconomicMap getInductionVariables() { * * @return a map from node to induction variable */ - private EconomicMap findInductionVariables() { + private EconomicMap findInductionVariables(boolean computeDeoptLoopExitIVs) { EconomicMap currentIvs = EconomicMap.create(Equivalence.IDENTITY); // first find basic induction variables @@ -554,7 +602,10 @@ private EconomicMap findInductionVariables() { ValueNode baseIvNode = baseIv.valueNode(); for (ValueNode op : baseIvNode.usages().filter(ValueNode.class)) { if (this.isOutsideLoop(op)) { - continue; + boolean needExitIVs = computeDeoptLoopExitIVs && hasDeoptUsage(op, this); + if (!needExitIVs) { + continue; + } } if (op.hasExactlyOneUsage() && op.usages().first() == baseIvNode) { /* @@ -597,6 +648,59 @@ private EconomicMap findInductionVariables() { return currentIvs; } + /** + * Maximum search depth for recursive application of deopt usage search. Used when state usages + * are found. See {@link #hasDeoptUsage(ValueNode, Loop)} for details. + */ + private static final int MAX_DEPTH_DEOPT_USAGES = 4; + + private static boolean hasDeoptUsage(ValueNode op, Loop loop) { + for (Node usage : op.usages()) { + if (hasDeoptUsage(usage, loop, 0)) { + return true; + } + } + return hasDeoptUsage(op, loop, 0); + } + + /** + * Determine if the given operation denoted as {@code op} represents an induction variable that + * is used in a deopt path inside the loop body. With Java bytecode liveness is always a + * problem. There is a discrepancy between bytecode level loops and actual, natural loop exits + * (see {@link CFGLoop#getNaturalExits()} for details). This value might only be used in a deopt + * path outside the loop == a natural exit path. Still create IVs if wanted for such cases so + * certain optimizations can use that information. + * + * If {@code depth} is reached this method returns {@code false} indicating no usage is found. + * This however is imprecise. A return value of {@code false} can indicate there was no usage or + * the depth filter was hit and we DO NOT KNOW. + */ + private static boolean hasDeoptUsage(Node usage, Loop loop, int depth) { + if (depth >= MAX_DEPTH_DEOPT_USAGES) { + return false; + } + if (usage instanceof FrameState fs) { + for (Node fsUsage : fs.usages()) { + if (fsUsage instanceof AbstractDeoptimizeNode deopt) { + HIRBlock deoptBlock = loop.getCFGLoop().getHeader().getCfg().blockFor(deopt); + while (deoptBlock != null) { + if (loop.getCFGLoop().getBlocks().contains(deoptBlock)) { + return true; + } + deoptBlock = deoptBlock.getDominator(); + } + } else if (fsUsage instanceof FrameState stateUsage && stateUsage.outerFrameState() == fs) { + // go into recursion another level + if (hasDeoptUsage(fsUsage, loop, depth + 1)) { + return true; + } + } + } + } + + return false; + } + /** * Determines if {@code op} is using {@code base} as an input. If so, determines if the other * input of {@code op} is loop invariant with respect to {@code loop}. This marks one diff --git a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/nodes/loop/LoopFragmentInside.java b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/nodes/loop/LoopFragmentInside.java index eebb7979d93a..9518f1c538f1 100644 --- a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/nodes/loop/LoopFragmentInside.java +++ b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/nodes/loop/LoopFragmentInside.java @@ -420,9 +420,21 @@ private void mergeRegularEarlyExit(FixedNode next, AbstractBeginNode exitBranchB } } - public static ValueNode patchProxyAtPhi(PhiNode phi, LoopExitNode lex, ValueNode proxyInput) { + public static ProxyNode patchProxyAtPhi(PhiNode phi, LoopExitNode lex, ValueNode proxyInput) { + return patchProxyAtPhi(phi, lex, proxyInput, false); + } + + public static ProxyNode patchProxyAtPhi(PhiNode phi, LoopExitNode lex, ValueNode proxyInput, boolean unrestrictedStamp) { if (phi instanceof ValuePhiNode) { - return phi.graph().addOrUnique(new ValueProxyNode(proxyInput, lex)); + if (unrestrictedStamp) { + /* + * Delay precise stamp injection to the first time #inferStamp is called on the + * value proxy. + */ + return phi.graph().addOrUnique(new ValueProxyNode(proxyInput.stamp(NodeView.DEFAULT).unrestricted(), proxyInput, lex)); + } else { + return phi.graph().addOrUnique(new ValueProxyNode(proxyInput, lex)); + } } else if (phi instanceof MemoryPhiNode) { return phi.graph().addOrUnique(new MemoryProxyNode((MemoryKill) proxyInput, lex, ((MemoryPhiNode) phi).getLocationIdentity())); } else if (phi instanceof GuardPhiNode) { diff --git a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/phases/common/LoopSafepointInsertionPhase.java b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/phases/common/LoopSafepointInsertionPhase.java index 137157df9bf3..7b1040431a04 100644 --- a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/phases/common/LoopSafepointInsertionPhase.java +++ b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/phases/common/LoopSafepointInsertionPhase.java @@ -63,10 +63,11 @@ protected void run(StructuredGraph graph, MidTierContext context) { if (GenLoopSafepoints.getValue(graph.getOptions())) { for (LoopBeginNode loopBeginNode : graph.getNodes(LoopBeginNode.TYPE)) { for (LoopEndNode loopEndNode : loopBeginNode.loopEnds().snapshot()) { - if (loopEndNode.canSafepoint()) { + if (loopEndNode.getSafepointState().canSafepoint()) { try (DebugCloseable s = loopEndNode.withNodeSourcePosition()) { SafepointNode safepointNode = graph.add(new SafepointNode(loopBeginNode)); graph.addBeforeFixed(loopEndNode, safepointNode); + safepointNode.setLoop(loopBeginNode); } } } diff --git a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/truffle/phases/TruffleLoopSafepointEliminationPhase.java b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/truffle/phases/TruffleLoopSafepointEliminationPhase.java index 236635ed1e06..b17636a4ad8b 100644 --- a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/truffle/phases/TruffleLoopSafepointEliminationPhase.java +++ b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/truffle/phases/TruffleLoopSafepointEliminationPhase.java @@ -29,6 +29,7 @@ import jdk.graal.compiler.nodes.CallTargetNode; import jdk.graal.compiler.nodes.FixedNode; import jdk.graal.compiler.nodes.Invoke; +import jdk.graal.compiler.nodes.LoopBeginNode.SafepointState; import jdk.graal.compiler.nodes.LoopEndNode; import jdk.graal.compiler.nodes.StructuredGraph; import jdk.graal.compiler.nodes.java.AbstractNewObjectNode; @@ -43,7 +44,7 @@ * safepoints when the loop contains truffle calls or when there is a plain counted loop without * allocations. * - * @see LoopEndNode#canGuestSafepoint + * @see LoopEndNode#getGuestSafepointState() */ public final class TruffleLoopSafepointEliminationPhase extends LoopSafepointEliminationPhase { @@ -55,10 +56,10 @@ public TruffleLoopSafepointEliminationPhase(KnownTruffleTypes types) { @Override protected void run(StructuredGraph graph, MidTierContext context) { - LoopSafepointEliminationPhase.Instance instance = new LoopSafepointEliminationPhase.Instance(graph, context) { + LoopSafepointEliminationPhase.SafepointOptimizer instance = new LoopSafepointEliminationPhase.SafepointOptimizer(graph, context) { @Override - protected void onSafepointDisabledLoopBegin(Loop loop) { + protected void onSafepointDisabledLoopBegin(LoopSafepointPlan safepointPlan, Loop loop) { for (Node node : loop.whole().nodes()) { if (node instanceof CommitAllocationNode || node instanceof AbstractNewObjectNode) { // we can disable truffle safepoints if there are no allocations @@ -66,14 +67,14 @@ protected void onSafepointDisabledLoopBegin(Loop loop) { return; } } - loop.loopBegin().disableGuestSafepoint(loop.loopBegin().getLoopEndsSafepointState()); + safepointPlan.setGuestEndStateAllEnds(loop, SafepointState.OPTIMIZER_DISABLED); } @Override - protected boolean onCallInLoop(LoopEndNode loopEnd, FixedNode currentCallNode) { + protected boolean onCallInLoop(LoopSafepointPlan safepointPlan, LoopEndNode loopEnd, FixedNode currentCallNode) { if (currentCallNode instanceof Invoke && isTruffleCall((Invoke) currentCallNode)) { // only truffle calls imply a truffle safepoint at method exits - loopEnd.disableGuestSafepoint(); + safepointPlan.setGuestEndState(loopEnd, SafepointState.OPTIMIZER_DISABLED); return true; } return false; @@ -84,7 +85,7 @@ protected boolean allowGuestSafepoints() { return true; } }; - instance.optimizeSafepoints(); + instance.optimizeSafepoints().apply(); } private boolean isTruffleCall(Invoke call) { diff --git a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/truffle/phases/TruffleSafepointInsertionPhase.java b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/truffle/phases/TruffleSafepointInsertionPhase.java index 90edaf90a879..533b529a2576 100644 --- a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/truffle/phases/TruffleSafepointInsertionPhase.java +++ b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/truffle/phases/TruffleSafepointInsertionPhase.java @@ -109,7 +109,7 @@ protected void run(StructuredGraph graph) { } for (LoopBeginNode loopBeginNode : graph.getNodes(LoopBeginNode.TYPE)) { for (LoopEndNode loopEndNode : loopBeginNode.loopEnds()) { - if (loopEndNode.canGuestSafepoint()) { + if (loopEndNode.getGuestSafepointState().canSafepoint()) { try (DebugCloseable s = loopEndNode.withNodeSourcePosition()) { insertSafepoint(graph, loopEndNode); }