1 /*
2 * Copyright (c) 2012, 2016, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 */
23 package org.graalvm.compiler.hotspot.replacements;
24
25 import static org.graalvm.compiler.core.common.GraalOptions.SnippetCounters;
26 import static org.graalvm.compiler.hotspot.GraalHotSpotVMConfig.INJECTED_VMCONFIG;
27 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.arrayBaseOffset;
28 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.arrayIndexScale;
29 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.cardTableShift;
30 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.dirtyCardValue;
31 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.g1CardQueueBufferOffset;
32 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.g1CardQueueIndexOffset;
33 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.g1SATBQueueBufferOffset;
34 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.g1SATBQueueIndexOffset;
35 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.g1SATBQueueMarkingOffset;
36 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.g1YoungCardValue;
37 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.registerAsWord;
38 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.verifyOop;
39 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.verifyOops;
40 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.wordSize;
41 import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.FREQUENT_PROBABILITY;
42 import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.LIKELY_PROBABILITY;
43 import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.NOT_FREQUENT_PROBABILITY;
44 import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.probability;
45 import static org.graalvm.compiler.replacements.SnippetTemplate.DEFAULT_REPLACER;
46 import static jdk.vm.ci.code.MemoryBarriers.STORE_LOAD;
47
48 import org.graalvm.compiler.api.replacements.Snippet;
49 import org.graalvm.compiler.api.replacements.Snippet.ConstantParameter;
50 import org.graalvm.compiler.core.common.GraalOptions;
51 import org.graalvm.compiler.core.common.LocationIdentity;
52 import org.graalvm.compiler.core.common.spi.ForeignCallDescriptor;
53 import org.graalvm.compiler.graph.Node.ConstantNodeParameter;
54 import org.graalvm.compiler.graph.Node.NodeIntrinsic;
55 import org.graalvm.compiler.hotspot.CompressEncoding;
56 import org.graalvm.compiler.hotspot.meta.HotSpotProviders;
57 import org.graalvm.compiler.hotspot.meta.HotSpotRegistersProvider;
58 import org.graalvm.compiler.hotspot.nodes.CompressionNode;
59 import org.graalvm.compiler.hotspot.nodes.G1ArrayRangePostWriteBarrier;
60 import org.graalvm.compiler.hotspot.nodes.G1ArrayRangePreWriteBarrier;
61 import org.graalvm.compiler.hotspot.nodes.G1PostWriteBarrier;
62 import org.graalvm.compiler.hotspot.nodes.G1PreWriteBarrier;
63 import org.graalvm.compiler.hotspot.nodes.G1ReferentFieldReadBarrier;
64 import org.graalvm.compiler.hotspot.nodes.GetObjectAddressNode;
65 import org.graalvm.compiler.hotspot.nodes.GraalHotSpotVMConfigNode;
66 import org.graalvm.compiler.hotspot.nodes.SerialArrayRangeWriteBarrier;
67 import org.graalvm.compiler.hotspot.nodes.SerialWriteBarrier;
68 import org.graalvm.compiler.hotspot.nodes.type.NarrowOopStamp;
69 import org.graalvm.compiler.nodes.NamedLocationIdentity;
70 import org.graalvm.compiler.nodes.StructuredGraph;
71 import org.graalvm.compiler.nodes.ValueNode;
72 import org.graalvm.compiler.nodes.extended.FixedValueAnchorNode;
73 import org.graalvm.compiler.nodes.extended.ForeignCallNode;
74 import org.graalvm.compiler.nodes.extended.MembarNode;
75 import org.graalvm.compiler.nodes.extended.NullCheckNode;
76 import org.graalvm.compiler.nodes.memory.HeapAccess.BarrierType;
77 import org.graalvm.compiler.nodes.memory.address.AddressNode;
78 import org.graalvm.compiler.nodes.memory.address.AddressNode.Address;
79 import org.graalvm.compiler.nodes.memory.address.OffsetAddressNode;
80 import org.graalvm.compiler.nodes.spi.LoweringTool;
81 import org.graalvm.compiler.replacements.Log;
82 import org.graalvm.compiler.replacements.SnippetCounter;
83 import org.graalvm.compiler.replacements.SnippetTemplate.AbstractTemplates;
84 import org.graalvm.compiler.replacements.SnippetTemplate.Arguments;
85 import org.graalvm.compiler.replacements.SnippetTemplate.SnippetInfo;
86 import org.graalvm.compiler.replacements.Snippets;
87 import org.graalvm.compiler.replacements.nodes.DirectObjectStoreNode;
88 import org.graalvm.compiler.replacements.nodes.DirectStoreNode;
89 import org.graalvm.compiler.word.Pointer;
90 import org.graalvm.compiler.word.Unsigned;
91 import org.graalvm.compiler.word.Word;
92
93 import jdk.vm.ci.code.Register;
94 import jdk.vm.ci.code.TargetDescription;
95 import jdk.vm.ci.meta.JavaKind;
96
97 public class WriteBarrierSnippets implements Snippets {
98
99 private static final SnippetCounter.Group countersWriteBarriers = SnippetCounters.getValue() ? new SnippetCounter.Group("WriteBarriers") : null;
100 private static final SnippetCounter serialWriteBarrierCounter = new SnippetCounter(countersWriteBarriers, "serialWriteBarrier", "Number of Serial Write Barriers");
101 private static final SnippetCounter g1AttemptedPreWriteBarrierCounter = new SnippetCounter(countersWriteBarriers, "g1AttemptedPreWriteBarrier", "Number of attempted G1 Pre Write Barriers");
102 private static final SnippetCounter g1EffectivePreWriteBarrierCounter = new SnippetCounter(countersWriteBarriers, "g1EffectivePreWriteBarrier", "Number of effective G1 Pre Write Barriers");
103 private static final SnippetCounter g1ExecutedPreWriteBarrierCounter = new SnippetCounter(countersWriteBarriers, "g1ExecutedPreWriteBarrier", "Number of executed G1 Pre Write Barriers");
104 private static final SnippetCounter g1AttemptedPostWriteBarrierCounter = new SnippetCounter(countersWriteBarriers, "g1AttemptedPostWriteBarrier", "Number of attempted G1 Post Write Barriers");
105 private static final SnippetCounter g1EffectiveAfterXORPostWriteBarrierCounter = new SnippetCounter(countersWriteBarriers, "g1EffectiveAfterXORPostWriteBarrier",
106 "Number of effective G1 Post Write Barriers (after passing the XOR test)");
107 private static final SnippetCounter g1EffectiveAfterNullPostWriteBarrierCounter = new SnippetCounter(countersWriteBarriers, "g1EffectiveAfterNullPostWriteBarrier",
108 "Number of effective G1 Post Write Barriers (after passing the NULL test)");
109 private static final SnippetCounter g1ExecutedPostWriteBarrierCounter = new SnippetCounter(countersWriteBarriers, "g1ExecutedPostWriteBarrier", "Number of executed G1 Post Write Barriers");
110
111 public static final LocationIdentity GC_CARD_LOCATION = NamedLocationIdentity.mutable("GC-Card");
112 public static final LocationIdentity GC_LOG_LOCATION = NamedLocationIdentity.mutable("GC-Log");
113 public static final LocationIdentity GC_INDEX_LOCATION = NamedLocationIdentity.mutable("GC-Index");
114
115 private static void serialWriteBarrier(Pointer ptr) {
116 serialWriteBarrierCounter.inc();
117 final long startAddress = GraalHotSpotVMConfigNode.cardTableAddress();
118 Word base = (Word) ptr.unsignedShiftRight(cardTableShift(INJECTED_VMCONFIG));
119 if (((int) startAddress) == startAddress && GraalHotSpotVMConfigNode.isCardTableAddressConstant()) {
120 base.writeByte((int) startAddress, (byte) 0, GC_CARD_LOCATION);
121 } else {
122 base.writeByte(Word.unsigned(startAddress), (byte) 0, GC_CARD_LOCATION);
123 }
124 }
125
126 @Snippet
127 public static void serialImpreciseWriteBarrier(Object object) {
128 serialWriteBarrier(Word.objectToTrackedPointer(object));
129 }
130
131 @Snippet
132 public static void serialPreciseWriteBarrier(Address address) {
133 serialWriteBarrier(Word.fromAddress(address));
134 }
135
136 @Snippet
137 public static void serialArrayRangeWriteBarrier(Object object, int startIndex, int length) {
138 if (length == 0) {
139 return;
140 }
141 Object dest = FixedValueAnchorNode.getObject(object);
142 int cardShift = cardTableShift(INJECTED_VMCONFIG);
143 final long cardStart = GraalHotSpotVMConfigNode.cardTableAddress();
144 final int scale = arrayIndexScale(JavaKind.Object);
145 int header = arrayBaseOffset(JavaKind.Object);
146 long dstAddr = GetObjectAddressNode.get(dest);
147 long start = (dstAddr + header + (long) startIndex * scale) >>> cardShift;
148 long end = (dstAddr + header + ((long) startIndex + length - 1) * scale) >>> cardShift;
149 long count = end - start + 1;
150 while (count-- > 0) {
151 DirectStoreNode.storeBoolean((start + cardStart) + count, false, JavaKind.Boolean);
152 }
153 }
154
155 @Snippet
156 public static void g1PreWriteBarrier(Address address, Object object, Object expectedObject, @ConstantParameter boolean doLoad, @ConstantParameter boolean nullCheck,
157 @ConstantParameter Register threadRegister, @ConstantParameter boolean trace) {
158 if (nullCheck) {
159 NullCheckNode.nullCheck(address);
160 }
161 Word thread = registerAsWord(threadRegister);
162 verifyOop(object);
163 Object fixedExpectedObject = FixedValueAnchorNode.getObject(expectedObject);
164 Pointer field = Word.fromAddress(address);
165 Pointer previousOop = Word.objectToTrackedPointer(fixedExpectedObject);
166 byte markingValue = thread.readByte(g1SATBQueueMarkingOffset(INJECTED_VMCONFIG));
167 int gcCycle = 0;
168 if (trace) {
169 gcCycle = (int) Word.unsigned(HotSpotReplacementsUtil.gcTotalCollectionsAddress(INJECTED_VMCONFIG)).readLong(0);
170 log(trace, "[%d] G1-Pre Thread %p Object %p\n", gcCycle, thread.rawValue(), Word.objectToTrackedPointer(object).rawValue());
171 log(trace, "[%d] G1-Pre Thread %p Expected Object %p\n", gcCycle, thread.rawValue(), Word.objectToTrackedPointer(fixedExpectedObject).rawValue());
172 log(trace, "[%d] G1-Pre Thread %p Field %p\n", gcCycle, thread.rawValue(), field.rawValue());
173 log(trace, "[%d] G1-Pre Thread %p Marking %d\n", gcCycle, thread.rawValue(), markingValue);
174 log(trace, "[%d] G1-Pre Thread %p DoLoad %d\n", gcCycle, thread.rawValue(), doLoad ? 1L : 0L);
175 }
176 g1AttemptedPreWriteBarrierCounter.inc();
177 // If the concurrent marker is enabled, the barrier is issued.
178 if (probability(NOT_FREQUENT_PROBABILITY, markingValue != (byte) 0)) {
179 // If the previous value has to be loaded (before the write), the load is issued.
180 // The load is always issued except the cases of CAS and referent field.
181 if (probability(LIKELY_PROBABILITY, doLoad)) {
182 previousOop = Word.objectToTrackedPointer(field.readObject(0, BarrierType.NONE));
183 if (trace) {
184 log(trace, "[%d] G1-Pre Thread %p Previous Object %p\n ", gcCycle, thread.rawValue(), previousOop.rawValue());
185 verifyOop(previousOop.toObject());
186 }
187 }
188 g1EffectivePreWriteBarrierCounter.inc();
189 // If the previous value is null the barrier should not be issued.
190 if (probability(FREQUENT_PROBABILITY, previousOop.notEqual(0))) {
191 g1ExecutedPreWriteBarrierCounter.inc();
192 // If the thread-local SATB buffer is full issue a native call which will
193 // initialize a new one and add the entry.
194 Word indexAddress = thread.add(g1SATBQueueIndexOffset(INJECTED_VMCONFIG));
195 Word indexValue = indexAddress.readWord(0);
196 if (probability(FREQUENT_PROBABILITY, indexValue.notEqual(0))) {
197 Word bufferAddress = thread.readWord(g1SATBQueueBufferOffset(INJECTED_VMCONFIG));
198 Word nextIndex = indexValue.subtract(wordSize());
199 Word logAddress = bufferAddress.add(nextIndex);
200 // Log the object to be marked as well as update the SATB's buffer next index.
201 logAddress.writeWord(0, previousOop, GC_LOG_LOCATION);
202 indexAddress.writeWord(0, nextIndex, GC_INDEX_LOCATION);
203 } else {
204 g1PreBarrierStub(G1WBPRECALL, previousOop.toObject());
205 }
206 }
207 }
208 }
209
210 @Snippet
211 public static void g1PostWriteBarrier(Address address, Object object, Object value, @ConstantParameter boolean usePrecise, @ConstantParameter Register threadRegister,
212 @ConstantParameter boolean trace) {
213 Word thread = registerAsWord(threadRegister);
214 Object fixedValue = FixedValueAnchorNode.getObject(value);
215 verifyOop(object);
216 verifyOop(fixedValue);
217 validateObject(object, fixedValue);
218 Pointer oop;
219 if (usePrecise) {
220 oop = Word.fromAddress(address);
221 } else {
222 oop = Word.objectToTrackedPointer(object);
223 }
224 int gcCycle = 0;
225 if (trace) {
226 gcCycle = (int) Word.unsigned(HotSpotReplacementsUtil.gcTotalCollectionsAddress(INJECTED_VMCONFIG)).readLong(0);
227 log(trace, "[%d] G1-Post Thread: %p Object: %p\n", gcCycle, thread.rawValue(), Word.objectToTrackedPointer(object).rawValue());
228 log(trace, "[%d] G1-Post Thread: %p Field: %p\n", gcCycle, thread.rawValue(), oop.rawValue());
229 }
230 Pointer writtenValue = Word.objectToTrackedPointer(fixedValue);
231 // The result of the xor reveals whether the installed pointer crosses heap regions.
232 // In case it does the write barrier has to be issued.
233 final int logOfHeapRegionGrainBytes = GraalHotSpotVMConfigNode.logOfHeapRegionGrainBytes();
234 Unsigned xorResult = (oop.xor(writtenValue)).unsignedShiftRight(logOfHeapRegionGrainBytes);
235
236 // Calculate the address of the card to be enqueued to the
237 // thread local card queue.
238 Unsigned cardBase = oop.unsignedShiftRight(cardTableShift(INJECTED_VMCONFIG));
239 final long startAddress = GraalHotSpotVMConfigNode.cardTableAddress();
240 int displacement = 0;
241 if (((int) startAddress) == startAddress && GraalHotSpotVMConfigNode.isCardTableAddressConstant()) {
242 displacement = (int) startAddress;
243 } else {
244 cardBase = cardBase.add(Word.unsigned(startAddress));
245 }
246 Word cardAddress = (Word) cardBase.add(displacement);
247
248 g1AttemptedPostWriteBarrierCounter.inc();
249 if (probability(FREQUENT_PROBABILITY, xorResult.notEqual(0))) {
250 g1EffectiveAfterXORPostWriteBarrierCounter.inc();
251
252 // If the written value is not null continue with the barrier addition.
253 if (probability(FREQUENT_PROBABILITY, writtenValue.notEqual(0))) {
254 byte cardByte = cardAddress.readByte(0, GC_CARD_LOCATION);
255 g1EffectiveAfterNullPostWriteBarrierCounter.inc();
256
257 // If the card is already dirty, (hence already enqueued) skip the insertion.
258 if (probability(NOT_FREQUENT_PROBABILITY, cardByte != g1YoungCardValue(INJECTED_VMCONFIG))) {
259 MembarNode.memoryBarrier(STORE_LOAD, GC_CARD_LOCATION);
260 byte cardByteReload = cardAddress.readByte(0, GC_CARD_LOCATION);
261 if (probability(NOT_FREQUENT_PROBABILITY, cardByteReload != dirtyCardValue(INJECTED_VMCONFIG))) {
262 log(trace, "[%d] G1-Post Thread: %p Card: %p \n", gcCycle, thread.rawValue(), Word.unsigned(cardByte).rawValue());
263 cardAddress.writeByte(0, (byte) 0, GC_CARD_LOCATION);
264 g1ExecutedPostWriteBarrierCounter.inc();
265
266 // If the thread local card queue is full, issue a native call which will
267 // initialize a new one and add the card entry.
268 Word indexAddress = thread.add(g1CardQueueIndexOffset(INJECTED_VMCONFIG));
269 Word indexValue = thread.readWord(g1CardQueueIndexOffset(INJECTED_VMCONFIG));
270 if (probability(FREQUENT_PROBABILITY, indexValue.notEqual(0))) {
271 Word bufferAddress = thread.readWord(g1CardQueueBufferOffset(INJECTED_VMCONFIG));
272 Word nextIndex = indexValue.subtract(wordSize());
273 Word logAddress = bufferAddress.add(nextIndex);
274 // Log the object to be scanned as well as update
275 // the card queue's next index.
276 logAddress.writeWord(0, cardAddress, GC_LOG_LOCATION);
277 indexAddress.writeWord(0, nextIndex, GC_INDEX_LOCATION);
278 } else {
279 g1PostBarrierStub(G1WBPOSTCALL, cardAddress);
280 }
281 }
282 }
283 }
284 }
285 }
286
287 @Snippet
288 public static void g1ArrayRangePreWriteBarrier(Object object, int startIndex, int length, @ConstantParameter Register threadRegister) {
289 Word thread = registerAsWord(threadRegister);
290 byte markingValue = thread.readByte(g1SATBQueueMarkingOffset(INJECTED_VMCONFIG));
291 // If the concurrent marker is not enabled or the vector length is zero, return.
292 if (markingValue == (byte) 0 || length == 0) {
293 return;
294 }
295 Object dest = FixedValueAnchorNode.getObject(object);
296 Word bufferAddress = thread.readWord(g1SATBQueueBufferOffset(INJECTED_VMCONFIG));
297 Word indexAddress = thread.add(g1SATBQueueIndexOffset(INJECTED_VMCONFIG));
298 long dstAddr = GetObjectAddressNode.get(dest);
299 long indexValue = indexAddress.readWord(0).rawValue();
300 final int scale = arrayIndexScale(JavaKind.Object);
301 int header = arrayBaseOffset(JavaKind.Object);
302
303 for (int i = startIndex; i < length; i++) {
304 long address = dstAddr + header + (i * scale);
305 Pointer oop = Word.objectToTrackedPointer(Word.unsigned(address).readObject(0, BarrierType.NONE));
306 verifyOop(oop.toObject());
307 if (oop.notEqual(0)) {
308 if (indexValue != 0) {
309 indexValue = indexValue - wordSize();
310 Word logAddress = bufferAddress.add(Word.unsigned(indexValue));
311 // Log the object to be marked as well as update the SATB's buffer next index.
312 logAddress.writeWord(0, oop, GC_LOG_LOCATION);
313 indexAddress.writeWord(0, Word.unsigned(indexValue), GC_INDEX_LOCATION);
314 } else {
315 g1PreBarrierStub(G1WBPRECALL, oop.toObject());
316 }
317 }
318 }
319 }
320
321 @Snippet
322 public static void g1ArrayRangePostWriteBarrier(Object object, int startIndex, int length, @ConstantParameter Register threadRegister) {
323 if (length == 0) {
324 return;
325 }
326 Object dest = FixedValueAnchorNode.getObject(object);
327 Word thread = registerAsWord(threadRegister);
328 Word bufferAddress = thread.readWord(g1CardQueueBufferOffset(INJECTED_VMCONFIG));
329 Word indexAddress = thread.add(g1CardQueueIndexOffset(INJECTED_VMCONFIG));
330 long indexValue = thread.readWord(g1CardQueueIndexOffset(INJECTED_VMCONFIG)).rawValue();
331
332 int cardShift = cardTableShift(INJECTED_VMCONFIG);
333 final long cardStart = GraalHotSpotVMConfigNode.cardTableAddress();
334 final int scale = arrayIndexScale(JavaKind.Object);
335 int header = arrayBaseOffset(JavaKind.Object);
336 long dstAddr = GetObjectAddressNode.get(dest);
337 long start = (dstAddr + header + (long) startIndex * scale) >>> cardShift;
338 long end = (dstAddr + header + ((long) startIndex + length - 1) * scale) >>> cardShift;
339 long count = end - start + 1;
340
341 while (count-- > 0) {
342 Word cardAddress = Word.unsigned((start + cardStart) + count);
343 byte cardByte = cardAddress.readByte(0, GC_CARD_LOCATION);
344 // If the card is already dirty, (hence already enqueued) skip the insertion.
345 if (probability(NOT_FREQUENT_PROBABILITY, cardByte != g1YoungCardValue(INJECTED_VMCONFIG))) {
346 MembarNode.memoryBarrier(STORE_LOAD, GC_CARD_LOCATION);
347 byte cardByteReload = cardAddress.readByte(0, GC_CARD_LOCATION);
348 if (probability(NOT_FREQUENT_PROBABILITY, cardByteReload != dirtyCardValue(INJECTED_VMCONFIG))) {
349 cardAddress.writeByte(0, (byte) 0, GC_CARD_LOCATION);
350 // If the thread local card queue is full, issue a native call which will
351 // initialize a new one and add the card entry.
352 if (indexValue != 0) {
353 indexValue = indexValue - wordSize();
354 Word logAddress = bufferAddress.add(Word.unsigned(indexValue));
355 // Log the object to be scanned as well as update
356 // the card queue's next index.
357 logAddress.writeWord(0, cardAddress, GC_LOG_LOCATION);
358 indexAddress.writeWord(0, Word.unsigned(indexValue), GC_INDEX_LOCATION);
359 } else {
360 g1PostBarrierStub(G1WBPOSTCALL, cardAddress);
361 }
362 }
363 }
364 }
365 }
366
367 public static final ForeignCallDescriptor G1WBPRECALL = new ForeignCallDescriptor("write_barrier_pre", void.class, Object.class);
368
369 @NodeIntrinsic(ForeignCallNode.class)
370 private static native void g1PreBarrierStub(@ConstantNodeParameter ForeignCallDescriptor descriptor, Object object);
371
372 public static final ForeignCallDescriptor G1WBPOSTCALL = new ForeignCallDescriptor("write_barrier_post", void.class, Word.class);
373
374 @NodeIntrinsic(ForeignCallNode.class)
375 public static native void g1PostBarrierStub(@ConstantNodeParameter ForeignCallDescriptor descriptor, Word card);
376
377 public static class Templates extends AbstractTemplates {
378
379 private final SnippetInfo serialImpreciseWriteBarrier = snippet(WriteBarrierSnippets.class, "serialImpreciseWriteBarrier", GC_CARD_LOCATION);
380 private final SnippetInfo serialPreciseWriteBarrier = snippet(WriteBarrierSnippets.class, "serialPreciseWriteBarrier", GC_CARD_LOCATION);
381 private final SnippetInfo serialArrayRangeWriteBarrier = snippet(WriteBarrierSnippets.class, "serialArrayRangeWriteBarrier");
382 private final SnippetInfo g1PreWriteBarrier = snippet(WriteBarrierSnippets.class, "g1PreWriteBarrier", GC_INDEX_LOCATION, GC_LOG_LOCATION);
383 private final SnippetInfo g1ReferentReadBarrier = snippet(WriteBarrierSnippets.class, "g1PreWriteBarrier", GC_INDEX_LOCATION, GC_LOG_LOCATION);
384 private final SnippetInfo g1PostWriteBarrier = snippet(WriteBarrierSnippets.class, "g1PostWriteBarrier", GC_CARD_LOCATION, GC_INDEX_LOCATION, GC_LOG_LOCATION);
385 private final SnippetInfo g1ArrayRangePreWriteBarrier = snippet(WriteBarrierSnippets.class, "g1ArrayRangePreWriteBarrier", GC_INDEX_LOCATION, GC_LOG_LOCATION);
386 private final SnippetInfo g1ArrayRangePostWriteBarrier = snippet(WriteBarrierSnippets.class, "g1ArrayRangePostWriteBarrier", GC_CARD_LOCATION, GC_INDEX_LOCATION, GC_LOG_LOCATION);
387
388 private final CompressEncoding oopEncoding;
389
390 public Templates(HotSpotProviders providers, TargetDescription target, CompressEncoding oopEncoding) {
391 super(providers, providers.getSnippetReflection(), target);
392 this.oopEncoding = oopEncoding;
393 }
394
395 public void lower(SerialWriteBarrier writeBarrier, LoweringTool tool) {
396 Arguments args;
397 if (writeBarrier.usePrecise()) {
398 args = new Arguments(serialPreciseWriteBarrier, writeBarrier.graph().getGuardsStage(), tool.getLoweringStage());
399 args.add("address", writeBarrier.getAddress());
400 } else {
401 args = new Arguments(serialImpreciseWriteBarrier, writeBarrier.graph().getGuardsStage(), tool.getLoweringStage());
402 OffsetAddressNode address = (OffsetAddressNode) writeBarrier.getAddress();
403 args.add("object", address.getBase());
404 }
405 template(args).instantiate(providers.getMetaAccess(), writeBarrier, DEFAULT_REPLACER, args);
406 }
407
408 public void lower(SerialArrayRangeWriteBarrier arrayRangeWriteBarrier, LoweringTool tool) {
409 Arguments args = new Arguments(serialArrayRangeWriteBarrier, arrayRangeWriteBarrier.graph().getGuardsStage(), tool.getLoweringStage());
410 args.add("object", arrayRangeWriteBarrier.getObject());
411 args.add("startIndex", arrayRangeWriteBarrier.getStartIndex());
412 args.add("length", arrayRangeWriteBarrier.getLength());
413 template(args).instantiate(providers.getMetaAccess(), arrayRangeWriteBarrier, DEFAULT_REPLACER, args);
414 }
415
416 public void lower(G1PreWriteBarrier writeBarrierPre, HotSpotRegistersProvider registers, LoweringTool tool) {
417 Arguments args = new Arguments(g1PreWriteBarrier, writeBarrierPre.graph().getGuardsStage(), tool.getLoweringStage());
418 AddressNode address = writeBarrierPre.getAddress();
419 args.add("address", address);
420 if (address instanceof OffsetAddressNode) {
421 args.add("object", ((OffsetAddressNode) address).getBase());
422 } else {
423 args.add("object", null);
424 }
425
426 ValueNode expected = writeBarrierPre.getExpectedObject();
427 if (expected != null && expected.stamp() instanceof NarrowOopStamp) {
428 assert oopEncoding != null;
429 expected = CompressionNode.uncompress(expected, oopEncoding);
430 }
431 args.add("expectedObject", expected);
432
433 args.addConst("doLoad", writeBarrierPre.doLoad());
434 args.addConst("nullCheck", writeBarrierPre.getNullCheck());
435 args.addConst("threadRegister", registers.getThreadRegister());
436 args.addConst("trace", traceBarrier());
437 template(args).instantiate(providers.getMetaAccess(), writeBarrierPre, DEFAULT_REPLACER, args);
438 }
439
440 public void lower(G1ReferentFieldReadBarrier readBarrier, HotSpotRegistersProvider registers, LoweringTool tool) {
441 Arguments args = new Arguments(g1ReferentReadBarrier, readBarrier.graph().getGuardsStage(), tool.getLoweringStage());
442 AddressNode address = readBarrier.getAddress();
443 args.add("address", address);
444 if (address instanceof OffsetAddressNode) {
445 args.add("object", ((OffsetAddressNode) address).getBase());
446 } else {
447 args.add("object", null);
448 }
449
450 ValueNode expected = readBarrier.getExpectedObject();
451 if (expected != null && expected.stamp() instanceof NarrowOopStamp) {
452 assert oopEncoding != null;
453 expected = CompressionNode.uncompress(expected, oopEncoding);
454 }
455
456 args.add("expectedObject", expected);
457 args.addConst("doLoad", readBarrier.doLoad());
458 args.addConst("nullCheck", false);
459 args.addConst("threadRegister", registers.getThreadRegister());
460 args.addConst("trace", traceBarrier());
461 template(args).instantiate(providers.getMetaAccess(), readBarrier, DEFAULT_REPLACER, args);
462 }
463
464 public void lower(G1PostWriteBarrier writeBarrierPost, HotSpotRegistersProvider registers, LoweringTool tool) {
465 StructuredGraph graph = writeBarrierPost.graph();
466 if (writeBarrierPost.alwaysNull()) {
467 graph.removeFixed(writeBarrierPost);
468 return;
469 }
470 Arguments args = new Arguments(g1PostWriteBarrier, graph.getGuardsStage(), tool.getLoweringStage());
471 AddressNode address = writeBarrierPost.getAddress();
472 args.add("address", address);
473 if (address instanceof OffsetAddressNode) {
474 args.add("object", ((OffsetAddressNode) address).getBase());
475 } else {
476 assert writeBarrierPost.usePrecise() : "found imprecise barrier that's not an object access " + writeBarrierPost;
477 args.add("object", null);
478 }
479
480 ValueNode value = writeBarrierPost.getValue();
481 if (value.stamp() instanceof NarrowOopStamp) {
482 assert oopEncoding != null;
483 value = CompressionNode.uncompress(value, oopEncoding);
484 }
485 args.add("value", value);
486
487 args.addConst("usePrecise", writeBarrierPost.usePrecise());
488 args.addConst("threadRegister", registers.getThreadRegister());
489 args.addConst("trace", traceBarrier());
490 template(args).instantiate(providers.getMetaAccess(), writeBarrierPost, DEFAULT_REPLACER, args);
491 }
492
493 public void lower(G1ArrayRangePreWriteBarrier arrayRangeWriteBarrier, HotSpotRegistersProvider registers, LoweringTool tool) {
494 Arguments args = new Arguments(g1ArrayRangePreWriteBarrier, arrayRangeWriteBarrier.graph().getGuardsStage(), tool.getLoweringStage());
495 args.add("object", arrayRangeWriteBarrier.getObject());
496 args.add("startIndex", arrayRangeWriteBarrier.getStartIndex());
497 args.add("length", arrayRangeWriteBarrier.getLength());
498 args.addConst("threadRegister", registers.getThreadRegister());
499 template(args).instantiate(providers.getMetaAccess(), arrayRangeWriteBarrier, DEFAULT_REPLACER, args);
500 }
501
502 public void lower(G1ArrayRangePostWriteBarrier arrayRangeWriteBarrier, HotSpotRegistersProvider registers, LoweringTool tool) {
503 Arguments args = new Arguments(g1ArrayRangePostWriteBarrier, arrayRangeWriteBarrier.graph().getGuardsStage(), tool.getLoweringStage());
504 args.add("object", arrayRangeWriteBarrier.getObject());
505 args.add("startIndex", arrayRangeWriteBarrier.getStartIndex());
506 args.add("length", arrayRangeWriteBarrier.getLength());
507 args.addConst("threadRegister", registers.getThreadRegister());
508 template(args).instantiate(providers.getMetaAccess(), arrayRangeWriteBarrier, DEFAULT_REPLACER, args);
509 }
510 }
511
512 /**
513 * Log method of debugging purposes.
514 */
515 public static void log(boolean enabled, String format, long value) {
516 if (enabled) {
517 Log.printf(format, value);
518 }
519 }
520
521 public static void log(boolean enabled, String format, long value1, long value2) {
522 if (enabled) {
523 Log.printf(format, value1, value2);
524 }
525 }
526
527 public static void log(boolean enabled, String format, long value1, long value2, long value3) {
528 if (enabled) {
529 Log.printf(format, value1, value2, value3);
530 }
531 }
532
533 public static boolean traceBarrier() {
534 return GraalOptions.GCDebugStartCycle.getValue() > 0 &&
535 ((int) Word.unsigned(HotSpotReplacementsUtil.gcTotalCollectionsAddress(INJECTED_VMCONFIG)).readLong(0) > GraalOptions.GCDebugStartCycle.getValue());
536 }
537
538 /**
539 * Validation helper method which performs sanity checks on write operations. The addresses of
540 * both the object and the value being written are checked in order to determine if they reside
541 * in a valid heap region. If an object is stale, an invalid access is performed in order to
542 * prematurely crash the VM and debug the stack trace of the faulty method.
543 */
544 public static void validateObject(Object parent, Object child) {
545 if (verifyOops(INJECTED_VMCONFIG) && child != null && !validateOop(VALIDATE_OBJECT, parent, child)) {
546 log(true, "Verification ERROR, Parent: %p Child: %p\n", Word.objectToTrackedPointer(parent).rawValue(), Word.objectToTrackedPointer(child).rawValue());
547 DirectObjectStoreNode.storeObject(null, 0, 0, null, LocationIdentity.any(), JavaKind.Object);
548 }
549 }
550
551 public static final ForeignCallDescriptor VALIDATE_OBJECT = new ForeignCallDescriptor("validate_object", boolean.class, Word.class, Word.class);
552
553 @NodeIntrinsic(ForeignCallNode.class)
554 private static native boolean validateOop(@ConstantNodeParameter ForeignCallDescriptor descriptor, Object parent, Object object);
555
556 }