1 /*
2 * Copyright (c) 2012, 2018, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 */
23
24
25 package org.graalvm.compiler.hotspot.replacements;
26
27 import static jdk.vm.ci.code.MemoryBarriers.STORE_LOAD;
28 import static org.graalvm.compiler.hotspot.GraalHotSpotVMConfig.INJECTED_VMCONFIG;
29 import static org.graalvm.compiler.hotspot.GraalHotSpotVMConfigBase.INJECTED_METAACCESS;
30 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.cardTableShift;
31 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.dirtyCardValue;
32 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.g1CardQueueBufferOffset;
33 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.g1CardQueueIndexOffset;
34 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.g1SATBQueueBufferOffset;
35 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.g1SATBQueueIndexOffset;
36 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.g1SATBQueueMarkingOffset;
37 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.g1YoungCardValue;
38 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.registerAsWord;
39 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.verifyOop;
40 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.verifyOops;
41 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.wordSize;
42 import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.FREQUENT_PROBABILITY;
43 import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.LIKELY_PROBABILITY;
44 import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.NOT_FREQUENT_PROBABILITY;
45 import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.probability;
46 import static org.graalvm.compiler.replacements.SnippetTemplate.DEFAULT_REPLACER;
47
48 import org.graalvm.compiler.api.replacements.Snippet;
49 import org.graalvm.compiler.api.replacements.Snippet.ConstantParameter;
50 import org.graalvm.compiler.core.common.CompressEncoding;
51 import org.graalvm.compiler.core.common.GraalOptions;
52 import org.graalvm.compiler.core.common.spi.ForeignCallDescriptor;
53 import org.graalvm.compiler.debug.DebugHandlersFactory;
54 import org.graalvm.compiler.graph.Node.ConstantNodeParameter;
55 import org.graalvm.compiler.graph.Node.NodeIntrinsic;
56 import org.graalvm.compiler.hotspot.GraalHotSpotVMConfig;
57 import org.graalvm.compiler.hotspot.gc.g1.G1ArrayRangePostWriteBarrier;
58 import org.graalvm.compiler.hotspot.gc.g1.G1ArrayRangePreWriteBarrier;
59 import org.graalvm.compiler.hotspot.gc.g1.G1PostWriteBarrier;
60 import org.graalvm.compiler.hotspot.gc.g1.G1PreWriteBarrier;
61 import org.graalvm.compiler.hotspot.gc.g1.G1ReferentFieldReadBarrier;
62 import org.graalvm.compiler.hotspot.gc.shared.SerialArrayRangeWriteBarrier;
63 import org.graalvm.compiler.hotspot.gc.shared.SerialWriteBarrier;
64 import org.graalvm.compiler.hotspot.meta.HotSpotProviders;
65 import org.graalvm.compiler.hotspot.meta.HotSpotRegistersProvider;
66 import org.graalvm.compiler.hotspot.nodes.GraalHotSpotVMConfigNode;
67 import org.graalvm.compiler.hotspot.nodes.HotSpotCompressionNode;
68 import org.graalvm.compiler.hotspot.nodes.VMErrorNode;
69 import org.graalvm.compiler.nodes.NamedLocationIdentity;
70 import org.graalvm.compiler.nodes.NodeView;
71 import org.graalvm.compiler.nodes.PiNode;
72 import org.graalvm.compiler.nodes.SnippetAnchorNode;
73 import org.graalvm.compiler.nodes.StructuredGraph;
74 import org.graalvm.compiler.nodes.ValueNode;
75 import org.graalvm.compiler.nodes.extended.FixedValueAnchorNode;
76 import org.graalvm.compiler.nodes.extended.ForeignCallNode;
77 import org.graalvm.compiler.nodes.extended.MembarNode;
78 import org.graalvm.compiler.nodes.extended.NullCheckNode;
79 import org.graalvm.compiler.nodes.memory.HeapAccess.BarrierType;
80 import org.graalvm.compiler.nodes.memory.address.AddressNode;
81 import org.graalvm.compiler.nodes.memory.address.AddressNode.Address;
82 import org.graalvm.compiler.nodes.memory.address.OffsetAddressNode;
83 import org.graalvm.compiler.nodes.spi.LoweringTool;
84 import org.graalvm.compiler.nodes.type.NarrowOopStamp;
85 import org.graalvm.compiler.options.OptionValues;
86 import org.graalvm.compiler.replacements.Log;
87 import org.graalvm.compiler.replacements.ReplacementsUtil;
88 import org.graalvm.compiler.replacements.SnippetCounter;
89 import org.graalvm.compiler.replacements.SnippetCounter.Group;
90 import org.graalvm.compiler.replacements.SnippetTemplate.AbstractTemplates;
91 import org.graalvm.compiler.replacements.SnippetTemplate.Arguments;
92 import org.graalvm.compiler.replacements.SnippetTemplate.SnippetInfo;
93 import org.graalvm.compiler.replacements.Snippets;
94 import org.graalvm.compiler.replacements.nodes.AssertionNode;
95 import org.graalvm.compiler.replacements.nodes.DirectStoreNode;
96 import org.graalvm.compiler.word.Word;
97 import jdk.internal.vm.compiler.word.LocationIdentity;
98 import jdk.internal.vm.compiler.word.Pointer;
99 import jdk.internal.vm.compiler.word.UnsignedWord;
100 import jdk.internal.vm.compiler.word.WordFactory;
101
102 import jdk.vm.ci.code.Register;
103 import jdk.vm.ci.code.TargetDescription;
104 import jdk.vm.ci.meta.JavaKind;
105
106 public class WriteBarrierSnippets implements Snippets {
107
108 static class Counters {
109 Counters(SnippetCounter.Group.Factory factory) {
110 Group countersWriteBarriers = factory.createSnippetCounterGroup("WriteBarriers");
111 serialWriteBarrierCounter = new SnippetCounter(countersWriteBarriers, "serialWriteBarrier", "Number of Serial Write Barriers");
112 g1AttemptedPreWriteBarrierCounter = new SnippetCounter(countersWriteBarriers, "g1AttemptedPreWriteBarrier", "Number of attempted G1 Pre Write Barriers");
113 g1EffectivePreWriteBarrierCounter = new SnippetCounter(countersWriteBarriers, "g1EffectivePreWriteBarrier", "Number of effective G1 Pre Write Barriers");
114 g1ExecutedPreWriteBarrierCounter = new SnippetCounter(countersWriteBarriers, "g1ExecutedPreWriteBarrier", "Number of executed G1 Pre Write Barriers");
115 g1AttemptedPostWriteBarrierCounter = new SnippetCounter(countersWriteBarriers, "g1AttemptedPostWriteBarrier", "Number of attempted G1 Post Write Barriers");
116 g1EffectiveAfterXORPostWriteBarrierCounter = new SnippetCounter(countersWriteBarriers, "g1EffectiveAfterXORPostWriteBarrier",
117 "Number of effective G1 Post Write Barriers (after passing the XOR test)");
118 g1EffectiveAfterNullPostWriteBarrierCounter = new SnippetCounter(countersWriteBarriers, "g1EffectiveAfterNullPostWriteBarrier",
119 "Number of effective G1 Post Write Barriers (after passing the NULL test)");
120 g1ExecutedPostWriteBarrierCounter = new SnippetCounter(countersWriteBarriers, "g1ExecutedPostWriteBarrier", "Number of executed G1 Post Write Barriers");
121
122 }
123
124 final SnippetCounter serialWriteBarrierCounter;
125 final SnippetCounter g1AttemptedPreWriteBarrierCounter;
126 final SnippetCounter g1EffectivePreWriteBarrierCounter;
127 final SnippetCounter g1ExecutedPreWriteBarrierCounter;
128 final SnippetCounter g1AttemptedPostWriteBarrierCounter;
129 final SnippetCounter g1EffectiveAfterXORPostWriteBarrierCounter;
130 final SnippetCounter g1EffectiveAfterNullPostWriteBarrierCounter;
131 final SnippetCounter g1ExecutedPostWriteBarrierCounter;
132 }
133
134 public static final LocationIdentity GC_CARD_LOCATION = NamedLocationIdentity.mutable("GC-Card");
135 public static final LocationIdentity GC_LOG_LOCATION = NamedLocationIdentity.mutable("GC-Log");
136 public static final LocationIdentity GC_INDEX_LOCATION = NamedLocationIdentity.mutable("GC-Index");
137
138 private static void serialWriteBarrier(Pointer ptr, Counters counters) {
139 counters.serialWriteBarrierCounter.inc();
140 final long startAddress = GraalHotSpotVMConfigNode.cardTableAddress();
141 Word base = (Word) ptr.unsignedShiftRight(cardTableShift(INJECTED_VMCONFIG));
142 if (((int) startAddress) == startAddress && GraalHotSpotVMConfigNode.isCardTableAddressConstant()) {
143 base.writeByte((int) startAddress, (byte) 0, GC_CARD_LOCATION);
144 } else {
145 base.writeByte(WordFactory.unsigned(startAddress), (byte) 0, GC_CARD_LOCATION);
146 }
147 }
148
149 @Snippet
150 public static void serialImpreciseWriteBarrier(Object object, @ConstantParameter boolean verifyBarrier, @ConstantParameter Counters counters) {
151 if (verifyBarrier) {
152 verifyNotArray(object);
153 }
154 serialWriteBarrier(Word.objectToTrackedPointer(object), counters);
155 }
156
157 @Snippet
158 public static void serialPreciseWriteBarrier(Address address, @ConstantParameter Counters counters) {
159 serialWriteBarrier(Word.fromAddress(address), counters);
160 }
161
162 @Snippet
163 public static void serialArrayRangeWriteBarrier(Address address, int length, @ConstantParameter int elementStride) {
164 if (length == 0) {
165 return;
166 }
167 int cardShift = cardTableShift(INJECTED_VMCONFIG);
168 final long cardStart = GraalHotSpotVMConfigNode.cardTableAddress();
169 long start = getPointerToFirstArrayElement(address, length, elementStride) >>> cardShift;
170 long end = getPointerToLastArrayElement(address, length, elementStride) >>> cardShift;
171 long count = end - start + 1;
172 while (count-- > 0) {
173 DirectStoreNode.storeBoolean((start + cardStart) + count, false, JavaKind.Boolean);
174 }
175 }
176
177 @Snippet
178 public static void g1PreWriteBarrier(Address address, Object object, Object expectedObject, @ConstantParameter boolean doLoad, @ConstantParameter boolean nullCheck,
179 @ConstantParameter Register threadRegister, @ConstantParameter boolean trace, @ConstantParameter Counters counters) {
180 if (nullCheck) {
181 NullCheckNode.nullCheck(address);
182 }
183 Word thread = registerAsWord(threadRegister);
184 verifyOop(object);
185 Object fixedExpectedObject = FixedValueAnchorNode.getObject(expectedObject);
186 Word field = Word.fromAddress(address);
187 Pointer previousOop = Word.objectToTrackedPointer(fixedExpectedObject);
188 byte markingValue = thread.readByte(g1SATBQueueMarkingOffset(INJECTED_VMCONFIG));
189 int gcCycle = 0;
190 if (trace) {
191 Pointer gcTotalCollectionsAddress = WordFactory.pointer(HotSpotReplacementsUtil.gcTotalCollectionsAddress(INJECTED_VMCONFIG));
192 gcCycle = (int) gcTotalCollectionsAddress.readLong(0);
193 log(trace, "[%d] G1-Pre Thread %p Object %p\n", gcCycle, thread.rawValue(), Word.objectToTrackedPointer(object).rawValue());
194 log(trace, "[%d] G1-Pre Thread %p Expected Object %p\n", gcCycle, thread.rawValue(), Word.objectToTrackedPointer(fixedExpectedObject).rawValue());
195 log(trace, "[%d] G1-Pre Thread %p Field %p\n", gcCycle, thread.rawValue(), field.rawValue());
196 log(trace, "[%d] G1-Pre Thread %p Marking %d\n", gcCycle, thread.rawValue(), markingValue);
197 log(trace, "[%d] G1-Pre Thread %p DoLoad %d\n", gcCycle, thread.rawValue(), doLoad ? 1L : 0L);
198 }
199 counters.g1AttemptedPreWriteBarrierCounter.inc();
200 // If the concurrent marker is enabled, the barrier is issued.
201 if (probability(NOT_FREQUENT_PROBABILITY, markingValue != (byte) 0)) {
202 // If the previous value has to be loaded (before the write), the load is issued.
203 // The load is always issued except the cases of CAS and referent field.
204 if (probability(LIKELY_PROBABILITY, doLoad)) {
205 previousOop = Word.objectToTrackedPointer(field.readObject(0, BarrierType.NONE));
206 if (trace) {
207 log(trace, "[%d] G1-Pre Thread %p Previous Object %p\n ", gcCycle, thread.rawValue(), previousOop.rawValue());
208 verifyOop(previousOop.toObject());
209 }
210 }
211 counters.g1EffectivePreWriteBarrierCounter.inc();
212 // If the previous value is null the barrier should not be issued.
213 if (probability(FREQUENT_PROBABILITY, previousOop.notEqual(0))) {
214 counters.g1ExecutedPreWriteBarrierCounter.inc();
215 // If the thread-local SATB buffer is full issue a native call which will
216 // initialize a new one and add the entry.
217 Word indexAddress = thread.add(g1SATBQueueIndexOffset(INJECTED_VMCONFIG));
218 Word indexValue = indexAddress.readWord(0);
219 if (probability(FREQUENT_PROBABILITY, indexValue.notEqual(0))) {
220 Word bufferAddress = thread.readWord(g1SATBQueueBufferOffset(INJECTED_VMCONFIG));
221 Word nextIndex = indexValue.subtract(wordSize());
222 Word logAddress = bufferAddress.add(nextIndex);
223 // Log the object to be marked as well as update the SATB's buffer next index.
224 logAddress.writeWord(0, previousOop, GC_LOG_LOCATION);
225 indexAddress.writeWord(0, nextIndex, GC_INDEX_LOCATION);
226 } else {
227 g1PreBarrierStub(G1WBPRECALL, previousOop.toObject());
228 }
229 }
230 }
231 }
232
233 @Snippet
234 public static void g1PostWriteBarrier(Address address, Object object, Object value, @ConstantParameter boolean usePrecise, @ConstantParameter boolean verifyBarrier,
235 @ConstantParameter Register threadRegister, @ConstantParameter boolean trace, @ConstantParameter Counters counters) {
236 Word thread = registerAsWord(threadRegister);
237 Object fixedValue = FixedValueAnchorNode.getObject(value);
238 verifyOop(object);
239 verifyOop(fixedValue);
240 validateObject(object, fixedValue);
241 Pointer oop;
242 if (usePrecise) {
243 oop = Word.fromAddress(address);
244 } else {
245 if (verifyBarrier) {
246 verifyNotArray(object);
247 }
248 oop = Word.objectToTrackedPointer(object);
249 }
250 int gcCycle = 0;
251 if (trace) {
252 Pointer gcTotalCollectionsAddress = WordFactory.pointer(HotSpotReplacementsUtil.gcTotalCollectionsAddress(INJECTED_VMCONFIG));
253 gcCycle = (int) gcTotalCollectionsAddress.readLong(0);
254 log(trace, "[%d] G1-Post Thread: %p Object: %p\n", gcCycle, thread.rawValue(), Word.objectToTrackedPointer(object).rawValue());
255 log(trace, "[%d] G1-Post Thread: %p Field: %p\n", gcCycle, thread.rawValue(), oop.rawValue());
256 }
257 Pointer writtenValue = Word.objectToTrackedPointer(fixedValue);
258 // The result of the xor reveals whether the installed pointer crosses heap regions.
259 // In case it does the write barrier has to be issued.
260 final int logOfHeapRegionGrainBytes = GraalHotSpotVMConfigNode.logOfHeapRegionGrainBytes();
261 UnsignedWord xorResult = (oop.xor(writtenValue)).unsignedShiftRight(logOfHeapRegionGrainBytes);
262
263 // Calculate the address of the card to be enqueued to the
264 // thread local card queue.
265 UnsignedWord cardBase = oop.unsignedShiftRight(cardTableShift(INJECTED_VMCONFIG));
266 final long startAddress = GraalHotSpotVMConfigNode.cardTableAddress();
267 int displacement = 0;
268 if (((int) startAddress) == startAddress && GraalHotSpotVMConfigNode.isCardTableAddressConstant()) {
269 displacement = (int) startAddress;
270 } else {
271 cardBase = cardBase.add(WordFactory.unsigned(startAddress));
272 }
273 Word cardAddress = (Word) cardBase.add(displacement);
274
275 counters.g1AttemptedPostWriteBarrierCounter.inc();
276 if (probability(FREQUENT_PROBABILITY, xorResult.notEqual(0))) {
277 counters.g1EffectiveAfterXORPostWriteBarrierCounter.inc();
278
279 // If the written value is not null continue with the barrier addition.
280 if (probability(FREQUENT_PROBABILITY, writtenValue.notEqual(0))) {
281 byte cardByte = cardAddress.readByte(0, GC_CARD_LOCATION);
282 counters.g1EffectiveAfterNullPostWriteBarrierCounter.inc();
283
284 // If the card is already dirty, (hence already enqueued) skip the insertion.
285 if (probability(NOT_FREQUENT_PROBABILITY, cardByte != g1YoungCardValue(INJECTED_VMCONFIG))) {
286 MembarNode.memoryBarrier(STORE_LOAD, GC_CARD_LOCATION);
287 byte cardByteReload = cardAddress.readByte(0, GC_CARD_LOCATION);
288 if (probability(NOT_FREQUENT_PROBABILITY, cardByteReload != dirtyCardValue(INJECTED_VMCONFIG))) {
289 log(trace, "[%d] G1-Post Thread: %p Card: %p \n", gcCycle, thread.rawValue(), WordFactory.unsigned((int) cardByte).rawValue());
290 cardAddress.writeByte(0, (byte) 0, GC_CARD_LOCATION);
291 counters.g1ExecutedPostWriteBarrierCounter.inc();
292
293 // If the thread local card queue is full, issue a native call which will
294 // initialize a new one and add the card entry.
295 Word indexAddress = thread.add(g1CardQueueIndexOffset(INJECTED_VMCONFIG));
296 Word indexValue = thread.readWord(g1CardQueueIndexOffset(INJECTED_VMCONFIG));
297 if (probability(FREQUENT_PROBABILITY, indexValue.notEqual(0))) {
298 Word bufferAddress = thread.readWord(g1CardQueueBufferOffset(INJECTED_VMCONFIG));
299 Word nextIndex = indexValue.subtract(wordSize());
300 Word logAddress = bufferAddress.add(nextIndex);
301 // Log the object to be scanned as well as update
302 // the card queue's next index.
303 logAddress.writeWord(0, cardAddress, GC_LOG_LOCATION);
304 indexAddress.writeWord(0, nextIndex, GC_INDEX_LOCATION);
305 } else {
306 g1PostBarrierStub(G1WBPOSTCALL, cardAddress);
307 }
308 }
309 }
310 }
311 }
312 }
313
314 private static void verifyNotArray(Object object) {
315 if (object != null) {
316 // Manually build the null check and cast because we're in snippet that's lowered late.
317 AssertionNode.assertion(false, !PiNode.piCastNonNull(object, SnippetAnchorNode.anchor()).getClass().isArray(), "imprecise card mark used with array");
318 }
319 }
320
321 @Snippet
322 public static void g1ArrayRangePreWriteBarrier(Address address, int length, @ConstantParameter int elementStride, @ConstantParameter Register threadRegister) {
323 Word thread = registerAsWord(threadRegister);
324 byte markingValue = thread.readByte(g1SATBQueueMarkingOffset(INJECTED_VMCONFIG));
325 // If the concurrent marker is not enabled or the vector length is zero, return.
326 if (markingValue == (byte) 0 || length == 0) {
327 return;
328 }
329 Word bufferAddress = thread.readWord(g1SATBQueueBufferOffset(INJECTED_VMCONFIG));
330 Word indexAddress = thread.add(g1SATBQueueIndexOffset(INJECTED_VMCONFIG));
331 long indexValue = indexAddress.readWord(0).rawValue();
332 final int scale = ReplacementsUtil.arrayIndexScale(INJECTED_METAACCESS, JavaKind.Object);
333 long start = getPointerToFirstArrayElement(address, length, elementStride);
334
335 for (int i = 0; i < length; i++) {
336 Word arrElemPtr = WordFactory.pointer(start + i * scale);
337 Pointer oop = Word.objectToTrackedPointer(arrElemPtr.readObject(0, BarrierType.NONE));
338 verifyOop(oop.toObject());
339 if (oop.notEqual(0)) {
340 if (indexValue != 0) {
341 indexValue = indexValue - wordSize();
342 Word logAddress = bufferAddress.add(WordFactory.unsigned(indexValue));
343 // Log the object to be marked as well as update the SATB's buffer next index.
344 logAddress.writeWord(0, oop, GC_LOG_LOCATION);
345 indexAddress.writeWord(0, WordFactory.unsigned(indexValue), GC_INDEX_LOCATION);
346 } else {
347 g1PreBarrierStub(G1WBPRECALL, oop.toObject());
348 }
349 }
350 }
351 }
352
353 @Snippet
354 public static void g1ArrayRangePostWriteBarrier(Address address, int length, @ConstantParameter int elementStride, @ConstantParameter Register threadRegister) {
355 if (length == 0) {
356 return;
357 }
358 Word thread = registerAsWord(threadRegister);
359 Word bufferAddress = thread.readWord(g1CardQueueBufferOffset(INJECTED_VMCONFIG));
360 Word indexAddress = thread.add(g1CardQueueIndexOffset(INJECTED_VMCONFIG));
361 long indexValue = thread.readWord(g1CardQueueIndexOffset(INJECTED_VMCONFIG)).rawValue();
362
363 int cardShift = cardTableShift(INJECTED_VMCONFIG);
364 final long cardStart = GraalHotSpotVMConfigNode.cardTableAddress();
365 long start = getPointerToFirstArrayElement(address, length, elementStride) >>> cardShift;
366 long end = getPointerToLastArrayElement(address, length, elementStride) >>> cardShift;
367 long count = end - start + 1;
368
369 while (count-- > 0) {
370 Word cardAddress = WordFactory.unsigned((start + cardStart) + count);
371 byte cardByte = cardAddress.readByte(0, GC_CARD_LOCATION);
372 // If the card is already dirty, (hence already enqueued) skip the insertion.
373 if (probability(NOT_FREQUENT_PROBABILITY, cardByte != g1YoungCardValue(INJECTED_VMCONFIG))) {
374 MembarNode.memoryBarrier(STORE_LOAD, GC_CARD_LOCATION);
375 byte cardByteReload = cardAddress.readByte(0, GC_CARD_LOCATION);
376 if (probability(NOT_FREQUENT_PROBABILITY, cardByteReload != dirtyCardValue(INJECTED_VMCONFIG))) {
377 cardAddress.writeByte(0, (byte) 0, GC_CARD_LOCATION);
378 // If the thread local card queue is full, issue a native call which will
379 // initialize a new one and add the card entry.
380 if (indexValue != 0) {
381 indexValue = indexValue - wordSize();
382 Word logAddress = bufferAddress.add(WordFactory.unsigned(indexValue));
383 // Log the object to be scanned as well as update
384 // the card queue's next index.
385 logAddress.writeWord(0, cardAddress, GC_LOG_LOCATION);
386 indexAddress.writeWord(0, WordFactory.unsigned(indexValue), GC_INDEX_LOCATION);
387 } else {
388 g1PostBarrierStub(G1WBPOSTCALL, cardAddress);
389 }
390 }
391 }
392 }
393 }
394
395 private static long getPointerToFirstArrayElement(Address address, int length, int elementStride) {
396 long result = Word.fromAddress(address).rawValue();
397 if (elementStride < 0) {
398 // the address points to the place after the last array element
399 result = result + elementStride * length;
400 }
401 return result;
402 }
403
404 private static long getPointerToLastArrayElement(Address address, int length, int elementStride) {
405 long result = Word.fromAddress(address).rawValue();
406 if (elementStride < 0) {
407 // the address points to the place after the last array element
408 result = result + elementStride;
409 } else {
410 result = result + (length - 1) * elementStride;
411 }
412 return result;
413 }
414
415 public static final ForeignCallDescriptor G1WBPRECALL = new ForeignCallDescriptor("write_barrier_pre", void.class, Object.class);
416
417 @NodeIntrinsic(ForeignCallNode.class)
418 private static native void g1PreBarrierStub(@ConstantNodeParameter ForeignCallDescriptor descriptor, Object object);
419
420 public static final ForeignCallDescriptor G1WBPOSTCALL = new ForeignCallDescriptor("write_barrier_post", void.class, Word.class);
421
422 @NodeIntrinsic(ForeignCallNode.class)
423 public static native void g1PostBarrierStub(@ConstantNodeParameter ForeignCallDescriptor descriptor, Word card);
424
425 public static class Templates extends AbstractTemplates {
426
427 private final SnippetInfo serialImpreciseWriteBarrier = snippet(WriteBarrierSnippets.class, "serialImpreciseWriteBarrier", GC_CARD_LOCATION);
428 private final SnippetInfo serialPreciseWriteBarrier = snippet(WriteBarrierSnippets.class, "serialPreciseWriteBarrier", GC_CARD_LOCATION);
429 private final SnippetInfo serialArrayRangeWriteBarrier = snippet(WriteBarrierSnippets.class, "serialArrayRangeWriteBarrier");
430 private final SnippetInfo g1PreWriteBarrier = snippet(WriteBarrierSnippets.class, "g1PreWriteBarrier", GC_INDEX_LOCATION, GC_LOG_LOCATION);
431 private final SnippetInfo g1ReferentReadBarrier = g1PreWriteBarrier;
432 private final SnippetInfo g1PostWriteBarrier = snippet(WriteBarrierSnippets.class, "g1PostWriteBarrier", GC_CARD_LOCATION, GC_INDEX_LOCATION, GC_LOG_LOCATION);
433 private final SnippetInfo g1ArrayRangePreWriteBarrier = snippet(WriteBarrierSnippets.class, "g1ArrayRangePreWriteBarrier", GC_INDEX_LOCATION, GC_LOG_LOCATION);
434 private final SnippetInfo g1ArrayRangePostWriteBarrier = snippet(WriteBarrierSnippets.class, "g1ArrayRangePostWriteBarrier", GC_CARD_LOCATION, GC_INDEX_LOCATION, GC_LOG_LOCATION);
435
436 private final CompressEncoding oopEncoding;
437 private final Counters counters;
438 private final boolean verifyBarrier;
439 private final long gcTotalCollectionsAddress;
440
441 public Templates(OptionValues options, Iterable<DebugHandlersFactory> factories, Group.Factory factory, HotSpotProviders providers, TargetDescription target,
442 GraalHotSpotVMConfig config) {
443 super(options, factories, providers, providers.getSnippetReflection(), target);
444 this.oopEncoding = config.useCompressedOops ? config.getOopEncoding() : null;
445 this.verifyBarrier = ReplacementsUtil.REPLACEMENTS_ASSERTIONS_ENABLED || config.verifyBeforeGC || config.verifyAfterGC;
446 this.gcTotalCollectionsAddress = config.gcTotalCollectionsAddress();
447 this.counters = new Counters(factory);
448 }
449
450 public boolean traceBarrier(StructuredGraph graph) {
451 long startCycle = GraalOptions.GCDebugStartCycle.getValue(graph.getOptions());
452 return startCycle > 0 && ((Pointer) WordFactory.pointer(gcTotalCollectionsAddress)).readLong(0) > startCycle;
453 }
454
455 public void lower(SerialWriteBarrier writeBarrier, LoweringTool tool) {
456 Arguments args;
457 if (writeBarrier.usePrecise()) {
458 args = new Arguments(serialPreciseWriteBarrier, writeBarrier.graph().getGuardsStage(), tool.getLoweringStage());
459 args.add("address", writeBarrier.getAddress());
460 } else {
461 args = new Arguments(serialImpreciseWriteBarrier, writeBarrier.graph().getGuardsStage(), tool.getLoweringStage());
462 OffsetAddressNode address = (OffsetAddressNode) writeBarrier.getAddress();
463 args.add("object", address.getBase());
464 args.addConst("verifyBarrier", verifyBarrier);
465 }
466 args.addConst("counters", counters);
467 template(writeBarrier, args).instantiate(providers.getMetaAccess(), writeBarrier, DEFAULT_REPLACER, args);
468 }
469
470 public void lower(SerialArrayRangeWriteBarrier arrayRangeWriteBarrier, LoweringTool tool) {
471 Arguments args = new Arguments(serialArrayRangeWriteBarrier, arrayRangeWriteBarrier.graph().getGuardsStage(), tool.getLoweringStage());
472 args.add("address", arrayRangeWriteBarrier.getAddress());
473 args.add("length", arrayRangeWriteBarrier.getLength());
474 args.addConst("elementStride", arrayRangeWriteBarrier.getElementStride());
475 template(arrayRangeWriteBarrier, args).instantiate(providers.getMetaAccess(), arrayRangeWriteBarrier, DEFAULT_REPLACER, args);
476 }
477
478 public void lower(G1PreWriteBarrier writeBarrierPre, HotSpotRegistersProvider registers, LoweringTool tool) {
479 Arguments args = new Arguments(g1PreWriteBarrier, writeBarrierPre.graph().getGuardsStage(), tool.getLoweringStage());
480 AddressNode address = writeBarrierPre.getAddress();
481 args.add("address", address);
482 if (address instanceof OffsetAddressNode) {
483 args.add("object", ((OffsetAddressNode) address).getBase());
484 } else {
485 args.add("object", null);
486 }
487
488 ValueNode expected = writeBarrierPre.getExpectedObject();
489 if (expected != null && expected.stamp(NodeView.DEFAULT) instanceof NarrowOopStamp) {
490 assert oopEncoding != null;
491 expected = HotSpotCompressionNode.uncompress(expected, oopEncoding);
492 }
493 args.add("expectedObject", expected);
494
495 args.addConst("doLoad", writeBarrierPre.doLoad());
496 args.addConst("nullCheck", writeBarrierPre.getNullCheck());
497 args.addConst("threadRegister", registers.getThreadRegister());
498 args.addConst("trace", traceBarrier(writeBarrierPre.graph()));
499 args.addConst("counters", counters);
500 template(writeBarrierPre, args).instantiate(providers.getMetaAccess(), writeBarrierPre, DEFAULT_REPLACER, args);
501 }
502
503 public void lower(G1ReferentFieldReadBarrier readBarrier, HotSpotRegistersProvider registers, LoweringTool tool) {
504 Arguments args = new Arguments(g1ReferentReadBarrier, readBarrier.graph().getGuardsStage(), tool.getLoweringStage());
505 AddressNode address = readBarrier.getAddress();
506 args.add("address", address);
507 if (address instanceof OffsetAddressNode) {
508 args.add("object", ((OffsetAddressNode) address).getBase());
509 } else {
510 args.add("object", null);
511 }
512
513 ValueNode expected = readBarrier.getExpectedObject();
514 if (expected != null && expected.stamp(NodeView.DEFAULT) instanceof NarrowOopStamp) {
515 assert oopEncoding != null;
516 expected = HotSpotCompressionNode.uncompress(expected, oopEncoding);
517 }
518
519 args.add("expectedObject", expected);
520 args.addConst("doLoad", readBarrier.doLoad());
521 args.addConst("nullCheck", false);
522 args.addConst("threadRegister", registers.getThreadRegister());
523 args.addConst("trace", traceBarrier(readBarrier.graph()));
524 args.addConst("counters", counters);
525 template(readBarrier, args).instantiate(providers.getMetaAccess(), readBarrier, DEFAULT_REPLACER, args);
526 }
527
528 public void lower(G1PostWriteBarrier writeBarrierPost, HotSpotRegistersProvider registers, LoweringTool tool) {
529 StructuredGraph graph = writeBarrierPost.graph();
530 if (writeBarrierPost.alwaysNull()) {
531 graph.removeFixed(writeBarrierPost);
532 return;
533 }
534 Arguments args = new Arguments(g1PostWriteBarrier, graph.getGuardsStage(), tool.getLoweringStage());
535 AddressNode address = writeBarrierPost.getAddress();
536 args.add("address", address);
537 if (address instanceof OffsetAddressNode) {
538 args.add("object", ((OffsetAddressNode) address).getBase());
539 } else {
540 assert writeBarrierPost.usePrecise() : "found imprecise barrier that's not an object access " + writeBarrierPost;
541 args.add("object", null);
542 }
543
544 ValueNode value = writeBarrierPost.getValue();
545 if (value.stamp(NodeView.DEFAULT) instanceof NarrowOopStamp) {
546 assert oopEncoding != null;
547 value = HotSpotCompressionNode.uncompress(value, oopEncoding);
548 }
549 args.add("value", value);
550
551 args.addConst("usePrecise", writeBarrierPost.usePrecise());
552 args.addConst("verifyBarrier", verifyBarrier);
553 args.addConst("threadRegister", registers.getThreadRegister());
554 args.addConst("trace", traceBarrier(writeBarrierPost.graph()));
555 args.addConst("counters", counters);
556 template(writeBarrierPost, args).instantiate(providers.getMetaAccess(), writeBarrierPost, DEFAULT_REPLACER, args);
557 }
558
559 public void lower(G1ArrayRangePreWriteBarrier arrayRangeWriteBarrier, HotSpotRegistersProvider registers, LoweringTool tool) {
560 Arguments args = new Arguments(g1ArrayRangePreWriteBarrier, arrayRangeWriteBarrier.graph().getGuardsStage(), tool.getLoweringStage());
561 args.add("address", arrayRangeWriteBarrier.getAddress());
562 args.add("length", arrayRangeWriteBarrier.getLength());
563 args.addConst("elementStride", arrayRangeWriteBarrier.getElementStride());
564 args.addConst("threadRegister", registers.getThreadRegister());
565 template(arrayRangeWriteBarrier, args).instantiate(providers.getMetaAccess(), arrayRangeWriteBarrier, DEFAULT_REPLACER, args);
566 }
567
568 public void lower(G1ArrayRangePostWriteBarrier arrayRangeWriteBarrier, HotSpotRegistersProvider registers, LoweringTool tool) {
569 Arguments args = new Arguments(g1ArrayRangePostWriteBarrier, arrayRangeWriteBarrier.graph().getGuardsStage(), tool.getLoweringStage());
570 args.add("address", arrayRangeWriteBarrier.getAddress());
571 args.add("length", arrayRangeWriteBarrier.getLength());
572 args.addConst("elementStride", arrayRangeWriteBarrier.getElementStride());
573 args.addConst("threadRegister", registers.getThreadRegister());
574 template(arrayRangeWriteBarrier, args).instantiate(providers.getMetaAccess(), arrayRangeWriteBarrier, DEFAULT_REPLACER, args);
575 }
576 }
577
578 /**
579 * Log method of debugging purposes.
580 */
581 public static void log(boolean enabled, String format, long value) {
582 if (enabled) {
583 Log.printf(format, value);
584 }
585 }
586
587 public static void log(boolean enabled, String format, long value1, long value2) {
588 if (enabled) {
589 Log.printf(format, value1, value2);
590 }
591 }
592
593 public static void log(boolean enabled, String format, long value1, long value2, long value3) {
594 if (enabled) {
595 Log.printf(format, value1, value2, value3);
596 }
597 }
598
599 /**
600 * Validation helper method which performs sanity checks on write operations. The addresses of
601 * both the object and the value being written are checked in order to determine if they reside
602 * in a valid heap region. If an object is stale, an invalid access is performed in order to
603 * prematurely crash the VM and debug the stack trace of the faulty method.
604 */
605 public static void validateObject(Object parent, Object child) {
606 if (verifyOops(INJECTED_VMCONFIG) && child != null) {
607 Word parentWord = Word.objectToTrackedPointer(parent);
608 Word childWord = Word.objectToTrackedPointer(child);
609 if (!validateOop(VALIDATE_OBJECT, parentWord, childWord)) {
610 log(true, "Verification ERROR, Parent: %p Child: %p\n", parentWord.rawValue(), childWord.rawValue());
611 VMErrorNode.vmError("Verification ERROR, Parent: %p\n", parentWord.rawValue());
612 }
613 }
614 }
615
616 public static final ForeignCallDescriptor VALIDATE_OBJECT = new ForeignCallDescriptor("validate_object", boolean.class, Word.class, Word.class);
617
618 @NodeIntrinsic(ForeignCallNode.class)
619 private static native boolean validateOop(@ConstantNodeParameter ForeignCallDescriptor descriptor, Word parent, Word object);
620
621 }