1 /*
2 * Copyright (c) 2018, Red Hat, Inc. and/or its affiliates.
3 *
4 * This code is free software; you can redistribute it and/or modify it
5 * under the terms of the GNU General Public License version 2 only, as
6 * published by the Free Software Foundation.
7 *
8 * This code is distributed in the hope that it will be useful, but WITHOUT
9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
10 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
11 * version 2 for more details (a copy is included in the LICENSE file that
12 * accompanied this code).
13 *
14 * You should have received a copy of the GNU General Public License version
15 * 2 along with this work; if not, write to the Free Software Foundation,
16 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
17 *
18 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
19 * or visit www.oracle.com if you need additional information or have any
20 * questions.
21 *
22 */
23
24 #include "precompiled.hpp"
25 #include "gc/shenandoah/shenandoahBarrierSetAssembler.hpp"
26 #include "gc/shenandoah/shenandoahHeap.hpp"
27 #include "gc/shenandoah/shenandoahHeapRegion.hpp"
28 #include "gc/shenandoah/shenandoahHeuristics.hpp"
29 #include "gc/shenandoah/shenandoahRuntime.hpp"
30 #include "gc/shenandoah/shenandoahThreadLocalData.hpp"
31 #include "interpreter/interpreter.hpp"
32 #include "interpreter/interp_masm.hpp"
33 #include "runtime/sharedRuntime.hpp"
34 #include "runtime/thread.hpp"
35 #ifdef COMPILER1
36 #include "c1/c1_LIRAssembler.hpp"
37 #include "c1/c1_MacroAssembler.hpp"
38 #include "gc/shenandoah/c1/shenandoahBarrierSetC1.hpp"
39 #endif
40
41 #define __ masm->
42
43 address ShenandoahBarrierSetAssembler::_shenandoah_wb = NULL;
44 address ShenandoahBarrierSetAssembler::_shenandoah_wb_C = NULL;
45
46 void ShenandoahBarrierSetAssembler::arraycopy_prologue(MacroAssembler* masm, DecoratorSet decorators, bool is_oop,
47 Register addr, Register count, RegSet saved_regs) {
48 if (is_oop) {
49 bool dest_uninitialized = (decorators & IS_DEST_UNINITIALIZED) != 0;
50 if (!dest_uninitialized && !ShenandoahHeap::heap()->heuristics()->can_do_traversal_gc()) {
51 __ push(saved_regs, sp);
52 if (count == c_rarg0) {
53 if (addr == c_rarg1) {
54 // exactly backwards!!
55 __ mov(rscratch1, c_rarg0);
56 __ mov(c_rarg0, c_rarg1);
57 __ mov(c_rarg1, rscratch1);
58 } else {
59 __ mov(c_rarg1, count);
60 __ mov(c_rarg0, addr);
61 }
62 } else {
63 __ mov(c_rarg0, addr);
64 __ mov(c_rarg1, count);
65 }
66 if (UseCompressedOops) {
67 __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_array_pre_narrow_oop_entry), 2);
68 } else {
69 __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_array_pre_oop_entry), 2);
70 }
71 __ pop(saved_regs, sp);
72 }
73 }
74 }
75
76 void ShenandoahBarrierSetAssembler::arraycopy_epilogue(MacroAssembler* masm, DecoratorSet decorators, bool is_oop,
77 Register start, Register end, Register scratch, RegSet saved_regs) {
78 if (is_oop) {
79 __ push(saved_regs, sp);
80 // must compute element count unless barrier set interface is changed (other platforms supply count)
81 assert_different_registers(start, end, scratch);
82 __ lea(scratch, Address(end, BytesPerHeapOop));
83 __ sub(scratch, scratch, start); // subtract start to get #bytes
84 __ lsr(scratch, scratch, LogBytesPerHeapOop); // convert to element count
85 __ mov(c_rarg0, start);
86 __ mov(c_rarg1, scratch);
87 __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_array_post_entry), 2);
88 __ pop(saved_regs, sp);
89 }
90 }
91
92 void ShenandoahBarrierSetAssembler::shenandoah_write_barrier_pre(MacroAssembler* masm,
93 Register obj,
94 Register pre_val,
95 Register thread,
96 Register tmp,
97 bool tosca_live,
98 bool expand_call) {
99 if (ShenandoahSATBBarrier) {
100 satb_write_barrier_pre(masm, obj, pre_val, thread, tmp, tosca_live, expand_call);
101 }
102 }
103
104 void ShenandoahBarrierSetAssembler::satb_write_barrier_pre(MacroAssembler* masm,
105 Register obj,
106 Register pre_val,
107 Register thread,
108 Register tmp,
109 bool tosca_live,
110 bool expand_call) {
111 // If expand_call is true then we expand the call_VM_leaf macro
112 // directly to skip generating the check by
113 // InterpreterMacroAssembler::call_VM_leaf_base that checks _last_sp.
114
115 assert(thread == rthread, "must be");
116
117 Label done;
118 Label runtime;
119
120 assert_different_registers(obj, pre_val, tmp, rscratch1);
121 assert(pre_val != noreg && tmp != noreg, "expecting a register");
122
123 Address in_progress(thread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_active_offset()));
124 Address index(thread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_index_offset()));
125 Address buffer(thread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_buffer_offset()));
126
127 // Is marking active?
128 if (in_bytes(SATBMarkQueue::byte_width_of_active()) == 4) {
129 __ ldrw(tmp, in_progress);
130 } else {
131 assert(in_bytes(SATBMarkQueue::byte_width_of_active()) == 1, "Assumption");
132 __ ldrb(tmp, in_progress);
133 }
134 __ cbzw(tmp, done);
135
136 // Do we need to load the previous value?
137 if (obj != noreg) {
138 __ load_heap_oop(pre_val, Address(obj, 0), noreg, noreg, AS_RAW);
139 }
140
141 // Is the previous value null?
142 __ cbz(pre_val, done);
143
144 // Can we store original value in the thread's buffer?
145 // Is index == 0?
146 // (The index field is typed as size_t.)
147
148 __ ldr(tmp, index); // tmp := *index_adr
149 __ cbz(tmp, runtime); // tmp == 0?
150 // If yes, goto runtime
151
152 __ sub(tmp, tmp, wordSize); // tmp := tmp - wordSize
153 __ str(tmp, index); // *index_adr := tmp
154 __ ldr(rscratch1, buffer);
155 __ add(tmp, tmp, rscratch1); // tmp := tmp + *buffer_adr
156
157 // Record the previous value
158 __ str(pre_val, Address(tmp, 0));
159 __ b(done);
160
161 __ bind(runtime);
162 // save the live input values
163 RegSet saved = RegSet::of(pre_val);
164 if (tosca_live) saved += RegSet::of(r0);
165 if (obj != noreg) saved += RegSet::of(obj);
166
167 __ push(saved, sp);
168
169 // Calling the runtime using the regular call_VM_leaf mechanism generates
170 // code (generated by InterpreterMacroAssember::call_VM_leaf_base)
171 // that checks that the *(rfp+frame::interpreter_frame_last_sp) == NULL.
172 //
173 // If we care generating the pre-barrier without a frame (e.g. in the
174 // intrinsified Reference.get() routine) then ebp might be pointing to
175 // the caller frame and so this check will most likely fail at runtime.
176 //
177 // Expanding the call directly bypasses the generation of the check.
178 // So when we do not have have a full interpreter frame on the stack
179 // expand_call should be passed true.
180
181 if (expand_call) {
182 assert(pre_val != c_rarg1, "smashed arg");
183 __ super_call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_field_pre_entry), pre_val, thread);
184 } else {
185 __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_field_pre_entry), pre_val, thread);
186 }
187
188 __ pop(saved, sp);
189
190 __ bind(done);
191 }
192
193 void ShenandoahBarrierSetAssembler::read_barrier(MacroAssembler* masm, Register dst) {
194 if (ShenandoahReadBarrier) {
195 read_barrier_impl(masm, dst);
196 }
197 }
198
199 void ShenandoahBarrierSetAssembler::read_barrier_impl(MacroAssembler* masm, Register dst) {
200 assert(UseShenandoahGC && (ShenandoahReadBarrier || ShenandoahStoreValReadBarrier || ShenandoahCASBarrier), "should be enabled");
201 Label is_null;
202 __ cbz(dst, is_null);
203 read_barrier_not_null_impl(masm, dst);
204 __ bind(is_null);
205 }
206
207 void ShenandoahBarrierSetAssembler::read_barrier_not_null(MacroAssembler* masm, Register dst) {
208 if (ShenandoahReadBarrier) {
209 read_barrier_not_null_impl(masm, dst);
210 }
211 }
212
213
214 void ShenandoahBarrierSetAssembler::read_barrier_not_null_impl(MacroAssembler* masm, Register dst) {
215 assert(UseShenandoahGC && (ShenandoahReadBarrier || ShenandoahStoreValReadBarrier || ShenandoahCASBarrier), "should be enabled");
216 __ ldr(dst, Address(dst, ShenandoahBrooksPointer::byte_offset()));
217 }
218
219 void ShenandoahBarrierSetAssembler::write_barrier(MacroAssembler* masm, Register dst) {
220 if (ShenandoahWriteBarrier) {
221 write_barrier_impl(masm, dst);
222 }
223 }
224
225 void ShenandoahBarrierSetAssembler::write_barrier_impl(MacroAssembler* masm, Register dst) {
226 assert(UseShenandoahGC && (ShenandoahWriteBarrier || ShenandoahStoreValEnqueueBarrier), "Should be enabled");
227 assert(dst != rscratch1, "need rscratch1");
228 assert(dst != rscratch2, "need rscratch2");
229
230 Label done;
231
232 Address gc_state(rthread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
233 __ ldrb(rscratch1, gc_state);
234
235 // Check for heap stability
236 __ mov(rscratch2, ShenandoahHeap::HAS_FORWARDED | ShenandoahHeap::EVACUATION | ShenandoahHeap::TRAVERSAL);
237 __ tst(rscratch1, rscratch2);
238 __ br(Assembler::EQ, done);
239
240 // Heap is unstable, need to perform the read-barrier even if WB is inactive
241 if (ShenandoahWriteBarrierRB) {
242 __ ldr(dst, Address(dst, ShenandoahBrooksPointer::byte_offset()));
243 }
244
245 // Check for evacuation-in-progress and jump to WB slow-path if needed
246 __ mov(rscratch2, ShenandoahHeap::EVACUATION | ShenandoahHeap::TRAVERSAL);
247 __ tst(rscratch1, rscratch2);
248 __ br(Assembler::EQ, done);
249
250 RegSet to_save = RegSet::of(r0);
251 if (dst != r0) {
252 __ push(to_save, sp);
253 __ mov(r0, dst);
254 }
255
256 __ far_call(RuntimeAddress(CAST_FROM_FN_PTR(address, ShenandoahBarrierSetAssembler::shenandoah_wb())));
257
258 if (dst != r0) {
259 __ mov(dst, r0);
260 __ pop(to_save, sp);
261 }
262
263 __ bind(done);
264 }
265
266 void ShenandoahBarrierSetAssembler::storeval_barrier(MacroAssembler* masm, Register dst, Register tmp) {
267 if (ShenandoahStoreValEnqueueBarrier) {
268 Label is_null;
269 __ cbz(dst, is_null);
270 write_barrier_impl(masm, dst);
271 __ bind(is_null);
272 // Save possibly live regs.
273 RegSet live_regs = RegSet::range(r0, r4) - dst;
274 __ push(live_regs, sp);
275 __ strd(v0, __ pre(sp, 2 * -wordSize));
276
277 satb_write_barrier_pre(masm, noreg, dst, rthread, tmp, true, false);
278
279 // Restore possibly live regs.
280 __ ldrd(v0, __ post(sp, 2 * wordSize));
281 __ pop(live_regs, sp);
282 }
283 if (ShenandoahStoreValReadBarrier) {
284 read_barrier_impl(masm, dst);
285 }
286 }
287
288 void ShenandoahBarrierSetAssembler::load_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
289 Register dst, Address src, Register tmp1, Register tmp_thread) {
290 bool on_oop = type == T_OBJECT || type == T_ARRAY;
291 bool in_heap = (decorators & IN_HEAP) != 0;
292 bool on_weak = (decorators & ON_WEAK_OOP_REF) != 0;
293 bool on_phantom = (decorators & ON_PHANTOM_OOP_REF) != 0;
294 bool on_reference = on_weak || on_phantom;
295
296 if (in_heap) {
297 read_barrier_not_null(masm, src.base());
298 }
299
300 BarrierSetAssembler::load_at(masm, decorators, type, dst, src, tmp1, tmp_thread);
301 if (ShenandoahKeepAliveBarrier && on_oop && on_reference) {
302 __ enter();
303 satb_write_barrier_pre(masm /* masm */,
304 noreg /* obj */,
305 dst /* pre_val */,
306 rthread /* thread */,
307 tmp1 /* tmp */,
308 true /* tosca_live */,
309 true /* expand_call */);
310 __ leave();
311 }
312 }
313
314 void ShenandoahBarrierSetAssembler::store_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
315 Address dst, Register val, Register tmp1, Register tmp2) {
316 bool on_oop = type == T_OBJECT || type == T_ARRAY;
317 bool in_heap = (decorators & IN_HEAP) != 0;
318 if (in_heap) {
319 write_barrier(masm, dst.base());
320 }
321 if (!on_oop) {
322 BarrierSetAssembler::store_at(masm, decorators, type, dst, val, tmp1, tmp2);
323 return;
324 }
325
326 // flatten object address if needed
327 if (dst.index() == noreg && dst.offset() == 0) {
328 if (dst.base() != r3) {
329 __ mov(r3, dst.base());
330 }
331 } else {
332 __ lea(r3, dst);
333 }
334
335 shenandoah_write_barrier_pre(masm,
336 r3 /* obj */,
337 tmp2 /* pre_val */,
338 rthread /* thread */,
339 tmp1 /* tmp */,
340 val != noreg /* tosca_live */,
341 false /* expand_call */);
342
343 if (val == noreg) {
344 BarrierSetAssembler::store_at(masm, decorators, type, Address(r3, 0), noreg, noreg, noreg);
345 } else {
346 storeval_barrier(masm, val, tmp1);
347 // G1 barrier needs uncompressed oop for region cross check.
348 Register new_val = val;
349 if (UseCompressedOops) {
350 new_val = rscratch2;
351 __ mov(new_val, val);
352 }
353 BarrierSetAssembler::store_at(masm, decorators, type, Address(r3, 0), val, noreg, noreg);
354 }
355
356 }
357
358 void ShenandoahBarrierSetAssembler::obj_equals(MacroAssembler* masm, Register op1, Register op2) {
359 __ cmp(op1, op2);
360 if (ShenandoahAcmpBarrier) {
361 Label done;
362 __ br(Assembler::EQ, done);
363 // The object may have been evacuated, but we won't see it without a
364 // membar here.
365 __ membar(Assembler::LoadStore| Assembler::LoadLoad);
366 read_barrier(masm, op1);
367 read_barrier(masm, op2);
368 __ cmp(op1, op2);
369 __ bind(done);
370 }
371 }
372
373 void ShenandoahBarrierSetAssembler::tlab_allocate(MacroAssembler* masm, Register obj,
374 Register var_size_in_bytes,
375 int con_size_in_bytes,
376 Register t1,
377 Register t2,
378 Label& slow_case) {
379
380 assert_different_registers(obj, t2);
381 assert_different_registers(obj, var_size_in_bytes);
382 Register end = t2;
383
384 __ ldr(obj, Address(rthread, JavaThread::tlab_top_offset()));
385 if (var_size_in_bytes == noreg) {
386 __ lea(end, Address(obj, (int) (con_size_in_bytes + ShenandoahBrooksPointer::byte_size())));
387 } else {
388 __ add(var_size_in_bytes, var_size_in_bytes, ShenandoahBrooksPointer::byte_size());
389 __ lea(end, Address(obj, var_size_in_bytes));
390 }
391 __ ldr(rscratch1, Address(rthread, JavaThread::tlab_end_offset()));
392 __ cmp(end, rscratch1);
393 __ br(Assembler::HI, slow_case);
394
395 // update the tlab top pointer
396 __ str(end, Address(rthread, JavaThread::tlab_top_offset()));
397
398 __ add(obj, obj, ShenandoahBrooksPointer::byte_size());
399 __ str(obj, Address(obj, ShenandoahBrooksPointer::byte_offset()));
400
401 // recover var_size_in_bytes if necessary
402 if (var_size_in_bytes == end) {
403 __ sub(var_size_in_bytes, var_size_in_bytes, obj);
404 }
405 }
406
407 void ShenandoahBarrierSetAssembler::resolve(MacroAssembler* masm, DecoratorSet decorators, Register obj) {
408 bool oop_not_null = (decorators & IS_NOT_NULL) != 0;
409 bool is_write = (decorators & ACCESS_WRITE) != 0;
410 if (is_write) {
411 if (oop_not_null) {
412 write_barrier(masm, obj);
413 } else {
414 Label done;
415 __ cbz(obj, done);
416 write_barrier(masm, obj);
417 __ bind(done);
418 }
419 } else {
420 if (oop_not_null) {
421 read_barrier_not_null(masm, obj);
422 } else {
423 read_barrier(masm, obj);
424 }
425 }
426 }
427
428 void ShenandoahBarrierSetAssembler::cmpxchg_oop(MacroAssembler* masm, Register addr, Register expected, Register new_val,
429 bool acquire, bool release, bool weak, bool encode,
430 Register tmp1, Register tmp2, Register tmp3,
431 Register result) {
432
433 if (!ShenandoahCASBarrier) {
434 if (UseCompressedOops) {
435 if (encode) {
436 __ encode_heap_oop(tmp1, expected);
437 expected = tmp1;
438 __ encode_heap_oop(tmp3, new_val);
439 new_val = tmp3;
440 }
441 __ cmpxchg(addr, expected, new_val, Assembler::word, /* acquire*/ true, /* release*/ true, /* weak*/ false, rscratch1);
442 __ membar(__ AnyAny);
443 } else {
444 __ cmpxchg(addr, expected, new_val, Assembler::xword, /* acquire*/ true, /* release*/ true, /* weak*/ false, rscratch1);
445 __ membar(__ AnyAny);
446 }
447 return;
448 }
449
450 if (encode) {
451 storeval_barrier(masm, new_val, tmp3);
452 }
453
454 if (UseCompressedOops) {
455 if (encode) {
456 __ encode_heap_oop(tmp1, expected);
457 expected = tmp1;
458 __ encode_heap_oop(tmp2, new_val);
459 new_val = tmp2;
460 }
461 }
462 bool is_cae = (result != noreg);
463 bool is_narrow = UseCompressedOops;
464 Assembler::operand_size size = is_narrow ? Assembler::word : Assembler::xword;
465 if (! is_cae) result = rscratch1;
466
467 assert_different_registers(addr, expected, new_val, result, tmp3);
468
469 Label retry, done, fail;
470
471 // CAS, using LL/SC pair.
472 __ bind(retry);
473 __ load_exclusive(result, addr, size, acquire);
474 if (is_narrow) {
475 __ cmpw(result, expected);
476 } else {
477 __ cmp(result, expected);
478 }
479 __ br(Assembler::NE, fail);
480 __ store_exclusive(tmp3, new_val, addr, size, release);
481 if (weak) {
482 __ cmpw(tmp3, 0u); // If the store fails, return NE to our caller
483 } else {
484 __ cbnzw(tmp3, retry);
485 }
486 __ b(done);
487
488 __ bind(fail);
489 // Check if rb(expected)==rb(result)
490 // Shuffle registers so that we have memory value ready for next expected.
491 __ mov(tmp3, expected);
492 __ mov(expected, result);
493 if (is_narrow) {
494 __ decode_heap_oop(result, result);
495 __ decode_heap_oop(tmp3, tmp3);
496 }
497 read_barrier_impl(masm, result);
498 read_barrier_impl(masm, tmp3);
499 __ cmp(result, tmp3);
500 // Retry with expected now being the value we just loaded from addr.
501 __ br(Assembler::EQ, retry);
502 if (is_narrow && is_cae) {
503 // For cmp-and-exchange and narrow oops, we need to restore
504 // the compressed old-value. We moved it to 'expected' a few lines up.
505 __ mov(result, expected);
506 }
507 __ bind(done);
508
509 }
510
511 #ifdef COMPILER1
512
513 #undef __
514 #define __ ce->masm()->
515
516 void ShenandoahBarrierSetAssembler::gen_pre_barrier_stub(LIR_Assembler* ce, ShenandoahPreBarrierStub* stub) {
517 ShenandoahBarrierSetC1* bs = (ShenandoahBarrierSetC1*)BarrierSet::barrier_set()->barrier_set_c1();
518 // At this point we know that marking is in progress.
519 // If do_load() is true then we have to emit the
520 // load of the previous value; otherwise it has already
521 // been loaded into _pre_val.
522
523 __ bind(*stub->entry());
524
525 assert(stub->pre_val()->is_register(), "Precondition.");
526
527 Register pre_val_reg = stub->pre_val()->as_register();
528
529 if (stub->do_load()) {
530 ce->mem2reg(stub->addr(), stub->pre_val(), T_OBJECT, stub->patch_code(), stub->info(), false /*wide*/, false /*unaligned*/);
531 }
532 __ cbz(pre_val_reg, *stub->continuation());
533 ce->store_parameter(stub->pre_val()->as_register(), 0);
534 __ far_call(RuntimeAddress(bs->pre_barrier_c1_runtime_code_blob()->code_begin()));
535 __ b(*stub->continuation());
536 }
537
538 void ShenandoahBarrierSetAssembler::gen_write_barrier_stub(LIR_Assembler* ce, ShenandoahWriteBarrierStub* stub) {
539
540 Register obj = stub->obj()->as_register();
541 Register res = stub->result()->as_register();
542
543 Label done;
544
545 __ bind(*stub->entry());
546
547 if (res != obj) {
548 __ mov(res, obj);
549 }
550 // Check for null.
551 if (stub->needs_null_check()) {
552 __ cbz(res, done);
553 }
554
555 write_barrier(ce->masm(), res);
556
557 __ bind(done);
558 __ b(*stub->continuation());
559 }
560
561 #undef __
562
563 #define __ sasm->
564
565 void ShenandoahBarrierSetAssembler::generate_c1_pre_barrier_runtime_stub(StubAssembler* sasm) {
566 __ prologue("shenandoah_pre_barrier", false);
567
568 // arg0 : previous value of memory
569
570 BarrierSet* bs = BarrierSet::barrier_set();
571
572 const Register pre_val = r0;
573 const Register thread = rthread;
574 const Register tmp = rscratch1;
575
576 Address queue_index(thread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_index_offset()));
577 Address buffer(thread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_buffer_offset()));
578
579 Label done;
580 Label runtime;
581
582 // Is marking still active?
583 Address gc_state(thread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
584 __ ldrb(tmp, gc_state);
585 __ mov(rscratch2, ShenandoahHeap::MARKING | ShenandoahHeap::TRAVERSAL);
586 __ tst(tmp, rscratch2);
587 __ br(Assembler::EQ, done);
588
589 // Can we store original value in the thread's buffer?
590 __ ldr(tmp, queue_index);
591 __ cbz(tmp, runtime);
592
593 __ sub(tmp, tmp, wordSize);
594 __ str(tmp, queue_index);
595 __ ldr(rscratch2, buffer);
596 __ add(tmp, tmp, rscratch2);
597 __ load_parameter(0, rscratch2);
598 __ str(rscratch2, Address(tmp, 0));
599 __ b(done);
600
601 __ bind(runtime);
602 __ push_call_clobbered_registers();
603 __ load_parameter(0, pre_val);
604 __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_field_pre_entry), pre_val, thread);
605 __ pop_call_clobbered_registers();
606 __ bind(done);
607
608 __ epilogue();
609 }
610
611 #undef __
612
613 #endif // COMPILER1
614
615 address ShenandoahBarrierSetAssembler::shenandoah_wb() {
616 assert(_shenandoah_wb != NULL, "need write barrier stub");
617 return _shenandoah_wb;
618 }
619
620 address ShenandoahBarrierSetAssembler::shenandoah_wb_C() {
621 assert(_shenandoah_wb_C != NULL, "need write barrier stub");
622 return _shenandoah_wb_C;
623 }
624
625 #define __ cgen->assembler()->
626
627 // Shenandoah write barrier.
628 //
629 // Input:
630 // r0: OOP to evacuate. Not null.
631 //
632 // Output:
633 // r0: Pointer to evacuated OOP.
634 //
635 // Trash rscratch1, rscratch2. Preserve everything else.
636 address ShenandoahBarrierSetAssembler::generate_shenandoah_wb(StubCodeGenerator* cgen, bool c_abi, bool do_cset_test) {
637
638 __ align(6);
639 StubCodeMark mark(cgen, "StubRoutines", "shenandoah_wb");
640 address start = __ pc();
641
642 if (do_cset_test) {
643 Label work;
644 __ mov(rscratch2, ShenandoahHeap::in_cset_fast_test_addr());
645 __ lsr(rscratch1, r0, ShenandoahHeapRegion::region_size_bytes_shift_jint());
646 __ ldrb(rscratch2, Address(rscratch2, rscratch1));
647 __ tbnz(rscratch2, 0, work);
648 __ ret(lr);
649 __ bind(work);
650 }
651
652 Register obj = r0;
653
654 __ enter(); // required for proper stackwalking of RuntimeStub frame
655
656 if (!c_abi) {
657 __ push_call_clobbered_registers();
658 } else {
659 __ push_call_clobbered_fp_registers();
660 }
661
662 __ mov(lr, CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_barrier_JRT));
663 __ blrt(lr, 1, 0, MacroAssembler::ret_type_integral);
664 if (!c_abi) {
665 __ mov(rscratch1, obj);
666 __ pop_call_clobbered_registers();
667 __ mov(obj, rscratch1);
668 } else {
669 __ pop_call_clobbered_fp_registers();
670 }
671
672 __ leave(); // required for proper stackwalking of RuntimeStub frame
673 __ ret(lr);
674
675 return start;
676 }
677
678 #undef __
679
680 void ShenandoahBarrierSetAssembler::barrier_stubs_init() {
681 if (ShenandoahWriteBarrier || ShenandoahStoreValEnqueueBarrier) {
682 int stub_code_size = 2048;
683 ResourceMark rm;
684 BufferBlob* bb = BufferBlob::create("shenandoah_barrier_stubs", stub_code_size);
685 CodeBuffer buf(bb);
686 StubCodeGenerator cgen(&buf);
687 _shenandoah_wb = generate_shenandoah_wb(&cgen, false, true);
688 _shenandoah_wb_C = generate_shenandoah_wb(&cgen, true, false);
689 }
690 }