1 /*
2 * Copyright (c) 2018, 2019, Red Hat, Inc. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "precompiled.hpp"
26 #include "gc/shenandoah/shenandoahBarrierSet.hpp"
27 #include "gc/shenandoah/shenandoahBarrierSetAssembler.hpp"
28 #include "gc/shenandoah/shenandoahForwarding.hpp"
29 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
30 #include "gc/shenandoah/shenandoahHeapRegion.hpp"
31 #include "gc/shenandoah/shenandoahHeuristics.hpp"
32 #include "gc/shenandoah/shenandoahRuntime.hpp"
33 #include "gc/shenandoah/shenandoahThreadLocalData.hpp"
34 #include "interpreter/interpreter.hpp"
35 #include "interpreter/interp_masm.hpp"
36 #include "runtime/sharedRuntime.hpp"
37 #include "runtime/thread.hpp"
38 #include "utilities/macros.hpp"
39 #ifdef COMPILER1
40 #include "c1/c1_LIRAssembler.hpp"
41 #include "c1/c1_MacroAssembler.hpp"
42 #include "gc/shenandoah/c1/shenandoahBarrierSetC1.hpp"
43 #endif
44
45 #define __ masm->
46
47 address ShenandoahBarrierSetAssembler::_shenandoah_lrb = NULL;
48
49 void ShenandoahBarrierSetAssembler::arraycopy_prologue(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
50 Register src, Register dst, Register count) {
51
52 bool dest_uninitialized = (decorators & IS_DEST_UNINITIALIZED) != 0;
53
54 if (is_reference_type(type)) {
55
56 if ((ShenandoahSATBBarrier && !dest_uninitialized) || ShenandoahLoadRefBarrier) {
57 #ifdef _LP64
58 Register thread = r15_thread;
59 #else
60 Register thread = rax;
61 if (thread == src || thread == dst || thread == count) {
62 thread = rbx;
63 }
64 if (thread == src || thread == dst || thread == count) {
65 thread = rcx;
66 }
67 if (thread == src || thread == dst || thread == count) {
68 thread = rdx;
69 }
70 __ push(thread);
71 __ get_thread(thread);
72 #endif
73 assert_different_registers(src, dst, count, thread);
74
75 Label done;
76 // Short-circuit if count == 0.
77 __ testptr(count, count);
78 __ jcc(Assembler::zero, done);
79
80 // Avoid runtime call when not marking.
81 Address gc_state(thread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
82 int flags = ShenandoahHeap::HAS_FORWARDED;
83 if (!dest_uninitialized) {
84 flags |= ShenandoahHeap::MARKING;
85 }
86 __ testb(gc_state, flags);
87 __ jcc(Assembler::zero, done);
88
89 __ pusha(); // push registers
90 #ifdef _LP64
91 assert(src == rdi, "expected");
92 assert(dst == rsi, "expected");
93 assert(count == rdx, "expected");
94 if (UseCompressedOops) {
95 if (dest_uninitialized) {
96 __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_array_pre_duinit_narrow_oop_entry), src, dst, count);
97 } else {
98 __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_array_pre_narrow_oop_entry), src, dst, count);
99 }
100 } else
101 #endif
102 {
103 if (dest_uninitialized) {
104 __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_array_pre_duinit_oop_entry), src, dst, count);
105 } else {
106 __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_array_pre_oop_entry), src, dst, count);
107 }
108 }
109 __ popa();
110 __ bind(done);
111 NOT_LP64(__ pop(thread);)
112 }
113 }
114
115 }
116
117 void ShenandoahBarrierSetAssembler::shenandoah_write_barrier_pre(MacroAssembler* masm,
118 Register obj,
119 Register pre_val,
120 Register thread,
121 Register tmp,
122 bool tosca_live,
123 bool expand_call) {
124
125 if (ShenandoahSATBBarrier) {
126 satb_write_barrier_pre(masm, obj, pre_val, thread, tmp, tosca_live, expand_call);
127 }
128 }
129
130 void ShenandoahBarrierSetAssembler::satb_write_barrier_pre(MacroAssembler* masm,
131 Register obj,
132 Register pre_val,
133 Register thread,
134 Register tmp,
135 bool tosca_live,
136 bool expand_call) {
137 // If expand_call is true then we expand the call_VM_leaf macro
138 // directly to skip generating the check by
139 // InterpreterMacroAssembler::call_VM_leaf_base that checks _last_sp.
140
141 #ifdef _LP64
142 assert(thread == r15_thread, "must be");
143 #endif // _LP64
144
145 Label done;
146 Label runtime;
147
148 assert(pre_val != noreg, "check this code");
149
150 if (obj != noreg) {
151 assert_different_registers(obj, pre_val, tmp);
152 assert(pre_val != rax, "check this code");
153 }
154
155 Address in_progress(thread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_active_offset()));
156 Address index(thread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_index_offset()));
157 Address buffer(thread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_buffer_offset()));
158
159 Address gc_state(thread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
160 __ testb(gc_state, ShenandoahHeap::MARKING | ShenandoahHeap::TRAVERSAL);
161 __ jcc(Assembler::zero, done);
162
163 // Do we need to load the previous value?
164 if (obj != noreg) {
165 __ load_heap_oop(pre_val, Address(obj, 0), noreg, noreg, AS_RAW);
166 }
167
168 // Is the previous value null?
169 __ cmpptr(pre_val, (int32_t) NULL_WORD);
170 __ jcc(Assembler::equal, done);
171
172 // Can we store original value in the thread's buffer?
173 // Is index == 0?
174 // (The index field is typed as size_t.)
175
176 __ movptr(tmp, index); // tmp := *index_adr
177 __ cmpptr(tmp, 0); // tmp == 0?
178 __ jcc(Assembler::equal, runtime); // If yes, goto runtime
179
180 __ subptr(tmp, wordSize); // tmp := tmp - wordSize
181 __ movptr(index, tmp); // *index_adr := tmp
182 __ addptr(tmp, buffer); // tmp := tmp + *buffer_adr
183
184 // Record the previous value
185 __ movptr(Address(tmp, 0), pre_val);
186 __ jmp(done);
187
188 __ bind(runtime);
189 // save the live input values
190 if(tosca_live) __ push(rax);
191
192 if (obj != noreg && obj != rax)
193 __ push(obj);
194
195 if (pre_val != rax)
196 __ push(pre_val);
197
198 // Calling the runtime using the regular call_VM_leaf mechanism generates
199 // code (generated by InterpreterMacroAssember::call_VM_leaf_base)
200 // that checks that the *(ebp+frame::interpreter_frame_last_sp) == NULL.
201 //
202 // If we care generating the pre-barrier without a frame (e.g. in the
203 // intrinsified Reference.get() routine) then ebp might be pointing to
204 // the caller frame and so this check will most likely fail at runtime.
205 //
206 // Expanding the call directly bypasses the generation of the check.
207 // So when we do not have have a full interpreter frame on the stack
208 // expand_call should be passed true.
209
210 NOT_LP64( __ push(thread); )
211
212 #ifdef _LP64
213 // We move pre_val into c_rarg0 early, in order to avoid smashing it, should
214 // pre_val be c_rarg1 (where the call prologue would copy thread argument).
215 // Note: this should not accidentally smash thread, because thread is always r15.
216 assert(thread != c_rarg0, "smashed arg");
217 if (c_rarg0 != pre_val) {
218 __ mov(c_rarg0, pre_val);
219 }
220 #endif
221
222 if (expand_call) {
223 LP64_ONLY( assert(pre_val != c_rarg1, "smashed arg"); )
224 #ifdef _LP64
225 if (c_rarg1 != thread) {
226 __ mov(c_rarg1, thread);
227 }
228 // Already moved pre_val into c_rarg0 above
229 #else
230 __ push(thread);
231 __ push(pre_val);
232 #endif
233 __ MacroAssembler::call_VM_leaf_base(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_field_pre_entry), 2);
234 } else {
235 __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_field_pre_entry), LP64_ONLY(c_rarg0) NOT_LP64(pre_val), thread);
236 }
237
238 NOT_LP64( __ pop(thread); )
239
240 // save the live input values
241 if (pre_val != rax)
242 __ pop(pre_val);
243
244 if (obj != noreg && obj != rax)
245 __ pop(obj);
246
247 if(tosca_live) __ pop(rax);
248
249 __ bind(done);
250 }
251
252 void ShenandoahBarrierSetAssembler::load_reference_barrier_not_null(MacroAssembler* masm, Register dst, Address src) {
253 assert(ShenandoahLoadRefBarrier, "Should be enabled");
254
255 Label done;
256
257 #ifdef _LP64
258 Register thread = r15_thread;
259 #else
260 Register thread = rcx;
261 if (thread == dst) {
262 thread = rbx;
263 }
264 __ push(thread);
265 __ get_thread(thread);
266 #endif
267
268 Address gc_state(thread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
269 __ testb(gc_state, ShenandoahHeap::HAS_FORWARDED);
270 __ jccb(Assembler::zero, done);
271
272 // Use rsi for src address
273 const Register src_addr = rsi;
274 // Setup address parameter first, if it does not clobber oop in dst
275 bool need_addr_setup = (src_addr != dst);
276
277 if (need_addr_setup) {
278 __ push(src_addr);
279 __ lea(src_addr, src);
280
281 if (dst != rax) {
282 // Move obj into rax and save rax
283 __ push(rax);
284 __ movptr(rax, dst);
285 }
286 } else {
287 // dst == rsi
288 __ push(rax);
289 __ movptr(rax, dst);
290
291 // we can clobber it, since it is outgoing register
292 __ lea(src_addr, src);
293 }
294
295 __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, ShenandoahBarrierSetAssembler::shenandoah_lrb())));
296
297 if (need_addr_setup) {
298 if (dst != rax) {
299 __ movptr(dst, rax);
300 __ pop(rax);
301 }
302 __ pop(src_addr);
303 } else {
304 __ movptr(dst, rax);
305 __ pop(rax);
306 }
307
308 __ bind(done);
309
310 #ifndef _LP64
311 __ pop(thread);
312 #endif
313 }
314
315 void ShenandoahBarrierSetAssembler::load_reference_barrier_native(MacroAssembler* masm, Register dst, Address src) {
316 if (!ShenandoahLoadRefBarrier) {
317 return;
318 }
319
320 Label done;
321 Label not_null;
322 Label slow_path;
323 __ block_comment("load_reference_barrier_native { ");
324
325 // null check
326 __ testptr(dst, dst);
327 __ jcc(Assembler::notZero, not_null);
328 __ jmp(done);
329 __ bind(not_null);
330
331
332 #ifdef _LP64
333 Register thread = r15_thread;
334 #else
335 Register thread = rcx;
336 if (thread == dst) {
337 thread = rbx;
338 }
339 __ push(thread);
340 __ get_thread(thread);
341 #endif
342 assert_different_registers(dst, thread);
343
344 Address gc_state(thread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
345 __ testb(gc_state, ShenandoahHeap::EVACUATION);
346 #ifndef _LP64
347 __ pop(thread);
348 #endif
349 __ jccb(Assembler::notZero, slow_path);
350 __ jmp(done);
351 __ bind(slow_path);
352
353 if (dst != rax) {
354 __ push(rax);
355 }
356 __ push(rcx);
357 __ push(rdx);
358 __ push(rdi);
359 __ push(rsi);
360 #ifdef _LP64
361 __ push(r8);
362 __ push(r9);
363 __ push(r10);
364 __ push(r11);
365 __ push(r12);
366 __ push(r13);
367 __ push(r14);
368 __ push(r15);
369 #endif
370
371 assert_different_registers(dst, rsi);
372 __ lea(rsi, src);
373 __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_native), dst, rsi);
374
375 #ifdef _LP64
376 __ pop(r15);
377 __ pop(r14);
378 __ pop(r13);
379 __ pop(r12);
380 __ pop(r11);
381 __ pop(r10);
382 __ pop(r9);
383 __ pop(r8);
384 #endif
385 __ pop(rsi);
386 __ pop(rdi);
387 __ pop(rdx);
388 __ pop(rcx);
389
390 if (dst != rax) {
391 __ movptr(dst, rax);
392 __ pop(rax);
393 }
394
395 __ bind(done);
396 __ block_comment("load_reference_barrier_native { ");
397 }
398
399 void ShenandoahBarrierSetAssembler::storeval_barrier(MacroAssembler* masm, Register dst, Register tmp) {
400 if (ShenandoahStoreValEnqueueBarrier) {
401 storeval_barrier_impl(masm, dst, tmp);
402 }
403 }
404
405 void ShenandoahBarrierSetAssembler::storeval_barrier_impl(MacroAssembler* masm, Register dst, Register tmp) {
406 assert(ShenandoahStoreValEnqueueBarrier, "should be enabled");
407
408 if (dst == noreg) return;
409
410 if (ShenandoahStoreValEnqueueBarrier) {
411 // The set of registers to be saved+restored is the same as in the write-barrier above.
412 // Those are the commonly used registers in the interpreter.
413 __ pusha();
414 // __ push_callee_saved_registers();
415 __ subptr(rsp, 2 * Interpreter::stackElementSize);
416 __ movdbl(Address(rsp, 0), xmm0);
417
418 #ifdef _LP64
419 Register thread = r15_thread;
420 #else
421 Register thread = rcx;
422 if (thread == dst || thread == tmp) {
423 thread = rdi;
424 }
425 if (thread == dst || thread == tmp) {
426 thread = rbx;
427 }
428 __ get_thread(thread);
429 #endif
430 assert_different_registers(dst, tmp, thread);
431
432 satb_write_barrier_pre(masm, noreg, dst, thread, tmp, true, false);
433 __ movdbl(xmm0, Address(rsp, 0));
434 __ addptr(rsp, 2 * Interpreter::stackElementSize);
435 //__ pop_callee_saved_registers();
436 __ popa();
437 }
438 }
439
440 void ShenandoahBarrierSetAssembler::load_reference_barrier(MacroAssembler* masm, Register dst, Address src) {
441 if (ShenandoahLoadRefBarrier) {
442 Label done;
443 __ testptr(dst, dst);
444 __ jcc(Assembler::zero, done);
445 load_reference_barrier_not_null(masm, dst, src);
446 __ bind(done);
447 }
448 }
449
450 //
451 // Arguments:
452 //
453 // Inputs:
454 // src: oop location, might be clobbered
455 // tmp1: scratch register, might not be valid.
456 //
457 // Output:
458 // dst: oop loaded from src location
459 //
460 // Kill:
461 // tmp1 (if it is valid)
462 //
463 void ShenandoahBarrierSetAssembler::load_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
464 Register dst, Address src, Register tmp1, Register tmp_thread) {
465 // 1: non-reference load, no additional barrier is needed
466 if (!is_reference_type(type)) {
467 BarrierSetAssembler::load_at(masm, decorators, type, dst, src, tmp1, tmp_thread);
468 return;
469 }
470
471 assert((decorators & ON_UNKNOWN_OOP_REF) == 0, "Not expected");
472
473 // 2: load a reference from src location and apply LRB if needed
474 if (ShenandoahBarrierSet::need_load_reference_barrier(decorators, type)) {
475 Register result_dst = dst;
476 bool use_tmp1_for_dst = false;
477
478 // Preserve src location for LRB
479 if (dst == src.base() || dst == src.index()) {
480 // Use tmp1 for dst if possible, as it is not used in BarrierAssembler::load_at()
481 if (tmp1->is_valid() && tmp1 != src.base() && tmp1 != src.index()) {
482 dst = tmp1;
483 use_tmp1_for_dst = true;
484 } else {
485 dst = rdi;
486 __ push(dst);
487 }
488 assert_different_registers(dst, src.base(), src.index());
489 }
490
491 BarrierSetAssembler::load_at(masm, decorators, type, dst, src, tmp1, tmp_thread);
492
493 if (ShenandoahBarrierSet::use_load_reference_barrier_native(decorators, type)) {
494 load_reference_barrier_native(masm, dst, src);
495 } else {
496 load_reference_barrier(masm, dst, src);
497 }
498
499 // Move loaded oop to final destination
500 if (dst != result_dst) {
501 __ movptr(result_dst, dst);
502
503 if (!use_tmp1_for_dst) {
504 __ pop(dst);
505 }
506
507 dst = result_dst;
508 }
509 } else {
510 BarrierSetAssembler::load_at(masm, decorators, type, dst, src, tmp1, tmp_thread);
511 }
512
513 // 3: apply keep-alive barrier if needed
514 if (ShenandoahBarrierSet::need_keep_alive_barrier(decorators, type)) {
515 __ push_IU_state();
516 Register thread = NOT_LP64(tmp_thread) LP64_ONLY(r15_thread);
517 assert_different_registers(dst, tmp1, tmp_thread);
518 if (!thread->is_valid()) {
519 thread = rdx;
520 }
521 NOT_LP64(__ get_thread(thread));
522 // Generate the SATB pre-barrier code to log the value of
523 // the referent field in an SATB buffer.
524 shenandoah_write_barrier_pre(masm /* masm */,
525 noreg /* obj */,
526 dst /* pre_val */,
527 thread /* thread */,
528 tmp1 /* tmp */,
529 true /* tosca_live */,
530 true /* expand_call */);
531 __ pop_IU_state();
532 }
533 }
534
535 void ShenandoahBarrierSetAssembler::store_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
536 Address dst, Register val, Register tmp1, Register tmp2) {
537
538 bool on_oop = is_reference_type(type);
539 bool in_heap = (decorators & IN_HEAP) != 0;
540 bool as_normal = (decorators & AS_NORMAL) != 0;
541 if (on_oop && in_heap) {
542 bool needs_pre_barrier = as_normal;
543
544 Register tmp3 = LP64_ONLY(r8) NOT_LP64(rsi);
545 Register rthread = LP64_ONLY(r15_thread) NOT_LP64(rcx);
546 // flatten object address if needed
547 // We do it regardless of precise because we need the registers
548 if (dst.index() == noreg && dst.disp() == 0) {
549 if (dst.base() != tmp1) {
550 __ movptr(tmp1, dst.base());
551 }
552 } else {
553 __ lea(tmp1, dst);
554 }
555
556 assert_different_registers(val, tmp1, tmp2, tmp3, rthread);
557
558 #ifndef _LP64
559 __ get_thread(rthread);
560 InterpreterMacroAssembler *imasm = static_cast<InterpreterMacroAssembler*>(masm);
561 imasm->save_bcp();
562 #endif
563
564 if (needs_pre_barrier) {
565 shenandoah_write_barrier_pre(masm /*masm*/,
566 tmp1 /* obj */,
567 tmp2 /* pre_val */,
568 rthread /* thread */,
569 tmp3 /* tmp */,
570 val != noreg /* tosca_live */,
571 false /* expand_call */);
572 }
573 if (val == noreg) {
574 BarrierSetAssembler::store_at(masm, decorators, type, Address(tmp1, 0), val, noreg, noreg);
575 } else {
576 storeval_barrier(masm, val, tmp3);
577 BarrierSetAssembler::store_at(masm, decorators, type, Address(tmp1, 0), val, noreg, noreg);
578 }
579 NOT_LP64(imasm->restore_bcp());
580 } else {
581 BarrierSetAssembler::store_at(masm, decorators, type, dst, val, tmp1, tmp2);
582 }
583 }
584
585 void ShenandoahBarrierSetAssembler::try_resolve_jobject_in_native(MacroAssembler* masm, Register jni_env,
586 Register obj, Register tmp, Label& slowpath) {
587 Label done;
588 // Resolve jobject
589 BarrierSetAssembler::try_resolve_jobject_in_native(masm, jni_env, obj, tmp, slowpath);
590
591 // Check for null.
592 __ testptr(obj, obj);
593 __ jcc(Assembler::zero, done);
594
595 Address gc_state(jni_env, ShenandoahThreadLocalData::gc_state_offset() - JavaThread::jni_environment_offset());
596 __ testb(gc_state, ShenandoahHeap::EVACUATION);
597 __ jccb(Assembler::notZero, slowpath);
598 __ bind(done);
599 }
600
601 // Special Shenandoah CAS implementation that handles false negatives
602 // due to concurrent evacuation.
603 void ShenandoahBarrierSetAssembler::cmpxchg_oop(MacroAssembler* masm,
604 Register res, Address addr, Register oldval, Register newval,
605 bool exchange, Register tmp1, Register tmp2) {
606 assert(ShenandoahCASBarrier, "Should only be used when CAS barrier is enabled");
607 assert(oldval == rax, "must be in rax for implicit use in cmpxchg");
608 assert_different_registers(oldval, newval, tmp1, tmp2);
609
610 Label L_success, L_failure;
611
612 // Remember oldval for retry logic below
613 #ifdef _LP64
614 if (UseCompressedOops) {
615 __ movl(tmp1, oldval);
616 } else
617 #endif
618 {
619 __ movptr(tmp1, oldval);
620 }
621
622 // Step 1. Fast-path.
623 //
624 // Try to CAS with given arguments. If successful, then we are done.
625
626 if (os::is_MP()) __ lock();
627 #ifdef _LP64
628 if (UseCompressedOops) {
629 __ cmpxchgl(newval, addr);
630 } else
631 #endif
632 {
633 __ cmpxchgptr(newval, addr);
634 }
635 __ jcc(Assembler::equal, L_success);
636
637 // Step 2. CAS had failed. This may be a false negative.
638 //
639 // The trouble comes when we compare the to-space pointer with the from-space
640 // pointer to the same object. To resolve this, it will suffice to resolve
641 // the value from memory -- this will give both to-space pointers.
642 // If they mismatch, then it was a legitimate failure.
643 //
644 // Before reaching to resolve sequence, see if we can avoid the whole shebang
645 // with filters.
646
647 // Filter: when offending in-memory value is NULL, the failure is definitely legitimate
648 __ testptr(oldval, oldval);
649 __ jcc(Assembler::zero, L_failure);
650
651 // Filter: when heap is stable, the failure is definitely legitimate
652 #ifdef _LP64
653 const Register thread = r15_thread;
654 #else
655 const Register thread = tmp2;
656 __ get_thread(thread);
657 #endif
658 Address gc_state(thread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
659 __ testb(gc_state, ShenandoahHeap::HAS_FORWARDED);
660 __ jcc(Assembler::zero, L_failure);
661
662 #ifdef _LP64
663 if (UseCompressedOops) {
664 __ movl(tmp2, oldval);
665 __ decode_heap_oop(tmp2);
666 } else
667 #endif
668 {
669 __ movptr(tmp2, oldval);
670 }
671
672 // Decode offending in-memory value.
673 // Test if-forwarded
674 __ testb(Address(tmp2, oopDesc::mark_offset_in_bytes()), markWord::marked_value);
675 __ jcc(Assembler::noParity, L_failure); // When odd number of bits, then not forwarded
676 __ jcc(Assembler::zero, L_failure); // When it is 00, then also not forwarded
677
678 // Load and mask forwarding pointer
679 __ movptr(tmp2, Address(tmp2, oopDesc::mark_offset_in_bytes()));
680 __ shrptr(tmp2, 2);
681 __ shlptr(tmp2, 2);
682
683 #ifdef _LP64
684 if (UseCompressedOops) {
685 __ decode_heap_oop(tmp1); // decode for comparison
686 }
687 #endif
688
689 // Now we have the forwarded offender in tmp2.
690 // Compare and if they don't match, we have legitimate failure
691 __ cmpptr(tmp1, tmp2);
692 __ jcc(Assembler::notEqual, L_failure);
693
694 // Step 3. Need to fix the memory ptr before continuing.
695 //
696 // At this point, we have from-space oldval in the register, and its to-space
697 // address is in tmp2. Let's try to update it into memory. We don't care if it
698 // succeeds or not. If it does, then the retrying CAS would see it and succeed.
699 // If this fixup fails, this means somebody else beat us to it, and necessarily
700 // with to-space ptr store. We still have to do the retry, because the GC might
701 // have updated the reference for us.
702
703 #ifdef _LP64
704 if (UseCompressedOops) {
705 __ encode_heap_oop(tmp2); // previously decoded at step 2.
706 }
707 #endif
708
709 if (os::is_MP()) __ lock();
710 #ifdef _LP64
711 if (UseCompressedOops) {
712 __ cmpxchgl(tmp2, addr);
713 } else
714 #endif
715 {
716 __ cmpxchgptr(tmp2, addr);
717 }
718
719 // Step 4. Try to CAS again.
720 //
721 // This is guaranteed not to have false negatives, because oldval is definitely
722 // to-space, and memory pointer is to-space as well. Nothing is able to store
723 // from-space ptr into memory anymore. Make sure oldval is restored, after being
724 // garbled during retries.
725 //
726 #ifdef _LP64
727 if (UseCompressedOops) {
728 __ movl(oldval, tmp2);
729 } else
730 #endif
731 {
732 __ movptr(oldval, tmp2);
733 }
734
735 if (os::is_MP()) __ lock();
736 #ifdef _LP64
737 if (UseCompressedOops) {
738 __ cmpxchgl(newval, addr);
739 } else
740 #endif
741 {
742 __ cmpxchgptr(newval, addr);
743 }
744 if (!exchange) {
745 __ jccb(Assembler::equal, L_success); // fastpath, peeking into Step 5, no need to jump
746 }
747
748 // Step 5. If we need a boolean result out of CAS, set the flag appropriately.
749 // and promote the result. Note that we handle the flag from both the 1st and 2nd CAS.
750 // Otherwise, failure witness for CAE is in oldval on all paths, and we can return.
751
752 if (exchange) {
753 __ bind(L_failure);
754 __ bind(L_success);
755 } else {
756 assert(res != NULL, "need result register");
757
758 Label exit;
759 __ bind(L_failure);
760 __ xorptr(res, res);
761 __ jmpb(exit);
762
763 __ bind(L_success);
764 __ movptr(res, 1);
765 __ bind(exit);
766 }
767 }
768
769 #undef __
770
771 #ifdef COMPILER1
772
773 #define __ ce->masm()->
774
775 void ShenandoahBarrierSetAssembler::gen_pre_barrier_stub(LIR_Assembler* ce, ShenandoahPreBarrierStub* stub) {
776 ShenandoahBarrierSetC1* bs = (ShenandoahBarrierSetC1*)BarrierSet::barrier_set()->barrier_set_c1();
777 // At this point we know that marking is in progress.
778 // If do_load() is true then we have to emit the
779 // load of the previous value; otherwise it has already
780 // been loaded into _pre_val.
781
782 __ bind(*stub->entry());
783 assert(stub->pre_val()->is_register(), "Precondition.");
784
785 Register pre_val_reg = stub->pre_val()->as_register();
786
787 if (stub->do_load()) {
788 ce->mem2reg(stub->addr(), stub->pre_val(), T_OBJECT, stub->patch_code(), stub->info(), false /*wide*/, false /*unaligned*/);
789 }
790
791 __ cmpptr(pre_val_reg, (int32_t)NULL_WORD);
792 __ jcc(Assembler::equal, *stub->continuation());
793 ce->store_parameter(stub->pre_val()->as_register(), 0);
794 __ call(RuntimeAddress(bs->pre_barrier_c1_runtime_code_blob()->code_begin()));
795 __ jmp(*stub->continuation());
796
797 }
798
799 void ShenandoahBarrierSetAssembler::gen_load_reference_barrier_stub(LIR_Assembler* ce, ShenandoahLoadReferenceBarrierStub* stub) {
800 ShenandoahBarrierSetC1* bs = (ShenandoahBarrierSetC1*)BarrierSet::barrier_set()->barrier_set_c1();
801 __ bind(*stub->entry());
802
803 Register obj = stub->obj()->as_register();
804 Register res = stub->result()->as_register();
805 Register addr = stub->addr()->as_register();
806 Register tmp1 = stub->tmp1()->as_register();
807 Register tmp2 = stub->tmp2()->as_register();
808 assert_different_registers(obj, res, addr, tmp1, tmp2);
809
810 Label slow_path;
811
812 assert(res == rax, "result must arrive in rax");
813
814 if (res != obj) {
815 __ mov(res, obj);
816 }
817
818 // Check for null.
819 __ testptr(res, res);
820 __ jcc(Assembler::zero, *stub->continuation());
821
822 // Check for object being in the collection set.
823 __ mov(tmp1, res);
824 __ shrptr(tmp1, ShenandoahHeapRegion::region_size_bytes_shift_jint());
825 __ movptr(tmp2, (intptr_t) ShenandoahHeap::in_cset_fast_test_addr());
826 #ifdef _LP64
827 __ movbool(tmp2, Address(tmp2, tmp1, Address::times_1));
828 __ testbool(tmp2);
829 #else
830 // On x86_32, C1 register allocator can give us the register without 8-bit support.
831 // Do the full-register access and test to avoid compilation failures.
832 __ movptr(tmp2, Address(tmp2, tmp1, Address::times_1));
833 __ testptr(tmp2, 0xFF);
834 #endif
835 __ jcc(Assembler::zero, *stub->continuation());
836
837 __ bind(slow_path);
838 ce->store_parameter(res, 0);
839 ce->store_parameter(addr, 1);
840 __ call(RuntimeAddress(bs->load_reference_barrier_rt_code_blob()->code_begin()));
841
842 __ jmp(*stub->continuation());
843 }
844
845 #undef __
846
847 #define __ sasm->
848
849 void ShenandoahBarrierSetAssembler::generate_c1_pre_barrier_runtime_stub(StubAssembler* sasm) {
850 __ prologue("shenandoah_pre_barrier", false);
851 // arg0 : previous value of memory
852
853 __ push(rax);
854 __ push(rdx);
855
856 const Register pre_val = rax;
857 const Register thread = NOT_LP64(rax) LP64_ONLY(r15_thread);
858 const Register tmp = rdx;
859
860 NOT_LP64(__ get_thread(thread);)
861
862 Address queue_index(thread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_index_offset()));
863 Address buffer(thread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_buffer_offset()));
864
865 Label done;
866 Label runtime;
867
868 // Is SATB still active?
869 Address gc_state(thread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
870 __ testb(gc_state, ShenandoahHeap::MARKING | ShenandoahHeap::TRAVERSAL);
871 __ jcc(Assembler::zero, done);
872
873 // Can we store original value in the thread's buffer?
874
875 __ movptr(tmp, queue_index);
876 __ testptr(tmp, tmp);
877 __ jcc(Assembler::zero, runtime);
878 __ subptr(tmp, wordSize);
879 __ movptr(queue_index, tmp);
880 __ addptr(tmp, buffer);
881
882 // prev_val (rax)
883 __ load_parameter(0, pre_val);
884 __ movptr(Address(tmp, 0), pre_val);
885 __ jmp(done);
886
887 __ bind(runtime);
888
889 __ save_live_registers_no_oop_map(true);
890
891 // load the pre-value
892 __ load_parameter(0, rcx);
893 __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_field_pre_entry), rcx, thread);
894
895 __ restore_live_registers(true);
896
897 __ bind(done);
898
899 __ pop(rdx);
900 __ pop(rax);
901
902 __ epilogue();
903 }
904
905 void ShenandoahBarrierSetAssembler::generate_c1_load_reference_barrier_runtime_stub(StubAssembler* sasm) {
906 __ prologue("shenandoah_load_reference_barrier", false);
907 // arg0 : object to be resolved
908
909 __ save_live_registers_no_oop_map(true);
910
911 #ifdef _LP64
912 __ load_parameter(0, c_rarg0);
913 __ load_parameter(1, c_rarg1);
914 if (UseCompressedOops) {
915 __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_narrow), c_rarg0, c_rarg1);
916 } else {
917 __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier), c_rarg0, c_rarg1);
918 }
919 #else
920 __ load_parameter(0, rax);
921 __ load_parameter(1, rbx);
922 __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier), rax, rbx);
923 #endif
924
925 __ restore_live_registers_except_rax(true);
926
927 __ epilogue();
928 }
929
930 #undef __
931
932 #endif // COMPILER1
933
934 address ShenandoahBarrierSetAssembler::shenandoah_lrb() {
935 assert(_shenandoah_lrb != NULL, "need load reference barrier stub");
936 return _shenandoah_lrb;
937 }
938
939 #define __ cgen->assembler()->
940
941 /*
942 * Incoming parameters:
943 * rax: oop
944 * rsi: load address
945 */
946 address ShenandoahBarrierSetAssembler::generate_shenandoah_lrb(StubCodeGenerator* cgen) {
947 __ align(CodeEntryAlignment);
948 StubCodeMark mark(cgen, "StubRoutines", "shenandoah_lrb");
949 address start = __ pc();
950
951 Label resolve_oop, slow_path;
952
953 // We use RDI, which also serves as argument register for slow call.
954 // RAX always holds the src object ptr, except after the slow call,
955 // then it holds the result. R8/RBX is used as temporary register.
956
957 Register tmp1 = rdi;
958 Register tmp2 = LP64_ONLY(r8) NOT_LP64(rbx);
959
960 __ push(tmp1);
961 __ push(tmp2);
962
963 // Check for object being in the collection set.
964 __ mov(tmp1, rax);
965 __ shrptr(tmp1, ShenandoahHeapRegion::region_size_bytes_shift_jint());
966 __ movptr(tmp2, (intptr_t) ShenandoahHeap::in_cset_fast_test_addr());
967 __ movbool(tmp2, Address(tmp2, tmp1, Address::times_1));
968 __ testbool(tmp2);
969 __ jccb(Assembler::notZero, resolve_oop);
970 __ pop(tmp2);
971 __ pop(tmp1);
972 __ ret(0);
973
974 // Test if object is already resolved.
975 __ bind(resolve_oop);
976 __ movptr(tmp2, Address(rax, oopDesc::mark_offset_in_bytes()));
977 // Test if both lowest bits are set. We trick it by negating the bits
978 // then test for both bits clear.
979 __ notptr(tmp2);
980 __ testb(tmp2, markWord::marked_value);
981 __ jccb(Assembler::notZero, slow_path);
982 // Clear both lower bits. It's still inverted, so set them, and then invert back.
983 __ orptr(tmp2, markWord::marked_value);
984 __ notptr(tmp2);
985 // At this point, tmp2 contains the decoded forwarding pointer.
986 __ mov(rax, tmp2);
987
988 __ pop(tmp2);
989 __ pop(tmp1);
990 __ ret(0);
991
992 __ bind(slow_path);
993
994 __ push(rcx);
995 __ push(rdx);
996 __ push(rdi);
997 #ifdef _LP64
998 __ push(r8);
999 __ push(r9);
1000 __ push(r10);
1001 __ push(r11);
1002 __ push(r12);
1003 __ push(r13);
1004 __ push(r14);
1005 __ push(r15);
1006 #endif
1007 __ push(rbp);
1008 __ movptr(rbp, rsp);
1009 __ andptr(rsp, -StackAlignmentInBytes);
1010 __ push_FPU_state();
1011 if (UseCompressedOops) {
1012 __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_narrow), rax, rsi);
1013 } else {
1014 __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier), rax, rsi);
1015 }
1016 __ pop_FPU_state();
1017 __ movptr(rsp, rbp);
1018 __ pop(rbp);
1019 #ifdef _LP64
1020 __ pop(r15);
1021 __ pop(r14);
1022 __ pop(r13);
1023 __ pop(r12);
1024 __ pop(r11);
1025 __ pop(r10);
1026 __ pop(r9);
1027 __ pop(r8);
1028 #endif
1029 __ pop(rdi);
1030 __ pop(rdx);
1031 __ pop(rcx);
1032
1033 __ pop(tmp2);
1034 __ pop(tmp1);
1035 __ ret(0);
1036
1037 return start;
1038 }
1039
1040 #undef __
1041
1042 void ShenandoahBarrierSetAssembler::barrier_stubs_init() {
1043 if (ShenandoahLoadRefBarrier) {
1044 int stub_code_size = 4096;
1045 ResourceMark rm;
1046 BufferBlob* bb = BufferBlob::create("shenandoah_barrier_stubs", stub_code_size);
1047 CodeBuffer buf(bb);
1048 StubCodeGenerator cgen(&buf);
1049 _shenandoah_lrb = generate_shenandoah_lrb(&cgen);
1050 }
1051 }