1 /*
2 * Copyright (c) 2018, Red Hat, Inc. and/or its affiliates.
3 *
4 * This code is free software; you can redistribute it and/or modify it
5 * under the terms of the GNU General Public License version 2 only, as
6 * published by the Free Software Foundation.
7 *
8 * This code is distributed in the hope that it will be useful, but WITHOUT
9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
10 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
11 * version 2 for more details (a copy is included in the LICENSE file that
12 * accompanied this code).
13 *
14 * You should have received a copy of the GNU General Public License version
15 * 2 along with this work; if not, write to the Free Software Foundation,
16 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
17 *
18 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
19 * or visit www.oracle.com if you need additional information or have any
20 * questions.
21 *
22 */
23
24 #include "precompiled.hpp"
25 #include "gc/shenandoah/shenandoahBarrierSetAssembler.hpp"
26 #include "gc/shenandoah/shenandoahHeap.hpp"
27 #include "gc/shenandoah/shenandoahHeapRegion.hpp"
28 #include "gc/shenandoah/shenandoahHeuristics.hpp"
29 #include "gc/shenandoah/shenandoahRuntime.hpp"
30 #include "gc/shenandoah/shenandoahThreadLocalData.hpp"
31 #include "interpreter/interpreter.hpp"
32 #include "interpreter/interp_masm.hpp"
33 #include "runtime/sharedRuntime.hpp"
34 #include "runtime/thread.hpp"
35 #include "utilities/macros.hpp"
36 #ifdef COMPILER1
37 #include "c1/c1_LIRAssembler.hpp"
38 #include "c1/c1_MacroAssembler.hpp"
39 #include "gc/shenandoah/c1/shenandoahBarrierSetC1.hpp"
40 #endif
41
42 #define __ masm->
43
44 address ShenandoahBarrierSetAssembler::_shenandoah_wb = NULL;
45
46 void ShenandoahBarrierSetAssembler::arraycopy_prologue(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
47 Register src, Register dst, Register count) {
48
49 bool checkcast = (decorators & ARRAYCOPY_CHECKCAST) != 0;
50 bool disjoint = (decorators & ARRAYCOPY_DISJOINT) != 0;
51 bool obj_int = type == T_OBJECT LP64_ONLY(&& UseCompressedOops);
52 bool dest_uninitialized = (decorators & IS_DEST_UNINITIALIZED) != 0;
53
54 if (type == T_OBJECT || type == T_ARRAY) {
55 #ifdef _LP64
56 if (!checkcast && !obj_int) {
57 // Save count for barrier
58 __ movptr(r11, count);
59 } else if (disjoint && obj_int) {
60 // Save dst in r11 in the disjoint case
61 __ movq(r11, dst);
62 }
63 #else
64 if (disjoint) {
65 __ mov(rdx, dst); // save 'to'
66 }
67 #endif
68
69 if (!dest_uninitialized && !ShenandoahHeap::heap()->heuristics()->can_do_traversal_gc()) {
70 Register thread = NOT_LP64(rax) LP64_ONLY(r15_thread);
71 #ifndef _LP64
72 __ push(thread);
73 __ get_thread(thread);
74 #endif
75
76 Label filtered;
77 Address in_progress(thread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_active_offset()));
78 // Is marking active?
79 if (in_bytes(SATBMarkQueue::byte_width_of_active()) == 4) {
80 __ cmpl(in_progress, 0);
81 } else {
82 assert(in_bytes(SATBMarkQueue::byte_width_of_active()) == 1, "Assumption");
83 __ cmpb(in_progress, 0);
84 }
85
86 NOT_LP64(__ pop(thread);)
87
88 __ jcc(Assembler::equal, filtered);
89
90 __ pusha(); // push registers
91 #ifdef _LP64
92 if (count == c_rarg0) {
93 if (dst == c_rarg1) {
94 // exactly backwards!!
95 __ xchgptr(c_rarg1, c_rarg0);
96 } else {
97 __ movptr(c_rarg1, count);
98 __ movptr(c_rarg0, dst);
99 }
100 } else {
101 __ movptr(c_rarg0, dst);
102 __ movptr(c_rarg1, count);
103 }
104 if (UseCompressedOops) {
105 __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_array_pre_narrow_oop_entry), 2);
106 } else {
107 __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_array_pre_oop_entry), 2);
108 }
109 #else
110 __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_array_pre_oop_entry),
111 dst, count);
112 #endif
113 __ popa();
114 __ bind(filtered);
115 }
116 }
117
118 }
119
120 void ShenandoahBarrierSetAssembler::arraycopy_epilogue(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
121 Register src, Register dst, Register count) {
122 bool checkcast = (decorators & ARRAYCOPY_CHECKCAST) != 0;
123 bool disjoint = (decorators & ARRAYCOPY_DISJOINT) != 0;
124 bool obj_int = type == T_OBJECT LP64_ONLY(&& UseCompressedOops);
125 Register tmp = rax;
126
127 if (type == T_OBJECT || type == T_ARRAY) {
128 #ifdef _LP64
129 if (!checkcast && !obj_int) {
130 // Save count for barrier
131 count = r11;
132 } else if (disjoint && obj_int) {
133 // Use the saved dst in the disjoint case
134 dst = r11;
135 } else if (checkcast) {
136 tmp = rscratch1;
137 }
138 #else
139 if (disjoint) {
140 __ mov(dst, rdx); // restore 'to'
141 }
142 #endif
143
144 __ pusha(); // push registers (overkill)
145 #ifdef _LP64
146 if (c_rarg0 == count) { // On win64 c_rarg0 == rcx
147 assert_different_registers(c_rarg1, dst);
148 __ mov(c_rarg1, count);
149 __ mov(c_rarg0, dst);
150 } else {
151 assert_different_registers(c_rarg0, count);
152 __ mov(c_rarg0, dst);
153 __ mov(c_rarg1, count);
154 }
155 __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_array_post_entry), 2);
156 #else
157 __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_array_post_entry),
158 dst, count);
159 #endif
160 __ popa();
161 }
162 }
163
164 void ShenandoahBarrierSetAssembler::shenandoah_write_barrier_pre(MacroAssembler* masm,
165 Register obj,
166 Register pre_val,
167 Register thread,
168 Register tmp,
169 bool tosca_live,
170 bool expand_call) {
171
172 if (ShenandoahSATBBarrier) {
173 satb_write_barrier_pre(masm, obj, pre_val, thread, tmp, tosca_live, expand_call);
174 }
175 }
176
177 void ShenandoahBarrierSetAssembler::satb_write_barrier_pre(MacroAssembler* masm,
178 Register obj,
179 Register pre_val,
180 Register thread,
181 Register tmp,
182 bool tosca_live,
183 bool expand_call) {
184 // If expand_call is true then we expand the call_VM_leaf macro
185 // directly to skip generating the check by
186 // InterpreterMacroAssembler::call_VM_leaf_base that checks _last_sp.
187
188 #ifdef _LP64
189 assert(thread == r15_thread, "must be");
190 #endif // _LP64
191
192 Label done;
193 Label runtime;
194
195 assert(pre_val != noreg, "check this code");
196
197 if (obj != noreg) {
198 assert_different_registers(obj, pre_val, tmp);
199 assert(pre_val != rax, "check this code");
200 }
201
202 Address in_progress(thread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_active_offset()));
203 Address index(thread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_index_offset()));
204 Address buffer(thread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_buffer_offset()));
205
206 Address gc_state(thread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
207 __ testb(gc_state, ShenandoahHeap::MARKING | ShenandoahHeap::TRAVERSAL);
208 __ jcc(Assembler::zero, done);
209
210 // Do we need to load the previous value?
211 if (obj != noreg) {
212 __ load_heap_oop(pre_val, Address(obj, 0), noreg, noreg, AS_RAW);
213 }
214
215 // Is the previous value null?
216 __ cmpptr(pre_val, (int32_t) NULL_WORD);
217 __ jcc(Assembler::equal, done);
218
219 // Can we store original value in the thread's buffer?
220 // Is index == 0?
221 // (The index field is typed as size_t.)
222
223 __ movptr(tmp, index); // tmp := *index_adr
224 __ cmpptr(tmp, 0); // tmp == 0?
225 __ jcc(Assembler::equal, runtime); // If yes, goto runtime
226
227 __ subptr(tmp, wordSize); // tmp := tmp - wordSize
228 __ movptr(index, tmp); // *index_adr := tmp
229 __ addptr(tmp, buffer); // tmp := tmp + *buffer_adr
230
231 // Record the previous value
232 __ movptr(Address(tmp, 0), pre_val);
233 __ jmp(done);
234
235 __ bind(runtime);
236 // save the live input values
237 if(tosca_live) __ push(rax);
238
239 if (obj != noreg && obj != rax)
240 __ push(obj);
241
242 if (pre_val != rax)
243 __ push(pre_val);
244
245 // Calling the runtime using the regular call_VM_leaf mechanism generates
246 // code (generated by InterpreterMacroAssember::call_VM_leaf_base)
247 // that checks that the *(ebp+frame::interpreter_frame_last_sp) == NULL.
248 //
249 // If we care generating the pre-barrier without a frame (e.g. in the
250 // intrinsified Reference.get() routine) then ebp might be pointing to
251 // the caller frame and so this check will most likely fail at runtime.
252 //
253 // Expanding the call directly bypasses the generation of the check.
254 // So when we do not have have a full interpreter frame on the stack
255 // expand_call should be passed true.
256
257 NOT_LP64( __ push(thread); )
258
259 #ifdef _LP64
260 // We move pre_val into c_rarg0 early, in order to avoid smashing it, should
261 // pre_val be c_rarg1 (where the call prologue would copy thread argument).
262 // Note: this should not accidentally smash thread, because thread is always r15.
263 assert(thread != c_rarg0, "smashed arg");
264 if (c_rarg0 != pre_val) {
265 __ mov(c_rarg0, pre_val);
266 }
267 #endif
268
269 if (expand_call) {
270 LP64_ONLY( assert(pre_val != c_rarg1, "smashed arg"); )
271 #ifdef _LP64
272 if (c_rarg1 != thread) {
273 __ mov(c_rarg1, thread);
274 }
275 // Already moved pre_val into c_rarg0 above
276 #else
277 __ push(thread);
278 __ push(pre_val);
279 #endif
280 __ MacroAssembler::call_VM_leaf_base(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_field_pre_entry), 2);
281 } else {
282 __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_field_pre_entry), LP64_ONLY(c_rarg0) NOT_LP64(pre_val), thread);
283 }
284
285 NOT_LP64( __ pop(thread); )
286
287 // save the live input values
288 if (pre_val != rax)
289 __ pop(pre_val);
290
291 if (obj != noreg && obj != rax)
292 __ pop(obj);
293
294 if(tosca_live) __ pop(rax);
295
296 __ bind(done);
297 }
298
299 void ShenandoahBarrierSetAssembler::read_barrier(MacroAssembler* masm, Register dst) {
300 if (ShenandoahReadBarrier) {
301 read_barrier_impl(masm, dst);
302 }
303 }
304
305 void ShenandoahBarrierSetAssembler::read_barrier_impl(MacroAssembler* masm, Register dst) {
306 assert(UseShenandoahGC && (ShenandoahReadBarrier || ShenandoahStoreValReadBarrier || ShenandoahCASBarrier), "should be enabled");
307 Label is_null;
308 __ testptr(dst, dst);
309 __ jcc(Assembler::zero, is_null);
310 read_barrier_not_null_impl(masm, dst);
311 __ bind(is_null);
312 }
313
314 void ShenandoahBarrierSetAssembler::read_barrier_not_null(MacroAssembler* masm, Register dst) {
315 if (ShenandoahReadBarrier) {
316 read_barrier_not_null_impl(masm, dst);
317 }
318 }
319
320 void ShenandoahBarrierSetAssembler::read_barrier_not_null_impl(MacroAssembler* masm, Register dst) {
321 assert(UseShenandoahGC && (ShenandoahReadBarrier || ShenandoahStoreValReadBarrier || ShenandoahCASBarrier), "should be enabled");
322 __ movptr(dst, Address(dst, ShenandoahBrooksPointer::byte_offset()));
323 }
324
325
326 void ShenandoahBarrierSetAssembler::write_barrier(MacroAssembler* masm, Register dst) {
327 if (ShenandoahWriteBarrier) {
328 write_barrier_impl(masm, dst);
329 }
330 }
331
332 void ShenandoahBarrierSetAssembler::write_barrier_impl(MacroAssembler* masm, Register dst) {
333 assert(UseShenandoahGC && (ShenandoahWriteBarrier || ShenandoahStoreValEnqueueBarrier), "Should be enabled");
334 #ifdef _LP64
335 Label done;
336
337 Address gc_state(r15_thread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
338 __ testb(gc_state, ShenandoahHeap::HAS_FORWARDED | ShenandoahHeap::EVACUATION | ShenandoahHeap::TRAVERSAL);
339 __ jccb(Assembler::zero, done);
340
341 // Heap is unstable, need to perform the read-barrier even if WB is inactive
342 if (ShenandoahWriteBarrierRB) {
343 read_barrier_not_null(masm, dst);
344 }
345
346 __ testb(gc_state, ShenandoahHeap::EVACUATION | ShenandoahHeap::TRAVERSAL);
347 __ jccb(Assembler::zero, done);
348
349 if (dst != rax) {
350 __ xchgptr(dst, rax); // Move obj into rax and save rax into obj.
351 }
352
353 __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, ShenandoahBarrierSetAssembler::shenandoah_wb())));
354
355 if (dst != rax) {
356 __ xchgptr(rax, dst); // Swap back obj with rax.
357 }
358
359 __ bind(done);
360 #else
361 Unimplemented();
362 #endif
363 }
364
365 void ShenandoahBarrierSetAssembler::storeval_barrier(MacroAssembler* masm, Register dst, Register tmp) {
366 if (ShenandoahStoreValReadBarrier || ShenandoahStoreValEnqueueBarrier) {
367 storeval_barrier_impl(masm, dst, tmp);
368 }
369 }
370
371 void ShenandoahBarrierSetAssembler::storeval_barrier_impl(MacroAssembler* masm, Register dst, Register tmp) {
372 assert(UseShenandoahGC && (ShenandoahStoreValReadBarrier || ShenandoahStoreValEnqueueBarrier), "should be enabled");
373
374 if (dst == noreg) return;
375
376 #ifdef _LP64
377 if (ShenandoahStoreValEnqueueBarrier) {
378 Label is_null;
379 __ testptr(dst, dst);
380 __ jcc(Assembler::zero, is_null);
381 write_barrier_impl(masm, dst);
382 __ bind(is_null);
383
384 // The set of registers to be saved+restored is the same as in the write-barrier above.
385 // Those are the commonly used registers in the interpreter.
386 __ pusha();
387 // __ push_callee_saved_registers();
388 __ subptr(rsp, 2 * Interpreter::stackElementSize);
389 __ movdbl(Address(rsp, 0), xmm0);
390
391 satb_write_barrier_pre(masm, noreg, dst, r15_thread, tmp, true, false);
392 __ movdbl(xmm0, Address(rsp, 0));
393 __ addptr(rsp, 2 * Interpreter::stackElementSize);
394 //__ pop_callee_saved_registers();
395 __ popa();
396 }
397 if (ShenandoahStoreValReadBarrier) {
398 read_barrier_impl(masm, dst);
399 }
400 #else
401 Unimplemented();
402 #endif
403 }
404
405 void ShenandoahBarrierSetAssembler::load_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
406 Register dst, Address src, Register tmp1, Register tmp_thread) {
407 bool on_oop = type == T_OBJECT || type == T_ARRAY;
408 bool in_heap = (decorators & IN_HEAP) != 0;
409 bool on_weak = (decorators & ON_WEAK_OOP_REF) != 0;
410 bool on_phantom = (decorators & ON_PHANTOM_OOP_REF) != 0;
411 bool on_reference = on_weak || on_phantom;
412 if (in_heap) {
413 read_barrier_not_null(masm, src.base());
414 }
415 BarrierSetAssembler::load_at(masm, decorators, type, dst, src, tmp1, tmp_thread);
416 if (ShenandoahKeepAliveBarrier && on_oop && on_reference) {
417 const Register thread = NOT_LP64(tmp_thread) LP64_ONLY(r15_thread);
418 NOT_LP64(__ get_thread(thread));
419
420 // Generate the SATB pre-barrier code to log the value of
421 // the referent field in an SATB buffer.
422 shenandoah_write_barrier_pre(masm /* masm */,
423 noreg /* obj */,
424 dst /* pre_val */,
425 thread /* thread */,
426 tmp1 /* tmp */,
427 true /* tosca_live */,
428 true /* expand_call */);
429 }
430 }
431
432 void ShenandoahBarrierSetAssembler::store_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
433 Address dst, Register val, Register tmp1, Register tmp2) {
434
435 bool in_heap = (decorators & IN_HEAP) != 0;
436 bool as_normal = (decorators & AS_NORMAL) != 0;
437 if (in_heap) {
438 write_barrier(masm, dst.base());
439 }
440 if (type == T_OBJECT || type == T_ARRAY) {
441 bool needs_pre_barrier = as_normal;
442
443 Register tmp3 = LP64_ONLY(r8) NOT_LP64(rsi);
444 Register rthread = LP64_ONLY(r15_thread) NOT_LP64(rcx);
445 // flatten object address if needed
446 // We do it regardless of precise because we need the registers
447 if (dst.index() == noreg && dst.disp() == 0) {
448 if (dst.base() != tmp1) {
449 __ movptr(tmp1, dst.base());
450 }
451 } else {
452 __ lea(tmp1, dst);
453 }
454
455 #ifndef _LP64
456 InterpreterMacroAssembler *imasm = static_cast<InterpreterMacroAssembler*>(masm);
457 #endif
458
459 NOT_LP64(__ get_thread(rcx));
460 NOT_LP64(imasm->save_bcp());
461
462 if (needs_pre_barrier) {
463 shenandoah_write_barrier_pre(masm /*masm*/,
464 tmp1 /* obj */,
465 tmp2 /* pre_val */,
466 rthread /* thread */,
467 tmp3 /* tmp */,
468 val != noreg /* tosca_live */,
469 false /* expand_call */);
470 }
471 if (val == noreg) {
472 BarrierSetAssembler::store_at(masm, decorators, type, Address(tmp1, 0), val, noreg, noreg);
473 } else {
474 storeval_barrier(masm, val, tmp3);
475 BarrierSetAssembler::store_at(masm, decorators, type, Address(tmp1, 0), val, noreg, noreg);
476 }
477 NOT_LP64(imasm->restore_bcp());
478 } else {
479 BarrierSetAssembler::store_at(masm, decorators, type, dst, val, tmp1, tmp2);
480 }
481 }
482
483 #ifndef _LP64
484 void ShenandoahBarrierSetAssembler::obj_equals(MacroAssembler* masm,
485 Address obj1, jobject obj2) {
486 Unimplemented();
487 }
488
489 void ShenandoahBarrierSetAssembler::obj_equals(MacroAssembler* masm,
490 Register obj1, jobject obj2) {
491 Unimplemented();
492 }
493 #endif
494
495
496 void ShenandoahBarrierSetAssembler::obj_equals(MacroAssembler* masm, Register op1, Register op2) {
497 __ cmpptr(op1, op2);
498 if (ShenandoahAcmpBarrier) {
499 Label done;
500 __ jccb(Assembler::equal, done);
501 read_barrier(masm, op1);
502 read_barrier(masm, op2);
503 __ cmpptr(op1, op2);
504 __ bind(done);
505 }
506 }
507
508 void ShenandoahBarrierSetAssembler::obj_equals(MacroAssembler* masm, Register src1, Address src2) {
509 __ cmpptr(src1, src2);
510 if (ShenandoahAcmpBarrier) {
511 Label done;
512 __ jccb(Assembler::equal, done);
513 __ movptr(rscratch2, src2);
514 read_barrier(masm, src1);
515 read_barrier(masm, rscratch2);
516 __ cmpptr(src1, rscratch2);
517 __ bind(done);
518 }
519 }
520
521 void ShenandoahBarrierSetAssembler::tlab_allocate(MacroAssembler* masm,
522 Register thread, Register obj,
523 Register var_size_in_bytes,
524 int con_size_in_bytes,
525 Register t1, Register t2,
526 Label& slow_case) {
527 assert_different_registers(obj, t1, t2);
528 assert_different_registers(obj, var_size_in_bytes, t1);
529 Register end = t2;
530 if (!thread->is_valid()) {
531 #ifdef _LP64
532 thread = r15_thread;
533 #else
534 assert(t1->is_valid(), "need temp reg");
535 thread = t1;
536 __ get_thread(thread);
537 #endif
538 }
539
540 __ verify_tlab();
541
542 __ movptr(obj, Address(thread, JavaThread::tlab_top_offset()));
543 if (var_size_in_bytes == noreg) {
544 __ lea(end, Address(obj, con_size_in_bytes + ShenandoahBrooksPointer::byte_size()));
545 } else {
546 __ addptr(var_size_in_bytes, ShenandoahBrooksPointer::byte_size());
547 __ lea(end, Address(obj, var_size_in_bytes, Address::times_1));
548 }
549 __ cmpptr(end, Address(thread, JavaThread::tlab_end_offset()));
550 __ jcc(Assembler::above, slow_case);
551
552 // update the tlab top pointer
553 __ movptr(Address(thread, JavaThread::tlab_top_offset()), end);
554
555 // Initialize brooks pointer
556 #ifdef _LP64
557 __ incrementq(obj, ShenandoahBrooksPointer::byte_size());
558 #else
559 __ incrementl(obj, ShenandoahBrooksPointer::byte_size());
560 #endif
561 __ movptr(Address(obj, ShenandoahBrooksPointer::byte_offset()), obj);
562
563 // recover var_size_in_bytes if necessary
564 if (var_size_in_bytes == end) {
565 __ subptr(var_size_in_bytes, obj);
566 }
567 __ verify_tlab();
568 }
569
570 void ShenandoahBarrierSetAssembler::resolve(MacroAssembler* masm, DecoratorSet decorators, Register obj) {
571 bool oop_not_null = (decorators & IS_NOT_NULL) != 0;
572 bool is_write = (decorators & ACCESS_WRITE) != 0;
573 if (is_write) {
574 if (oop_not_null) {
575 write_barrier(masm, obj);
576 } else {
577 Label done;
578 __ testptr(obj, obj);
579 __ jcc(Assembler::zero, done);
580 write_barrier(masm, obj);
581 __ bind(done);
582 }
583 } else {
584 if (oop_not_null) {
585 read_barrier_not_null(masm, obj);
586 } else {
587 read_barrier(masm, obj);
588 }
589 }
590 }
591
592 // Special Shenandoah CAS implementation that handles false negatives
593 // due to concurrent evacuation.
594 #ifndef _LP64
595 void ShenandoahBarrierSetAssembler::cmpxchg_oop(MacroAssembler* masm,
596 Register res, Address addr, Register oldval, Register newval,
597 bool exchange, bool encode, Register tmp1, Register tmp2) {
598 // Shenandoah has no 32-bit version for this.
599 Unimplemented();
600 }
601 #else
602 void ShenandoahBarrierSetAssembler::cmpxchg_oop(MacroAssembler* masm,
603 Register res, Address addr, Register oldval, Register newval,
604 bool exchange, bool encode, Register tmp1, Register tmp2) {
605 if (!ShenandoahCASBarrier) {
606 #ifdef _LP64
607 if (UseCompressedOops) {
608 if (encode) {
609 __ encode_heap_oop(oldval);
610 __ mov(rscratch1, newval);
611 __ encode_heap_oop(rscratch1);
612 newval = rscratch1;
613 }
614 if (os::is_MP()) {
615 __ lock();
616 }
617 // oldval (rax) is implicitly used by this instruction
618 __ cmpxchgl(newval, addr);
619 } else
620 #endif
621 {
622 if (os::is_MP()) {
623 __ lock();
624 }
625 __ cmpxchgptr(newval, addr);
626 }
627
628 if (!exchange) {
629 assert(res != NULL, "need result register");
630 __ setb(Assembler::equal, res);
631 __ movzbl(res, res);
632 }
633 return;
634 }
635
636 assert(ShenandoahCASBarrier, "Should only be used when CAS barrier is enabled");
637 assert(oldval == rax, "must be in rax for implicit use in cmpxchg");
638
639 Label retry, done;
640
641 // Apply storeval barrier to newval.
642 if (encode) {
643 storeval_barrier(masm, newval, tmp1);
644 }
645
646 if (UseCompressedOops) {
647 if (encode) {
648 __ encode_heap_oop(oldval);
649 __ mov(rscratch1, newval);
650 __ encode_heap_oop(rscratch1);
651 newval = rscratch1;
652 }
653 }
654
655 // Remember oldval for retry logic below
656 if (UseCompressedOops) {
657 __ movl(tmp1, oldval);
658 } else {
659 __ movptr(tmp1, oldval);
660 }
661
662 // Step 1. Try to CAS with given arguments. If successful, then we are done,
663 // and can safely return.
664 if (os::is_MP()) __ lock();
665 if (UseCompressedOops) {
666 __ cmpxchgl(newval, addr);
667 } else {
668 __ cmpxchgptr(newval, addr);
669 }
670 __ jcc(Assembler::equal, done, true);
671
672 // Step 2. CAS had failed. This may be a false negative.
673 //
674 // The trouble comes when we compare the to-space pointer with the from-space
675 // pointer to the same object. To resolve this, it will suffice to read both
676 // oldval and the value from memory through the read barriers -- this will give
677 // both to-space pointers. If they mismatch, then it was a legitimate failure.
678 //
679 if (UseCompressedOops) {
680 __ decode_heap_oop(tmp1);
681 }
682 read_barrier_impl(masm, tmp1);
683
684 if (UseCompressedOops) {
685 __ movl(tmp2, oldval);
686 __ decode_heap_oop(tmp2);
687 } else {
688 __ movptr(tmp2, oldval);
689 }
690 read_barrier_impl(masm, tmp2);
691
692 __ cmpptr(tmp1, tmp2);
693 __ jcc(Assembler::notEqual, done, true);
694
695 // Step 3. Try to CAS again with resolved to-space pointers.
696 //
697 // Corner case: it may happen that somebody stored the from-space pointer
698 // to memory while we were preparing for retry. Therefore, we can fail again
699 // on retry, and so need to do this in loop, always re-reading the failure
700 // witness through the read barrier.
701 __ bind(retry);
702 if (os::is_MP()) __ lock();
703 if (UseCompressedOops) {
704 __ cmpxchgl(newval, addr);
705 } else {
706 __ cmpxchgptr(newval, addr);
707 }
708 __ jcc(Assembler::equal, done, true);
709
710 if (UseCompressedOops) {
711 __ movl(tmp2, oldval);
712 __ decode_heap_oop(tmp2);
713 } else {
714 __ movptr(tmp2, oldval);
715 }
716 read_barrier_impl(masm, tmp2);
717
718 __ cmpptr(tmp1, tmp2);
719 __ jcc(Assembler::equal, retry, true);
720
721 // Step 4. If we need a boolean result out of CAS, check the flag again,
722 // and promote the result. Note that we handle the flag from both the CAS
723 // itself and from the retry loop.
724 __ bind(done);
725 if (!exchange) {
726 assert(res != NULL, "need result register");
727 __ setb(Assembler::equal, res);
728 __ movzbl(res, res);
729 }
730 }
731 #endif // LP64
732
733 void ShenandoahBarrierSetAssembler::save_vector_registers(MacroAssembler* masm) {
734 int num_xmm_regs = LP64_ONLY(16) NOT_LP64(8);
735 if (UseAVX > 2) {
736 num_xmm_regs = LP64_ONLY(32) NOT_LP64(8);
737 }
738
739 if (UseSSE == 1) {
740 __ subptr(rsp, sizeof(jdouble)*8);
741 for (int n = 0; n < 8; n++) {
742 __ movflt(Address(rsp, n*sizeof(jdouble)), as_XMMRegister(n));
743 }
744 } else if (UseSSE >= 2) {
745 if (UseAVX > 2) {
746 __ push(rbx);
747 __ movl(rbx, 0xffff);
748 __ kmovwl(k1, rbx);
749 __ pop(rbx);
750 }
751 #ifdef COMPILER2
752 if (MaxVectorSize > 16) {
753 if(UseAVX > 2) {
754 // Save upper half of ZMM registers
755 __ subptr(rsp, 32*num_xmm_regs);
756 for (int n = 0; n < num_xmm_regs; n++) {
757 __ vextractf64x4_high(Address(rsp, n*32), as_XMMRegister(n));
758 }
759 }
760 assert(UseAVX > 0, "256 bit vectors are supported only with AVX");
761 // Save upper half of YMM registers
762 __ subptr(rsp, 16*num_xmm_regs);
763 for (int n = 0; n < num_xmm_regs; n++) {
764 __ vextractf128_high(Address(rsp, n*16), as_XMMRegister(n));
765 }
766 }
767 #endif
768 // Save whole 128bit (16 bytes) XMM registers
769 __ subptr(rsp, 16*num_xmm_regs);
770 #ifdef _LP64
771 if (VM_Version::supports_evex()) {
772 for (int n = 0; n < num_xmm_regs; n++) {
773 __ vextractf32x4(Address(rsp, n*16), as_XMMRegister(n), 0);
774 }
775 } else {
776 for (int n = 0; n < num_xmm_regs; n++) {
777 __ movdqu(Address(rsp, n*16), as_XMMRegister(n));
778 }
779 }
780 #else
781 for (int n = 0; n < num_xmm_regs; n++) {
782 __ movdqu(Address(rsp, n*16), as_XMMRegister(n));
783 }
784 #endif
785 }
786 }
787
788 void ShenandoahBarrierSetAssembler::restore_vector_registers(MacroAssembler* masm) {
789 int num_xmm_regs = LP64_ONLY(16) NOT_LP64(8);
790 if (UseAVX > 2) {
791 num_xmm_regs = LP64_ONLY(32) NOT_LP64(8);
792 }
793 if (UseSSE == 1) {
794 for (int n = 0; n < 8; n++) {
795 __ movflt(as_XMMRegister(n), Address(rsp, n*sizeof(jdouble)));
796 }
797 __ addptr(rsp, sizeof(jdouble)*8);
798 } else if (UseSSE >= 2) {
799 // Restore whole 128bit (16 bytes) XMM registers
800 #ifdef _LP64
801 if (VM_Version::supports_evex()) {
802 for (int n = 0; n < num_xmm_regs; n++) {
803 __ vinsertf32x4(as_XMMRegister(n), as_XMMRegister(n), Address(rsp, n*16), 0);
804 }
805 } else {
806 for (int n = 0; n < num_xmm_regs; n++) {
807 __ movdqu(as_XMMRegister(n), Address(rsp, n*16));
808 }
809 }
810 #else
811 for (int n = 0; n < num_xmm_regs; n++) {
812 __ movdqu(as_XMMRegister(n), Address(rsp, n*16));
813 }
814 #endif
815 __ addptr(rsp, 16*num_xmm_regs);
816
817 #ifdef COMPILER2
818 if (MaxVectorSize > 16) {
819 // Restore upper half of YMM registers.
820 for (int n = 0; n < num_xmm_regs; n++) {
821 __ vinsertf128_high(as_XMMRegister(n), Address(rsp, n*16));
822 }
823 __ addptr(rsp, 16*num_xmm_regs);
824 if (UseAVX > 2) {
825 for (int n = 0; n < num_xmm_regs; n++) {
826 __ vinsertf64x4_high(as_XMMRegister(n), Address(rsp, n*32));
827 }
828 __ addptr(rsp, 32*num_xmm_regs);
829 }
830 }
831 #endif
832 }
833 }
834
835 #ifdef COMPILER1
836
837 #undef __
838 #define __ ce->masm()->
839
840 void ShenandoahBarrierSetAssembler::gen_pre_barrier_stub(LIR_Assembler* ce, ShenandoahPreBarrierStub* stub) {
841 ShenandoahBarrierSetC1* bs = (ShenandoahBarrierSetC1*)BarrierSet::barrier_set()->barrier_set_c1();
842 // At this point we know that marking is in progress.
843 // If do_load() is true then we have to emit the
844 // load of the previous value; otherwise it has already
845 // been loaded into _pre_val.
846
847 __ bind(*stub->entry());
848 assert(stub->pre_val()->is_register(), "Precondition.");
849
850 Register pre_val_reg = stub->pre_val()->as_register();
851
852 if (stub->do_load()) {
853 ce->mem2reg(stub->addr(), stub->pre_val(), T_OBJECT, stub->patch_code(), stub->info(), false /*wide*/, false /*unaligned*/);
854 }
855
856 __ cmpptr(pre_val_reg, (int32_t)NULL_WORD);
857 __ jcc(Assembler::equal, *stub->continuation());
858 ce->store_parameter(stub->pre_val()->as_register(), 0);
859 __ call(RuntimeAddress(bs->pre_barrier_c1_runtime_code_blob()->code_begin()));
860 __ jmp(*stub->continuation());
861
862 }
863
864 void ShenandoahBarrierSetAssembler::gen_write_barrier_stub(LIR_Assembler* ce, ShenandoahWriteBarrierStub* stub) {
865 __ bind(*stub->entry());
866
867 Label done;
868 Register obj = stub->obj()->as_register();
869 Register res = stub->result()->as_register();
870
871 if (res != obj) {
872 __ mov(res, obj);
873 }
874
875 // Check for null.
876 if (stub->needs_null_check()) {
877 __ testptr(res, res);
878 __ jcc(Assembler::zero, done);
879 }
880
881 write_barrier(ce->masm(), res);
882
883 __ bind(done);
884 __ jmp(*stub->continuation());
885 }
886
887 #undef __
888
889 #define __ sasm->
890
891 void ShenandoahBarrierSetAssembler::generate_c1_pre_barrier_runtime_stub(StubAssembler* sasm) {
892 __ prologue("shenandoah_pre_barrier", false);
893 // arg0 : previous value of memory
894
895 __ push(rax);
896 __ push(rdx);
897
898 const Register pre_val = rax;
899 const Register thread = NOT_LP64(rax) LP64_ONLY(r15_thread);
900 const Register tmp = rdx;
901
902 NOT_LP64(__ get_thread(thread);)
903
904 Address queue_index(thread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_index_offset()));
905 Address buffer(thread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_buffer_offset()));
906
907 Label done;
908 Label runtime;
909
910 // Is SATB still active?
911 Address gc_state(thread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
912 __ testb(gc_state, ShenandoahHeap::MARKING | ShenandoahHeap::TRAVERSAL);
913 __ jcc(Assembler::zero, done);
914
915 // Can we store original value in the thread's buffer?
916
917 __ movptr(tmp, queue_index);
918 __ testptr(tmp, tmp);
919 __ jcc(Assembler::zero, runtime);
920 __ subptr(tmp, wordSize);
921 __ movptr(queue_index, tmp);
922 __ addptr(tmp, buffer);
923
924 // prev_val (rax)
925 __ load_parameter(0, pre_val);
926 __ movptr(Address(tmp, 0), pre_val);
927 __ jmp(done);
928
929 __ bind(runtime);
930
931 __ save_live_registers_no_oop_map(true);
932
933 // load the pre-value
934 __ load_parameter(0, rcx);
935 __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_field_pre_entry), rcx, thread);
936
937 __ restore_live_registers(true);
938
939 __ bind(done);
940
941 __ pop(rdx);
942 __ pop(rax);
943
944 __ epilogue();
945 }
946
947 #undef __
948
949 #endif // COMPILER1
950
951 address ShenandoahBarrierSetAssembler::shenandoah_wb() {
952 assert(_shenandoah_wb != NULL, "need write barrier stub");
953 return _shenandoah_wb;
954 }
955
956 #define __ cgen->assembler()->
957
958 address ShenandoahBarrierSetAssembler::generate_shenandoah_wb(StubCodeGenerator* cgen) {
959 __ align(CodeEntryAlignment);
960 StubCodeMark mark(cgen, "StubRoutines", "shenandoah_wb");
961 address start = __ pc();
962
963 #ifdef _LP64
964 Label not_done;
965
966 // We use RDI, which also serves as argument register for slow call.
967 // RAX always holds the src object ptr, except after the slow call and
968 // the cmpxchg, then it holds the result.
969 // R8 and RCX are used as temporary registers.
970 __ push(rdi);
971 __ push(r8);
972
973 // Check for object beeing in the collection set.
974 // TODO: Can we use only 1 register here?
975 // The source object arrives here in rax.
976 // live: rax
977 // live: rdi
978 __ mov(rdi, rax);
979 __ shrptr(rdi, ShenandoahHeapRegion::region_size_bytes_shift_jint());
980 // live: r8
981 __ movptr(r8, (intptr_t) ShenandoahHeap::in_cset_fast_test_addr());
982 __ movbool(r8, Address(r8, rdi, Address::times_1));
983 // unlive: rdi
984 __ testbool(r8);
985 // unlive: r8
986 __ jccb(Assembler::notZero, not_done);
987
988 __ pop(r8);
989 __ pop(rdi);
990 __ ret(0);
991
992 __ bind(not_done);
993
994 // Save general purpose registers
995 __ subq(rsp, 6 * wordSize);
996 // Skip rax because it's used for return anyway
997 __ movq(Address(rsp, 5 * wordSize), rcx);
998 __ movq(Address(rsp, 4 * wordSize), rdx);
999 // Skip rbx and rbp because they are callee saved in all calling conventions
1000 // skip rsp
1001 __ movq(Address(rsp, 3 * wordSize), rsi);
1002 // Skip r8 and rdi, we already pushed them above
1003 __ movq(Address(rsp, 2 * wordSize), r9);
1004 __ movq(Address(rsp, 1 * wordSize), r10);
1005 __ movq(Address(rsp, 0 * wordSize), r11);
1006 // Skip r12..r15 because they are callee saved in all calling conventions
1007
1008 save_vector_registers(cgen->assembler());
1009 __ movptr(c_rarg0, rax);
1010 __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_barrier_JRT), c_rarg0);
1011 restore_vector_registers(cgen->assembler());
1012
1013 // Restore general purpose registers
1014 // Skip r12..r15
1015 __ movq(r11, Address(rsp, 0 * wordSize));
1016 __ movq(r10, Address(rsp, 1 * wordSize));
1017 __ movq(r9, Address(rsp, 2 * wordSize));
1018 // Skip r8 and rdi
1019 __ movq(rsi, Address(rsp, 3 * wordSize));
1020 // skip rdb, rsp and rbx
1021 __ movq(rdx, Address(rsp, 4 * wordSize));
1022 __ movq(rcx, Address(rsp, 5 * wordSize));
1023
1024 // Restore r8 and rdi from outermost pushes
1025 __ movq(r8, Address(rsp, 6 * wordSize));
1026 __ movq(rdi, Address(rsp, 7 * wordSize));
1027 __ addq(rsp, 8 * wordSize);
1028
1029 __ ret(0);
1030 #else
1031 ShouldNotReachHere();
1032 #endif
1033 return start;
1034 }
1035
1036 #undef __
1037
1038 void ShenandoahBarrierSetAssembler::barrier_stubs_init() {
1039 if (ShenandoahWriteBarrier || ShenandoahStoreValEnqueueBarrier) {
1040 int stub_code_size = 4096;
1041 ResourceMark rm;
1042 BufferBlob* bb = BufferBlob::create("shenandoah_barrier_stubs", stub_code_size);
1043 CodeBuffer buf(bb);
1044 StubCodeGenerator cgen(&buf);
1045 _shenandoah_wb = generate_shenandoah_wb(&cgen);
1046 }
1047 }