1 /*
2 * Copyright (c) 2018, Red Hat, Inc. All rights reserved.
3 *
4 * This code is free software; you can redistribute it and/or modify it
5 * under the terms of the GNU General Public License version 2 only, as
6 * published by the Free Software Foundation.
7 *
8 * This code is distributed in the hope that it will be useful, but WITHOUT
9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
10 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
11 * version 2 for more details (a copy is included in the LICENSE file that
12 * accompanied this code).
13 *
14 * You should have received a copy of the GNU General Public License version
15 * 2 along with this work; if not, write to the Free Software Foundation,
16 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
17 *
18 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
19 * or visit www.oracle.com if you need additional information or have any
20 * questions.
21 *
22 */
23
24 #include "precompiled.hpp"
25 #include "gc/shenandoah/shenandoahBarrierSetAssembler.hpp"
26 #include "gc/shenandoah/shenandoahHeap.hpp"
27 #include "gc/shenandoah/shenandoahHeapRegion.hpp"
28 #include "gc/shenandoah/shenandoahHeuristics.hpp"
29 #include "gc/shenandoah/shenandoahRuntime.hpp"
30 #include "gc/shenandoah/shenandoahThreadLocalData.hpp"
31 #include "interpreter/interpreter.hpp"
32 #include "interpreter/interp_masm.hpp"
33 #include "runtime/sharedRuntime.hpp"
34 #include "runtime/thread.hpp"
35 #include "utilities/macros.hpp"
36 #ifdef COMPILER1
37 #include "c1/c1_LIRAssembler.hpp"
38 #include "c1/c1_MacroAssembler.hpp"
39 #include "gc/shenandoah/c1/shenandoahBarrierSetC1.hpp"
40 #endif
41
42 #define __ masm->
43
44 address ShenandoahBarrierSetAssembler::_shenandoah_wb = NULL;
45
46 void ShenandoahBarrierSetAssembler::arraycopy_prologue(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
47 Register src, Register dst, Register count) {
48
49 bool checkcast = (decorators & ARRAYCOPY_CHECKCAST) != 0;
50 bool disjoint = (decorators & ARRAYCOPY_DISJOINT) != 0;
51 bool obj_int = type == T_OBJECT LP64_ONLY(&& UseCompressedOops);
52 bool dest_uninitialized = (decorators & IS_DEST_UNINITIALIZED) != 0;
53
54 if (type == T_OBJECT || type == T_ARRAY) {
55 #ifdef _LP64
56 if (!checkcast && !obj_int) {
57 // Save count for barrier
58 __ movptr(r11, count);
59 } else if (disjoint && obj_int) {
60 // Save dst in r11 in the disjoint case
61 __ movq(r11, dst);
62 }
63 #else
64 if (disjoint) {
65 __ mov(rdx, dst); // save 'to'
66 }
67 #endif
68
69 if (!dest_uninitialized && !ShenandoahHeap::heap()->heuristics()->can_do_traversal_gc()) {
70 Register thread = NOT_LP64(rax) LP64_ONLY(r15_thread);
71 #ifndef _LP64
72 __ push(thread);
73 __ get_thread(thread);
74 #endif
75
76 Label filtered;
77 Address in_progress(thread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_active_offset()));
78 // Is marking active?
79 if (in_bytes(SATBMarkQueue::byte_width_of_active()) == 4) {
80 __ cmpl(in_progress, 0);
81 } else {
82 assert(in_bytes(SATBMarkQueue::byte_width_of_active()) == 1, "Assumption");
83 __ cmpb(in_progress, 0);
84 }
85
86 NOT_LP64(__ pop(thread);)
87
88 __ jcc(Assembler::equal, filtered);
89
90 __ pusha(); // push registers
91 #ifdef _LP64
92 if (count == c_rarg0) {
93 if (dst == c_rarg1) {
94 // exactly backwards!!
95 __ xchgptr(c_rarg1, c_rarg0);
96 } else {
97 __ movptr(c_rarg1, count);
98 __ movptr(c_rarg0, dst);
99 }
100 } else {
101 __ movptr(c_rarg0, dst);
102 __ movptr(c_rarg1, count);
103 }
104 if (UseCompressedOops) {
105 __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_array_pre_narrow_oop_entry), 2);
106 } else {
107 __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_array_pre_oop_entry), 2);
108 }
109 #else
110 __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_array_pre_oop_entry),
111 dst, count);
112 #endif
113 __ popa();
114 __ bind(filtered);
115 }
116 }
117
118 }
119
120 void ShenandoahBarrierSetAssembler::arraycopy_epilogue(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
121 Register src, Register dst, Register count) {
122 bool checkcast = (decorators & ARRAYCOPY_CHECKCAST) != 0;
123 bool disjoint = (decorators & ARRAYCOPY_DISJOINT) != 0;
124 bool obj_int = type == T_OBJECT LP64_ONLY(&& UseCompressedOops);
125 Register tmp = rax;
126
127 if (type == T_OBJECT || type == T_ARRAY) {
128 #ifdef _LP64
129 if (!checkcast && !obj_int) {
130 // Save count for barrier
131 count = r11;
132 } else if (disjoint && obj_int) {
133 // Use the saved dst in the disjoint case
134 dst = r11;
135 } else if (checkcast) {
136 tmp = rscratch1;
137 }
138 #else
139 if (disjoint) {
140 __ mov(dst, rdx); // restore 'to'
141 }
142 #endif
143
144 __ pusha(); // push registers (overkill)
145 #ifdef _LP64
146 if (c_rarg0 == count) { // On win64 c_rarg0 == rcx
147 assert_different_registers(c_rarg1, dst);
148 __ mov(c_rarg1, count);
149 __ mov(c_rarg0, dst);
150 } else {
151 assert_different_registers(c_rarg0, count);
152 __ mov(c_rarg0, dst);
153 __ mov(c_rarg1, count);
154 }
155 __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_array_post_entry), 2);
156 #else
157 __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_array_post_entry),
158 dst, count);
159 #endif
160 __ popa();
161 }
162 }
163
164 void ShenandoahBarrierSetAssembler::shenandoah_write_barrier_pre(MacroAssembler* masm,
165 Register obj,
166 Register pre_val,
167 Register thread,
168 Register tmp,
169 bool tosca_live,
170 bool expand_call) {
171
172 if (ShenandoahSATBBarrier) {
173 satb_write_barrier_pre(masm, obj, pre_val, thread, tmp, tosca_live, expand_call);
174 }
175 }
176
177 void ShenandoahBarrierSetAssembler::satb_write_barrier_pre(MacroAssembler* masm,
178 Register obj,
179 Register pre_val,
180 Register thread,
181 Register tmp,
182 bool tosca_live,
183 bool expand_call) {
184 // If expand_call is true then we expand the call_VM_leaf macro
185 // directly to skip generating the check by
186 // InterpreterMacroAssembler::call_VM_leaf_base that checks _last_sp.
187
188 #ifdef _LP64
189 assert(thread == r15_thread, "must be");
190 #endif // _LP64
191
192 Label done;
193 Label runtime;
194
195 assert(pre_val != noreg, "check this code");
196
197 if (obj != noreg) {
198 assert_different_registers(obj, pre_val, tmp);
199 assert(pre_val != rax, "check this code");
200 }
201
202 Address in_progress(thread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_active_offset()));
203 Address index(thread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_index_offset()));
204 Address buffer(thread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_buffer_offset()));
205
206 Address gc_state(thread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
207 __ testb(gc_state, ShenandoahHeap::MARKING | ShenandoahHeap::TRAVERSAL);
208 __ jcc(Assembler::zero, done);
209
210 // Do we need to load the previous value?
211 if (obj != noreg) {
212 __ load_heap_oop(pre_val, Address(obj, 0), noreg, noreg, AS_RAW);
213 }
214
215 // Is the previous value null?
216 __ cmpptr(pre_val, (int32_t) NULL_WORD);
217 __ jcc(Assembler::equal, done);
218
219 // Can we store original value in the thread's buffer?
220 // Is index == 0?
221 // (The index field is typed as size_t.)
222
223 __ movptr(tmp, index); // tmp := *index_adr
224 __ cmpptr(tmp, 0); // tmp == 0?
225 __ jcc(Assembler::equal, runtime); // If yes, goto runtime
226
227 __ subptr(tmp, wordSize); // tmp := tmp - wordSize
228 __ movptr(index, tmp); // *index_adr := tmp
229 __ addptr(tmp, buffer); // tmp := tmp + *buffer_adr
230
231 // Record the previous value
232 __ movptr(Address(tmp, 0), pre_val);
233 __ jmp(done);
234
235 __ bind(runtime);
236 // save the live input values
237 if(tosca_live) __ push(rax);
238
239 if (obj != noreg && obj != rax)
240 __ push(obj);
241
242 if (pre_val != rax)
243 __ push(pre_val);
244
245 // Calling the runtime using the regular call_VM_leaf mechanism generates
246 // code (generated by InterpreterMacroAssember::call_VM_leaf_base)
247 // that checks that the *(ebp+frame::interpreter_frame_last_sp) == NULL.
248 //
249 // If we care generating the pre-barrier without a frame (e.g. in the
250 // intrinsified Reference.get() routine) then ebp might be pointing to
251 // the caller frame and so this check will most likely fail at runtime.
252 //
253 // Expanding the call directly bypasses the generation of the check.
254 // So when we do not have have a full interpreter frame on the stack
255 // expand_call should be passed true.
256
257 NOT_LP64( __ push(thread); )
258
259 #ifdef _LP64
260 // We move pre_val into c_rarg0 early, in order to avoid smashing it, should
261 // pre_val be c_rarg1 (where the call prologue would copy thread argument).
262 // Note: this should not accidentally smash thread, because thread is always r15.
263 assert(thread != c_rarg0, "smashed arg");
264 if (c_rarg0 != pre_val) {
265 __ mov(c_rarg0, pre_val);
266 }
267 #endif
268
269 if (expand_call) {
270 LP64_ONLY( assert(pre_val != c_rarg1, "smashed arg"); )
271 #ifdef _LP64
272 if (c_rarg1 != thread) {
273 __ mov(c_rarg1, thread);
274 }
275 // Already moved pre_val into c_rarg0 above
276 #else
277 __ push(thread);
278 __ push(pre_val);
279 #endif
280 __ MacroAssembler::call_VM_leaf_base(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_field_pre_entry), 2);
281 } else {
282 __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_field_pre_entry), LP64_ONLY(c_rarg0) NOT_LP64(pre_val), thread);
283 }
284
285 NOT_LP64( __ pop(thread); )
286
287 // save the live input values
288 if (pre_val != rax)
289 __ pop(pre_val);
290
291 if (obj != noreg && obj != rax)
292 __ pop(obj);
293
294 if(tosca_live) __ pop(rax);
295
296 __ bind(done);
297 }
298
299 void ShenandoahBarrierSetAssembler::read_barrier(MacroAssembler* masm, Register dst) {
300 if (ShenandoahReadBarrier) {
301 read_barrier_impl(masm, dst);
302 }
303 }
304
305 void ShenandoahBarrierSetAssembler::read_barrier_impl(MacroAssembler* masm, Register dst) {
306 assert(UseShenandoahGC && (ShenandoahReadBarrier || ShenandoahStoreValReadBarrier || ShenandoahCASBarrier), "should be enabled");
307 Label is_null;
308 __ testptr(dst, dst);
309 __ jcc(Assembler::zero, is_null);
310 read_barrier_not_null_impl(masm, dst);
311 __ bind(is_null);
312 }
313
314 void ShenandoahBarrierSetAssembler::read_barrier_not_null(MacroAssembler* masm, Register dst) {
315 if (ShenandoahReadBarrier) {
316 read_barrier_not_null_impl(masm, dst);
317 }
318 }
319
320 void ShenandoahBarrierSetAssembler::read_barrier_not_null_impl(MacroAssembler* masm, Register dst) {
321 assert(UseShenandoahGC && (ShenandoahReadBarrier || ShenandoahStoreValReadBarrier || ShenandoahCASBarrier), "should be enabled");
322 __ movptr(dst, Address(dst, ShenandoahBrooksPointer::byte_offset()));
323 }
324
325
326 void ShenandoahBarrierSetAssembler::write_barrier(MacroAssembler* masm, Register dst) {
327 if (ShenandoahWriteBarrier) {
328 write_barrier_impl(masm, dst);
329 }
330 }
331
332 void ShenandoahBarrierSetAssembler::write_barrier_impl(MacroAssembler* masm, Register dst) {
333 assert(UseShenandoahGC && (ShenandoahWriteBarrier || ShenandoahStoreValEnqueueBarrier), "Should be enabled");
334 #ifdef _LP64
335 Label done;
336
337 Address gc_state(r15_thread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
338 __ testb(gc_state, ShenandoahHeap::HAS_FORWARDED | ShenandoahHeap::EVACUATION | ShenandoahHeap::TRAVERSAL);
339 __ jccb(Assembler::zero, done);
340
341 // Heap is unstable, need to perform the read-barrier even if WB is inactive
342 read_barrier_not_null(masm, dst);
343
344 __ testb(gc_state, ShenandoahHeap::EVACUATION | ShenandoahHeap::TRAVERSAL);
345 __ jccb(Assembler::zero, done);
346
347 if (dst != rax) {
348 __ xchgptr(dst, rax); // Move obj into rax and save rax into obj.
349 }
350
351 __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, ShenandoahBarrierSetAssembler::shenandoah_wb())));
352
353 if (dst != rax) {
354 __ xchgptr(rax, dst); // Swap back obj with rax.
355 }
356
357 __ bind(done);
358 #else
359 Unimplemented();
360 #endif
361 }
362
363 void ShenandoahBarrierSetAssembler::storeval_barrier(MacroAssembler* masm, Register dst, Register tmp) {
364 if (ShenandoahStoreValReadBarrier || ShenandoahStoreValEnqueueBarrier) {
365 storeval_barrier_impl(masm, dst, tmp);
366 }
367 }
368
369 void ShenandoahBarrierSetAssembler::storeval_barrier_impl(MacroAssembler* masm, Register dst, Register tmp) {
370 assert(UseShenandoahGC && (ShenandoahStoreValReadBarrier || ShenandoahStoreValEnqueueBarrier), "should be enabled");
371
372 if (dst == noreg) return;
373
374 #ifdef _LP64
375 if (ShenandoahStoreValEnqueueBarrier) {
376 Label is_null;
377 __ testptr(dst, dst);
378 __ jcc(Assembler::zero, is_null);
379 write_barrier_impl(masm, dst);
380 __ bind(is_null);
381
382 // The set of registers to be saved+restored is the same as in the write-barrier above.
383 // Those are the commonly used registers in the interpreter.
384 __ pusha();
385 // __ push_callee_saved_registers();
386 __ subptr(rsp, 2 * Interpreter::stackElementSize);
387 __ movdbl(Address(rsp, 0), xmm0);
388
389 satb_write_barrier_pre(masm, noreg, dst, r15_thread, tmp, true, false);
390 __ movdbl(xmm0, Address(rsp, 0));
391 __ addptr(rsp, 2 * Interpreter::stackElementSize);
392 //__ pop_callee_saved_registers();
393 __ popa();
394 }
395 if (ShenandoahStoreValReadBarrier) {
396 read_barrier_impl(masm, dst);
397 }
398 #else
399 Unimplemented();
400 #endif
401 }
402
403 void ShenandoahBarrierSetAssembler::load_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
404 Register dst, Address src, Register tmp1, Register tmp_thread) {
405 bool on_oop = type == T_OBJECT || type == T_ARRAY;
406 bool in_heap = (decorators & IN_HEAP) != 0;
407 bool on_weak = (decorators & ON_WEAK_OOP_REF) != 0;
408 bool on_phantom = (decorators & ON_PHANTOM_OOP_REF) != 0;
409 bool on_reference = on_weak || on_phantom;
410 if (in_heap) {
411 read_barrier_not_null(masm, src.base());
412 }
413 BarrierSetAssembler::load_at(masm, decorators, type, dst, src, tmp1, tmp_thread);
414 if (ShenandoahKeepAliveBarrier && on_oop && on_reference) {
415 const Register thread = NOT_LP64(tmp_thread) LP64_ONLY(r15_thread);
416 NOT_LP64(__ get_thread(thread));
417
418 // Generate the SATB pre-barrier code to log the value of
419 // the referent field in an SATB buffer.
420 shenandoah_write_barrier_pre(masm /* masm */,
421 noreg /* obj */,
422 dst /* pre_val */,
423 thread /* thread */,
424 tmp1 /* tmp */,
425 true /* tosca_live */,
426 true /* expand_call */);
427 }
428 }
429
430 void ShenandoahBarrierSetAssembler::store_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
431 Address dst, Register val, Register tmp1, Register tmp2) {
432
433 bool in_heap = (decorators & IN_HEAP) != 0;
434 bool as_normal = (decorators & AS_NORMAL) != 0;
435 if (in_heap) {
436 write_barrier(masm, dst.base());
437 }
438 if (type == T_OBJECT || type == T_ARRAY) {
439 bool needs_pre_barrier = as_normal;
440
441 Register tmp3 = LP64_ONLY(r8) NOT_LP64(rsi);
442 Register rthread = LP64_ONLY(r15_thread) NOT_LP64(rcx);
443 // flatten object address if needed
444 // We do it regardless of precise because we need the registers
445 if (dst.index() == noreg && dst.disp() == 0) {
446 if (dst.base() != tmp1) {
447 __ movptr(tmp1, dst.base());
448 }
449 } else {
450 __ lea(tmp1, dst);
451 }
452
453 #ifndef _LP64
454 InterpreterMacroAssembler *imasm = static_cast<InterpreterMacroAssembler*>(masm);
455 #endif
456
457 NOT_LP64(__ get_thread(rcx));
458 NOT_LP64(imasm->save_bcp());
459
460 if (needs_pre_barrier) {
461 shenandoah_write_barrier_pre(masm /*masm*/,
462 tmp1 /* obj */,
463 tmp2 /* pre_val */,
464 rthread /* thread */,
465 tmp3 /* tmp */,
466 val != noreg /* tosca_live */,
467 false /* expand_call */);
468 }
469 if (val == noreg) {
470 BarrierSetAssembler::store_at(masm, decorators, type, Address(tmp1, 0), val, noreg, noreg);
471 } else {
472 storeval_barrier(masm, val, tmp3);
473 BarrierSetAssembler::store_at(masm, decorators, type, Address(tmp1, 0), val, noreg, noreg);
474 }
475 NOT_LP64(imasm->restore_bcp());
476 } else {
477 BarrierSetAssembler::store_at(masm, decorators, type, dst, val, tmp1, tmp2);
478 }
479 }
480
481 #ifndef _LP64
482 void ShenandoahBarrierSetAssembler::obj_equals(MacroAssembler* masm,
483 Address obj1, jobject obj2) {
484 Unimplemented();
485 }
486
487 void ShenandoahBarrierSetAssembler::obj_equals(MacroAssembler* masm,
488 Register obj1, jobject obj2) {
489 Unimplemented();
490 }
491 #endif
492
493
494 void ShenandoahBarrierSetAssembler::obj_equals(MacroAssembler* masm, Register op1, Register op2) {
495 __ cmpptr(op1, op2);
496 if (ShenandoahAcmpBarrier) {
497 Label done;
498 __ jccb(Assembler::equal, done);
499 read_barrier(masm, op1);
500 read_barrier(masm, op2);
501 __ cmpptr(op1, op2);
502 __ bind(done);
503 }
504 }
505
506 void ShenandoahBarrierSetAssembler::obj_equals(MacroAssembler* masm, Register src1, Address src2) {
507 __ cmpptr(src1, src2);
508 if (ShenandoahAcmpBarrier) {
509 Label done;
510 __ jccb(Assembler::equal, done);
511 __ movptr(rscratch2, src2);
512 read_barrier(masm, src1);
513 read_barrier(masm, rscratch2);
514 __ cmpptr(src1, rscratch2);
515 __ bind(done);
516 }
517 }
518
519 void ShenandoahBarrierSetAssembler::tlab_allocate(MacroAssembler* masm,
520 Register thread, Register obj,
521 Register var_size_in_bytes,
522 int con_size_in_bytes,
523 Register t1, Register t2,
524 Label& slow_case) {
525 assert_different_registers(obj, t1, t2);
526 assert_different_registers(obj, var_size_in_bytes, t1);
527 Register end = t2;
528 if (!thread->is_valid()) {
529 #ifdef _LP64
530 thread = r15_thread;
531 #else
532 assert(t1->is_valid(), "need temp reg");
533 thread = t1;
534 __ get_thread(thread);
535 #endif
536 }
537
538 __ verify_tlab();
539
540 __ movptr(obj, Address(thread, JavaThread::tlab_top_offset()));
541 if (var_size_in_bytes == noreg) {
542 __ lea(end, Address(obj, con_size_in_bytes + ShenandoahBrooksPointer::byte_size()));
543 } else {
544 __ addptr(var_size_in_bytes, ShenandoahBrooksPointer::byte_size());
545 __ lea(end, Address(obj, var_size_in_bytes, Address::times_1));
546 }
547 __ cmpptr(end, Address(thread, JavaThread::tlab_end_offset()));
548 __ jcc(Assembler::above, slow_case);
549
550 // update the tlab top pointer
551 __ movptr(Address(thread, JavaThread::tlab_top_offset()), end);
552
553 // Initialize brooks pointer
554 #ifdef _LP64
555 __ incrementq(obj, ShenandoahBrooksPointer::byte_size());
556 #else
557 __ incrementl(obj, ShenandoahBrooksPointer::byte_size());
558 #endif
559 __ movptr(Address(obj, ShenandoahBrooksPointer::byte_offset()), obj);
560
561 // recover var_size_in_bytes if necessary
562 if (var_size_in_bytes == end) {
563 __ subptr(var_size_in_bytes, obj);
564 }
565 __ verify_tlab();
566 }
567
568 void ShenandoahBarrierSetAssembler::resolve(MacroAssembler* masm, DecoratorSet decorators, Register obj) {
569 bool oop_not_null = (decorators & IS_NOT_NULL) != 0;
570 bool is_write = (decorators & ACCESS_WRITE) != 0;
571 if (is_write) {
572 if (oop_not_null) {
573 write_barrier(masm, obj);
574 } else {
575 Label done;
576 __ testptr(obj, obj);
577 __ jcc(Assembler::zero, done);
578 write_barrier(masm, obj);
579 __ bind(done);
580 }
581 } else {
582 if (oop_not_null) {
583 read_barrier_not_null(masm, obj);
584 } else {
585 read_barrier(masm, obj);
586 }
587 }
588 }
589
590 // Special Shenandoah CAS implementation that handles false negatives
591 // due to concurrent evacuation.
592 #ifndef _LP64
593 void ShenandoahBarrierSetAssembler::cmpxchg_oop(MacroAssembler* masm,
594 Register res, Address addr, Register oldval, Register newval,
595 bool exchange, bool encode, Register tmp1, Register tmp2) {
596 // Shenandoah has no 32-bit version for this.
597 Unimplemented();
598 }
599 #else
600 void ShenandoahBarrierSetAssembler::cmpxchg_oop(MacroAssembler* masm,
601 Register res, Address addr, Register oldval, Register newval,
602 bool exchange, bool encode, Register tmp1, Register tmp2) {
603 if (!ShenandoahCASBarrier) {
604 #ifdef _LP64
605 if (UseCompressedOops) {
606 if (encode) {
607 __ encode_heap_oop(oldval);
608 __ mov(rscratch1, newval);
609 __ encode_heap_oop(rscratch1);
610 newval = rscratch1;
611 }
612 if (os::is_MP()) {
613 __ lock();
614 }
615 // oldval (rax) is implicitly used by this instruction
616 __ cmpxchgl(newval, addr);
617 } else
618 #endif
619 {
620 if (os::is_MP()) {
621 __ lock();
622 }
623 __ cmpxchgptr(newval, addr);
624 }
625
626 if (!exchange) {
627 assert(res != NULL, "need result register");
628 __ setb(Assembler::equal, res);
629 __ movzbl(res, res);
630 }
631 return;
632 }
633
634 assert(ShenandoahCASBarrier, "Should only be used when CAS barrier is enabled");
635 assert(oldval == rax, "must be in rax for implicit use in cmpxchg");
636
637 Label retry, done;
638
639 // Apply storeval barrier to newval.
640 if (encode) {
641 storeval_barrier(masm, newval, tmp1);
642 }
643
644 if (UseCompressedOops) {
645 if (encode) {
646 __ encode_heap_oop(oldval);
647 __ mov(rscratch1, newval);
648 __ encode_heap_oop(rscratch1);
649 newval = rscratch1;
650 }
651 }
652
653 // Remember oldval for retry logic below
654 if (UseCompressedOops) {
655 __ movl(tmp1, oldval);
656 } else {
657 __ movptr(tmp1, oldval);
658 }
659
660 // Step 1. Try to CAS with given arguments. If successful, then we are done,
661 // and can safely return.
662 if (os::is_MP()) __ lock();
663 if (UseCompressedOops) {
664 __ cmpxchgl(newval, addr);
665 } else {
666 __ cmpxchgptr(newval, addr);
667 }
668 __ jcc(Assembler::equal, done, true);
669
670 // Step 2. CAS had failed. This may be a false negative.
671 //
672 // The trouble comes when we compare the to-space pointer with the from-space
673 // pointer to the same object. To resolve this, it will suffice to read both
674 // oldval and the value from memory through the read barriers -- this will give
675 // both to-space pointers. If they mismatch, then it was a legitimate failure.
676 //
677 if (UseCompressedOops) {
678 __ decode_heap_oop(tmp1);
679 }
680 read_barrier_impl(masm, tmp1);
681
682 if (UseCompressedOops) {
683 __ movl(tmp2, oldval);
684 __ decode_heap_oop(tmp2);
685 } else {
686 __ movptr(tmp2, oldval);
687 }
688 read_barrier_impl(masm, tmp2);
689
690 __ cmpptr(tmp1, tmp2);
691 __ jcc(Assembler::notEqual, done, true);
692
693 // Step 3. Try to CAS again with resolved to-space pointers.
694 //
695 // Corner case: it may happen that somebody stored the from-space pointer
696 // to memory while we were preparing for retry. Therefore, we can fail again
697 // on retry, and so need to do this in loop, always re-reading the failure
698 // witness through the read barrier.
699 __ bind(retry);
700 if (os::is_MP()) __ lock();
701 if (UseCompressedOops) {
702 __ cmpxchgl(newval, addr);
703 } else {
704 __ cmpxchgptr(newval, addr);
705 }
706 __ jcc(Assembler::equal, done, true);
707
708 if (UseCompressedOops) {
709 __ movl(tmp2, oldval);
710 __ decode_heap_oop(tmp2);
711 } else {
712 __ movptr(tmp2, oldval);
713 }
714 read_barrier_impl(masm, tmp2);
715
716 __ cmpptr(tmp1, tmp2);
717 __ jcc(Assembler::equal, retry, true);
718
719 // Step 4. If we need a boolean result out of CAS, check the flag again,
720 // and promote the result. Note that we handle the flag from both the CAS
721 // itself and from the retry loop.
722 __ bind(done);
723 if (!exchange) {
724 assert(res != NULL, "need result register");
725 __ setb(Assembler::equal, res);
726 __ movzbl(res, res);
727 }
728 }
729 #endif // LP64
730
731 void ShenandoahBarrierSetAssembler::save_vector_registers(MacroAssembler* masm) {
732 int num_xmm_regs = LP64_ONLY(16) NOT_LP64(8);
733 if (UseAVX > 2) {
734 num_xmm_regs = LP64_ONLY(32) NOT_LP64(8);
735 }
736
737 if (UseSSE == 1) {
738 __ subptr(rsp, sizeof(jdouble)*8);
739 for (int n = 0; n < 8; n++) {
740 __ movflt(Address(rsp, n*sizeof(jdouble)), as_XMMRegister(n));
741 }
742 } else if (UseSSE >= 2) {
743 if (UseAVX > 2) {
744 __ push(rbx);
745 __ movl(rbx, 0xffff);
746 __ kmovwl(k1, rbx);
747 __ pop(rbx);
748 }
749 #ifdef COMPILER2
750 if (MaxVectorSize > 16) {
751 if(UseAVX > 2) {
752 // Save upper half of ZMM registers
753 __ subptr(rsp, 32*num_xmm_regs);
754 for (int n = 0; n < num_xmm_regs; n++) {
755 __ vextractf64x4_high(Address(rsp, n*32), as_XMMRegister(n));
756 }
757 }
758 assert(UseAVX > 0, "256 bit vectors are supported only with AVX");
759 // Save upper half of YMM registers
760 __ subptr(rsp, 16*num_xmm_regs);
761 for (int n = 0; n < num_xmm_regs; n++) {
762 __ vextractf128_high(Address(rsp, n*16), as_XMMRegister(n));
763 }
764 }
765 #endif
766 // Save whole 128bit (16 bytes) XMM registers
767 __ subptr(rsp, 16*num_xmm_regs);
768 #ifdef _LP64
769 if (VM_Version::supports_evex()) {
770 for (int n = 0; n < num_xmm_regs; n++) {
771 __ vextractf32x4(Address(rsp, n*16), as_XMMRegister(n), 0);
772 }
773 } else {
774 for (int n = 0; n < num_xmm_regs; n++) {
775 __ movdqu(Address(rsp, n*16), as_XMMRegister(n));
776 }
777 }
778 #else
779 for (int n = 0; n < num_xmm_regs; n++) {
780 __ movdqu(Address(rsp, n*16), as_XMMRegister(n));
781 }
782 #endif
783 }
784 }
785
786 void ShenandoahBarrierSetAssembler::restore_vector_registers(MacroAssembler* masm) {
787 int num_xmm_regs = LP64_ONLY(16) NOT_LP64(8);
788 if (UseAVX > 2) {
789 num_xmm_regs = LP64_ONLY(32) NOT_LP64(8);
790 }
791 if (UseSSE == 1) {
792 for (int n = 0; n < 8; n++) {
793 __ movflt(as_XMMRegister(n), Address(rsp, n*sizeof(jdouble)));
794 }
795 __ addptr(rsp, sizeof(jdouble)*8);
796 } else if (UseSSE >= 2) {
797 // Restore whole 128bit (16 bytes) XMM registers
798 #ifdef _LP64
799 if (VM_Version::supports_evex()) {
800 for (int n = 0; n < num_xmm_regs; n++) {
801 __ vinsertf32x4(as_XMMRegister(n), as_XMMRegister(n), Address(rsp, n*16), 0);
802 }
803 } else {
804 for (int n = 0; n < num_xmm_regs; n++) {
805 __ movdqu(as_XMMRegister(n), Address(rsp, n*16));
806 }
807 }
808 #else
809 for (int n = 0; n < num_xmm_regs; n++) {
810 __ movdqu(as_XMMRegister(n), Address(rsp, n*16));
811 }
812 #endif
813 __ addptr(rsp, 16*num_xmm_regs);
814
815 #ifdef COMPILER2
816 if (MaxVectorSize > 16) {
817 // Restore upper half of YMM registers.
818 for (int n = 0; n < num_xmm_regs; n++) {
819 __ vinsertf128_high(as_XMMRegister(n), Address(rsp, n*16));
820 }
821 __ addptr(rsp, 16*num_xmm_regs);
822 if (UseAVX > 2) {
823 for (int n = 0; n < num_xmm_regs; n++) {
824 __ vinsertf64x4_high(as_XMMRegister(n), Address(rsp, n*32));
825 }
826 __ addptr(rsp, 32*num_xmm_regs);
827 }
828 }
829 #endif
830 }
831 }
832
833 #ifdef COMPILER1
834
835 #undef __
836 #define __ ce->masm()->
837
838 void ShenandoahBarrierSetAssembler::gen_pre_barrier_stub(LIR_Assembler* ce, ShenandoahPreBarrierStub* stub) {
839 ShenandoahBarrierSetC1* bs = (ShenandoahBarrierSetC1*)BarrierSet::barrier_set()->barrier_set_c1();
840 // At this point we know that marking is in progress.
841 // If do_load() is true then we have to emit the
842 // load of the previous value; otherwise it has already
843 // been loaded into _pre_val.
844
845 __ bind(*stub->entry());
846 assert(stub->pre_val()->is_register(), "Precondition.");
847
848 Register pre_val_reg = stub->pre_val()->as_register();
849
850 if (stub->do_load()) {
851 ce->mem2reg(stub->addr(), stub->pre_val(), T_OBJECT, stub->patch_code(), stub->info(), false /*wide*/, false /*unaligned*/);
852 }
853
854 __ cmpptr(pre_val_reg, (int32_t)NULL_WORD);
855 __ jcc(Assembler::equal, *stub->continuation());
856 ce->store_parameter(stub->pre_val()->as_register(), 0);
857 __ call(RuntimeAddress(bs->pre_barrier_c1_runtime_code_blob()->code_begin()));
858 __ jmp(*stub->continuation());
859
860 }
861
862 void ShenandoahBarrierSetAssembler::gen_write_barrier_stub(LIR_Assembler* ce, ShenandoahWriteBarrierStub* stub) {
863 __ bind(*stub->entry());
864
865 Label done;
866 Register obj = stub->obj()->as_register();
867 Register res = stub->result()->as_register();
868
869 if (res != obj) {
870 __ mov(res, obj);
871 }
872
873 // Check for null.
874 if (stub->needs_null_check()) {
875 __ testptr(res, res);
876 __ jcc(Assembler::zero, done);
877 }
878
879 write_barrier(ce->masm(), res);
880
881 __ bind(done);
882 __ jmp(*stub->continuation());
883 }
884
885 #undef __
886
887 #define __ sasm->
888
889 void ShenandoahBarrierSetAssembler::generate_c1_pre_barrier_runtime_stub(StubAssembler* sasm) {
890 __ prologue("shenandoah_pre_barrier", false);
891 // arg0 : previous value of memory
892
893 __ push(rax);
894 __ push(rdx);
895
896 const Register pre_val = rax;
897 const Register thread = NOT_LP64(rax) LP64_ONLY(r15_thread);
898 const Register tmp = rdx;
899
900 NOT_LP64(__ get_thread(thread);)
901
902 Address queue_index(thread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_index_offset()));
903 Address buffer(thread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_buffer_offset()));
904
905 Label done;
906 Label runtime;
907
908 // Is SATB still active?
909 Address gc_state(thread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
910 __ testb(gc_state, ShenandoahHeap::MARKING | ShenandoahHeap::TRAVERSAL);
911 __ jcc(Assembler::zero, done);
912
913 // Can we store original value in the thread's buffer?
914
915 __ movptr(tmp, queue_index);
916 __ testptr(tmp, tmp);
917 __ jcc(Assembler::zero, runtime);
918 __ subptr(tmp, wordSize);
919 __ movptr(queue_index, tmp);
920 __ addptr(tmp, buffer);
921
922 // prev_val (rax)
923 __ load_parameter(0, pre_val);
924 __ movptr(Address(tmp, 0), pre_val);
925 __ jmp(done);
926
927 __ bind(runtime);
928
929 __ save_live_registers_no_oop_map(true);
930
931 // load the pre-value
932 __ load_parameter(0, rcx);
933 __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_field_pre_entry), rcx, thread);
934
935 __ restore_live_registers(true);
936
937 __ bind(done);
938
939 __ pop(rdx);
940 __ pop(rax);
941
942 __ epilogue();
943 }
944
945 #undef __
946
947 #endif // COMPILER1
948
949 address ShenandoahBarrierSetAssembler::shenandoah_wb() {
950 assert(_shenandoah_wb != NULL, "need write barrier stub");
951 return _shenandoah_wb;
952 }
953
954 #define __ cgen->assembler()->
955
956 address ShenandoahBarrierSetAssembler::generate_shenandoah_wb(StubCodeGenerator* cgen) {
957 __ align(CodeEntryAlignment);
958 StubCodeMark mark(cgen, "StubRoutines", "shenandoah_wb");
959 address start = __ pc();
960
961 #ifdef _LP64
962 Label not_done;
963
964 // We use RDI, which also serves as argument register for slow call.
965 // RAX always holds the src object ptr, except after the slow call and
966 // the cmpxchg, then it holds the result.
967 // R8 and RCX are used as temporary registers.
968 __ push(rdi);
969 __ push(r8);
970
971 // Check for object beeing in the collection set.
972 // TODO: Can we use only 1 register here?
973 // The source object arrives here in rax.
974 // live: rax
975 // live: rdi
976 __ mov(rdi, rax);
977 __ shrptr(rdi, ShenandoahHeapRegion::region_size_bytes_shift_jint());
978 // live: r8
979 __ movptr(r8, (intptr_t) ShenandoahHeap::in_cset_fast_test_addr());
980 __ movbool(r8, Address(r8, rdi, Address::times_1));
981 // unlive: rdi
982 __ testbool(r8);
983 // unlive: r8
984 __ jccb(Assembler::notZero, not_done);
985
986 __ pop(r8);
987 __ pop(rdi);
988 __ ret(0);
989
990 __ bind(not_done);
991
992 __ push(rcx);
993 __ push(rdx);
994 __ push(rdi);
995 __ push(rsi);
996 __ push(r8);
997 __ push(r9);
998 __ push(r10);
999 __ push(r11);
1000 __ push(r12);
1001 __ push(r13);
1002 __ push(r14);
1003 __ push(r15);
1004 save_vector_registers(cgen->assembler());
1005 __ movptr(rdi, rax);
1006 __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_barrier_JRT), rdi);
1007 restore_vector_registers(cgen->assembler());
1008 __ pop(r15);
1009 __ pop(r14);
1010 __ pop(r13);
1011 __ pop(r12);
1012 __ pop(r11);
1013 __ pop(r10);
1014 __ pop(r9);
1015 __ pop(r8);
1016 __ pop(rsi);
1017 __ pop(rdi);
1018 __ pop(rdx);
1019 __ pop(rcx);
1020
1021 __ pop(r8);
1022 __ pop(rdi);
1023 __ ret(0);
1024 #else
1025 ShouldNotReachHere();
1026 #endif
1027 return start;
1028 }
1029
1030 #undef __
1031
1032 void ShenandoahBarrierSetAssembler::barrier_stubs_init() {
1033 if (ShenandoahWriteBarrier || ShenandoahStoreValEnqueueBarrier) {
1034 int stub_code_size = 4096;
1035 ResourceMark rm;
1036 BufferBlob* bb = BufferBlob::create("shenandoah_barrier_stubs", stub_code_size);
1037 CodeBuffer buf(bb);
1038 StubCodeGenerator cgen(&buf);
1039 _shenandoah_wb = generate_shenandoah_wb(&cgen);
1040 }
1041 }