87 if (dest_uninitialized) {
88 __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_array_pre_duinit_narrow_oop_entry), src, dst, count);
89 } else {
90 __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_array_pre_narrow_oop_entry), src, dst, count);
91 }
92 } else
93 #endif
94 {
95 if (dest_uninitialized) {
96 __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_array_pre_duinit_oop_entry), src, dst, count);
97 } else {
98 __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_array_pre_oop_entry), src, dst, count);
99 }
100 }
101 __ popa();
102 __ bind(done);
103 NOT_LP64(__ pop(thread);)
104 }
105 }
106
107 void ShenandoahBarrierSetAssembler::resolve_forward_pointer(MacroAssembler* masm, Register dst, Register tmp) {
108 assert(ShenandoahCASBarrier, "should be enabled");
109 Label is_null;
110 __ testptr(dst, dst);
111 __ jcc(Assembler::zero, is_null);
112 resolve_forward_pointer_not_null(masm, dst, tmp);
113 __ bind(is_null);
114 }
115
116 void ShenandoahBarrierSetAssembler::resolve_forward_pointer_not_null(MacroAssembler* masm, Register dst, Register tmp) {
117 assert(ShenandoahCASBarrier || ShenandoahLoadRefBarrier, "should be enabled");
118 // The below loads the mark word, checks if the lowest two bits are
119 // set, and if so, clear the lowest two bits and copy the result
120 // to dst. Otherwise it leaves dst alone.
121 // Implementing this is surprisingly awkward. I do it here by:
122 // - Inverting the mark word
123 // - Test lowest two bits == 0
124 // - If so, set the lowest two bits
125 // - Invert the result back, and copy to dst
126
127 bool borrow_reg = (tmp == noreg);
128 if (borrow_reg) {
129 // No free registers available. Make one useful.
130 tmp = LP64_ONLY(rscratch1) NOT_LP64(rdx);
131 if (tmp == dst) {
132 tmp = LP64_ONLY(rscratch2) NOT_LP64(rcx);
133 }
134 __ push(tmp);
135 }
136
137 assert_different_registers(dst, tmp);
138
139 Label done;
140 __ movptr(tmp, Address(dst, oopDesc::mark_offset_in_bytes()));
141 __ notptr(tmp);
142 __ testb(tmp, markOopDesc::marked_value);
143 __ jccb(Assembler::notZero, done);
144 __ orptr(tmp, markOopDesc::marked_value);
145 __ notptr(tmp);
146 __ mov(dst, tmp);
147 __ bind(done);
148
149 if (borrow_reg) {
150 __ pop(tmp);
151 }
152 }
153
154 void ShenandoahBarrierSetAssembler::load_reference_barrier_not_null(MacroAssembler* masm, Register dst) {
155 assert(ShenandoahLoadRefBarrier, "Should be enabled");
156
157 Label done;
158
159 #ifdef _LP64
160 Register thread = r15_thread;
161 #else
162 Register thread = rcx;
163 if (thread == dst) {
164 thread = rbx;
165 }
166 __ push(thread);
167 __ get_thread(thread);
168 #endif
169 assert_different_registers(dst, thread);
170
171 Address gc_state(thread, in_bytes(JavaThread::gc_state_offset()));
172 __ testb(gc_state, ShenandoahHeap::HAS_FORWARDED);
173 __ jcc(Assembler::zero, done);
271 }
272 }
273
274 void ShenandoahBarrierSetAssembler::load_reference_barrier(MacroAssembler* masm, Register dst) {
275 if (ShenandoahLoadRefBarrier) {
276 Label done;
277 __ testptr(dst, dst);
278 __ jcc(Assembler::zero, done);
279 load_reference_barrier_not_null(masm, dst);
280 __ bind(done);
281 }
282 }
283
284 // Special Shenandoah CAS implementation that handles false negatives
285 // due to concurrent evacuation.
286 void ShenandoahBarrierSetAssembler::cmpxchg_oop(MacroAssembler* masm,
287 Register res, Address addr, Register oldval, Register newval,
288 bool exchange, Register tmp1, Register tmp2) {
289 assert(ShenandoahCASBarrier, "Should only be used when CAS barrier is enabled");
290 assert(oldval == rax, "must be in rax for implicit use in cmpxchg");
291
292 Label retry, done;
293
294 // Remember oldval for retry logic below
295 #ifdef _LP64
296 if (UseCompressedOops) {
297 __ movl(tmp1, oldval);
298 } else
299 #endif
300 {
301 __ movptr(tmp1, oldval);
302 }
303
304 // Step 1. Try to CAS with given arguments. If successful, then we are done,
305 // and can safely return.
306 if (os::is_MP()) __ lock();
307 #ifdef _LP64
308 if (UseCompressedOops) {
309 __ cmpxchgl(newval, addr);
310 } else
311 #endif
312 {
313 __ cmpxchgptr(newval, addr);
314 }
315 __ jcc(Assembler::equal, done, true);
316
317 // Step 2. CAS had failed. This may be a false negative.
318 //
319 // The trouble comes when we compare the to-space pointer with the from-space
320 // pointer to the same object. To resolve this, it will suffice to resolve both
321 // oldval and the value from memory -- this will give both to-space pointers.
322 // If they mismatch, then it was a legitimate failure.
323 //
324 #ifdef _LP64
325 if (UseCompressedOops) {
326 __ decode_heap_oop(tmp1);
327 }
328 #endif
329 resolve_forward_pointer(masm, tmp1);
330
331 #ifdef _LP64
332 if (UseCompressedOops) {
333 __ movl(tmp2, oldval);
334 __ decode_heap_oop(tmp2);
335 } else
336 #endif
337 {
338 __ movptr(tmp2, oldval);
339 }
340 resolve_forward_pointer(masm, tmp2);
341
342 __ cmpptr(tmp1, tmp2);
343 __ jcc(Assembler::notEqual, done, true);
344
345 // Step 3. Try to CAS again with resolved to-space pointers.
346 //
347 // Corner case: it may happen that somebody stored the from-space pointer
348 // to memory while we were preparing for retry. Therefore, we can fail again
349 // on retry, and so need to do this in loop, always resolving the failure
350 // witness.
351 __ bind(retry);
352 if (os::is_MP()) __ lock();
353 #ifdef _LP64
354 if (UseCompressedOops) {
355 __ cmpxchgl(newval, addr);
356 } else
357 #endif
358 {
359 __ cmpxchgptr(newval, addr);
360 }
361 __ jcc(Assembler::equal, done, true);
362
363 #ifdef _LP64
364 if (UseCompressedOops) {
365 __ movl(tmp2, oldval);
366 __ decode_heap_oop(tmp2);
367 } else
368 #endif
369 {
370 __ movptr(tmp2, oldval);
371 }
372 resolve_forward_pointer(masm, tmp2);
373
374 __ cmpptr(tmp1, tmp2);
375 __ jcc(Assembler::equal, retry, true);
376
377 // Step 4. If we need a boolean result out of CAS, check the flag again,
378 // and promote the result. Note that we handle the flag from both the CAS
379 // itself and from the retry loop.
380 __ bind(done);
381 if (!exchange) {
382 assert(res != NULL, "need result register");
383 #ifdef _LP64
384 __ setb(Assembler::equal, res);
385 __ movzbl(res, res);
386 #else
387 // Need something else to clean the result, because some registers
388 // do not have byte encoding that movzbl wants. Cannot do the xor first,
389 // because it modifies the flags.
390 Label res_non_zero;
391 __ movptr(res, 1);
392 __ jcc(Assembler::equal, res_non_zero, true);
393 __ xorptr(res, res);
394 __ bind(res_non_zero);
395 #endif
396 }
397 }
398
399 #undef __
400
401 #ifdef COMPILER1
402
403 #define __ ce->masm()->
404
405 void ShenandoahBarrierSetAssembler::gen_load_reference_barrier_stub(LIR_Assembler* ce, ShenandoahLoadReferenceBarrierStub* stub) {
406 __ bind(*stub->entry());
407
408 Label done;
409 Register obj = stub->obj()->as_register();
410 Register res = stub->result()->as_register();
411
412 if (res != obj) {
413 __ mov(res, obj);
414 }
415
|
87 if (dest_uninitialized) {
88 __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_array_pre_duinit_narrow_oop_entry), src, dst, count);
89 } else {
90 __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_array_pre_narrow_oop_entry), src, dst, count);
91 }
92 } else
93 #endif
94 {
95 if (dest_uninitialized) {
96 __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_array_pre_duinit_oop_entry), src, dst, count);
97 } else {
98 __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_array_pre_oop_entry), src, dst, count);
99 }
100 }
101 __ popa();
102 __ bind(done);
103 NOT_LP64(__ pop(thread);)
104 }
105 }
106
107 void ShenandoahBarrierSetAssembler::load_reference_barrier_not_null(MacroAssembler* masm, Register dst) {
108 assert(ShenandoahLoadRefBarrier, "Should be enabled");
109
110 Label done;
111
112 #ifdef _LP64
113 Register thread = r15_thread;
114 #else
115 Register thread = rcx;
116 if (thread == dst) {
117 thread = rbx;
118 }
119 __ push(thread);
120 __ get_thread(thread);
121 #endif
122 assert_different_registers(dst, thread);
123
124 Address gc_state(thread, in_bytes(JavaThread::gc_state_offset()));
125 __ testb(gc_state, ShenandoahHeap::HAS_FORWARDED);
126 __ jcc(Assembler::zero, done);
224 }
225 }
226
227 void ShenandoahBarrierSetAssembler::load_reference_barrier(MacroAssembler* masm, Register dst) {
228 if (ShenandoahLoadRefBarrier) {
229 Label done;
230 __ testptr(dst, dst);
231 __ jcc(Assembler::zero, done);
232 load_reference_barrier_not_null(masm, dst);
233 __ bind(done);
234 }
235 }
236
237 // Special Shenandoah CAS implementation that handles false negatives
238 // due to concurrent evacuation.
239 void ShenandoahBarrierSetAssembler::cmpxchg_oop(MacroAssembler* masm,
240 Register res, Address addr, Register oldval, Register newval,
241 bool exchange, Register tmp1, Register tmp2) {
242 assert(ShenandoahCASBarrier, "Should only be used when CAS barrier is enabled");
243 assert(oldval == rax, "must be in rax for implicit use in cmpxchg");
244 assert_different_registers(oldval, newval, tmp1, tmp2);
245
246 Label L_success, L_failure;
247
248 // Remember oldval for retry logic below
249 #ifdef _LP64
250 if (UseCompressedOops) {
251 __ movl(tmp1, oldval);
252 } else
253 #endif
254 {
255 __ movptr(tmp1, oldval);
256 }
257
258 // Step 1. Fast-path.
259 //
260 // Try to CAS with given arguments. If successful, then we are done.
261
262 if (os::is_MP()) __ lock();
263 #ifdef _LP64
264 if (UseCompressedOops) {
265 __ cmpxchgl(newval, addr);
266 } else
267 #endif
268 {
269 __ cmpxchgptr(newval, addr);
270 }
271 __ jcc(Assembler::equal, L_success);
272
273 // Step 2. CAS had failed. This may be a false negative.
274 //
275 // The trouble comes when we compare the to-space pointer with the from-space
276 // pointer to the same object. To resolve this, it will suffice to resolve
277 // the value from memory -- this will give both to-space pointers.
278 // If they mismatch, then it was a legitimate failure.
279 //
280 // Before reaching to resolve sequence, see if we can avoid the whole shebang
281 // with filters.
282
283 // Filter: when offending in-memory value is NULL, the failure is definitely legitimate
284 __ testptr(oldval, oldval);
285 __ jcc(Assembler::zero, L_failure);
286
287 // Filter: when heap is stable, the failure is definitely legitimate
288 #ifdef _LP64
289 const Register thread = r15_thread;
290 #else
291 const Register thread = tmp2;
292 __ get_thread(thread);
293 #endif
294 Address gc_state(thread, in_bytes(JavaThread::gc_state_offset()));
295 __ testb(gc_state, ShenandoahHeap::HAS_FORWARDED);
296 __ jcc(Assembler::zero, L_failure);
297
298 #ifdef _LP64
299 if (UseCompressedOops) {
300 __ movl(tmp2, oldval);
301 __ decode_heap_oop(tmp2);
302 } else
303 #endif
304 {
305 __ movptr(tmp2, oldval);
306 }
307
308 // Decode offending in-memory value.
309 // Test if-forwarded
310 __ testb(Address(tmp2, oopDesc::mark_offset_in_bytes()), markOopDesc::marked_value);
311 __ jcc(Assembler::noParity, L_failure); // When odd number of bits, then not forwarded
312 __ jcc(Assembler::zero, L_failure); // When it is 00, then also not forwarded
313
314 // Load and mask forwarding pointer
315 __ movptr(tmp2, Address(tmp2, oopDesc::mark_offset_in_bytes()));
316 __ shrptr(tmp2, 2);
317 __ shlptr(tmp2, 2);
318
319 #ifdef _LP64
320 if (UseCompressedOops) {
321 __ decode_heap_oop(tmp1); // decode for comparison
322 }
323 #endif
324
325 // Now we have the forwarded offender in tmp2.
326 // Compare and if they don't match, we have legitimate failure
327 __ cmpptr(tmp1, tmp2);
328 __ jcc(Assembler::notEqual, L_failure);
329
330 // Step 3. Need to fix the memory ptr before continuing.
331 //
332 // At this point, we have from-space oldval in the register, and its to-space
333 // address is in tmp2. Let's try to update it into memory. We don't care if it
334 // succeeds or not. If it does, then the retrying CAS would see it and succeed.
335 // If this fixup fails, this means somebody else beat us to it, and necessarily
336 // with to-space ptr store. We still have to do the retry, because the GC might
337 // have updated the reference for us.
338
339 #ifdef _LP64
340 if (UseCompressedOops) {
341 __ encode_heap_oop(tmp2); // previously decoded at step 2.
342 }
343 #endif
344
345 if (os::is_MP()) __ lock();
346 #ifdef _LP64
347 if (UseCompressedOops) {
348 __ cmpxchgl(tmp2, addr);
349 } else
350 #endif
351 {
352 __ cmpxchgptr(tmp2, addr);
353 }
354
355 // Step 4. Try to CAS again.
356 //
357 // This is guaranteed not to have false negatives, because oldval is definitely
358 // to-space, and memory pointer is to-space as well. Nothing is able to store
359 // from-space ptr into memory anymore. Make sure oldval is restored, after being
360 // garbled during retries.
361 //
362 #ifdef _LP64
363 if (UseCompressedOops) {
364 __ movl(oldval, tmp2);
365 } else
366 #endif
367 {
368 __ movptr(oldval, tmp2);
369 }
370
371 if (os::is_MP()) __ lock();
372 #ifdef _LP64
373 if (UseCompressedOops) {
374 __ cmpxchgl(newval, addr);
375 } else
376 #endif
377 {
378 __ cmpxchgptr(newval, addr);
379 }
380 if (!exchange) {
381 __ jccb(Assembler::equal, L_success); // fastpath, peeking into Step 5, no need to jump
382 }
383
384 // Step 5. If we need a boolean result out of CAS, set the flag appropriately.
385 // and promote the result. Note that we handle the flag from both the 1st and 2nd CAS.
386 // Otherwise, failure witness for CAE is in oldval on all paths, and we can return.
387
388 if (exchange) {
389 __ bind(L_failure);
390 __ bind(L_success);
391 } else {
392 assert(res != NULL, "need result register");
393
394 Label exit;
395 __ bind(L_failure);
396 __ xorptr(res, res);
397 __ jmpb(exit);
398
399 __ bind(L_success);
400 __ movptr(res, 1);
401 __ bind(exit);
402 }
403 }
404
405 #undef __
406
407 #ifdef COMPILER1
408
409 #define __ ce->masm()->
410
411 void ShenandoahBarrierSetAssembler::gen_load_reference_barrier_stub(LIR_Assembler* ce, ShenandoahLoadReferenceBarrierStub* stub) {
412 __ bind(*stub->entry());
413
414 Label done;
415 Register obj = stub->obj()->as_register();
416 Register res = stub->result()->as_register();
417
418 if (res != obj) {
419 __ mov(res, obj);
420 }
421
|