274 /* out */
275 : [old_value] "=&r" (old_value),
276 "=m" (*dest)
277 /* in */
278 : [dest] "b" (dest),
279 [zero] "r" (zero),
280 [exchange_value] "r" (exchange_value),
281 "m" (*dest)
282 /* clobber */
283 : "cc",
284 "memory"
285 );
286
287 return (intptr_t) old_value;
288 }
289
290 inline void* Atomic::xchg_ptr(void* exchange_value, volatile void* dest) {
291 return (void*)xchg_ptr((intptr_t)exchange_value, (volatile intptr_t*)dest);
292 }
293
294 #define VM_HAS_SPECIALIZED_CMPXCHG_BYTE
295 inline jbyte Atomic::cmpxchg(jbyte exchange_value, volatile jbyte* dest, jbyte compare_value) {
296
297 // Note that cmpxchg guarantees a two-way memory barrier across
298 // the cmpxchg, so it's really a a 'fence_cmpxchg_acquire'
299 // (see atomic.hpp).
300
301 // Using 32 bit internally.
302 volatile int *dest_base = (volatile int*)((uintptr_t)dest & ~3);
303
304 #ifdef VM_LITTLE_ENDIAN
305 const unsigned int shift_amount = ((uintptr_t)dest & 3) * 8;
306 #else
307 const unsigned int shift_amount = ((~(uintptr_t)dest) & 3) * 8;
308 #endif
309 const unsigned int masked_compare_val = ((unsigned int)(unsigned char)compare_value),
310 masked_exchange_val = ((unsigned int)(unsigned char)exchange_value),
311 xor_value = (masked_compare_val ^ masked_exchange_val) << shift_amount;
312
313 unsigned int old_value, value32;
314
315 __asm__ __volatile__ (
316 /* fence */
317 strasm_sync
318 /* simple guard */
319 " lbz %[old_value], 0(%[dest]) \n"
320 " cmpw %[masked_compare_val], %[old_value] \n"
321 " bne- 2f \n"
322 /* atomic loop */
323 "1: \n"
324 " lwarx %[value32], 0, %[dest_base] \n"
325 /* extract byte and compare */
326 " srd %[old_value], %[value32], %[shift_amount] \n"
327 " clrldi %[old_value], %[old_value], 56 \n"
328 " cmpw %[masked_compare_val], %[old_value] \n"
329 " bne- 2f \n"
330 /* replace byte and try to store */
331 " xor %[value32], %[xor_value], %[value32] \n"
332 " stwcx. %[value32], 0, %[dest_base] \n"
333 " bne- 1b \n"
334 /* acquire */
335 strasm_sync
336 /* exit */
337 "2: \n"
338 /* out */
339 : [old_value] "=&r" (old_value),
340 [value32] "=&r" (value32),
341 "=m" (*dest),
342 "=m" (*dest_base)
343 /* in */
344 : [dest] "b" (dest),
345 [dest_base] "b" (dest_base),
346 [shift_amount] "r" (shift_amount),
347 [masked_compare_val] "r" (masked_compare_val),
348 [xor_value] "r" (xor_value),
349 "m" (*dest),
350 "m" (*dest_base)
351 /* clobber */
352 : "cc",
353 "memory"
354 );
355
356 return (jbyte)(unsigned char)old_value;
357 }
358
359 inline jint Atomic::cmpxchg(jint exchange_value, volatile jint* dest, jint compare_value) {
360
361 // Note that cmpxchg guarantees a two-way memory barrier across
362 // the cmpxchg, so it's really a a 'fence_cmpxchg_acquire'
363 // (see atomic.hpp).
364
365 unsigned int old_value;
366 const uint64_t zero = 0;
367
368 __asm__ __volatile__ (
369 /* fence */
370 strasm_sync
371 /* simple guard */
372 " lwz %[old_value], 0(%[dest]) \n"
373 " cmpw %[compare_value], %[old_value] \n"
374 " bne- 2f \n"
375 /* atomic loop */
376 "1: \n"
377 " lwarx %[old_value], %[dest], %[zero] \n"
378 " cmpw %[compare_value], %[old_value] \n"
379 " bne- 2f \n"
380 " stwcx. %[exchange_value], %[dest], %[zero] \n"
381 " bne- 1b \n"
382 /* acquire */
383 strasm_sync
384 /* exit */
385 "2: \n"
386 /* out */
387 : [old_value] "=&r" (old_value),
388 "=m" (*dest)
389 /* in */
390 : [dest] "b" (dest),
391 [zero] "r" (zero),
392 [compare_value] "r" (compare_value),
393 [exchange_value] "r" (exchange_value),
394 "m" (*dest)
395 /* clobber */
396 : "cc",
397 "memory"
398 );
399
400 return (jint) old_value;
401 }
402
403 inline jlong Atomic::cmpxchg(jlong exchange_value, volatile jlong* dest, jlong compare_value) {
404
405 // Note that cmpxchg guarantees a two-way memory barrier across
406 // the cmpxchg, so it's really a a 'fence_cmpxchg_acquire'
407 // (see atomic.hpp).
408
409 long old_value;
410 const uint64_t zero = 0;
411
412 __asm__ __volatile__ (
413 /* fence */
414 strasm_sync
415 /* simple guard */
416 " ld %[old_value], 0(%[dest]) \n"
417 " cmpd %[compare_value], %[old_value] \n"
418 " bne- 2f \n"
419 /* atomic loop */
420 "1: \n"
421 " ldarx %[old_value], %[dest], %[zero] \n"
422 " cmpd %[compare_value], %[old_value] \n"
423 " bne- 2f \n"
424 " stdcx. %[exchange_value], %[dest], %[zero] \n"
425 " bne- 1b \n"
426 /* acquire */
427 strasm_sync
428 /* exit */
429 "2: \n"
430 /* out */
431 : [old_value] "=&r" (old_value),
432 "=m" (*dest)
433 /* in */
434 : [dest] "b" (dest),
435 [zero] "r" (zero),
436 [compare_value] "r" (compare_value),
437 [exchange_value] "r" (exchange_value),
438 "m" (*dest)
439 /* clobber */
440 : "cc",
441 "memory"
442 );
443
444 return (jlong) old_value;
445 }
446
447 inline intptr_t Atomic::cmpxchg_ptr(intptr_t exchange_value, volatile intptr_t* dest, intptr_t compare_value) {
448 return (intptr_t)cmpxchg((jlong)exchange_value, (volatile jlong*)dest, (jlong)compare_value);
449 }
450
451 inline void* Atomic::cmpxchg_ptr(void* exchange_value, volatile void* dest, void* compare_value) {
452 return (void*)cmpxchg((jlong)exchange_value, (volatile jlong*)dest, (jlong)compare_value);
453 }
454
455 #undef strasm_sync
456 #undef strasm_lwsync
457 #undef strasm_isync
458 #undef strasm_release
459 #undef strasm_acquire
460 #undef strasm_fence
461 #undef strasm_nobarrier
462 #undef strasm_nobarrier_clobber_memory
463
464 #endif // OS_CPU_LINUX_PPC_VM_ATOMIC_LINUX_PPC_INLINE_HPP
|
274 /* out */
275 : [old_value] "=&r" (old_value),
276 "=m" (*dest)
277 /* in */
278 : [dest] "b" (dest),
279 [zero] "r" (zero),
280 [exchange_value] "r" (exchange_value),
281 "m" (*dest)
282 /* clobber */
283 : "cc",
284 "memory"
285 );
286
287 return (intptr_t) old_value;
288 }
289
290 inline void* Atomic::xchg_ptr(void* exchange_value, volatile void* dest) {
291 return (void*)xchg_ptr((intptr_t)exchange_value, (volatile intptr_t*)dest);
292 }
293
294 inline void cmpxchg_pre_membar(memory_order order) {
295 if (order == memory_order_seq_cst) {
296 __asm__ __volatile__ (
297 /* fence */
298 strasm_sync
299 );
300 } else if (order == memory_order_release || order == memory_order_acq_rel) {
301 __asm__ __volatile__ (
302 /* release */
303 strasm_lwsync
304 );
305 }
306 }
307
308 inline void cmpxchg_post_membar(memory_order order) {
309 if (order == memory_order_seq_cst) {
310 __asm__ __volatile__ (
311 /* fence */
312 strasm_sync
313 );
314 } else if (order == memory_order_acquire || order == memory_order_acq_rel) {
315 __asm__ __volatile__ (
316 /* acquire */
317 strasm_isync
318 );
319 }
320 }
321
322 #define VM_HAS_SPECIALIZED_CMPXCHG_BYTE
323 inline jbyte Atomic::cmpxchg(jbyte exchange_value, volatile jbyte* dest, jbyte compare_value, memory_order order) {
324
325 // Note that cmpxchg guarantees a two-way memory barrier across
326 // the cmpxchg, so it's really a a 'fence_cmpxchg_acquire'
327 // (see atomic.hpp).
328
329 // Using 32 bit internally.
330 volatile int *dest_base = (volatile int*)((uintptr_t)dest & ~3);
331
332 #ifdef VM_LITTLE_ENDIAN
333 const unsigned int shift_amount = ((uintptr_t)dest & 3) * 8;
334 #else
335 const unsigned int shift_amount = ((~(uintptr_t)dest) & 3) * 8;
336 #endif
337 const unsigned int masked_compare_val = ((unsigned int)(unsigned char)compare_value),
338 masked_exchange_val = ((unsigned int)(unsigned char)exchange_value),
339 xor_value = (masked_compare_val ^ masked_exchange_val) << shift_amount;
340
341 unsigned int old_value, value32;
342
343 cmpxchg_pre_membar(order);
344
345 __asm__ __volatile__ (
346 /* simple guard */
347 " lbz %[old_value], 0(%[dest]) \n"
348 " cmpw %[masked_compare_val], %[old_value] \n"
349 " bne- 2f \n"
350 /* atomic loop */
351 "1: \n"
352 " lwarx %[value32], 0, %[dest_base] \n"
353 /* extract byte and compare */
354 " srd %[old_value], %[value32], %[shift_amount] \n"
355 " clrldi %[old_value], %[old_value], 56 \n"
356 " cmpw %[masked_compare_val], %[old_value] \n"
357 " bne- 2f \n"
358 /* replace byte and try to store */
359 " xor %[value32], %[xor_value], %[value32] \n"
360 " stwcx. %[value32], 0, %[dest_base] \n"
361 " bne- 1b \n"
362 /* exit */
363 "2: \n"
364 /* out */
365 : [old_value] "=&r" (old_value),
366 [value32] "=&r" (value32),
367 "=m" (*dest),
368 "=m" (*dest_base)
369 /* in */
370 : [dest] "b" (dest),
371 [dest_base] "b" (dest_base),
372 [shift_amount] "r" (shift_amount),
373 [masked_compare_val] "r" (masked_compare_val),
374 [xor_value] "r" (xor_value),
375 "m" (*dest),
376 "m" (*dest_base)
377 /* clobber */
378 : "cc",
379 "memory"
380 );
381
382 cmpxchg_post_membar(order);
383
384 return (jbyte)(unsigned char)old_value;
385 }
386
387 inline jint Atomic::cmpxchg(jint exchange_value, volatile jint* dest, jint compare_value, memory_order order) {
388
389 // Note that cmpxchg guarantees a two-way memory barrier across
390 // the cmpxchg, so it's really a a 'fence_cmpxchg_acquire'
391 // (see atomic.hpp).
392
393 unsigned int old_value;
394 const uint64_t zero = 0;
395
396 cmpxchg_pre_membar(order);
397
398 __asm__ __volatile__ (
399 /* simple guard */
400 " lwz %[old_value], 0(%[dest]) \n"
401 " cmpw %[compare_value], %[old_value] \n"
402 " bne- 2f \n"
403 /* atomic loop */
404 "1: \n"
405 " lwarx %[old_value], %[dest], %[zero] \n"
406 " cmpw %[compare_value], %[old_value] \n"
407 " bne- 2f \n"
408 " stwcx. %[exchange_value], %[dest], %[zero] \n"
409 " bne- 1b \n"
410 /* exit */
411 "2: \n"
412 /* out */
413 : [old_value] "=&r" (old_value),
414 "=m" (*dest)
415 /* in */
416 : [dest] "b" (dest),
417 [zero] "r" (zero),
418 [compare_value] "r" (compare_value),
419 [exchange_value] "r" (exchange_value),
420 "m" (*dest)
421 /* clobber */
422 : "cc",
423 "memory"
424 );
425
426 cmpxchg_post_membar(order);
427
428 return (jint) old_value;
429 }
430
431 inline jlong Atomic::cmpxchg(jlong exchange_value, volatile jlong* dest, jlong compare_value, memory_order order) {
432
433 // Note that cmpxchg guarantees a two-way memory barrier across
434 // the cmpxchg, so it's really a a 'fence_cmpxchg_acquire'
435 // (see atomic.hpp).
436
437 long old_value;
438 const uint64_t zero = 0;
439
440 cmpxchg_pre_membar(order);
441
442 __asm__ __volatile__ (
443 /* simple guard */
444 " ld %[old_value], 0(%[dest]) \n"
445 " cmpd %[compare_value], %[old_value] \n"
446 " bne- 2f \n"
447 /* atomic loop */
448 "1: \n"
449 " ldarx %[old_value], %[dest], %[zero] \n"
450 " cmpd %[compare_value], %[old_value] \n"
451 " bne- 2f \n"
452 " stdcx. %[exchange_value], %[dest], %[zero] \n"
453 " bne- 1b \n"
454 /* exit */
455 "2: \n"
456 /* out */
457 : [old_value] "=&r" (old_value),
458 "=m" (*dest)
459 /* in */
460 : [dest] "b" (dest),
461 [zero] "r" (zero),
462 [compare_value] "r" (compare_value),
463 [exchange_value] "r" (exchange_value),
464 "m" (*dest)
465 /* clobber */
466 : "cc",
467 "memory"
468 );
469
470 cmpxchg_post_membar(order);
471
472 return (jlong) old_value;
473 }
474
475 inline intptr_t Atomic::cmpxchg_ptr(intptr_t exchange_value, volatile intptr_t* dest, intptr_t compare_value, memory_order order) {
476 return (intptr_t)cmpxchg((jlong)exchange_value, (volatile jlong*)dest, (jlong)compare_value, order);
477 }
478
479 inline void* Atomic::cmpxchg_ptr(void* exchange_value, volatile void* dest, void* compare_value, memory_order order) {
480 return (void*)cmpxchg((jlong)exchange_value, (volatile jlong*)dest, (jlong)compare_value, order);
481 }
482
483 #undef strasm_sync
484 #undef strasm_lwsync
485 #undef strasm_isync
486 #undef strasm_release
487 #undef strasm_acquire
488 #undef strasm_fence
489 #undef strasm_nobarrier
490 #undef strasm_nobarrier_clobber_memory
491
492 #endif // OS_CPU_LINUX_PPC_VM_ATOMIC_LINUX_PPC_INLINE_HPP
|