289 }
290
291 inline void cmpxchg_pre_membar(cmpxchg_memory_order order) {
292 if (order != memory_order_relaxed) {
293 __asm__ __volatile__ (
294 /* fence */
295 strasm_sync
296 );
297 }
298 }
299
300 inline void cmpxchg_post_membar(cmpxchg_memory_order order) {
301 if (order != memory_order_relaxed) {
302 __asm__ __volatile__ (
303 /* fence */
304 strasm_sync
305 );
306 }
307 }
308
309 #define VM_HAS_SPECIALIZED_CMPXCHG_BYTE
310 inline jbyte Atomic::cmpxchg(jbyte exchange_value, volatile jbyte* dest, jbyte compare_value, cmpxchg_memory_order order) {
311
312 // Note that cmpxchg guarantees a two-way memory barrier across
313 // the cmpxchg, so it's really a a 'fence_cmpxchg_fence' if not
314 // specified otherwise (see atomic.hpp).
315
316 // Using 32 bit internally.
317 volatile int *dest_base = (volatile int*)((uintptr_t)dest & ~3);
318
319 #ifdef VM_LITTLE_ENDIAN
320 const unsigned int shift_amount = ((uintptr_t)dest & 3) * 8;
321 #else
322 const unsigned int shift_amount = ((~(uintptr_t)dest) & 3) * 8;
323 #endif
324 const unsigned int masked_compare_val = ((unsigned int)(unsigned char)compare_value),
325 masked_exchange_val = ((unsigned int)(unsigned char)exchange_value),
326 xor_value = (masked_compare_val ^ masked_exchange_val) << shift_amount;
327
328 unsigned int old_value, value32;
329
330 cmpxchg_pre_membar(order);
351 /* out */
352 : [old_value] "=&r" (old_value),
353 [value32] "=&r" (value32),
354 "=m" (*dest),
355 "=m" (*dest_base)
356 /* in */
357 : [dest] "b" (dest),
358 [dest_base] "b" (dest_base),
359 [shift_amount] "r" (shift_amount),
360 [masked_compare_val] "r" (masked_compare_val),
361 [xor_value] "r" (xor_value),
362 "m" (*dest),
363 "m" (*dest_base)
364 /* clobber */
365 : "cc",
366 "memory"
367 );
368
369 cmpxchg_post_membar(order);
370
371 return (jbyte)(unsigned char)old_value;
372 }
373
374 inline jint Atomic::cmpxchg(jint exchange_value, volatile jint* dest, jint compare_value, cmpxchg_memory_order order) {
375
376 // Note that cmpxchg guarantees a two-way memory barrier across
377 // the cmpxchg, so it's really a a 'fence_cmpxchg_fence' if not
378 // specified otherwise (see atomic.hpp).
379
380 unsigned int old_value;
381 const uint64_t zero = 0;
382
383 cmpxchg_pre_membar(order);
384
385 __asm__ __volatile__ (
386 /* simple guard */
387 " lwz %[old_value], 0(%[dest]) \n"
388 " cmpw %[compare_value], %[old_value] \n"
389 " bne- 2f \n"
390 /* atomic loop */
391 "1: \n"
392 " lwarx %[old_value], %[dest], %[zero] \n"
393 " cmpw %[compare_value], %[old_value] \n"
394 " bne- 2f \n"
395 " stwcx. %[exchange_value], %[dest], %[zero] \n"
396 " bne- 1b \n"
397 /* exit */
398 "2: \n"
399 /* out */
400 : [old_value] "=&r" (old_value),
401 "=m" (*dest)
402 /* in */
403 : [dest] "b" (dest),
404 [zero] "r" (zero),
405 [compare_value] "r" (compare_value),
406 [exchange_value] "r" (exchange_value),
407 "m" (*dest)
408 /* clobber */
409 : "cc",
410 "memory"
411 );
412
413 cmpxchg_post_membar(order);
414
415 return (jint) old_value;
416 }
417
418 inline jlong Atomic::cmpxchg(jlong exchange_value, volatile jlong* dest, jlong compare_value, cmpxchg_memory_order order) {
419
420 // Note that cmpxchg guarantees a two-way memory barrier across
421 // the cmpxchg, so it's really a a 'fence_cmpxchg_fence' if not
422 // specified otherwise (see atomic.hpp).
423
424 long old_value;
425 const uint64_t zero = 0;
426
427 cmpxchg_pre_membar(order);
428
429 __asm__ __volatile__ (
430 /* simple guard */
431 " ld %[old_value], 0(%[dest]) \n"
432 " cmpd %[compare_value], %[old_value] \n"
433 " bne- 2f \n"
434 /* atomic loop */
435 "1: \n"
436 " ldarx %[old_value], %[dest], %[zero] \n"
437 " cmpd %[compare_value], %[old_value] \n"
438 " bne- 2f \n"
439 " stdcx. %[exchange_value], %[dest], %[zero] \n"
440 " bne- 1b \n"
441 /* exit */
442 "2: \n"
443 /* out */
444 : [old_value] "=&r" (old_value),
445 "=m" (*dest)
446 /* in */
447 : [dest] "b" (dest),
448 [zero] "r" (zero),
449 [compare_value] "r" (compare_value),
450 [exchange_value] "r" (exchange_value),
451 "m" (*dest)
452 /* clobber */
453 : "cc",
454 "memory"
455 );
456
457 cmpxchg_post_membar(order);
458
459 return (jlong) old_value;
460 }
461
462 inline intptr_t Atomic::cmpxchg_ptr(intptr_t exchange_value, volatile intptr_t* dest, intptr_t compare_value, cmpxchg_memory_order order) {
463 return (intptr_t)cmpxchg((jlong)exchange_value, (volatile jlong*)dest, (jlong)compare_value, order);
464 }
465
466 inline void* Atomic::cmpxchg_ptr(void* exchange_value, volatile void* dest, void* compare_value, cmpxchg_memory_order order) {
467 return (void*)cmpxchg((jlong)exchange_value, (volatile jlong*)dest, (jlong)compare_value, order);
468 }
469
470 #undef strasm_sync
471 #undef strasm_lwsync
472 #undef strasm_isync
473 #undef strasm_release
474 #undef strasm_acquire
475 #undef strasm_fence
476 #undef strasm_nobarrier
477 #undef strasm_nobarrier_clobber_memory
478
479 #endif // OS_CPU_LINUX_PPC_VM_ATOMIC_LINUX_PPC_HPP
|
289 }
290
291 inline void cmpxchg_pre_membar(cmpxchg_memory_order order) {
292 if (order != memory_order_relaxed) {
293 __asm__ __volatile__ (
294 /* fence */
295 strasm_sync
296 );
297 }
298 }
299
300 inline void cmpxchg_post_membar(cmpxchg_memory_order order) {
301 if (order != memory_order_relaxed) {
302 __asm__ __volatile__ (
303 /* fence */
304 strasm_sync
305 );
306 }
307 }
308
309 template<>
310 template<typename T>
311 inline T Atomic::PlatformCmpxchg<1>::operator()(T exchange_value,
312 T volatile* dest,
313 T compare_value,
314 cmpxchg_memory_order order) const {
315 STATIC_ASSERT(1 == sizeof(T));
316
317 // Note that cmpxchg guarantees a two-way memory barrier across
318 // the cmpxchg, so it's really a a 'fence_cmpxchg_fence' if not
319 // specified otherwise (see atomic.hpp).
320
321 // Using 32 bit internally.
322 volatile int *dest_base = (volatile int*)((uintptr_t)dest & ~3);
323
324 #ifdef VM_LITTLE_ENDIAN
325 const unsigned int shift_amount = ((uintptr_t)dest & 3) * 8;
326 #else
327 const unsigned int shift_amount = ((~(uintptr_t)dest) & 3) * 8;
328 #endif
329 const unsigned int masked_compare_val = ((unsigned int)(unsigned char)compare_value),
330 masked_exchange_val = ((unsigned int)(unsigned char)exchange_value),
331 xor_value = (masked_compare_val ^ masked_exchange_val) << shift_amount;
332
333 unsigned int old_value, value32;
334
335 cmpxchg_pre_membar(order);
356 /* out */
357 : [old_value] "=&r" (old_value),
358 [value32] "=&r" (value32),
359 "=m" (*dest),
360 "=m" (*dest_base)
361 /* in */
362 : [dest] "b" (dest),
363 [dest_base] "b" (dest_base),
364 [shift_amount] "r" (shift_amount),
365 [masked_compare_val] "r" (masked_compare_val),
366 [xor_value] "r" (xor_value),
367 "m" (*dest),
368 "m" (*dest_base)
369 /* clobber */
370 : "cc",
371 "memory"
372 );
373
374 cmpxchg_post_membar(order);
375
376 return IntegerTypes::cast<T>((unsigned char)old_value);
377 }
378
379 template<>
380 template<typename T>
381 inline T Atomic::PlatformCmpxchg<4>::operator()(T exchange_value,
382 T volatile* dest,
383 T compare_value,
384 cmpxchg_memory_order order) const {
385 STATIC_ASSERT(4 == sizeof(T));
386
387 // Note that cmpxchg guarantees a two-way memory barrier across
388 // the cmpxchg, so it's really a a 'fence_cmpxchg_fence' if not
389 // specified otherwise (see atomic.hpp).
390
391 T old_value;
392 const uint64_t zero = 0;
393
394 cmpxchg_pre_membar(order);
395
396 __asm__ __volatile__ (
397 /* simple guard */
398 " lwz %[old_value], 0(%[dest]) \n"
399 " cmpw %[compare_value], %[old_value] \n"
400 " bne- 2f \n"
401 /* atomic loop */
402 "1: \n"
403 " lwarx %[old_value], %[dest], %[zero] \n"
404 " cmpw %[compare_value], %[old_value] \n"
405 " bne- 2f \n"
406 " stwcx. %[exchange_value], %[dest], %[zero] \n"
407 " bne- 1b \n"
408 /* exit */
409 "2: \n"
410 /* out */
411 : [old_value] "=&r" (old_value),
412 "=m" (*dest)
413 /* in */
414 : [dest] "b" (dest),
415 [zero] "r" (zero),
416 [compare_value] "r" (compare_value),
417 [exchange_value] "r" (exchange_value),
418 "m" (*dest)
419 /* clobber */
420 : "cc",
421 "memory"
422 );
423
424 cmpxchg_post_membar(order);
425
426 return old_value;
427 }
428
429 template<>
430 template<typename T>
431 inline T Atomic::PlatformCmpxchg<8>::operator()(T exchange_value,
432 T volatile* dest,
433 T compare_value,
434 cmpxchg_memory_order order) const {
435 STATIC_ASSERT(8 == sizeof(T));
436
437 // Note that cmpxchg guarantees a two-way memory barrier across
438 // the cmpxchg, so it's really a a 'fence_cmpxchg_fence' if not
439 // specified otherwise (see atomic.hpp).
440
441 T old_value;
442 const uint64_t zero = 0;
443
444 cmpxchg_pre_membar(order);
445
446 __asm__ __volatile__ (
447 /* simple guard */
448 " ld %[old_value], 0(%[dest]) \n"
449 " cmpd %[compare_value], %[old_value] \n"
450 " bne- 2f \n"
451 /* atomic loop */
452 "1: \n"
453 " ldarx %[old_value], %[dest], %[zero] \n"
454 " cmpd %[compare_value], %[old_value] \n"
455 " bne- 2f \n"
456 " stdcx. %[exchange_value], %[dest], %[zero] \n"
457 " bne- 1b \n"
458 /* exit */
459 "2: \n"
460 /* out */
461 : [old_value] "=&r" (old_value),
462 "=m" (*dest)
463 /* in */
464 : [dest] "b" (dest),
465 [zero] "r" (zero),
466 [compare_value] "r" (compare_value),
467 [exchange_value] "r" (exchange_value),
468 "m" (*dest)
469 /* clobber */
470 : "cc",
471 "memory"
472 );
473
474 cmpxchg_post_membar(order);
475
476 return old_value;
477 }
478
479 #undef strasm_sync
480 #undef strasm_lwsync
481 #undef strasm_isync
482 #undef strasm_release
483 #undef strasm_acquire
484 #undef strasm_fence
485 #undef strasm_nobarrier
486 #undef strasm_nobarrier_clobber_memory
487
488 #endif // OS_CPU_LINUX_PPC_VM_ATOMIC_LINUX_PPC_HPP
|