274 /* out */
275 : [old_value] "=&r" (old_value),
276 "=m" (*dest)
277 /* in */
278 : [dest] "b" (dest),
279 [zero] "r" (zero),
280 [exchange_value] "r" (exchange_value),
281 "m" (*dest)
282 /* clobber */
283 : "cc",
284 "memory"
285 );
286
287 return (intptr_t) old_value;
288 }
289
290 inline void* Atomic::xchg_ptr(void* exchange_value, volatile void* dest) {
291 return (void*)xchg_ptr((intptr_t)exchange_value, (volatile intptr_t*)dest);
292 }
293
294 #define VM_HAS_SPECIALIZED_CMPXCHG_BYTE
295 inline jbyte Atomic::cmpxchg(jbyte exchange_value, volatile jbyte* dest, jbyte compare_value) {
296
297 // Note that cmpxchg guarantees a two-way memory barrier across
298 // the cmpxchg, so it's really a a 'fence_cmpxchg_acquire'
299 // (see atomic.hpp).
300
301 // Using 32 bit internally.
302 volatile int *dest_base = (volatile int*)((uintptr_t)dest & ~3);
303
304 #ifdef VM_LITTLE_ENDIAN
305 const unsigned int shift_amount = ((uintptr_t)dest & 3) * 8;
306 #else
307 const unsigned int shift_amount = ((~(uintptr_t)dest) & 3) * 8;
308 #endif
309 const unsigned int masked_compare_val = ((unsigned int)(unsigned char)compare_value),
310 masked_exchange_val = ((unsigned int)(unsigned char)exchange_value),
311 xor_value = (masked_compare_val ^ masked_exchange_val) << shift_amount;
312
313 unsigned int old_value, value32;
314
315 __asm__ __volatile__ (
316 /* fence */
317 strasm_sync
318 /* simple guard */
319 " lbz %[old_value], 0(%[dest]) \n"
320 " cmpw %[masked_compare_val], %[old_value] \n"
321 " bne- 2f \n"
322 /* atomic loop */
323 "1: \n"
324 " lwarx %[value32], 0, %[dest_base] \n"
325 /* extract byte and compare */
326 " srd %[old_value], %[value32], %[shift_amount] \n"
327 " clrldi %[old_value], %[old_value], 56 \n"
328 " cmpw %[masked_compare_val], %[old_value] \n"
329 " bne- 2f \n"
330 /* replace byte and try to store */
331 " xor %[value32], %[xor_value], %[value32] \n"
332 " stwcx. %[value32], 0, %[dest_base] \n"
333 " bne- 1b \n"
334 /* acquire */
335 strasm_sync
336 /* exit */
337 "2: \n"
338 /* out */
339 : [old_value] "=&r" (old_value),
340 [value32] "=&r" (value32),
341 "=m" (*dest),
342 "=m" (*dest_base)
343 /* in */
344 : [dest] "b" (dest),
345 [dest_base] "b" (dest_base),
346 [shift_amount] "r" (shift_amount),
347 [masked_compare_val] "r" (masked_compare_val),
348 [xor_value] "r" (xor_value),
349 "m" (*dest),
350 "m" (*dest_base)
351 /* clobber */
352 : "cc",
353 "memory"
354 );
355
356 return (jbyte)(unsigned char)old_value;
357 }
358
359 inline jint Atomic::cmpxchg(jint exchange_value, volatile jint* dest, jint compare_value) {
360
361 // Note that cmpxchg guarantees a two-way memory barrier across
362 // the cmpxchg, so it's really a a 'fence_cmpxchg_acquire'
363 // (see atomic.hpp).
364
365 unsigned int old_value;
366 const uint64_t zero = 0;
367
368 __asm__ __volatile__ (
369 /* fence */
370 strasm_sync
371 /* simple guard */
372 " lwz %[old_value], 0(%[dest]) \n"
373 " cmpw %[compare_value], %[old_value] \n"
374 " bne- 2f \n"
375 /* atomic loop */
376 "1: \n"
377 " lwarx %[old_value], %[dest], %[zero] \n"
378 " cmpw %[compare_value], %[old_value] \n"
379 " bne- 2f \n"
380 " stwcx. %[exchange_value], %[dest], %[zero] \n"
381 " bne- 1b \n"
382 /* acquire */
383 strasm_sync
384 /* exit */
385 "2: \n"
386 /* out */
387 : [old_value] "=&r" (old_value),
388 "=m" (*dest)
389 /* in */
390 : [dest] "b" (dest),
391 [zero] "r" (zero),
392 [compare_value] "r" (compare_value),
393 [exchange_value] "r" (exchange_value),
394 "m" (*dest)
395 /* clobber */
396 : "cc",
397 "memory"
398 );
399
400 return (jint) old_value;
401 }
402
403 inline jlong Atomic::cmpxchg(jlong exchange_value, volatile jlong* dest, jlong compare_value) {
404
405 // Note that cmpxchg guarantees a two-way memory barrier across
406 // the cmpxchg, so it's really a a 'fence_cmpxchg_acquire'
407 // (see atomic.hpp).
408
409 long old_value;
410 const uint64_t zero = 0;
411
412 __asm__ __volatile__ (
413 /* fence */
414 strasm_sync
415 /* simple guard */
416 " ld %[old_value], 0(%[dest]) \n"
417 " cmpd %[compare_value], %[old_value] \n"
418 " bne- 2f \n"
419 /* atomic loop */
420 "1: \n"
421 " ldarx %[old_value], %[dest], %[zero] \n"
422 " cmpd %[compare_value], %[old_value] \n"
423 " bne- 2f \n"
424 " stdcx. %[exchange_value], %[dest], %[zero] \n"
425 " bne- 1b \n"
426 /* acquire */
427 strasm_sync
428 /* exit */
429 "2: \n"
430 /* out */
431 : [old_value] "=&r" (old_value),
432 "=m" (*dest)
433 /* in */
434 : [dest] "b" (dest),
435 [zero] "r" (zero),
436 [compare_value] "r" (compare_value),
437 [exchange_value] "r" (exchange_value),
438 "m" (*dest)
439 /* clobber */
440 : "cc",
441 "memory"
442 );
443
444 return (jlong) old_value;
445 }
446
447 inline intptr_t Atomic::cmpxchg_ptr(intptr_t exchange_value, volatile intptr_t* dest, intptr_t compare_value) {
448 return (intptr_t)cmpxchg((jlong)exchange_value, (volatile jlong*)dest, (jlong)compare_value);
449 }
450
451 inline void* Atomic::cmpxchg_ptr(void* exchange_value, volatile void* dest, void* compare_value) {
452 return (void*)cmpxchg((jlong)exchange_value, (volatile jlong*)dest, (jlong)compare_value);
453 }
454
455 #undef strasm_sync
456 #undef strasm_lwsync
457 #undef strasm_isync
458 #undef strasm_release
459 #undef strasm_acquire
460 #undef strasm_fence
461 #undef strasm_nobarrier
462 #undef strasm_nobarrier_clobber_memory
463
464 #endif // OS_CPU_LINUX_PPC_VM_ATOMIC_LINUX_PPC_INLINE_HPP
|
274 /* out */
275 : [old_value] "=&r" (old_value),
276 "=m" (*dest)
277 /* in */
278 : [dest] "b" (dest),
279 [zero] "r" (zero),
280 [exchange_value] "r" (exchange_value),
281 "m" (*dest)
282 /* clobber */
283 : "cc",
284 "memory"
285 );
286
287 return (intptr_t) old_value;
288 }
289
290 inline void* Atomic::xchg_ptr(void* exchange_value, volatile void* dest) {
291 return (void*)xchg_ptr((intptr_t)exchange_value, (volatile intptr_t*)dest);
292 }
293
294 inline void cmpxchg_pre_membar(cmpxchg_memory_order order) {
295 if (order != memory_order_relaxed) {
296 __asm__ __volatile__ (
297 /* fence */
298 strasm_sync
299 );
300 }
301 }
302
303 inline void cmpxchg_post_membar(cmpxchg_memory_order order) {
304 if (order != memory_order_relaxed) {
305 __asm__ __volatile__ (
306 /* fence */
307 strasm_sync
308 );
309 }
310 }
311
312 #define VM_HAS_SPECIALIZED_CMPXCHG_BYTE
313 inline jbyte Atomic::cmpxchg(jbyte exchange_value, volatile jbyte* dest, jbyte compare_value, cmpxchg_memory_order order) {
314
315 // Note that cmpxchg guarantees a two-way memory barrier across
316 // the cmpxchg, so it's really a a 'fence_cmpxchg_acquire'
317 // (see atomic.hpp).
318
319 // Using 32 bit internally.
320 volatile int *dest_base = (volatile int*)((uintptr_t)dest & ~3);
321
322 #ifdef VM_LITTLE_ENDIAN
323 const unsigned int shift_amount = ((uintptr_t)dest & 3) * 8;
324 #else
325 const unsigned int shift_amount = ((~(uintptr_t)dest) & 3) * 8;
326 #endif
327 const unsigned int masked_compare_val = ((unsigned int)(unsigned char)compare_value),
328 masked_exchange_val = ((unsigned int)(unsigned char)exchange_value),
329 xor_value = (masked_compare_val ^ masked_exchange_val) << shift_amount;
330
331 unsigned int old_value, value32;
332
333 cmpxchg_pre_membar(order);
334
335 __asm__ __volatile__ (
336 /* simple guard */
337 " lbz %[old_value], 0(%[dest]) \n"
338 " cmpw %[masked_compare_val], %[old_value] \n"
339 " bne- 2f \n"
340 /* atomic loop */
341 "1: \n"
342 " lwarx %[value32], 0, %[dest_base] \n"
343 /* extract byte and compare */
344 " srd %[old_value], %[value32], %[shift_amount] \n"
345 " clrldi %[old_value], %[old_value], 56 \n"
346 " cmpw %[masked_compare_val], %[old_value] \n"
347 " bne- 2f \n"
348 /* replace byte and try to store */
349 " xor %[value32], %[xor_value], %[value32] \n"
350 " stwcx. %[value32], 0, %[dest_base] \n"
351 " bne- 1b \n"
352 /* exit */
353 "2: \n"
354 /* out */
355 : [old_value] "=&r" (old_value),
356 [value32] "=&r" (value32),
357 "=m" (*dest),
358 "=m" (*dest_base)
359 /* in */
360 : [dest] "b" (dest),
361 [dest_base] "b" (dest_base),
362 [shift_amount] "r" (shift_amount),
363 [masked_compare_val] "r" (masked_compare_val),
364 [xor_value] "r" (xor_value),
365 "m" (*dest),
366 "m" (*dest_base)
367 /* clobber */
368 : "cc",
369 "memory"
370 );
371
372 cmpxchg_post_membar(order);
373
374 return (jbyte)(unsigned char)old_value;
375 }
376
377 inline jint Atomic::cmpxchg(jint exchange_value, volatile jint* dest, jint compare_value, cmpxchg_memory_order order) {
378
379 // Note that cmpxchg guarantees a two-way memory barrier across
380 // the cmpxchg, so it's really a a 'fence_cmpxchg_acquire'
381 // (see atomic.hpp).
382
383 unsigned int old_value;
384 const uint64_t zero = 0;
385
386 cmpxchg_pre_membar(order);
387
388 __asm__ __volatile__ (
389 /* simple guard */
390 " lwz %[old_value], 0(%[dest]) \n"
391 " cmpw %[compare_value], %[old_value] \n"
392 " bne- 2f \n"
393 /* atomic loop */
394 "1: \n"
395 " lwarx %[old_value], %[dest], %[zero] \n"
396 " cmpw %[compare_value], %[old_value] \n"
397 " bne- 2f \n"
398 " stwcx. %[exchange_value], %[dest], %[zero] \n"
399 " bne- 1b \n"
400 /* exit */
401 "2: \n"
402 /* out */
403 : [old_value] "=&r" (old_value),
404 "=m" (*dest)
405 /* in */
406 : [dest] "b" (dest),
407 [zero] "r" (zero),
408 [compare_value] "r" (compare_value),
409 [exchange_value] "r" (exchange_value),
410 "m" (*dest)
411 /* clobber */
412 : "cc",
413 "memory"
414 );
415
416 cmpxchg_post_membar(order);
417
418 return (jint) old_value;
419 }
420
421 inline jlong Atomic::cmpxchg(jlong exchange_value, volatile jlong* dest, jlong compare_value, cmpxchg_memory_order order) {
422
423 // Note that cmpxchg guarantees a two-way memory barrier across
424 // the cmpxchg, so it's really a a 'fence_cmpxchg_acquire'
425 // (see atomic.hpp).
426
427 long old_value;
428 const uint64_t zero = 0;
429
430 cmpxchg_pre_membar(order);
431
432 __asm__ __volatile__ (
433 /* simple guard */
434 " ld %[old_value], 0(%[dest]) \n"
435 " cmpd %[compare_value], %[old_value] \n"
436 " bne- 2f \n"
437 /* atomic loop */
438 "1: \n"
439 " ldarx %[old_value], %[dest], %[zero] \n"
440 " cmpd %[compare_value], %[old_value] \n"
441 " bne- 2f \n"
442 " stdcx. %[exchange_value], %[dest], %[zero] \n"
443 " bne- 1b \n"
444 /* exit */
445 "2: \n"
446 /* out */
447 : [old_value] "=&r" (old_value),
448 "=m" (*dest)
449 /* in */
450 : [dest] "b" (dest),
451 [zero] "r" (zero),
452 [compare_value] "r" (compare_value),
453 [exchange_value] "r" (exchange_value),
454 "m" (*dest)
455 /* clobber */
456 : "cc",
457 "memory"
458 );
459
460 cmpxchg_post_membar(order);
461
462 return (jlong) old_value;
463 }
464
465 inline intptr_t Atomic::cmpxchg_ptr(intptr_t exchange_value, volatile intptr_t* dest, intptr_t compare_value, cmpxchg_memory_order order) {
466 return (intptr_t)cmpxchg((jlong)exchange_value, (volatile jlong*)dest, (jlong)compare_value, order);
467 }
468
469 inline void* Atomic::cmpxchg_ptr(void* exchange_value, volatile void* dest, void* compare_value, cmpxchg_memory_order order) {
470 return (void*)cmpxchg((jlong)exchange_value, (volatile jlong*)dest, (jlong)compare_value, order);
471 }
472
473 #undef strasm_sync
474 #undef strasm_lwsync
475 #undef strasm_isync
476 #undef strasm_release
477 #undef strasm_acquire
478 #undef strasm_fence
479 #undef strasm_nobarrier
480 #undef strasm_nobarrier_clobber_memory
481
482 #endif // OS_CPU_LINUX_PPC_VM_ATOMIC_LINUX_PPC_INLINE_HPP
|