2350 bind(nope);
2351 membar(AnyAny);
2352 mov(oldv, tmp);
2353 }
2354 if (fail)
2355 b(*fail);
2356 }
2357
2358 // A generic CAS; success or failure is in the EQ flag. A weak CAS
2359 // doesn't retry and may fail spuriously. If the oldval is wanted,
2360 // Pass a register for the result, otherwise pass noreg.
2361
2362 // Clobbers rscratch1
2363 void MacroAssembler::cmpxchg(Register addr, Register expected,
2364 Register new_val,
2365 enum operand_size size,
2366 bool acquire, bool release,
2367 bool weak,
2368 Register result) {
2369 if (result == noreg) result = rscratch1;
2370 if (UseLSE) {
2371 mov(result, expected);
2372 lse_cas(result, new_val, addr, size, acquire, release, /*not_pair*/ true);
2373 cmp(result, expected);
2374 } else {
2375 BLOCK_COMMENT("cmpxchg {");
2376 Label retry_load, done;
2377 if ((VM_Version::features() & VM_Version::CPU_STXR_PREFETCH))
2378 prfm(Address(addr), PSTL1STRM);
2379 bind(retry_load);
2380 load_exclusive(result, addr, size, acquire);
2381 if (size == xword)
2382 cmp(result, expected);
2383 else
2384 cmpw(result, expected);
2385 br(Assembler::NE, done);
2386 store_exclusive(rscratch1, new_val, addr, size, release);
2387 if (weak) {
2388 cmpw(rscratch1, 0u); // If the store fails, return NE to our caller.
2389 } else {
2390 cbnzw(rscratch1, retry_load);
2391 }
2392 bind(done);
2393 BLOCK_COMMENT("} cmpxchg");
2394 }
2395 }
2396
2397 static bool different(Register a, RegisterOrConstant b, Register c) {
2398 if (b.is_constant())
2399 return a != c;
2400 else
2401 return a != b.as_register() && a != c && b.as_register() != c;
2402 }
2403
2404 #define ATOMIC_OP(NAME, LDXR, OP, IOP, AOP, STXR, sz) \
2405 void MacroAssembler::atomic_##NAME(Register prev, RegisterOrConstant incr, Register addr) { \
2406 if (UseLSE) { \
2407 prev = prev->is_valid() ? prev : zr; \
2408 if (incr.is_register()) { \
2409 AOP(sz, incr.as_register(), prev, addr); \
2410 } else { \
2411 mov(rscratch2, incr.as_constant()); \
2412 AOP(sz, rscratch2, prev, addr); \
2413 } \
2414 return; \
2415 } \
|
2350 bind(nope);
2351 membar(AnyAny);
2352 mov(oldv, tmp);
2353 }
2354 if (fail)
2355 b(*fail);
2356 }
2357
2358 // A generic CAS; success or failure is in the EQ flag. A weak CAS
2359 // doesn't retry and may fail spuriously. If the oldval is wanted,
2360 // Pass a register for the result, otherwise pass noreg.
2361
2362 // Clobbers rscratch1
2363 void MacroAssembler::cmpxchg(Register addr, Register expected,
2364 Register new_val,
2365 enum operand_size size,
2366 bool acquire, bool release,
2367 bool weak,
2368 Register result) {
2369 if (result == noreg) result = rscratch1;
2370 BLOCK_COMMENT("cmpxchg {");
2371 if (UseLSE) {
2372 mov(result, expected);
2373 lse_cas(result, new_val, addr, size, acquire, release, /*not_pair*/ true);
2374 compare_eq(result, expected, size);
2375 } else {
2376 Label retry_load, done;
2377 if ((VM_Version::features() & VM_Version::CPU_STXR_PREFETCH))
2378 prfm(Address(addr), PSTL1STRM);
2379 bind(retry_load);
2380 load_exclusive(result, addr, size, acquire);
2381 compare_eq(result, expected, size);
2382 br(Assembler::NE, done);
2383 store_exclusive(rscratch1, new_val, addr, size, release);
2384 if (weak) {
2385 cmpw(rscratch1, 0u); // If the store fails, return NE to our caller.
2386 } else {
2387 cbnzw(rscratch1, retry_load);
2388 }
2389 bind(done);
2390 }
2391 BLOCK_COMMENT("} cmpxchg");
2392 }
2393
2394 // A generic comparison. Only compares for equality, clobbers rscratch1.
2395 void MacroAssembler::compare_eq(Register rm, Register rn, enum operand_size size) {
2396 if (size == xword) {
2397 cmp(rm, rn);
2398 } else if (size == word) {
2399 cmpw(rm, rn);
2400 } else if (size == halfword) {
2401 eorw(rscratch1, rm, rn);
2402 ands(zr, rscratch1, 0xffff);
2403 } else if (size == byte) {
2404 eorw(rscratch1, rm, rn);
2405 ands(zr, rscratch1, 0xff);
2406 } else {
2407 ShouldNotReachHere();
2408 }
2409 }
2410
2411
2412 static bool different(Register a, RegisterOrConstant b, Register c) {
2413 if (b.is_constant())
2414 return a != c;
2415 else
2416 return a != b.as_register() && a != c && b.as_register() != c;
2417 }
2418
2419 #define ATOMIC_OP(NAME, LDXR, OP, IOP, AOP, STXR, sz) \
2420 void MacroAssembler::atomic_##NAME(Register prev, RegisterOrConstant incr, Register addr) { \
2421 if (UseLSE) { \
2422 prev = prev->is_valid() ? prev : zr; \
2423 if (incr.is_register()) { \
2424 AOP(sz, incr.as_register(), prev, addr); \
2425 } else { \
2426 mov(rscratch2, incr.as_constant()); \
2427 AOP(sz, rscratch2, prev, addr); \
2428 } \
2429 return; \
2430 } \
|