2395 }
2396 }
2397
2398 void MacroAssembler::atomic_ori_int(Register addr, Register result, int uimm16) {
2399 Label retry;
2400 bind(retry);
2401 lwarx(result, addr, /*hint*/ false);
2402 ori(result, result, uimm16);
2403 stwcx_(result, addr);
2404 if (UseStaticBranchPredictionInCompareAndSwapPPC64) {
2405 bne_predict_not_taken(CCR0, retry); // stXcx_ sets CCR0
2406 } else {
2407 bne( CCR0, retry); // stXcx_ sets CCR0
2408 }
2409 }
2410
2411 #if INCLUDE_RTM_OPT
2412
2413 // Update rtm_counters based on abort status
2414 // input: abort_status
2415 // rtm_counters (RTMLockingCounters*)
2416 void MacroAssembler::rtm_counters_update(Register abort_status, Register rtm_counters_Reg) {
2417 // Mapping to keep PreciseRTMLockingStatistics similar to x86.
2418 // x86 ppc (! means inverted, ? means not the same)
2419 // 0 31 Set if abort caused by XABORT instruction.
2420 // 1 ! 7 If set, the transaction may succeed on a retry. This bit is always clear if bit 0 is set.
2421 // 2 13 Set if another logical processor conflicted with a memory address that was part of the transaction that aborted.
2422 // 3 10 Set if an internal buffer overflowed.
2423 // 4 ?12 Set if a debug breakpoint was hit.
2424 // 5 ?32 Set if an abort occurred during execution of a nested transaction.
2425 const int tm_failure_bit[] = {Assembler::tm_tabort, // Note: Seems like signal handler sets this, too.
2426 Assembler::tm_failure_persistent, // inverted: transient
2427 Assembler::tm_trans_cf,
2428 Assembler::tm_footprint_of,
2429 Assembler::tm_non_trans_cf,
2430 Assembler::tm_suspended};
2431 const bool tm_failure_inv[] = {false, true, false, false, false, false};
2432 assert(sizeof(tm_failure_bit)/sizeof(int) == RTMLockingCounters::ABORT_STATUS_LIMIT, "adapt mapping!");
2433
2434 const Register addr_Reg = R0;
2435 // Keep track of offset to where rtm_counters_Reg had pointed to.
2436 int counters_offs = RTMLockingCounters::abort_count_offset();
2437 addi(addr_Reg, rtm_counters_Reg, counters_offs);
2438 const Register temp_Reg = rtm_counters_Reg;
2439
2440 //atomic_inc_ptr(addr_Reg, temp_Reg); We don't increment atomically
2441 ldx(temp_Reg, addr_Reg);
2442 addi(temp_Reg, temp_Reg, 1);
2443 stdx(temp_Reg, addr_Reg);
2444
2445 if (PrintPreciseRTMLockingStatistics) {
2446 int counters_offs_delta = RTMLockingCounters::abortX_count_offset() - counters_offs;
2447
2448 //mftexasr(abort_status); done by caller
2449 for (int i = 0; i < RTMLockingCounters::ABORT_STATUS_LIMIT; i++) {
2450 counters_offs += counters_offs_delta;
2451 li(temp_Reg, counters_offs_delta); // can't use addi with R0
2452 add(addr_Reg, addr_Reg, temp_Reg); // point to next counter
2453 counters_offs_delta = sizeof(uintx);
2454
2455 Label check_abort;
2456 rldicr_(temp_Reg, abort_status, tm_failure_bit[i], 0);
2457 if (tm_failure_inv[i]) {
2458 bne(CCR0, check_abort);
2459 } else {
2460 beq(CCR0, check_abort);
2461 }
2462 //atomic_inc_ptr(addr_Reg, temp_Reg); We don't increment atomically
2463 ldx(temp_Reg, addr_Reg);
2464 addi(temp_Reg, temp_Reg, 1);
2465 stdx(temp_Reg, addr_Reg);
2466 bind(check_abort);
2467 }
2468 }
2469 li(temp_Reg, -counters_offs); // can't use addi with R0
2470 add(rtm_counters_Reg, addr_Reg, temp_Reg); // restore
2471 }
2472
2473 // Branch if (random & (count-1) != 0), count is 2^n
2474 // tmp and CR0 are killed
2475 void MacroAssembler::branch_on_random_using_tb(Register tmp, int count, Label& brLabel) {
2476 mftb(tmp);
2477 andi_(tmp, tmp, count-1);
2478 bne(CCR0, brLabel);
2479 }
2480
2481 // Perform abort ratio calculation, set no_rtm bit if high ratio.
2482 // input: rtm_counters_Reg (RTMLockingCounters* address) - KILLED
2483 void MacroAssembler::rtm_abort_ratio_calculation(Register rtm_counters_Reg,
2484 RTMLockingCounters* rtm_counters,
2485 Metadata* method_data) {
2486 Label L_done, L_check_always_rtm1, L_check_always_rtm2;
2487
2488 if (RTMLockingCalculationDelay > 0) {
2489 // Delay calculation.
2490 ld(rtm_counters_Reg, (RegisterOrConstant)(intptr_t)RTMLockingCounters::rtm_calculation_flag_addr());
|
2395 }
2396 }
2397
2398 void MacroAssembler::atomic_ori_int(Register addr, Register result, int uimm16) {
2399 Label retry;
2400 bind(retry);
2401 lwarx(result, addr, /*hint*/ false);
2402 ori(result, result, uimm16);
2403 stwcx_(result, addr);
2404 if (UseStaticBranchPredictionInCompareAndSwapPPC64) {
2405 bne_predict_not_taken(CCR0, retry); // stXcx_ sets CCR0
2406 } else {
2407 bne( CCR0, retry); // stXcx_ sets CCR0
2408 }
2409 }
2410
2411 #if INCLUDE_RTM_OPT
2412
2413 // Update rtm_counters based on abort status
2414 // input: abort_status
2415 // rtm_counters_Reg (RTMLockingCounters*)
2416 void MacroAssembler::rtm_counters_update(Register abort_status, Register rtm_counters_Reg) {
2417 // Mapping to keep PreciseRTMLockingStatistics similar to x86.
2418 // x86 ppc (! means inverted, ? means not the same)
2419 // 0 31 Set if abort caused by XABORT instruction.
2420 // 1 ! 7 If set, the transaction may succeed on a retry. This bit is always clear if bit 0 is set.
2421 // 2 13 Set if another logical processor conflicted with a memory address that was part of the transaction that aborted.
2422 // 3 10 Set if an internal buffer overflowed.
2423 // 4 ?12 Set if a debug breakpoint was hit.
2424 // 5 ?32 Set if an abort occurred during execution of a nested transaction.
2425 const int failure_bit[] = {tm_tabort, // Signal handler will set this too.
2426 tm_failure_persistent, // Inverted: transient.
2427 tm_non_trans_cf,
2428 tm_trans_cf,
2429 tm_footprint_of,
2430 tm_failure_code,
2431 tm_transaction_level};
2432
2433 const bool failure_logic_inv[] = {false, // tabort
2434 true, // failure_persistent
2435 false, // non_trans_cf
2436 false, // trans_cf
2437 false, // footprint_of
2438 true, // failure_code
2439 false}; // transaction_level
2440
2441 const int num_failure_bits = sizeof(failure_bit) / sizeof(int);
2442 const int num_counters = RTMLockingCounters::ABORT_STATUS_LIMIT;
2443
2444 bool bit2counter_map[][num_counters] =
2445 // counters:
2446 // 0 1 2 3 4 5
2447 // abort , persist, conflict, overflow, debug , nested bits:
2448 {{ true , false , false , false , false , false }, // abort
2449 { false , true , false , false , false , false }, // failure_persistent
2450 { false , false , true , false , false , false }, // non_trans_cf
2451 { false , false , true , false , false , false }, // trans_cf
2452 { false , false , false , true , false , false }, // footprint_of
2453 { false , false , false , false , true , false }, // failure_code = 0xD4
2454 { false , false , false , false , false , true }}; // transaction_level > 1
2455 // ...
2456
2457 // Move abort_status value to R0 and use abort_status register as a
2458 // temporary register because R0 as third operand in ld/std is treated
2459 // as base address zero (value). Likewise, R0 as second operand in addi
2460 // is problematic because it amounts to li.
2461 const Register temp_Reg = abort_status;
2462 const Register abort_status_R0 = R0;
2463 mr(abort_status_R0, abort_status);
2464
2465 // Keep track of offsets added to rtm_counters_Reg to restore it back.
2466 int counters_offs = RTMLockingCounters::abort_count_offset();
2467 addi(rtm_counters_Reg, rtm_counters_Reg, counters_offs);
2468
2469 // Increment total abort counter.
2470 // atomic_inc_ptr(addr_Reg, temp_Reg); We don't increment atomically.
2471 ldx(temp_Reg, rtm_counters_Reg);
2472 addi(temp_Reg, temp_Reg, 1);
2473 stdx(temp_Reg, rtm_counters_Reg);
2474
2475 // Increment specific abort counters.
2476 if (PrintPreciseRTMLockingStatistics) {
2477
2478 int abort_offs;
2479
2480 abort_offs = RTMLockingCounters::abortX_count_offset() - counters_offs;
2481 addi(rtm_counters_Reg, rtm_counters_Reg, abort_offs);
2482
2483 // Keep track of offsets added to rtm_counters_Reg.
2484 counters_offs += abort_offs;
2485
2486 for (int nbit = 0; nbit < num_failure_bits; nbit++) {
2487 for (int ncounter = 0; ncounter < num_counters; ncounter++) {
2488 if (bit2counter_map[nbit][ncounter] == true) {
2489
2490 Label check_abort;
2491
2492 // Counter offset based on counter number (counter * 8 bytes).
2493 abort_offs = ncounter << 3;
2494
2495 if (failure_bit[nbit] == tm_transaction_level) {
2496 // Don't check outer transaction, TL = 1 (bit 63). Hence only
2497 // 11 bits in the TL field are checked to find out if failure
2498 // occured in a nested transaction. This check also matches
2499 // the case when nesting_of = 1 (nesting overflow).
2500 rldicr_(temp_Reg, abort_status_R0, failure_bit[nbit], 10);
2501 } else if (failure_bit[nbit] == tm_failure_code) {
2502 // Check failure code for trap or illegal caught in TM.
2503 // Bits 0:7 are tested as bit 7 (persistent) is copied from
2504 // tabort or treclaim source operand.
2505 // On Linux: trap or illegal is TM_CAUSE_SIGNAL (0xD4).
2506 rldicl(temp_Reg, abort_status_R0, 8, 56);
2507 cmpdi(CCR0, temp_Reg, 0xD4);
2508 } else {
2509 rldicr_(temp_Reg, abort_status_R0, failure_bit[nbit], 0);
2510 }
2511
2512 if (failure_logic_inv[nbit] == true) {
2513 bne(CCR0, check_abort);
2514 } else {
2515 beq(CCR0, check_abort);
2516 }
2517
2518 // We don't increment atomically.
2519 ld(temp_Reg, abort_offs, rtm_counters_Reg);
2520 addi(temp_Reg, temp_Reg, 1);
2521 std(temp_Reg, abort_offs, rtm_counters_Reg);
2522
2523 bind(check_abort);
2524 }
2525 }
2526 }
2527 }
2528
2529 // Restore rtm_counters_Reg and abort_status.
2530 addi(rtm_counters_Reg, rtm_counters_Reg, -counters_offs);
2531 mr(abort_status, abort_status_R0);
2532 }
2533
2534 // Branch if (random & (count-1) != 0), count is 2^n
2535 // tmp and CR0 are killed
2536 void MacroAssembler::branch_on_random_using_tb(Register tmp, int count, Label& brLabel) {
2537 mftb(tmp);
2538 andi_(tmp, tmp, count-1);
2539 bne(CCR0, brLabel);
2540 }
2541
2542 // Perform abort ratio calculation, set no_rtm bit if high ratio.
2543 // input: rtm_counters_Reg (RTMLockingCounters* address) - KILLED
2544 void MacroAssembler::rtm_abort_ratio_calculation(Register rtm_counters_Reg,
2545 RTMLockingCounters* rtm_counters,
2546 Metadata* method_data) {
2547 Label L_done, L_check_always_rtm1, L_check_always_rtm2;
2548
2549 if (RTMLockingCalculationDelay > 0) {
2550 // Delay calculation.
2551 ld(rtm_counters_Reg, (RegisterOrConstant)(intptr_t)RTMLockingCounters::rtm_calculation_flag_addr());
|