2395 }
2396 }
2397
2398 void MacroAssembler::atomic_ori_int(Register addr, Register result, int uimm16) {
2399 Label retry;
2400 bind(retry);
2401 lwarx(result, addr, /*hint*/ false);
2402 ori(result, result, uimm16);
2403 stwcx_(result, addr);
2404 if (UseStaticBranchPredictionInCompareAndSwapPPC64) {
2405 bne_predict_not_taken(CCR0, retry); // stXcx_ sets CCR0
2406 } else {
2407 bne( CCR0, retry); // stXcx_ sets CCR0
2408 }
2409 }
2410
2411 #if INCLUDE_RTM_OPT
2412
2413 // Update rtm_counters based on abort status
2414 // input: abort_status
2415 // rtm_counters (RTMLockingCounters*)
2416 void MacroAssembler::rtm_counters_update(Register abort_status, Register rtm_counters_Reg) {
2417 // Mapping to keep PreciseRTMLockingStatistics similar to x86.
2418 // x86 ppc (! means inverted, ? means not the same)
2419 // 0 31 Set if abort caused by XABORT instruction.
2420 // 1 ! 7 If set, the transaction may succeed on a retry. This bit is always clear if bit 0 is set.
2421 // 2 13 Set if another logical processor conflicted with a memory address that was part of the transaction that aborted.
2422 // 3 10 Set if an internal buffer overflowed.
2423 // 4 ?12 Set if a debug breakpoint was hit.
2424 // 5 ?32 Set if an abort occurred during execution of a nested transaction.
2425 const int tm_failure_bit[] = {Assembler::tm_tabort, // Note: Seems like signal handler sets this, too.
2426 Assembler::tm_failure_persistent, // inverted: transient
2427 Assembler::tm_trans_cf,
2428 Assembler::tm_footprint_of,
2429 Assembler::tm_non_trans_cf,
2430 Assembler::tm_suspended};
2431 const bool tm_failure_inv[] = {false, true, false, false, false, false};
2432 assert(sizeof(tm_failure_bit)/sizeof(int) == RTMLockingCounters::ABORT_STATUS_LIMIT, "adapt mapping!");
2433
2434 const Register addr_Reg = R0;
2435 // Keep track of offset to where rtm_counters_Reg had pointed to.
2436 int counters_offs = RTMLockingCounters::abort_count_offset();
2437 addi(addr_Reg, rtm_counters_Reg, counters_offs);
2438 const Register temp_Reg = rtm_counters_Reg;
2439
2440 //atomic_inc_ptr(addr_Reg, temp_Reg); We don't increment atomically
2441 ldx(temp_Reg, addr_Reg);
2442 addi(temp_Reg, temp_Reg, 1);
2443 stdx(temp_Reg, addr_Reg);
2444
2445 if (PrintPreciseRTMLockingStatistics) {
2446 int counters_offs_delta = RTMLockingCounters::abortX_count_offset() - counters_offs;
2447
2448 //mftexasr(abort_status); done by caller
2449 for (int i = 0; i < RTMLockingCounters::ABORT_STATUS_LIMIT; i++) {
2450 counters_offs += counters_offs_delta;
2451 li(temp_Reg, counters_offs_delta); // can't use addi with R0
2452 add(addr_Reg, addr_Reg, temp_Reg); // point to next counter
2453 counters_offs_delta = sizeof(uintx);
2454
2455 Label check_abort;
2456 rldicr_(temp_Reg, abort_status, tm_failure_bit[i], 0);
2457 if (tm_failure_inv[i]) {
2458 bne(CCR0, check_abort);
2459 } else {
2460 beq(CCR0, check_abort);
2461 }
2462 //atomic_inc_ptr(addr_Reg, temp_Reg); We don't increment atomically
2463 ldx(temp_Reg, addr_Reg);
2464 addi(temp_Reg, temp_Reg, 1);
2465 stdx(temp_Reg, addr_Reg);
2466 bind(check_abort);
2467 }
2468 }
2469 li(temp_Reg, -counters_offs); // can't use addi with R0
2470 add(rtm_counters_Reg, addr_Reg, temp_Reg); // restore
2471 }
2472
2473 // Branch if (random & (count-1) != 0), count is 2^n
2474 // tmp and CR0 are killed
2475 void MacroAssembler::branch_on_random_using_tb(Register tmp, int count, Label& brLabel) {
2476 mftb(tmp);
2477 andi_(tmp, tmp, count-1);
2478 bne(CCR0, brLabel);
2479 }
2480
2481 // Perform abort ratio calculation, set no_rtm bit if high ratio.
2482 // input: rtm_counters_Reg (RTMLockingCounters* address) - KILLED
2483 void MacroAssembler::rtm_abort_ratio_calculation(Register rtm_counters_Reg,
2484 RTMLockingCounters* rtm_counters,
2485 Metadata* method_data) {
2486 Label L_done, L_check_always_rtm1, L_check_always_rtm2;
2487
2488 if (RTMLockingCalculationDelay > 0) {
2489 // Delay calculation.
2490 ld(rtm_counters_Reg, (RegisterOrConstant)(intptr_t)RTMLockingCounters::rtm_calculation_flag_addr());
|
2395 }
2396 }
2397
2398 void MacroAssembler::atomic_ori_int(Register addr, Register result, int uimm16) {
2399 Label retry;
2400 bind(retry);
2401 lwarx(result, addr, /*hint*/ false);
2402 ori(result, result, uimm16);
2403 stwcx_(result, addr);
2404 if (UseStaticBranchPredictionInCompareAndSwapPPC64) {
2405 bne_predict_not_taken(CCR0, retry); // stXcx_ sets CCR0
2406 } else {
2407 bne( CCR0, retry); // stXcx_ sets CCR0
2408 }
2409 }
2410
2411 #if INCLUDE_RTM_OPT
2412
2413 // Update rtm_counters based on abort status
2414 // input: abort_status
2415 // rtm_counters_Reg (RTMLockingCounters*)
2416 void MacroAssembler::rtm_counters_update(Register abort_status, Register rtm_counters_Reg) {
2417 // Mapping to keep PreciseRTMLockingStatistics similar to x86.
2418 // x86 ppc (! means inverted, ? means not the same)
2419 // 0 31 Set if abort caused by XABORT instruction.
2420 // 1 ! 7 If set, the transaction may succeed on a retry. This bit is always clear if bit 0 is set.
2421 // 2 13 Set if another logical processor conflicted with a memory address that was part of the transaction that aborted.
2422 // 3 10 Set if an internal buffer overflowed.
2423 // 4 ?12 Set if a debug breakpoint was hit.
2424 // 5 ?32 Set if an abort occurred during execution of a nested transaction.
2425 const int failure_bit[] = {tm_tabort, // Signal handler will set this too.
2426 tm_failure_persistent,
2427 tm_non_trans_cf,
2428 tm_trans_cf,
2429 tm_footprint_of,
2430 tm_failure_code,
2431 tm_transaction_level};
2432
2433 const int num_failure_bits = sizeof(failure_bit) / sizeof(int);
2434 const int num_counters = RTMLockingCounters::ABORT_STATUS_LIMIT;
2435
2436 const int bit2counter_map[][num_counters] =
2437 // 0 = no map; 1 = mapped, no inverted logic; -1 = mapped, inverted logic
2438 // Inverted logic means that if a bit is set don't count it, or vice-versa.
2439 // Care must be taken when mapping bits to counters as bits for a given
2440 // counter must be mutually exclusive. Otherwise, counter will be increment
2441 // more than once.
2442 // counters:
2443 // 0 1 2 3 4 5
2444 // abort , persist, conflict, overflow, debug , nested bits:
2445 {{ 1 , 0 , 0 , 0 , 0 , 0 }, // abort
2446 { 0 , -1 , 0 , 0 , 0 , 0 }, // failure_persistent
2447 { 0 , 0 , 1 , 0 , 0 , 0 }, // non_trans_cf
2448 { 0 , 0 , 1 , 0 , 0 , 0 }, // trans_cf
2449 { 0 , 0 , 0 , 1 , 0 , 0 }, // footprint_of
2450 { 0 , 0 , 0 , 0 , -1 , 0 }, // failure_code = 0xD4
2451 { 0 , 0 , 0 , 0 , 0 , 1 }}; // transaction_level > 1
2452 // ...
2453
2454 // Move abort_status value to R0 and use abort_status register as a
2455 // temporary register because R0 as third operand in ld/std is treated
2456 // as base address zero (value). Likewise, R0 as second operand in addi
2457 // is problematic because it amounts to li.
2458 const Register temp_Reg = abort_status;
2459 const Register abort_status_R0 = R0;
2460 mr(abort_status_R0, abort_status);
2461
2462 // Increment total abort counter.
2463 int counters_offs = RTMLockingCounters::abort_count_offset();
2464 ld(temp_Reg, counters_offs, rtm_counters_Reg);
2465 addi(temp_Reg, temp_Reg, 1);
2466 std(temp_Reg, counters_offs, rtm_counters_Reg);
2467
2468 // Increment specific abort counters.
2469 if (PrintPreciseRTMLockingStatistics) {
2470
2471 // #0 counter offset.
2472 int abortX_offs;
2473 abortX_offs = RTMLockingCounters::abortX_count_offset();
2474
2475 for (int nbit = 0; nbit < num_failure_bits; nbit++) {
2476 for (int ncounter = 0; ncounter < num_counters; ncounter++) {
2477 if (bit2counter_map[nbit][ncounter] != 0) {
2478 Label check_abort;
2479 int abort_counter_offs;
2480
2481 abort_counter_offs = abortX_offs + (ncounter << 3);
2482
2483 if (failure_bit[nbit] == tm_transaction_level) {
2484 // Don't check outer transaction, TL = 1 (bit 63). Hence only
2485 // 11 bits in the TL field are checked to find out if failure
2486 // occured in a nested transaction. This check also matches
2487 // the case when nesting_of = 1 (nesting overflow).
2488 rldicr_(temp_Reg, abort_status_R0, failure_bit[nbit], 10);
2489 } else if (failure_bit[nbit] == tm_failure_code) {
2490 // Check failure code for trap or illegal caught in TM.
2491 // Bits 0:7 are tested as bit 7 (persistent) is copied from
2492 // tabort or treclaim source operand.
2493 // On Linux: trap or illegal is TM_CAUSE_SIGNAL (0xD4).
2494 rldicl(temp_Reg, abort_status_R0, 8, 56);
2495 cmpdi(CCR0, temp_Reg, 0xD4);
2496 } else {
2497 rldicr_(temp_Reg, abort_status_R0, failure_bit[nbit], 0);
2498 }
2499
2500 if (bit2counter_map[nbit][ncounter] == 1) {
2501 beq(CCR0, check_abort);
2502 } else {
2503 bne(CCR0, check_abort);
2504 }
2505
2506 // We don't increment atomically.
2507 ld(temp_Reg, abort_counter_offs, rtm_counters_Reg);
2508 addi(temp_Reg, temp_Reg, 1);
2509 std(temp_Reg, abort_counter_offs, rtm_counters_Reg);
2510
2511 bind(check_abort);
2512 }
2513 }
2514 }
2515 }
2516 // Restore abort_status.
2517 mr(abort_status, abort_status_R0);
2518 }
2519
2520 // Branch if (random & (count-1) != 0), count is 2^n
2521 // tmp and CR0 are killed
2522 void MacroAssembler::branch_on_random_using_tb(Register tmp, int count, Label& brLabel) {
2523 mftb(tmp);
2524 andi_(tmp, tmp, count-1);
2525 bne(CCR0, brLabel);
2526 }
2527
2528 // Perform abort ratio calculation, set no_rtm bit if high ratio.
2529 // input: rtm_counters_Reg (RTMLockingCounters* address) - KILLED
2530 void MacroAssembler::rtm_abort_ratio_calculation(Register rtm_counters_Reg,
2531 RTMLockingCounters* rtm_counters,
2532 Metadata* method_data) {
2533 Label L_done, L_check_always_rtm1, L_check_always_rtm2;
2534
2535 if (RTMLockingCalculationDelay > 0) {
2536 // Delay calculation.
2537 ld(rtm_counters_Reg, (RegisterOrConstant)(intptr_t)RTMLockingCounters::rtm_calculation_flag_addr());
|