src/cpu/x86/vm/c1_LIRAssembler_x86.cpp

rev 8802 : G1 performance improvements: card batching, joining, sorting, prefetching and write barrier fence elision and simplification based on a global syncrhonization using handshakes piggybacking on thread-local safepoints.
rev 8804 : Further JPRT improvements

*** 24,33 ****
--- 24,34 ----
  
  #include "precompiled.hpp"
  #include "asm/macroAssembler.hpp"
  #include "asm/macroAssembler.inline.hpp"
  #include "c1/c1_Compilation.hpp"
+ #include "c1/c1_CodeStubs.hpp"
  #include "c1/c1_LIRAssembler.hpp"
  #include "c1/c1_MacroAssembler.hpp"
  #include "c1/c1_Runtime1.hpp"
  #include "c1/c1_ValueStack.hpp"
  #include "ci/ciArrayKlass.hpp"
*** 511,521 ****
  }
  
  
  // This is the fast version of java.lang.String.compare; it has not
  // OSR-entry and therefore, we generate a slow version for OSR's
! void LIR_Assembler::emit_string_compare(LIR_Opr arg0, LIR_Opr arg1, LIR_Opr dst, CodeEmitInfo* info) {
    __ movptr (rbx, rcx); // receiver is in rcx
    __ movptr (rax, arg1->as_register());
  
    // Get addresses of first characters from both Strings
    __ load_heap_oop(rsi, Address(rax, java_lang_String::value_offset_in_bytes()));
--- 512,522 ----
  }
  
  
  // This is the fast version of java.lang.String.compare; it has not
  // OSR-entry and therefore, we generate a slow version for OSR's
! void LIR_Assembler::emit_string_compare(LIR_Opr arg0, LIR_Opr arg1, LIR_Opr dst, C1ThreadLocalSafepoint *tls_stub, CodeEmitInfo* info) {
    __ movptr (rbx, rcx); // receiver is in rcx
    __ movptr (rax, arg1->as_register());
  
    // Get addresses of first characters from both Strings
    __ load_heap_oop(rsi, Address(rax, java_lang_String::value_offset_in_bytes()));
*** 581,599 ****
  
    // strings are equal up to min length
  
    __ bind(noLoop);
    __ pop(rax);
!   return_op(LIR_OprFact::illegalOpr);
  
    __ bind(haveResult);
    // leave instruction is going to discard the TOS value
    __ mov (rax, rcx); // result of call is in rax,
  }
  
  
! void LIR_Assembler::return_op(LIR_Opr result) {
    assert(result->is_illegal() || !result->is_single_cpu() || result->as_register() == rax, "word returns are in rax,");
    if (!result->is_illegal() && result->is_float_kind() && !result->is_xmm_register()) {
      assert(result->fpu() == 0, "result must already be on TOS");
    }
  
--- 582,600 ----
  
    // strings are equal up to min length
  
    __ bind(noLoop);
    __ pop(rax);
!   return_op(LIR_OprFact::illegalOpr, tls_stub);
  
    __ bind(haveResult);
    // leave instruction is going to discard the TOS value
    __ mov (rax, rcx); // result of call is in rax,
  }
  
  
! void LIR_Assembler::return_op(LIR_Opr result, C1ThreadLocalSafepoint *code_stub) {
    assert(result->is_illegal() || !result->is_single_cpu() || result->as_register() == rax, "word returns are in rax,");
    if (!result->is_illegal() && result->is_float_kind() && !result->is_xmm_register()) {
      assert(result->fpu() == 0, "result must already be on TOS");
    }
  
*** 602,638 ****
  
    bool result_is_oop = result->is_valid() ? result->is_oop() : false;
  
    // Note: we do not need to round double result; float result has the right precision
    // the poll sets the condition code, but no data registers
    AddressLiteral polling_page(os::get_polling_page(), relocInfo::poll_return_type);
  
    if (Assembler::is_polling_page_far()) {
      __ lea(rscratch1, polling_page);
      __ relocate(relocInfo::poll_return_type);
      __ testl(rax, Address(rscratch1, 0));
    } else {
      __ testl(rax, polling_page);
    }
    __ ret(0);
  }
  
  
! int LIR_Assembler::safepoint_poll(LIR_Opr tmp, CodeEmitInfo* info) {
    AddressLiteral polling_page(os::get_polling_page(), relocInfo::poll_type);
    guarantee(info != NULL, "Shouldn't be NULL");
    int offset = __ offset();
    if (Assembler::is_polling_page_far()) {
      __ lea(rscratch1, polling_page);
      offset = __ offset();
      add_debug_info_for_branch(info);
      __ relocate(relocInfo::poll_type);
      __ testl(rax, Address(rscratch1, 0));
    } else {
      add_debug_info_for_branch(info);
      __ testl(rax, polling_page);
    }
    return offset;
  }
  
  
  void LIR_Assembler::move_regs(Register from_reg, Register to_reg) {
--- 603,664 ----
  
    bool result_is_oop = result->is_valid() ? result->is_oop() : false;
  
    // Note: we do not need to round double result; float result has the right precision
    // the poll sets the condition code, but no data registers
+ 
+   if (!ThreadLocalSafepoints) {
      AddressLiteral polling_page(os::get_polling_page(), relocInfo::poll_return_type);
  
      if (Assembler::is_polling_page_far()) {
        __ lea(rscratch1, polling_page);
        __ relocate(relocInfo::poll_return_type);
        __ testl(rax, Address(rscratch1, 0));
      } else {
        __ testl(rax, polling_page);
      }
+   } else {
+ #ifdef _LP64
+     code_stub->set_safepoint_pc(__ pc());
+     __ relocate(relocInfo::poll_return_type);
+     __ testb(Address(r15_thread, Thread::yieldpoint_offset()), 2);
+     __ jcc(Assembler::equal, *code_stub->entry());
+ #else
+     ShouldNotReachHere();
+ #endif
+   }
    __ ret(0);
  }
  
  
! int LIR_Assembler::safepoint_poll(LIR_Opr tmp, C1ThreadLocalSafepoint *code_stub, CodeEmitInfo* info) {
    AddressLiteral polling_page(os::get_polling_page(), relocInfo::poll_type);
    guarantee(info != NULL, "Shouldn't be NULL");
    int offset = __ offset();
+   if (!ThreadLocalSafepoints) {
      if (Assembler::is_polling_page_far()) {
        __ lea(rscratch1, polling_page);
        offset = __ offset();
        add_debug_info_for_branch(info);
        __ relocate(relocInfo::poll_type);
        __ testl(rax, Address(rscratch1, 0));
      } else {
        add_debug_info_for_branch(info);
        __ testl(rax, polling_page);
      }
+   } else {
+ #ifdef _LP64
+     add_debug_info_for_branch(info);
+     code_stub->set_safepoint_pc(__ pc());
+     __ relocate(relocInfo::poll_type);
+     __ testb(Address(r15_thread, Thread::yieldpoint_offset()), 1);
+     __ jcc(Assembler::equal, *code_stub->entry());
+ #else
+     ShouldNotReachHere();
+ #endif
+   }
+ 
    return offset;
  }
  
  
  void LIR_Assembler::move_regs(Register from_reg, Register to_reg) {