src/cpu/x86/vm/x86_64.ad

rev 8802 : G1 performance improvements: card batching, joining, sorting, prefetching and write barrier fence elision and simplification based on a global syncrhonization using handshakes piggybacking on thread-local safepoints.

*** 936,946 ****
    }
  
    st->print_cr("popq   rbp");
    if (do_polling() && C->is_method_compilation()) {
      st->print("\t");
!     if (Assembler::is_polling_page_far()) {
        st->print_cr("movq   rscratch1, #polling_page_address\n\t"
                     "testl  rax, [rscratch1]\t"
                     "# Safepoint: poll for GC");
      } else {
        st->print_cr("testl  rax, [rip + #offset_to_poll_page]\t"
--- 936,950 ----
    }
  
    st->print_cr("popq   rbp");
    if (do_polling() && C->is_method_compilation()) {
      st->print("\t");
!     if (ThreadLocalSafepoints) {
!       st->print_cr("testb  $1, [r15]\t"
!                    "# Safepoint: poll for GC\n\t");
!       st->print_cr("je  #slow_safepoint_runtime");
!     } else if (Assembler::is_polling_page_far()) {
        st->print_cr("movq   rscratch1, #polling_page_address\n\t"
                     "testl  rax, [rscratch1]\t"
                     "# Safepoint: poll for GC");
      } else {
        st->print_cr("testl  rax, [rip + #offset_to_poll_page]\t"
*** 985,995 ****
    emit_opcode(cbuf, 0x58 | RBP_enc);
  
    if (do_polling() && C->is_method_compilation()) {
      MacroAssembler _masm(&cbuf);
      AddressLiteral polling_page(os::get_polling_page(), relocInfo::poll_return_type);
!     if (Assembler::is_polling_page_far()) {
        __ lea(rscratch1, polling_page);
        __ relocate(relocInfo::poll_return_type);
        __ testl(rax, Address(rscratch1, 0));
      } else {
        __ testl(rax, polling_page);
--- 989,1006 ----
    emit_opcode(cbuf, 0x58 | RBP_enc);
  
    if (do_polling() && C->is_method_compilation()) {
      MacroAssembler _masm(&cbuf);
      AddressLiteral polling_page(os::get_polling_page(), relocInfo::poll_return_type);
!     if (ThreadLocalSafepoints) {
!       InternalAddress safepoint_pc(__ pc());
!       Label dummy_label;
!       Label &code_stub = &cbuf == C->code_buffer() ? C->tls_table()->add_safepoint(safepoint_pc, true) : dummy_label;
!       __ relocate(relocInfo::poll_return_type);
!       __ testb(Address(r15_thread, Thread::yieldpoint_offset()), 2);
!       __ jcc(Assembler::equal, code_stub);
!     } else if (Assembler::is_polling_page_far()) {
        __ lea(rscratch1, polling_page);
        __ relocate(relocInfo::poll_return_type);
        __ testl(rax, Address(rscratch1, 0));
      } else {
        __ testl(rax, polling_page);
*** 1003,1013 ****
                                // the hard way
  }
  
  int MachEpilogNode::reloc() const
  {
!   return 2; // a large enough number
  }
  
  const Pipeline* MachEpilogNode::pipeline() const
  {
    return MachNode::pipeline_class();
--- 1014,1024 ----
                                // the hard way
  }
  
  int MachEpilogNode::reloc() const
  {
!   return 3; // a large enough number
  }
  
  const Pipeline* MachEpilogNode::pipeline() const
  {
    return MachNode::pipeline_class();
*** 11550,11576 ****
  
  // ============================================================================
  // Safepoint Instructions
  instruct safePoint_poll(rFlagsReg cr)
  %{
!   predicate(!Assembler::is_polling_page_far());
    match(SafePoint);
    effect(KILL cr);
  
    format %{ "testl  rax, [rip + #offset_to_poll_page]\t"
              "# Safepoint: poll for GC" %}
    ins_cost(125);
    ins_encode %{
      AddressLiteral addr(os::get_polling_page(), relocInfo::poll_type);
      __ testl(rax, addr);
    %}
    ins_pipe(ialu_reg_mem);
  %}
  
  instruct safePoint_poll_far(rFlagsReg cr, rRegP poll)
  %{
!   predicate(Assembler::is_polling_page_far());
    match(SafePoint poll);
    effect(KILL cr, USE poll);
  
    format %{ "testl  rax, [$poll]\t"
              "# Safepoint: poll for GC" %}
--- 11561,11597 ----
  
  // ============================================================================
  // Safepoint Instructions
  instruct safePoint_poll(rFlagsReg cr)
  %{
!   predicate(!Assembler::is_polling_page_far() || ThreadLocalSafepoints);
    match(SafePoint);
    effect(KILL cr);
  
    format %{ "testl  rax, [rip + #offset_to_poll_page]\t"
              "# Safepoint: poll for GC" %}
    ins_cost(125);
    ins_encode %{
+     if (ThreadLocalSafepoints) {
+       Compile* C = ra_->C;
+       InternalAddress safepoint_pc(__ pc());
+       Label dummy_label;
+       Label &code_stub = &cbuf == C->code_buffer() ? C->tls_table()->add_safepoint(safepoint_pc, false) : dummy_label;
+       __ relocate(relocInfo::poll_type);
+       __ testb(Address(r15_thread, Thread::yieldpoint_offset()), 1);
+       __ jcc(Assembler::equal, code_stub);
+     } else {
        AddressLiteral addr(os::get_polling_page(), relocInfo::poll_type);
        __ testl(rax, addr);
+     }
    %}
    ins_pipe(ialu_reg_mem);
  %}
  
  instruct safePoint_poll_far(rFlagsReg cr, rRegP poll)
  %{
!   predicate(Assembler::is_polling_page_far() && !ThreadLocalSafepoints);
    match(SafePoint poll);
    effect(KILL cr, USE poll);
  
    format %{ "testl  rax, [$poll]\t"
              "# Safepoint: poll for GC" %}