src/share/vm/runtime/sharedRuntime.cpp

rev 8802 : G1 performance improvements: card batching, joining, sorting, prefetching and write barrier fence elision and simplification based on a global syncrhonization using handshakes piggybacking on thread-local safepoints.

@@ -92,26 +92,42 @@
   _ic_miss_blob                        = generate_resolve_blob(CAST_FROM_FN_PTR(address, SharedRuntime::handle_wrong_method_ic_miss),  "ic_miss_stub");
   _resolve_opt_virtual_call_blob       = generate_resolve_blob(CAST_FROM_FN_PTR(address, SharedRuntime::resolve_opt_virtual_call_C),   "resolve_opt_virtual_call");
   _resolve_virtual_call_blob           = generate_resolve_blob(CAST_FROM_FN_PTR(address, SharedRuntime::resolve_virtual_call_C),       "resolve_virtual_call");
   _resolve_static_call_blob            = generate_resolve_blob(CAST_FROM_FN_PTR(address, SharedRuntime::resolve_static_call_C),        "resolve_static_call");
 
+  address runtime_exception_handler;
+
+  if (ThreadLocalSafepoints) {
+    runtime_exception_handler = CAST_FROM_FN_PTR(address, SharedRuntime::thread_local_safepoint);
+  } else {
+    runtime_exception_handler = CAST_FROM_FN_PTR(address, SafepointSynchronize::handle_polling_page_exception);
+  }
+
 #ifdef COMPILER2
   // Vectors are generated only by C2.
   if (is_wide_vector(MaxVectorSize)) {
-    _polling_page_vectors_safepoint_handler_blob = generate_handler_blob(CAST_FROM_FN_PTR(address, SafepointSynchronize::handle_polling_page_exception), POLL_AT_VECTOR_LOOP);
+    _polling_page_vectors_safepoint_handler_blob = generate_handler_blob(runtime_exception_handler, POLL_AT_VECTOR_LOOP);
   }
 #endif // COMPILER2
-  _polling_page_safepoint_handler_blob = generate_handler_blob(CAST_FROM_FN_PTR(address, SafepointSynchronize::handle_polling_page_exception), POLL_AT_LOOP);
-  _polling_page_return_handler_blob    = generate_handler_blob(CAST_FROM_FN_PTR(address, SafepointSynchronize::handle_polling_page_exception), POLL_AT_RETURN);
+  _polling_page_safepoint_handler_blob = generate_handler_blob(runtime_exception_handler, POLL_AT_LOOP);
+  _polling_page_return_handler_blob    = generate_handler_blob(runtime_exception_handler, POLL_AT_RETURN);
 
   generate_deopt_blob();
 
 #ifdef COMPILER2
   generate_uncommon_trap_blob();
 #endif // COMPILER2
 }
 
+void SharedRuntime::thread_local_safepoint(JavaThread *thread) {
+  thread->set_yieldpoint(false);
+  thread->update_serialized_memory_version();
+  if (SafepointSynchronize::is_synchronizing()) {
+    SafepointSynchronize::handle_polling_page_exception(thread);
+  }
+}
+
 #include <math.h>
 
 // Implementation of SharedRuntime
 
 #ifndef PRODUCT

@@ -194,10 +210,11 @@
 
 #if INCLUDE_ALL_GCS
 
 // G1 write-barrier pre: executed before a pointer store.
 JRT_LEAF(void, SharedRuntime::g1_wb_pre(oopDesc* orig, JavaThread *thread))
+  thread->update_serialized_memory_version();
   if (orig == NULL) {
     assert(false, "should be optimized out");
     return;
   }
   assert(orig->is_oop(true /* ignore mark word */), "Error");

@@ -205,10 +222,11 @@
   thread->satb_mark_queue().enqueue(orig);
 JRT_END
 
 // G1 write-barrier post: executed after a pointer store.
 JRT_LEAF(void, SharedRuntime::g1_wb_post(void* card_addr, JavaThread* thread))
+  thread->update_serialized_memory_version();
   thread->dirty_card_queue().enqueue(card_addr);
 JRT_END
 
 #endif // INCLUDE_ALL_GCS
 

@@ -518,17 +536,10 @@
 address SharedRuntime::get_poll_stub(address pc) {
   address stub;
   // Look up the code blob
   CodeBlob *cb = CodeCache::find_blob(pc);
 
-  // Should be an nmethod
-  assert(cb && cb->is_nmethod(), "safepoint polling: pc must refer to an nmethod");
-
-  // Look up the relocation information
-  assert(((nmethod*)cb)->is_at_poll_or_poll_return(pc),
-    "safepoint polling: type must be poll");
-
   assert(((NativeInstruction*)pc)->is_safepoint_poll(),
     "Only polling locations are used for safepoint");
 
   bool at_poll_return = ((nmethod*)cb)->is_at_poll_return(pc);
   bool has_wide_vectors = ((nmethod*)cb)->has_wide_vectors();