src/share/vm/code/codeCache.cpp

*** 21,30 ****
--- 21,31 ----
   * questions.
   *
   */
  
  #include "precompiled.hpp"
+ #include "aot/aotLoader.hpp"
  #include "code/codeBlob.hpp"
  #include "code/codeCache.hpp"
  #include "code/compiledIC.hpp"
  #include "code/dependencies.hpp"
  #include "code/icBuffer.hpp"
*** 126,135 ****
--- 127,138 ----
    }
  };
  
  // Iterate over all CodeHeaps
  #define FOR_ALL_HEAPS(heap) for (GrowableArrayIterator<CodeHeap*> heap = _heaps->begin(); heap != _heaps->end(); ++heap)
+ #define FOR_ALL_NMETHOD_HEAPS(heap) for (GrowableArrayIterator<CodeHeap*> heap = _nmethod_heaps->begin(); heap != _nmethod_heaps->end(); ++heap)
+ 
  // Iterate over all CodeBlobs (cb) on the given CodeHeap
  #define FOR_ALL_BLOBS(cb, heap) for (CodeBlob* cb = first_blob(heap); cb != NULL; cb = next_blob(heap, cb))
  
  address CodeCache::_low_bound = 0;
  address CodeCache::_high_bound = 0;
*** 137,146 ****
--- 140,151 ----
  bool CodeCache::_needs_cache_clean = false;
  nmethod* CodeCache::_scavenge_root_nmethods = NULL;
  
  // Initialize array of CodeHeaps
  GrowableArray<CodeHeap*>* CodeCache::_heaps = new(ResourceObj::C_HEAP, mtCode) GrowableArray<CodeHeap*> (CodeBlobType::All, true);
+ GrowableArray<CodeHeap*>* CodeCache::_compiled_heaps = new(ResourceObj::C_HEAP, mtCode) GrowableArray<CodeHeap*> (CodeBlobType::All, true);
+ GrowableArray<CodeHeap*>* CodeCache::_nmethod_heaps = new(ResourceObj::C_HEAP, mtCode) GrowableArray<CodeHeap*> (CodeBlobType::All, true);
  
  void CodeCache::check_heap_sizes(size_t non_nmethod_size, size_t profiled_size, size_t non_profiled_size, size_t cache_size, bool all_set) {
    size_t total_size = non_nmethod_size + profiled_size + non_profiled_size;
    // Prepare error message
    const char* error = "Invalid code heap sizes";
*** 363,381 ****
    }
    ShouldNotReachHere();
    return NULL;
  }
  
  void CodeCache::add_heap(ReservedSpace rs, const char* name, int code_blob_type) {
    // Check if heap is needed
    if (!heap_available(code_blob_type)) {
      return;
    }
  
    // Create CodeHeap
    CodeHeap* heap = new CodeHeap(name, code_blob_type);
!   _heaps->append(heap);
  
    // Reserve Space
    size_t size_initial = MIN2(InitialCodeCacheSize, rs.size());
    size_initial = round_to(size_initial, os::vm_page_size());
    if (!heap->reserve(rs, size_initial, CodeCacheSegmentSize)) {
--- 368,408 ----
    }
    ShouldNotReachHere();
    return NULL;
  }
  
+ int CodeCache::code_heap_compare(CodeHeap* const &lhs, CodeHeap* const &rhs) {
+   if (lhs->code_blob_type() == rhs->code_blob_type()) {
+     return (lhs > rhs) ? 1 : ((lhs < rhs) ? -1 : 0);
+   } else {
+     return lhs->code_blob_type() - rhs->code_blob_type();
+   }
+ }
+ 
+ void CodeCache::add_heap(CodeHeap* heap) {
+   assert(!Universe::is_fully_initialized(), "late heap addition?");
+ 
+   _heaps->insert_sorted<code_heap_compare>(heap);
+ 
+   int type = heap->code_blob_type();
+   if (code_blob_type_accepts_compiled(type)) {
+     _compiled_heaps->insert_sorted<code_heap_compare>(heap);
+   }
+   if (code_blob_type_accepts_nmethod(type)) {
+     _nmethod_heaps->insert_sorted<code_heap_compare>(heap);
+   }
+ }
+ 
  void CodeCache::add_heap(ReservedSpace rs, const char* name, int code_blob_type) {
    // Check if heap is needed
    if (!heap_available(code_blob_type)) {
      return;
    }
  
    // Create CodeHeap
    CodeHeap* heap = new CodeHeap(name, code_blob_type);
!   add_heap(heap);
  
    // Reserve Space
    size_t size_initial = MIN2(InitialCodeCacheSize, rs.size());
    size_initial = round_to(size_initial, os::vm_page_size());
    if (!heap->reserve(rs, size_initial, CodeCacheSegmentSize)) {
*** 387,397 ****
  }
  
  CodeHeap* CodeCache::get_code_heap(const CodeBlob* cb) {
    assert(cb != NULL, "CodeBlob is null");
    FOR_ALL_HEAPS(heap) {
!     if ((*heap)->contains(cb)) {
        return *heap;
      }
    }
    ShouldNotReachHere();
    return NULL;
--- 414,424 ----
  }
  
  CodeHeap* CodeCache::get_code_heap(const CodeBlob* cb) {
    assert(cb != NULL, "CodeBlob is null");
    FOR_ALL_HEAPS(heap) {
!     if ((*heap)->contains(cb->code_begin())) {
        return *heap;
      }
    }
    ShouldNotReachHere();
    return NULL;
*** 424,437 ****
    assert_locked_or_safepoint(CodeCache_lock);
    assert(heap != NULL, "heap is null");
    return (CodeBlob*)heap->next(cb);
  }
  
- CodeBlob* CodeCache::next_blob(CodeBlob* cb) {
-   return next_blob(get_code_heap(cb), cb);
- }
- 
  /**
   * Do not seize the CodeCache lock here--if the caller has not
   * already done so, we are going to lose bigtime, since the code
   * cache will contain a garbage CodeBlob until the caller can
   * run the constructor for the CodeBlob subclass he is busy
--- 451,460 ----
*** 492,502 ****
        CompileBroker::handle_full_code_cache(orig_code_blob_type);
        return NULL;
      }
      if (PrintCodeCacheExtension) {
        ResourceMark rm;
!       if (_heaps->length() >= 1) {
          tty->print("%s", heap->name());
        } else {
          tty->print("CodeCache");
        }
        tty->print_cr(" extended to [" INTPTR_FORMAT ", " INTPTR_FORMAT "] (" SSIZE_FORMAT " bytes)",
--- 515,525 ----
        CompileBroker::handle_full_code_cache(orig_code_blob_type);
        return NULL;
      }
      if (PrintCodeCacheExtension) {
        ResourceMark rm;
!       if (_nmethod_heaps->length() >= 1) {
          tty->print("%s", heap->name());
        } else {
          tty->print("CodeCache");
        }
        tty->print_cr(" extended to [" INTPTR_FORMAT ", " INTPTR_FORMAT "] (" SSIZE_FORMAT " bytes)",
*** 557,566 ****
--- 580,593 ----
      }
    }
    return false;
  }
  
+ bool CodeCache::contains(nmethod *nm) {
+   return contains((void *)nm);
+ }
+ 
  // This method is safe to call without holding the CodeCache_lock, as long as a dead CodeBlob is not
  // looked up (i.e., one that has been marked for deletion). It only depends on the _segmap to contain
  // valid indices, which it will always do, as long as the CodeBlob is not in the process of being recycled.
  CodeBlob* CodeCache::find_blob(void* start) {
    CodeBlob* result = find_blob_unsafe(start);
*** 573,584 ****
  // what you are doing)
  CodeBlob* CodeCache::find_blob_unsafe(void* start) {
    // NMT can walk the stack before code cache is created
    if (_heaps != NULL && !_heaps->is_empty()) {
      FOR_ALL_HEAPS(heap) {
!       CodeBlob* result = (CodeBlob*) (*heap)->find_start(start);
!       if (result != NULL && result->blob_contains((address)start)) {
          return result;
        }
      }
    }
    return NULL;
--- 600,611 ----
  // what you are doing)
  CodeBlob* CodeCache::find_blob_unsafe(void* start) {
    // NMT can walk the stack before code cache is created
    if (_heaps != NULL && !_heaps->is_empty()) {
      FOR_ALL_HEAPS(heap) {
!       CodeBlob* result = (*heap)->find_blob_unsafe(start);
!       if (result != NULL) {
          return result;
        }
      }
    }
    return NULL;
*** 590,600 ****
    return (nmethod*)cb;
  }
  
  void CodeCache::blobs_do(void f(CodeBlob* nm)) {
    assert_locked_or_safepoint(CodeCache_lock);
!   FOR_ALL_HEAPS(heap) {
      FOR_ALL_BLOBS(cb, *heap) {
        f(cb);
      }
    }
  }
--- 617,627 ----
    return (nmethod*)cb;
  }
  
  void CodeCache::blobs_do(void f(CodeBlob* nm)) {
    assert_locked_or_safepoint(CodeCache_lock);
!   FOR_ALL_NMETHOD_HEAPS(heap) {
      FOR_ALL_BLOBS(cb, *heap) {
        f(cb);
      }
    }
  }
*** 611,620 ****
--- 638,648 ----
    assert_locked_or_safepoint(CodeCache_lock);
    NMethodIterator iter;
    while(iter.next_alive()) {
      iter.method()->metadata_do(f);
    }
+   AOTLoader::metadata_do(f);
  }
  
  int CodeCache::alignment_unit() {
    return (int)_heaps->first()->alignment_unit();
  }
*** 632,646 ****
    }
  }
  
  void CodeCache::blobs_do(CodeBlobClosure* f) {
    assert_locked_or_safepoint(CodeCache_lock);
!   FOR_ALL_HEAPS(heap) {
      FOR_ALL_BLOBS(cb, *heap) {
        if (cb->is_alive()) {
          f->do_code_blob(cb);
- 
  #ifdef ASSERT
          if (cb->is_nmethod())
          ((nmethod*)cb)->verify_scavenge_root_oops();
  #endif //ASSERT
        }
--- 660,673 ----
    }
  }
  
  void CodeCache::blobs_do(CodeBlobClosure* f) {
    assert_locked_or_safepoint(CodeCache_lock);
!   FOR_ALL_NMETHOD_HEAPS(heap) {
      FOR_ALL_BLOBS(cb, *heap) {
        if (cb->is_alive()) {
          f->do_code_blob(cb);
  #ifdef ASSERT
          if (cb->is_nmethod())
          ((nmethod*)cb)->verify_scavenge_root_oops();
  #endif //ASSERT
        }
*** 833,849 ****
  #ifdef ASSERT
    // make sure that we aren't leaking icholders
    int count = 0;
    FOR_ALL_HEAPS(heap) {
      FOR_ALL_BLOBS(cb, *heap) {
!       if (cb->is_nmethod()) {
!         nmethod* nm = (nmethod*)cb;
          count += nm->verify_icholder_relocations();
        }
      }
    }
- 
    assert(count + InlineCacheBuffer::pending_icholder_count() + CompiledICHolder::live_not_claimed_count() ==
           CompiledICHolder::live_count(), "must agree");
  #endif
  }
  
--- 860,875 ----
  #ifdef ASSERT
    // make sure that we aren't leaking icholders
    int count = 0;
    FOR_ALL_HEAPS(heap) {
      FOR_ALL_BLOBS(cb, *heap) {
!       CompiledMethod *nm = cb->as_compiled_method_or_null();
!       if (nm != NULL) {
          count += nm->verify_icholder_relocations();
        }
      }
    }
    assert(count + InlineCacheBuffer::pending_icholder_count() + CompiledICHolder::live_not_claimed_count() ==
           CompiledICHolder::live_count(), "must agree");
  #endif
  }
  
*** 900,910 ****
    return (heap != NULL) ? heap->nmethod_count() : 0;
  }
  
  int CodeCache::nmethod_count() {
    int count = 0;
!   FOR_ALL_HEAPS(heap) {
      count += (*heap)->nmethod_count();
    }
    return count;
  }
  
--- 926,936 ----
    return (heap != NULL) ? heap->nmethod_count() : 0;
  }
  
  int CodeCache::nmethod_count() {
    int count = 0;
!   FOR_ALL_NMETHOD_HEAPS(heap) {
      count += (*heap)->nmethod_count();
    }
    return count;
  }
  
*** 931,941 ****
    return (heap != NULL) ? (address)heap->high_boundary() : NULL;
  }
  
  size_t CodeCache::capacity() {
    size_t cap = 0;
!   FOR_ALL_HEAPS(heap) {
      cap += (*heap)->capacity();
    }
    return cap;
  }
  
--- 957,967 ----
    return (heap != NULL) ? (address)heap->high_boundary() : NULL;
  }
  
  size_t CodeCache::capacity() {
    size_t cap = 0;
!   FOR_ALL_NMETHOD_HEAPS(heap) {
      cap += (*heap)->capacity();
    }
    return cap;
  }
  
*** 944,962 ****
    return (heap != NULL) ? heap->unallocated_capacity() : 0;
  }
  
  size_t CodeCache::unallocated_capacity() {
    size_t unallocated_cap = 0;
!   FOR_ALL_HEAPS(heap) {
      unallocated_cap += (*heap)->unallocated_capacity();
    }
    return unallocated_cap;
  }
  
  size_t CodeCache::max_capacity() {
    size_t max_cap = 0;
!   FOR_ALL_HEAPS(heap) {
      max_cap += (*heap)->max_capacity();
    }
    return max_cap;
  }
  
--- 970,988 ----
    return (heap != NULL) ? heap->unallocated_capacity() : 0;
  }
  
  size_t CodeCache::unallocated_capacity() {
    size_t unallocated_cap = 0;
!   FOR_ALL_NMETHOD_HEAPS(heap) {
      unallocated_cap += (*heap)->unallocated_capacity();
    }
    return unallocated_cap;
  }
  
  size_t CodeCache::max_capacity() {
    size_t max_cap = 0;
!   FOR_ALL_NMETHOD_HEAPS(heap) {
      max_cap += (*heap)->max_capacity();
    }
    return max_cap;
  }
  
*** 978,1004 ****
    return result;
  }
  
  size_t CodeCache::bytes_allocated_in_freelists() {
    size_t allocated_bytes = 0;
!   FOR_ALL_HEAPS(heap) {
      allocated_bytes += (*heap)->allocated_in_freelist();
    }
    return allocated_bytes;
  }
  
  int CodeCache::allocated_segments() {
    int number_of_segments = 0;
!   FOR_ALL_HEAPS(heap) {
      number_of_segments += (*heap)->allocated_segments();
    }
    return number_of_segments;
  }
  
  size_t CodeCache::freelists_length() {
    size_t length = 0;
!   FOR_ALL_HEAPS(heap) {
      length += (*heap)->freelist_length();
    }
    return length;
  }
  
--- 1004,1030 ----
    return result;
  }
  
  size_t CodeCache::bytes_allocated_in_freelists() {
    size_t allocated_bytes = 0;
!   FOR_ALL_NMETHOD_HEAPS(heap) {
      allocated_bytes += (*heap)->allocated_in_freelist();
    }
    return allocated_bytes;
  }
  
  int CodeCache::allocated_segments() {
    int number_of_segments = 0;
!   FOR_ALL_NMETHOD_HEAPS(heap) {
      number_of_segments += (*heap)->allocated_segments();
    }
    return number_of_segments;
  }
  
  size_t CodeCache::freelists_length() {
    size_t length = 0;
!   FOR_ALL_NMETHOD_HEAPS(heap) {
      length += (*heap)->freelist_length();
    }
    return length;
  }
  
*** 1037,1046 ****
--- 1063,1074 ----
    os::register_code_area((char*)low_bound(), (char*)high_bound());
  }
  
  void codeCache_init() {
    CodeCache::initialize();
+   // Load AOT libraries and add AOT code heaps.
+   AOTLoader::initialize();
  }
  
  //------------------------------------------------------------------------------------------------
  
  int CodeCache::number_of_nmethods_with_dependencies() {
*** 1100,1109 ****
--- 1128,1146 ----
    CodeBlob *cb = find_blob(start);
    assert(cb == NULL || cb->is_compiled(), "did not find an compiled_method");
    return (CompiledMethod*)cb;
  }
  
+ bool CodeCache::is_far_target(address target) {
+ #if INCLUDE_AOT
+   return NativeCall::is_far_call(_low_bound,  target) ||
+          NativeCall::is_far_call(_high_bound, target);
+ #else
+   return false;
+ #endif
+ }
+ 
  #ifdef HOTSWAP
  int CodeCache::mark_for_evol_deoptimization(instanceKlassHandle dependee) {
    MutexLockerEx mu(CodeCache_lock, Mutex::_no_safepoint_check_flag);
    int number_of_marked_CodeBlobs = 0;
  
*** 1202,1212 ****
  #ifdef HOTSWAP
  // Flushes compiled methods dependent on dependee in the evolutionary sense
  void CodeCache::flush_evol_dependents_on(instanceKlassHandle ev_k_h) {
    // --- Compile_lock is not held. However we are at a safepoint.
    assert_locked_or_safepoint(Compile_lock);
!   if (number_of_nmethods_with_dependencies() == 0) return;
  
    // CodeCache can only be updated by a thread_in_VM and they will all be
    // stopped during the safepoint so CodeCache will be safe to update without
    // holding the CodeCache_lock.
  
--- 1239,1249 ----
  #ifdef HOTSWAP
  // Flushes compiled methods dependent on dependee in the evolutionary sense
  void CodeCache::flush_evol_dependents_on(instanceKlassHandle ev_k_h) {
    // --- Compile_lock is not held. However we are at a safepoint.
    assert_locked_or_safepoint(Compile_lock);
!   if (number_of_nmethods_with_dependencies() == 0 && !UseAOT) return;
  
    // CodeCache can only be updated by a thread_in_VM and they will all be
    // stopped during the safepoint so CodeCache will be safe to update without
    // holding the CodeCache_lock.
  
*** 1314,1324 ****
    }
  }
  
  void CodeCache::print_memory_overhead() {
    size_t wasted_bytes = 0;
!   FOR_ALL_HEAPS(heap) {
        CodeHeap* curr_heap = *heap;
        for (CodeBlob* cb = (CodeBlob*)curr_heap->first(); cb != NULL; cb = (CodeBlob*)curr_heap->next(cb)) {
          HeapBlock* heap_block = ((HeapBlock*)cb) - 1;
          wasted_bytes += heap_block->length() * CodeCacheSegmentSize - cb->size();
        }
--- 1351,1361 ----
    }
  }
  
  void CodeCache::print_memory_overhead() {
    size_t wasted_bytes = 0;
!   FOR_ALL_NMETHOD_HEAPS(heap) {
        CodeHeap* curr_heap = *heap;
        for (CodeBlob* cb = (CodeBlob*)curr_heap->first(); cb != NULL; cb = (CodeBlob*)curr_heap->next(cb)) {
          HeapBlock* heap_block = ((HeapBlock*)cb) - 1;
          wasted_bytes += heap_block->length() * CodeCacheSegmentSize - cb->size();
        }
*** 1360,1371 ****
    int nmethodNative = 0;
    int max_nm_size = 0;
    ResourceMark rm;
  
    int i = 0;
!   FOR_ALL_HEAPS(heap) {
!     if ((_heaps->length() >= 1) && Verbose) {
        tty->print_cr("-- %s --", (*heap)->name());
      }
      FOR_ALL_BLOBS(cb, *heap) {
        total++;
        if (cb->is_nmethod()) {
--- 1397,1408 ----
    int nmethodNative = 0;
    int max_nm_size = 0;
    ResourceMark rm;
  
    int i = 0;
!   FOR_ALL_NMETHOD_HEAPS(heap) {
!     if ((_nmethod_heaps->length() >= 1) && Verbose) {
        tty->print_cr("-- %s --", (*heap)->name());
      }
      FOR_ALL_BLOBS(cb, *heap) {
        total++;
        if (cb->is_nmethod()) {
*** 1457,1467 ****
    if (!Verbose) return;
  
    CodeBlob_sizes live;
    CodeBlob_sizes dead;
  
!   FOR_ALL_HEAPS(heap) {
      FOR_ALL_BLOBS(cb, *heap) {
        if (!cb->is_alive()) {
          dead.add(cb);
        } else {
          live.add(cb);
--- 1494,1504 ----
    if (!Verbose) return;
  
    CodeBlob_sizes live;
    CodeBlob_sizes dead;
  
!   FOR_ALL_NMETHOD_HEAPS(heap) {
      FOR_ALL_BLOBS(cb, *heap) {
        if (!cb->is_alive()) {
          dead.add(cb);
        } else {
          live.add(cb);
*** 1483,1493 ****
       // print the oop_map usage
      int code_size = 0;
      int number_of_blobs = 0;
      int number_of_oop_maps = 0;
      int map_size = 0;
!     FOR_ALL_HEAPS(heap) {
        FOR_ALL_BLOBS(cb, *heap) {
          if (cb->is_alive()) {
            number_of_blobs++;
            code_size += cb->code_size();
            ImmutableOopMapSet* set = cb->oop_maps();
--- 1520,1530 ----
       // print the oop_map usage
      int code_size = 0;
      int number_of_blobs = 0;
      int number_of_oop_maps = 0;
      int map_size = 0;
!     FOR_ALL_NMETHOD_HEAPS(heap) {
        FOR_ALL_BLOBS(cb, *heap) {
          if (cb->is_alive()) {
            number_of_blobs++;
            code_size += cb->code_size();
            ImmutableOopMapSet* set = cb->oop_maps();
*** 1566,1602 ****
              " adapters='" UINT32_FORMAT "' free_code_cache='" SIZE_FORMAT "'",
              blob_count(), nmethod_count(), adapter_count(),
              unallocated_capacity());
  }
  
- // Initialize iterator to given compiled method
- void CompiledMethodIterator::initialize(CompiledMethod* cm) {
-   _code_blob = (CodeBlob*)cm;
-   if (!SegmentedCodeCache) {
-     // Iterate over all CodeBlobs
-     _code_blob_type = CodeBlobType::All;
-   } else if (cm != NULL) {
-     _code_blob_type = CodeCache::get_code_blob_type(cm);
-   } else {
-     // Only iterate over method code heaps, starting with non-profiled
-     _code_blob_type = CodeBlobType::MethodNonProfiled;
-   }
- }
- 
- // Advance iterator to the next compiled method in the current code heap
- bool CompiledMethodIterator::next_compiled_method() {
-   // Get first method CodeBlob
-   if (_code_blob == NULL) {
-     _code_blob = CodeCache::first_blob(_code_blob_type);
-     if (_code_blob == NULL) {
-       return false;
-     } else if (_code_blob->is_nmethod()) {
-       return true;
-     }
-   }
-   // Search for next method CodeBlob
-   _code_blob = CodeCache::next_blob(_code_blob);
-   while (_code_blob != NULL && !_code_blob->is_compiled()) {
-     _code_blob = CodeCache::next_blob(_code_blob);
-   }
-   return _code_blob != NULL;
- }
--- 1603,1607 ----