1 /*
2 * Copyright (c) 2015, 2020, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 */
23
24 #include "precompiled.hpp"
25 #include "gc/shared/locationPrinter.hpp"
26 #include "gc/z/zAddress.inline.hpp"
27 #include "gc/z/zGlobals.hpp"
28 #include "gc/z/zHeap.inline.hpp"
29 #include "gc/z/zHeapIterator.hpp"
30 #include "gc/z/zHeuristics.hpp"
31 #include "gc/z/zMark.inline.hpp"
32 #include "gc/z/zPage.inline.hpp"
33 #include "gc/z/zPageTable.inline.hpp"
34 #include "gc/z/zRelocationSet.inline.hpp"
35 #include "gc/z/zRelocationSetSelector.inline.hpp"
36 #include "gc/z/zResurrection.hpp"
37 #include "gc/z/zStat.hpp"
38 #include "gc/z/zThread.inline.hpp"
39 #include "gc/z/zVerify.hpp"
40 #include "gc/z/zWorkers.inline.hpp"
41 #include "logging/log.hpp"
42 #include "memory/iterator.hpp"
43 #include "memory/resourceArea.hpp"
44 #include "runtime/handshake.hpp"
45 #include "runtime/safepoint.hpp"
46 #include "runtime/thread.hpp"
47 #include "utilities/debug.hpp"
48
49 static const ZStatSampler ZSamplerHeapUsedBeforeMark("Memory", "Heap Used Before Mark", ZStatUnitBytes);
50 static const ZStatSampler ZSamplerHeapUsedAfterMark("Memory", "Heap Used After Mark", ZStatUnitBytes);
51 static const ZStatSampler ZSamplerHeapUsedBeforeRelocation("Memory", "Heap Used Before Relocation", ZStatUnitBytes);
52 static const ZStatSampler ZSamplerHeapUsedAfterRelocation("Memory", "Heap Used After Relocation", ZStatUnitBytes);
53 static const ZStatCounter ZCounterUndoPageAllocation("Memory", "Undo Page Allocation", ZStatUnitOpsPerSecond);
54 static const ZStatCounter ZCounterOutOfMemory("Memory", "Out Of Memory", ZStatUnitOpsPerSecond);
55
56 ZHeap* ZHeap::_heap = NULL;
57
58 ZHeap::ZHeap() :
59 _workers(),
60 _object_allocator(),
61 _page_allocator(&_workers, MinHeapSize, InitialHeapSize, MaxHeapSize, ZHeuristics::max_reserve()),
62 _page_table(),
63 _forwarding_table(),
64 _mark(&_workers, &_page_table),
65 _reference_processor(&_workers),
66 _weak_roots_processor(&_workers),
67 _relocate(&_workers),
68 _relocation_set(),
69 _unload(&_workers),
70 _serviceability(min_capacity(), max_capacity()) {
71 // Install global heap instance
72 assert(_heap == NULL, "Already initialized");
73 _heap = this;
74
75 // Update statistics
76 ZStatHeap::set_at_initialize(min_capacity(), max_capacity(), max_reserve());
77 }
78
79 bool ZHeap::is_initialized() const {
80 return _page_allocator.is_initialized() && _mark.is_initialized();
81 }
82
83 size_t ZHeap::min_capacity() const {
84 return _page_allocator.min_capacity();
85 }
86
87 size_t ZHeap::max_capacity() const {
88 return _page_allocator.max_capacity();
89 }
90
91 size_t ZHeap::soft_max_capacity() const {
92 return _page_allocator.soft_max_capacity();
93 }
94
95 size_t ZHeap::capacity() const {
96 return _page_allocator.capacity();
97 }
98
99 size_t ZHeap::max_reserve() const {
100 return _page_allocator.max_reserve();
101 }
102
103 size_t ZHeap::used_high() const {
104 return _page_allocator.used_high();
105 }
106
107 size_t ZHeap::used_low() const {
108 return _page_allocator.used_low();
109 }
110
111 size_t ZHeap::used() const {
112 return _page_allocator.used();
113 }
114
115 size_t ZHeap::unused() const {
116 return _page_allocator.unused();
117 }
118
119 size_t ZHeap::allocated() const {
120 return _page_allocator.allocated();
121 }
122
123 size_t ZHeap::reclaimed() const {
124 return _page_allocator.reclaimed();
125 }
126
127 size_t ZHeap::tlab_capacity() const {
128 return capacity();
129 }
130
131 size_t ZHeap::tlab_used() const {
132 return _object_allocator.used();
133 }
134
135 size_t ZHeap::max_tlab_size() const {
136 return ZObjectSizeLimitSmall;
137 }
138
139 size_t ZHeap::unsafe_max_tlab_alloc() const {
140 size_t size = _object_allocator.remaining();
141
142 if (size < MinTLABSize) {
143 // The remaining space in the allocator is not enough to
144 // fit the smallest possible TLAB. This means that the next
145 // TLAB allocation will force the allocator to get a new
146 // backing page anyway, which in turn means that we can then
147 // fit the largest possible TLAB.
148 size = max_tlab_size();
149 }
150
151 return MIN2(size, max_tlab_size());
152 }
153
154 bool ZHeap::is_in(uintptr_t addr) const {
155 // An address is considered to be "in the heap" if it points into
156 // the allocated part of a page, regardless of which heap view is
157 // used. Note that an address with the finalizable metadata bit set
158 // is not pointing into a heap view, and therefore not considered
159 // to be "in the heap".
160
161 if (ZAddress::is_in(addr)) {
162 const ZPage* const page = _page_table.get(addr);
163 if (page != NULL) {
164 return page->is_in(addr);
165 }
166 }
167
168 return false;
169 }
170
171 uint ZHeap::nconcurrent_worker_threads() const {
172 return _workers.nconcurrent();
173 }
174
175 uint ZHeap::nconcurrent_no_boost_worker_threads() const {
176 return _workers.nconcurrent_no_boost();
177 }
178
179 void ZHeap::set_boost_worker_threads(bool boost) {
180 _workers.set_boost(boost);
181 }
182
183 void ZHeap::worker_threads_do(ThreadClosure* tc) const {
184 _workers.threads_do(tc);
185 }
186
187 void ZHeap::print_worker_threads_on(outputStream* st) const {
188 _workers.print_threads_on(st);
189 }
190
191 void ZHeap::out_of_memory() {
192 ResourceMark rm;
193
194 ZStatInc(ZCounterOutOfMemory);
195 log_info(gc)("Out Of Memory (%s)", Thread::current()->name());
196 }
197
198 ZPage* ZHeap::alloc_page(uint8_t type, size_t size, ZAllocationFlags flags) {
199 ZPage* const page = _page_allocator.alloc_page(type, size, flags);
200 if (page != NULL) {
201 // Insert page table entry
202 _page_table.insert(page);
203 }
204
205 return page;
206 }
207
208 void ZHeap::undo_alloc_page(ZPage* page) {
209 assert(page->is_allocating(), "Invalid page state");
210
211 ZStatInc(ZCounterUndoPageAllocation);
212 log_trace(gc)("Undo page allocation, thread: " PTR_FORMAT " (%s), page: " PTR_FORMAT ", size: " SIZE_FORMAT,
213 ZThread::id(), ZThread::name(), p2i(page), page->size());
214
215 free_page(page, false /* reclaimed */);
216 }
217
218 void ZHeap::free_page(ZPage* page, bool reclaimed) {
219 // Remove page table entry
220 _page_table.remove(page);
221
222 // Free page
223 _page_allocator.free_page(page, reclaimed);
224 }
225
226 void ZHeap::uncommit_run() {
227 _page_allocator.uncommit_run();
228 }
229
230 void ZHeap::uncommit_stop() {
231 _page_allocator.uncommit_stop();
232 }
233
234 void ZHeap::flip_to_marked() {
235 ZVerifyViewsFlip flip(&_page_allocator);
236 ZAddress::flip_to_marked();
237 }
238
239 void ZHeap::flip_to_remapped() {
240 ZVerifyViewsFlip flip(&_page_allocator);
241 ZAddress::flip_to_remapped();
242 }
243
244 void ZHeap::mark_start() {
245 assert(SafepointSynchronize::is_at_safepoint(), "Should be at safepoint");
246
247 // Update statistics
248 ZStatSample(ZSamplerHeapUsedBeforeMark, used());
249
250 // Flip address view
251 flip_to_marked();
252
253 // Retire allocating pages
254 _object_allocator.retire_pages();
255
256 // Reset allocated/reclaimed/used statistics
257 _page_allocator.reset_statistics();
258
259 // Reset encountered/dropped/enqueued statistics
260 _reference_processor.reset_statistics();
261
262 // Enter mark phase
263 ZGlobalPhase = ZPhaseMark;
264
265 // Reset marking information and mark roots
266 _mark.start();
267
268 // Update statistics
269 ZStatHeap::set_at_mark_start(soft_max_capacity(), capacity(), used());
270 }
271
272 void ZHeap::mark(bool initial) {
273 _mark.mark(initial);
274 }
275
276 void ZHeap::mark_flush_and_free(Thread* thread) {
277 _mark.flush_and_free(thread);
278 }
279
280 bool ZHeap::mark_end() {
281 assert(SafepointSynchronize::is_at_safepoint(), "Should be at safepoint");
282
283 // Try end marking
284 if (!_mark.end()) {
285 // Marking not completed, continue concurrent mark
286 return false;
287 }
288
289 // Enter mark completed phase
290 ZGlobalPhase = ZPhaseMarkCompleted;
291
292 // Verify after mark
293 ZVerify::after_mark();
294
295 // Update statistics
296 ZStatSample(ZSamplerHeapUsedAfterMark, used());
297 ZStatHeap::set_at_mark_end(capacity(), allocated(), used());
298
299 // Block resurrection of weak/phantom references
300 ZResurrection::block();
301
302 // Process weak roots
303 _weak_roots_processor.process_weak_roots();
304
305 // Prepare to unload stale metadata and nmethods
306 _unload.prepare();
307
308 return true;
309 }
310
311 void ZHeap::keep_alive(oop obj) {
312 ZBarrier::keep_alive_barrier_on_oop(obj);
313 }
314
315 void ZHeap::set_soft_reference_policy(bool clear) {
316 _reference_processor.set_soft_reference_policy(clear);
317 }
318
319 class ZRendezvousClosure : public HandshakeClosure {
320 public:
321 ZRendezvousClosure() :
322 HandshakeClosure("ZRendezvous") {}
323
324 void do_thread(Thread* thread) {}
325 };
326
327 void ZHeap::process_non_strong_references() {
328 // Process Soft/Weak/Final/PhantomReferences
329 _reference_processor.process_references();
330
331 // Process concurrent weak roots
332 _weak_roots_processor.process_concurrent_weak_roots();
333
334 // Unlink stale metadata and nmethods
335 _unload.unlink();
336
337 // Perform a handshake. This is needed 1) to make sure that stale
338 // metadata and nmethods are no longer observable. And 2), to
339 // prevent the race where a mutator first loads an oop, which is
340 // logically null but not yet cleared. Then this oop gets cleared
341 // by the reference processor and resurrection is unblocked. At
342 // this point the mutator could see the unblocked state and pass
343 // this invalid oop through the normal barrier path, which would
344 // incorrectly try to mark the oop.
345 ZRendezvousClosure cl;
346 Handshake::execute(&cl);
347
348 // Unblock resurrection of weak/phantom references
349 ZResurrection::unblock();
350
351 // Purge stale metadata and nmethods that were unlinked
352 _unload.purge();
353
354 // Enqueue Soft/Weak/Final/PhantomReferences. Note that this
355 // must be done after unblocking resurrection. Otherwise the
356 // Finalizer thread could call Reference.get() on the Finalizers
357 // that were just enqueued, which would incorrectly return null
358 // during the resurrection block window, since such referents
359 // are only Finalizable marked.
360 _reference_processor.enqueue_references();
361 }
362
363 void ZHeap::select_relocation_set() {
364 // Do not allow pages to be deleted
365 _page_allocator.enable_deferred_delete();
366
367 // Register relocatable pages with selector
368 ZRelocationSetSelector selector;
369 ZPageTableIterator pt_iter(&_page_table);
370 for (ZPage* page; pt_iter.next(&page);) {
371 if (!page->is_relocatable()) {
372 // Not relocatable, don't register
373 continue;
374 }
375
376 if (page->is_marked()) {
377 // Register live page
378 selector.register_live_page(page);
379 } else {
380 // Register garbage page
381 selector.register_garbage_page(page);
382
383 // Reclaim page immediately
384 free_page(page, true /* reclaimed */);
385 }
386 }
387
388 // Allow pages to be deleted
389 _page_allocator.disable_deferred_delete();
390
391 // Select pages to relocate
392 selector.select(&_relocation_set);
393
394 // Setup forwarding table
395 ZRelocationSetIterator rs_iter(&_relocation_set);
396 for (ZForwarding* forwarding; rs_iter.next(&forwarding);) {
397 _forwarding_table.insert(forwarding);
398 }
399
400 // Update statistics
401 ZStatRelocation::set_at_select_relocation_set(selector.stats());
402 ZStatHeap::set_at_select_relocation_set(selector.stats(), reclaimed());
403 }
404
405 void ZHeap::reset_relocation_set() {
406 // Reset forwarding table
407 ZRelocationSetIterator iter(&_relocation_set);
408 for (ZForwarding* forwarding; iter.next(&forwarding);) {
409 _forwarding_table.remove(forwarding);
410 }
411
412 // Reset relocation set
413 _relocation_set.reset();
414 }
415
416 void ZHeap::relocate_start() {
417 assert(SafepointSynchronize::is_at_safepoint(), "Should be at safepoint");
418
419 // Finish unloading stale metadata and nmethods
420 _unload.finish();
421
422 // Flip address view
423 flip_to_remapped();
424
425 // Enter relocate phase
426 ZGlobalPhase = ZPhaseRelocate;
427
428 // Update statistics
429 ZStatSample(ZSamplerHeapUsedBeforeRelocation, used());
430 ZStatHeap::set_at_relocate_start(capacity(), allocated(), used());
431
432 // Remap/Relocate roots
433 _relocate.start();
434 }
435
436 void ZHeap::relocate() {
437 // Relocate relocation set
438 const bool success = _relocate.relocate(&_relocation_set);
439
440 // Update statistics
441 ZStatSample(ZSamplerHeapUsedAfterRelocation, used());
442 ZStatRelocation::set_at_relocate_end(success);
443 ZStatHeap::set_at_relocate_end(capacity(), allocated(), reclaimed(),
444 used(), used_high(), used_low());
445 }
446
447 void ZHeap::object_iterate(ObjectClosure* cl, bool visit_weaks) {
448 assert(SafepointSynchronize::is_at_safepoint(), "Should be at safepoint");
449
450 ZHeapIterator iter;
451 iter.objects_do(cl, visit_weaks);
452 }
453
454 void ZHeap::pages_do(ZPageClosure* cl) {
455 ZPageTableIterator iter(&_page_table);
456 for (ZPage* page; iter.next(&page);) {
457 cl->do_page(page);
458 }
459 _page_allocator.pages_do(cl);
460 }
461
462 void ZHeap::serviceability_initialize() {
463 _serviceability.initialize();
464 }
465
466 GCMemoryManager* ZHeap::serviceability_memory_manager() {
467 return _serviceability.memory_manager();
468 }
469
470 MemoryPool* ZHeap::serviceability_memory_pool() {
471 return _serviceability.memory_pool();
472 }
473
474 ZServiceabilityCounters* ZHeap::serviceability_counters() {
475 return _serviceability.counters();
476 }
477
478 void ZHeap::print_on(outputStream* st) const {
479 st->print_cr(" ZHeap used " SIZE_FORMAT "M, capacity " SIZE_FORMAT "M, max capacity " SIZE_FORMAT "M",
480 used() / M,
481 capacity() / M,
482 max_capacity() / M);
483 MetaspaceUtils::print_on(st);
484 }
485
486 void ZHeap::print_extended_on(outputStream* st) const {
487 print_on(st);
488 st->cr();
489
490 // Do not allow pages to be deleted
491 _page_allocator.enable_deferred_delete();
492
493 // Print all pages
494 ZPageTableIterator iter(&_page_table);
495 for (ZPage* page; iter.next(&page);) {
496 page->print_on(st);
497 }
498
499 // Allow pages to be deleted
500 _page_allocator.enable_deferred_delete();
501
502 st->cr();
503 }
504
505 bool ZHeap::print_location(outputStream* st, uintptr_t addr) const {
506 if (LocationPrinter::is_valid_obj((void*)addr)) {
507 st->print(PTR_FORMAT " is a %s oop: ", addr, ZAddress::is_good(addr) ? "good" : "bad");
508 ZOop::from_address(addr)->print_on(st);
509 return true;
510 }
511
512 return false;
513 }
514
515 void ZHeap::verify() {
516 // Heap verification can only be done between mark end and
517 // relocate start. This is the only window where all oop are
518 // good and the whole heap is in a consistent state.
519 guarantee(ZGlobalPhase == ZPhaseMarkCompleted, "Invalid phase");
520
521 ZVerify::after_weak_processing();
522 }