1 /*
2 * Copyright (c) 2001, 2019, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "precompiled.hpp"
26 #include "code/nmethod.hpp"
27 #include "gc/g1/g1BlockOffsetTable.inline.hpp"
28 #include "gc/g1/g1CollectedHeap.inline.hpp"
29 #include "gc/g1/g1CollectionSet.hpp"
30 #include "gc/g1/g1HeapRegionTraceType.hpp"
31 #include "gc/g1/g1NUMA.hpp"
32 #include "gc/g1/g1OopClosures.inline.hpp"
33 #include "gc/g1/heapRegion.inline.hpp"
34 #include "gc/g1/heapRegionBounds.inline.hpp"
35 #include "gc/g1/heapRegionManager.inline.hpp"
36 #include "gc/g1/heapRegionRemSet.hpp"
37 #include "gc/g1/heapRegionTracer.hpp"
38 #include "gc/shared/genOopClosures.inline.hpp"
39 #include "logging/log.hpp"
40 #include "logging/logStream.hpp"
41 #include "memory/iterator.inline.hpp"
42 #include "memory/resourceArea.hpp"
43 #include "oops/access.inline.hpp"
44 #include "oops/compressedOops.inline.hpp"
45 #include "oops/oop.inline.hpp"
46
47 int HeapRegion::LogOfHRGrainBytes = 0;
48 int HeapRegion::LogOfHRGrainWords = 0;
49 int HeapRegion::LogCardsPerRegion = 0;
50 size_t HeapRegion::GrainBytes = 0;
51 size_t HeapRegion::GrainWords = 0;
52 size_t HeapRegion::CardsPerRegion = 0;
53
54 size_t HeapRegion::max_region_size() {
55 return HeapRegionBounds::max_size();
56 }
57
58 size_t HeapRegion::min_region_size_in_words() {
59 return HeapRegionBounds::min_size() >> LogHeapWordSize;
60 }
61
62 void HeapRegion::setup_heap_region_size(size_t initial_heap_size, size_t max_heap_size) {
63 size_t region_size = G1HeapRegionSize;
64 if (FLAG_IS_DEFAULT(G1HeapRegionSize)) {
65 size_t average_heap_size = (initial_heap_size + max_heap_size) / 2;
66 region_size = MAX2(average_heap_size / HeapRegionBounds::target_number(),
67 HeapRegionBounds::min_size());
68 }
69
70 int region_size_log = log2_long((jlong) region_size);
71 // Recalculate the region size to make sure it's a power of
72 // 2. This means that region_size is the largest power of 2 that's
73 // <= what we've calculated so far.
74 region_size = ((size_t)1 << region_size_log);
75
76 // Now make sure that we don't go over or under our limits.
77 if (region_size < HeapRegionBounds::min_size()) {
78 region_size = HeapRegionBounds::min_size();
79 } else if (region_size > HeapRegionBounds::max_size()) {
80 region_size = HeapRegionBounds::max_size();
81 }
82
83 // And recalculate the log.
84 region_size_log = log2_long((jlong) region_size);
85
86 // Now, set up the globals.
87 guarantee(LogOfHRGrainBytes == 0, "we should only set it once");
88 LogOfHRGrainBytes = region_size_log;
89
90 guarantee(LogOfHRGrainWords == 0, "we should only set it once");
91 LogOfHRGrainWords = LogOfHRGrainBytes - LogHeapWordSize;
92
93 guarantee(GrainBytes == 0, "we should only set it once");
94 // The cast to int is safe, given that we've bounded region_size by
95 // MIN_REGION_SIZE and MAX_REGION_SIZE.
96 GrainBytes = region_size;
97 log_info(gc, heap)("Heap region size: " SIZE_FORMAT "M", GrainBytes / M);
98
99 guarantee(GrainWords == 0, "we should only set it once");
100 GrainWords = GrainBytes >> LogHeapWordSize;
101 guarantee((size_t) 1 << LogOfHRGrainWords == GrainWords, "sanity");
102
103 guarantee(CardsPerRegion == 0, "we should only set it once");
104 CardsPerRegion = GrainBytes >> G1CardTable::card_shift;
105
106 LogCardsPerRegion = log2_long((jlong) CardsPerRegion);
107
108 if (G1HeapRegionSize != GrainBytes) {
109 FLAG_SET_ERGO(G1HeapRegionSize, GrainBytes);
110 }
111 }
112
113 void HeapRegion::handle_evacuation_failure() {
114 uninstall_surv_rate_group();
115 clear_young_index_in_cset();
116 set_evacuation_failed(false);
117 set_old();
118 }
119
120 void HeapRegion::unlink_from_list() {
121 set_next(NULL);
122 set_prev(NULL);
123 set_containing_set(NULL);
124 }
125
126 void HeapRegion::hr_clear(bool clear_space) {
127 assert(_humongous_start_region == NULL,
128 "we should have already filtered out humongous regions");
129 assert(!in_collection_set(),
130 "Should not clear heap region %u in the collection set", hrm_index());
131
132 clear_young_index_in_cset();
133 clear_index_in_opt_cset();
134 uninstall_surv_rate_group();
135 set_free();
136 reset_pre_dummy_top();
137
138 rem_set()->clear_locked();
139
140 zero_marked_bytes();
141
142 init_top_at_mark_start();
143 if (clear_space) clear(SpaceDecorator::Mangle);
144
145 _evacuation_failed = false;
146 _gc_efficiency = 0.0;
147 }
148
149 void HeapRegion::clear_cardtable() {
150 G1CardTable* ct = G1CollectedHeap::heap()->card_table();
151 ct->clear(MemRegion(bottom(), end()));
152 }
153
154 void HeapRegion::calc_gc_efficiency() {
155 // GC efficiency is the ratio of how much space would be
156 // reclaimed over how long we predict it would take to reclaim it.
157 G1Policy* policy = G1CollectedHeap::heap()->policy();
158
159 // Retrieve a prediction of the elapsed time for this region for
160 // a mixed gc because the region will only be evacuated during a
161 // mixed gc.
162 double region_elapsed_time_ms = policy->predict_region_total_time_ms(this, false /* for_young_gc */);
163 _gc_efficiency = (double) reclaimable_bytes() / region_elapsed_time_ms;
164 }
165
166 void HeapRegion::set_free() {
167 report_region_type_change(G1HeapRegionTraceType::Free);
168 _type.set_free();
169 }
170
171 void HeapRegion::set_eden() {
172 report_region_type_change(G1HeapRegionTraceType::Eden);
173 _type.set_eden();
174 }
175
176 void HeapRegion::set_eden_pre_gc() {
177 report_region_type_change(G1HeapRegionTraceType::Eden);
178 _type.set_eden_pre_gc();
179 }
180
181 void HeapRegion::set_survivor() {
182 report_region_type_change(G1HeapRegionTraceType::Survivor);
183 _type.set_survivor();
184 }
185
186 void HeapRegion::move_to_old() {
187 if (_type.relabel_as_old()) {
188 report_region_type_change(G1HeapRegionTraceType::Old);
189 }
190 }
191
192 void HeapRegion::set_old() {
193 report_region_type_change(G1HeapRegionTraceType::Old);
194 _type.set_old();
195 }
196
197 void HeapRegion::set_open_archive() {
198 report_region_type_change(G1HeapRegionTraceType::OpenArchive);
199 _type.set_open_archive();
200 }
201
202 void HeapRegion::set_closed_archive() {
203 report_region_type_change(G1HeapRegionTraceType::ClosedArchive);
204 _type.set_closed_archive();
205 }
206
207 void HeapRegion::set_starts_humongous(HeapWord* obj_top, size_t fill_size) {
208 assert(!is_humongous(), "sanity / pre-condition");
209 assert(top() == bottom(), "should be empty");
210
211 report_region_type_change(G1HeapRegionTraceType::StartsHumongous);
212 _type.set_starts_humongous();
213 _humongous_start_region = this;
214
215 _bot_part.set_for_starts_humongous(obj_top, fill_size);
216 }
217
218 void HeapRegion::set_continues_humongous(HeapRegion* first_hr) {
219 assert(!is_humongous(), "sanity / pre-condition");
220 assert(top() == bottom(), "should be empty");
221 assert(first_hr->is_starts_humongous(), "pre-condition");
222
223 report_region_type_change(G1HeapRegionTraceType::ContinuesHumongous);
224 _type.set_continues_humongous();
225 _humongous_start_region = first_hr;
226
227 _bot_part.set_object_can_span(true);
228 }
229
230 void HeapRegion::clear_humongous() {
231 assert(is_humongous(), "pre-condition");
232
233 assert(capacity() == HeapRegion::GrainBytes, "pre-condition");
234 _humongous_start_region = NULL;
235
236 _bot_part.set_object_can_span(false);
237 }
238
239 HeapRegion::HeapRegion(uint hrm_index,
240 G1BlockOffsetTable* bot,
241 MemRegion mr) :
242 _bottom(mr.start()),
243 _end(mr.end()),
244 _top(NULL),
245 _compaction_top(NULL),
246 _bot_part(bot, this),
247 _par_alloc_lock(Mutex::leaf, "HeapRegion par alloc lock", true),
248 _pre_dummy_top(NULL),
249 _rem_set(NULL),
250 _hrm_index(hrm_index),
251 _type(),
252 _humongous_start_region(NULL),
253 _evacuation_failed(false),
254 _index_in_opt_cset(InvalidCSetIndex),
255 _next(NULL), _prev(NULL),
256 #ifdef ASSERT
257 _containing_set(NULL),
258 #endif
259 _prev_top_at_mark_start(NULL), _next_top_at_mark_start(NULL),
260 _prev_marked_bytes(0), _next_marked_bytes(0),
261 _young_index_in_cset(-1),
262 _surv_rate_group(NULL), _age_index(G1SurvRateGroup::InvalidAgeIndex), _gc_efficiency(0.0),
263 _node_index(G1NUMA::UnknownNodeIndex)
264 {
265 assert(Universe::on_page_boundary(mr.start()) && Universe::on_page_boundary(mr.end()),
266 "invalid space boundaries");
267
268 _rem_set = new HeapRegionRemSet(bot, this);
269 initialize();
270 }
271
272 void HeapRegion::initialize(bool clear_space, bool mangle_space) {
273 assert(_rem_set->is_empty(), "Remembered set must be empty");
274
275 if (clear_space) {
276 clear(mangle_space);
277 }
278
279 set_top(bottom());
280 set_compaction_top(bottom());
281 reset_bot();
282
283 hr_clear(false /*clear_space*/);
284 }
285
286 void HeapRegion::report_region_type_change(G1HeapRegionTraceType::Type to) {
287 HeapRegionTracer::send_region_type_change(_hrm_index,
288 get_trace_type(),
289 to,
290 (uintptr_t)bottom(),
291 used());
292 }
293
294 void HeapRegion::note_self_forwarding_removal_start(bool during_initial_mark,
295 bool during_conc_mark) {
296 // We always recreate the prev marking info and we'll explicitly
297 // mark all objects we find to be self-forwarded on the prev
298 // bitmap. So all objects need to be below PTAMS.
299 _prev_marked_bytes = 0;
300
301 if (during_initial_mark) {
302 // During initial-mark, we'll also explicitly mark all objects
303 // we find to be self-forwarded on the next bitmap. So all
304 // objects need to be below NTAMS.
305 _next_top_at_mark_start = top();
306 _next_marked_bytes = 0;
307 } else if (during_conc_mark) {
308 // During concurrent mark, all objects in the CSet (including
309 // the ones we find to be self-forwarded) are implicitly live.
310 // So all objects need to be above NTAMS.
311 _next_top_at_mark_start = bottom();
312 _next_marked_bytes = 0;
313 }
314 }
315
316 void HeapRegion::note_self_forwarding_removal_end(size_t marked_bytes) {
317 assert(marked_bytes <= used(),
318 "marked: " SIZE_FORMAT " used: " SIZE_FORMAT, marked_bytes, used());
319 _prev_top_at_mark_start = top();
320 _prev_marked_bytes = marked_bytes;
321 }
322
323 // Code roots support
324
325 void HeapRegion::add_strong_code_root(nmethod* nm) {
326 HeapRegionRemSet* hrrs = rem_set();
327 hrrs->add_strong_code_root(nm);
328 }
329
330 void HeapRegion::add_strong_code_root_locked(nmethod* nm) {
331 assert_locked_or_safepoint(CodeCache_lock);
332 HeapRegionRemSet* hrrs = rem_set();
333 hrrs->add_strong_code_root_locked(nm);
334 }
335
336 void HeapRegion::remove_strong_code_root(nmethod* nm) {
337 HeapRegionRemSet* hrrs = rem_set();
338 hrrs->remove_strong_code_root(nm);
339 }
340
341 void HeapRegion::strong_code_roots_do(CodeBlobClosure* blk) const {
342 HeapRegionRemSet* hrrs = rem_set();
343 hrrs->strong_code_roots_do(blk);
344 }
345
346 class VerifyStrongCodeRootOopClosure: public OopClosure {
347 const HeapRegion* _hr;
348 bool _failures;
349 bool _has_oops_in_region;
350
351 template <class T> void do_oop_work(T* p) {
352 T heap_oop = RawAccess<>::oop_load(p);
353 if (!CompressedOops::is_null(heap_oop)) {
354 oop obj = CompressedOops::decode_not_null(heap_oop);
355
356 // Note: not all the oops embedded in the nmethod are in the
357 // current region. We only look at those which are.
358 if (_hr->is_in(obj)) {
359 // Object is in the region. Check that its less than top
360 if (_hr->top() <= (HeapWord*)obj) {
361 // Object is above top
362 log_error(gc, verify)("Object " PTR_FORMAT " in region " HR_FORMAT " is above top ",
363 p2i(obj), HR_FORMAT_PARAMS(_hr));
364 _failures = true;
365 return;
366 }
367 // Nmethod has at least one oop in the current region
368 _has_oops_in_region = true;
369 }
370 }
371 }
372
373 public:
374 VerifyStrongCodeRootOopClosure(const HeapRegion* hr):
375 _hr(hr), _failures(false), _has_oops_in_region(false) {}
376
377 void do_oop(narrowOop* p) { do_oop_work(p); }
378 void do_oop(oop* p) { do_oop_work(p); }
379
380 bool failures() { return _failures; }
381 bool has_oops_in_region() { return _has_oops_in_region; }
382 };
383
384 class VerifyStrongCodeRootCodeBlobClosure: public CodeBlobClosure {
385 const HeapRegion* _hr;
386 bool _failures;
387 public:
388 VerifyStrongCodeRootCodeBlobClosure(const HeapRegion* hr) :
389 _hr(hr), _failures(false) {}
390
391 void do_code_blob(CodeBlob* cb) {
392 nmethod* nm = (cb == NULL) ? NULL : cb->as_compiled_method()->as_nmethod_or_null();
393 if (nm != NULL) {
394 // Verify that the nemthod is live
395 if (!nm->is_alive()) {
396 log_error(gc, verify)("region [" PTR_FORMAT "," PTR_FORMAT "] has dead nmethod " PTR_FORMAT " in its strong code roots",
397 p2i(_hr->bottom()), p2i(_hr->end()), p2i(nm));
398 _failures = true;
399 } else {
400 VerifyStrongCodeRootOopClosure oop_cl(_hr);
401 nm->oops_do(&oop_cl);
402 if (!oop_cl.has_oops_in_region()) {
403 log_error(gc, verify)("region [" PTR_FORMAT "," PTR_FORMAT "] has nmethod " PTR_FORMAT " in its strong code roots with no pointers into region",
404 p2i(_hr->bottom()), p2i(_hr->end()), p2i(nm));
405 _failures = true;
406 } else if (oop_cl.failures()) {
407 log_error(gc, verify)("region [" PTR_FORMAT "," PTR_FORMAT "] has other failures for nmethod " PTR_FORMAT,
408 p2i(_hr->bottom()), p2i(_hr->end()), p2i(nm));
409 _failures = true;
410 }
411 }
412 }
413 }
414
415 bool failures() { return _failures; }
416 };
417
418 void HeapRegion::verify_strong_code_roots(VerifyOption vo, bool* failures) const {
419 if (!G1VerifyHeapRegionCodeRoots) {
420 // We're not verifying code roots.
421 return;
422 }
423 if (vo == VerifyOption_G1UseFullMarking) {
424 // Marking verification during a full GC is performed after class
425 // unloading, code cache unloading, etc so the strong code roots
426 // attached to each heap region are in an inconsistent state. They won't
427 // be consistent until the strong code roots are rebuilt after the
428 // actual GC. Skip verifying the strong code roots in this particular
429 // time.
430 assert(VerifyDuringGC, "only way to get here");
431 return;
432 }
433
434 HeapRegionRemSet* hrrs = rem_set();
435 size_t strong_code_roots_length = hrrs->strong_code_roots_list_length();
436
437 // if this region is empty then there should be no entries
438 // on its strong code root list
439 if (is_empty()) {
440 if (strong_code_roots_length > 0) {
441 log_error(gc, verify)("region " HR_FORMAT " is empty but has " SIZE_FORMAT " code root entries",
442 HR_FORMAT_PARAMS(this), strong_code_roots_length);
443 *failures = true;
444 }
445 return;
446 }
447
448 if (is_continues_humongous()) {
449 if (strong_code_roots_length > 0) {
450 log_error(gc, verify)("region " HR_FORMAT " is a continuation of a humongous region but has " SIZE_FORMAT " code root entries",
451 HR_FORMAT_PARAMS(this), strong_code_roots_length);
452 *failures = true;
453 }
454 return;
455 }
456
457 VerifyStrongCodeRootCodeBlobClosure cb_cl(this);
458 strong_code_roots_do(&cb_cl);
459
460 if (cb_cl.failures()) {
461 *failures = true;
462 }
463 }
464
465 void HeapRegion::print() const { print_on(tty); }
466
467 void HeapRegion::print_on(outputStream* st) const {
468 st->print("|%4u", this->_hrm_index);
469 st->print("|" PTR_FORMAT ", " PTR_FORMAT ", " PTR_FORMAT,
470 p2i(bottom()), p2i(top()), p2i(end()));
471 st->print("|%3d%%", (int) ((double) used() * 100 / capacity()));
472 st->print("|%2s", get_short_type_str());
473 if (in_collection_set()) {
474 st->print("|CS");
475 } else {
476 st->print("| ");
477 }
478 st->print("|TAMS " PTR_FORMAT ", " PTR_FORMAT "| %s ",
479 p2i(prev_top_at_mark_start()), p2i(next_top_at_mark_start()), rem_set()->get_state_str());
480 if (UseNUMA) {
481 G1NUMA* numa = G1NUMA::numa();
482 if (node_index() < numa->num_active_nodes()) {
483 st->print("|%d", numa->numa_id(node_index()));
484 } else {
485 st->print("|-");
486 }
487 }
488 st->print_cr("");
489 }
490
491 class G1VerificationClosure : public BasicOopIterateClosure {
492 protected:
493 G1CollectedHeap* _g1h;
494 G1CardTable *_ct;
495 oop _containing_obj;
496 bool _failures;
497 int _n_failures;
498 VerifyOption _vo;
499 public:
500 // _vo == UsePrevMarking -> use "prev" marking information,
501 // _vo == UseNextMarking -> use "next" marking information,
502 // _vo == UseFullMarking -> use "next" marking bitmap but no TAMS.
503 G1VerificationClosure(G1CollectedHeap* g1h, VerifyOption vo) :
504 _g1h(g1h), _ct(g1h->card_table()),
505 _containing_obj(NULL), _failures(false), _n_failures(0), _vo(vo) {
506 }
507
508 void set_containing_obj(oop obj) {
509 _containing_obj = obj;
510 }
511
512 bool failures() { return _failures; }
513 int n_failures() { return _n_failures; }
514
515 void print_object(outputStream* out, oop obj) {
516 #ifdef PRODUCT
517 Klass* k = obj->klass();
518 const char* class_name = k->external_name();
519 out->print_cr("class name %s", class_name);
520 #else // PRODUCT
521 obj->print_on(out);
522 #endif // PRODUCT
523 }
524 };
525
526 class VerifyLiveClosure : public G1VerificationClosure {
527 public:
528 VerifyLiveClosure(G1CollectedHeap* g1h, VerifyOption vo) : G1VerificationClosure(g1h, vo) {}
529 virtual void do_oop(narrowOop* p) { do_oop_work(p); }
530 virtual void do_oop(oop* p) { do_oop_work(p); }
531
532 template <class T>
533 void do_oop_work(T* p) {
534 assert(_containing_obj != NULL, "Precondition");
535 assert(!_g1h->is_obj_dead_cond(_containing_obj, _vo),
536 "Precondition");
537 verify_liveness(p);
538 }
539
540 template <class T>
541 void verify_liveness(T* p) {
542 T heap_oop = RawAccess<>::oop_load(p);
543 Log(gc, verify) log;
544 if (!CompressedOops::is_null(heap_oop)) {
545 oop obj = CompressedOops::decode_not_null(heap_oop);
546 bool failed = false;
547 if (!_g1h->is_in(obj) || _g1h->is_obj_dead_cond(obj, _vo)) {
548 MutexLocker x(ParGCRareEvent_lock,
549 Mutex::_no_safepoint_check_flag);
550
551 if (!_failures) {
552 log.error("----------");
553 }
554 ResourceMark rm;
555 if (!_g1h->is_in(obj)) {
556 HeapRegion* from = _g1h->heap_region_containing((HeapWord*)p);
557 log.error("Field " PTR_FORMAT " of live obj " PTR_FORMAT " in region " HR_FORMAT,
558 p2i(p), p2i(_containing_obj), HR_FORMAT_PARAMS(from));
559 LogStream ls(log.error());
560 print_object(&ls, _containing_obj);
561 HeapRegion* const to = _g1h->heap_region_containing(obj);
562 log.error("points to obj " PTR_FORMAT " in region " HR_FORMAT " remset %s",
563 p2i(obj), HR_FORMAT_PARAMS(to), to->rem_set()->get_state_str());
564 } else {
565 HeapRegion* from = _g1h->heap_region_containing((HeapWord*)p);
566 HeapRegion* to = _g1h->heap_region_containing((HeapWord*)obj);
567 log.error("Field " PTR_FORMAT " of live obj " PTR_FORMAT " in region " HR_FORMAT,
568 p2i(p), p2i(_containing_obj), HR_FORMAT_PARAMS(from));
569 LogStream ls(log.error());
570 print_object(&ls, _containing_obj);
571 log.error("points to dead obj " PTR_FORMAT " in region " HR_FORMAT,
572 p2i(obj), HR_FORMAT_PARAMS(to));
573 print_object(&ls, obj);
574 }
575 log.error("----------");
576 _failures = true;
577 failed = true;
578 _n_failures++;
579 }
580 }
581 }
582 };
583
584 class VerifyRemSetClosure : public G1VerificationClosure {
585 public:
586 VerifyRemSetClosure(G1CollectedHeap* g1h, VerifyOption vo) : G1VerificationClosure(g1h, vo) {}
587 virtual void do_oop(narrowOop* p) { do_oop_work(p); }
588 virtual void do_oop(oop* p) { do_oop_work(p); }
589
590 template <class T>
591 void do_oop_work(T* p) {
592 assert(_containing_obj != NULL, "Precondition");
593 assert(!_g1h->is_obj_dead_cond(_containing_obj, _vo),
594 "Precondition");
595 verify_remembered_set(p);
596 }
597
598 template <class T>
599 void verify_remembered_set(T* p) {
600 T heap_oop = RawAccess<>::oop_load(p);
601 Log(gc, verify) log;
602 if (!CompressedOops::is_null(heap_oop)) {
603 oop obj = CompressedOops::decode_not_null(heap_oop);
604 HeapRegion* from = _g1h->heap_region_containing((HeapWord*)p);
605 HeapRegion* to = _g1h->heap_region_containing(obj);
606 if (from != NULL && to != NULL &&
607 from != to &&
608 !to->is_pinned() &&
609 to->rem_set()->is_complete()) {
610 jbyte cv_obj = *_ct->byte_for_const(_containing_obj);
611 jbyte cv_field = *_ct->byte_for_const(p);
612 const jbyte dirty = G1CardTable::dirty_card_val();
613
614 bool is_bad = !(from->is_young()
615 || to->rem_set()->contains_reference(p)
616 || (_containing_obj->is_objArray() ?
617 cv_field == dirty :
618 cv_obj == dirty || cv_field == dirty));
619 if (is_bad) {
620 MutexLocker x(ParGCRareEvent_lock,
621 Mutex::_no_safepoint_check_flag);
622
623 if (!_failures) {
624 log.error("----------");
625 }
626 log.error("Missing rem set entry:");
627 log.error("Field " PTR_FORMAT " of obj " PTR_FORMAT " in region " HR_FORMAT,
628 p2i(p), p2i(_containing_obj), HR_FORMAT_PARAMS(from));
629 ResourceMark rm;
630 LogStream ls(log.error());
631 _containing_obj->print_on(&ls);
632 log.error("points to obj " PTR_FORMAT " in region " HR_FORMAT " remset %s",
633 p2i(obj), HR_FORMAT_PARAMS(to), to->rem_set()->get_state_str());
634 if (oopDesc::is_oop(obj)) {
635 obj->print_on(&ls);
636 }
637 log.error("Obj head CTE = %d, field CTE = %d.", cv_obj, cv_field);
638 log.error("----------");
639 _failures = true;
640 _n_failures++;
641 }
642 }
643 }
644 }
645 };
646
647 // Closure that applies the given two closures in sequence.
648 class G1Mux2Closure : public BasicOopIterateClosure {
649 OopClosure* _c1;
650 OopClosure* _c2;
651 public:
652 G1Mux2Closure(OopClosure *c1, OopClosure *c2) { _c1 = c1; _c2 = c2; }
653 template <class T> inline void do_oop_work(T* p) {
654 // Apply first closure; then apply the second.
655 _c1->do_oop(p);
656 _c2->do_oop(p);
657 }
658 virtual inline void do_oop(oop* p) { do_oop_work(p); }
659 virtual inline void do_oop(narrowOop* p) { do_oop_work(p); }
660 };
661
662 void HeapRegion::verify(VerifyOption vo,
663 bool* failures) const {
664 G1CollectedHeap* g1h = G1CollectedHeap::heap();
665 *failures = false;
666 HeapWord* p = bottom();
667 HeapWord* prev_p = NULL;
668 VerifyLiveClosure vl_cl(g1h, vo);
669 VerifyRemSetClosure vr_cl(g1h, vo);
670 bool is_region_humongous = is_humongous();
671 size_t object_num = 0;
672 while (p < top()) {
673 oop obj = oop(p);
674 size_t obj_size = block_size(p);
675 object_num += 1;
676
677 if (!g1h->is_obj_dead_cond(obj, this, vo)) {
678 if (oopDesc::is_oop(obj)) {
679 Klass* klass = obj->klass();
680 bool is_metaspace_object = Metaspace::contains(klass);
681 if (!is_metaspace_object) {
682 log_error(gc, verify)("klass " PTR_FORMAT " of object " PTR_FORMAT " "
683 "not metadata", p2i(klass), p2i(obj));
684 *failures = true;
685 return;
686 } else if (!klass->is_klass()) {
687 log_error(gc, verify)("klass " PTR_FORMAT " of object " PTR_FORMAT " "
688 "not a klass", p2i(klass), p2i(obj));
689 *failures = true;
690 return;
691 } else {
692 vl_cl.set_containing_obj(obj);
693 if (!g1h->collector_state()->in_full_gc() || G1VerifyRSetsDuringFullGC) {
694 // verify liveness and rem_set
695 vr_cl.set_containing_obj(obj);
696 G1Mux2Closure mux(&vl_cl, &vr_cl);
697 obj->oop_iterate(&mux);
698
699 if (vr_cl.failures()) {
700 *failures = true;
701 }
702 if (G1MaxVerifyFailures >= 0 &&
703 vr_cl.n_failures() >= G1MaxVerifyFailures) {
704 return;
705 }
706 } else {
707 // verify only liveness
708 obj->oop_iterate(&vl_cl);
709 }
710 if (vl_cl.failures()) {
711 *failures = true;
712 }
713 if (G1MaxVerifyFailures >= 0 &&
714 vl_cl.n_failures() >= G1MaxVerifyFailures) {
715 return;
716 }
717 }
718 } else {
719 log_error(gc, verify)(PTR_FORMAT " not an oop", p2i(obj));
720 *failures = true;
721 return;
722 }
723 }
724 prev_p = p;
725 p += obj_size;
726 }
727
728 if (!is_young() && !is_empty()) {
729 _bot_part.verify();
730 }
731
732 if (is_region_humongous) {
733 oop obj = oop(this->humongous_start_region()->bottom());
734 if ((HeapWord*)obj > bottom() || (HeapWord*)obj + obj->size() < bottom()) {
735 log_error(gc, verify)("this humongous region is not part of its' humongous object " PTR_FORMAT, p2i(obj));
736 *failures = true;
737 return;
738 }
739 }
740
741 if (!is_region_humongous && p != top()) {
742 log_error(gc, verify)("end of last object " PTR_FORMAT " "
743 "does not match top " PTR_FORMAT, p2i(p), p2i(top()));
744 *failures = true;
745 return;
746 }
747
748 HeapWord* the_end = end();
749 // Do some extra BOT consistency checking for addresses in the
750 // range [top, end). BOT look-ups in this range should yield
751 // top. No point in doing that if top == end (there's nothing there).
752 if (p < the_end) {
753 // Look up top
754 HeapWord* addr_1 = p;
755 HeapWord* b_start_1 = block_start_const(addr_1);
756 if (b_start_1 != p) {
757 log_error(gc, verify)("BOT look up for top: " PTR_FORMAT " "
758 " yielded " PTR_FORMAT ", expecting " PTR_FORMAT,
759 p2i(addr_1), p2i(b_start_1), p2i(p));
760 *failures = true;
761 return;
762 }
763
764 // Look up top + 1
765 HeapWord* addr_2 = p + 1;
766 if (addr_2 < the_end) {
767 HeapWord* b_start_2 = block_start_const(addr_2);
768 if (b_start_2 != p) {
769 log_error(gc, verify)("BOT look up for top + 1: " PTR_FORMAT " "
770 " yielded " PTR_FORMAT ", expecting " PTR_FORMAT,
771 p2i(addr_2), p2i(b_start_2), p2i(p));
772 *failures = true;
773 return;
774 }
775 }
776
777 // Look up an address between top and end
778 size_t diff = pointer_delta(the_end, p) / 2;
779 HeapWord* addr_3 = p + diff;
780 if (addr_3 < the_end) {
781 HeapWord* b_start_3 = block_start_const(addr_3);
782 if (b_start_3 != p) {
783 log_error(gc, verify)("BOT look up for top + diff: " PTR_FORMAT " "
784 " yielded " PTR_FORMAT ", expecting " PTR_FORMAT,
785 p2i(addr_3), p2i(b_start_3), p2i(p));
786 *failures = true;
787 return;
788 }
789 }
790
791 // Look up end - 1
792 HeapWord* addr_4 = the_end - 1;
793 HeapWord* b_start_4 = block_start_const(addr_4);
794 if (b_start_4 != p) {
795 log_error(gc, verify)("BOT look up for end - 1: " PTR_FORMAT " "
796 " yielded " PTR_FORMAT ", expecting " PTR_FORMAT,
797 p2i(addr_4), p2i(b_start_4), p2i(p));
798 *failures = true;
799 return;
800 }
801 }
802
803 verify_strong_code_roots(vo, failures);
804 }
805
806 void HeapRegion::verify() const {
807 bool dummy = false;
808 verify(VerifyOption_G1UsePrevMarking, /* failures */ &dummy);
809 }
810
811 void HeapRegion::verify_rem_set(VerifyOption vo, bool* failures) const {
812 G1CollectedHeap* g1h = G1CollectedHeap::heap();
813 *failures = false;
814 HeapWord* p = bottom();
815 HeapWord* prev_p = NULL;
816 VerifyRemSetClosure vr_cl(g1h, vo);
817 while (p < top()) {
818 oop obj = oop(p);
819 size_t obj_size = block_size(p);
820
821 if (!g1h->is_obj_dead_cond(obj, this, vo)) {
822 if (oopDesc::is_oop(obj)) {
823 vr_cl.set_containing_obj(obj);
824 obj->oop_iterate(&vr_cl);
825
826 if (vr_cl.failures()) {
827 *failures = true;
828 }
829 if (G1MaxVerifyFailures >= 0 &&
830 vr_cl.n_failures() >= G1MaxVerifyFailures) {
831 return;
832 }
833 } else {
834 log_error(gc, verify)(PTR_FORMAT " not an oop", p2i(obj));
835 *failures = true;
836 return;
837 }
838 }
839
840 prev_p = p;
841 p += obj_size;
842 }
843 }
844
845 void HeapRegion::verify_rem_set() const {
846 bool failures = false;
847 verify_rem_set(VerifyOption_G1UsePrevMarking, &failures);
848 guarantee(!failures, "HeapRegion RemSet verification failed");
849 }
850
851 void HeapRegion::clear(bool mangle_space) {
852 set_top(bottom());
853 set_compaction_top(bottom());
854
855 if (ZapUnusedHeapArea && mangle_space) {
856 mangle_unused_area();
857 }
858 reset_bot();
859 }
860
861 #ifndef PRODUCT
862 void HeapRegion::mangle_unused_area() {
863 SpaceMangler::mangle_region(MemRegion(top(), end()));
864 }
865 #endif
866
867 HeapWord* HeapRegion::initialize_threshold() {
868 return _bot_part.initialize_threshold();
869 }
870
871 HeapWord* HeapRegion::cross_threshold(HeapWord* start, HeapWord* end) {
872 _bot_part.alloc_block(start, end);
873 return _bot_part.threshold();
874 }
875
876 void HeapRegion::object_iterate(ObjectClosure* blk) {
877 HeapWord* p = bottom();
878 while (p < top()) {
879 if (block_is_obj(p)) {
880 blk->do_object(oop(p));
881 }
882 p += block_size(p);
883 }
884 }