1 /* 2 * Copyright (c) 2015, 2019, Oracle and/or its affiliates. All rights reserved. 3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 4 * 5 * This code is free software; you can redistribute it and/or modify it 6 * under the terms of the GNU General Public License version 2 only, as 7 * published by the Free Software Foundation. 8 * 9 * This code is distributed in the hope that it will be useful, but WITHOUT 10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 12 * version 2 for more details (a copy is included in the LICENSE file that 13 * accompanied this code). 14 * 15 * You should have received a copy of the GNU General Public License version 16 * 2 along with this work; if not, write to the Free Software Foundation, 17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. 18 * 19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 20 * or visit www.oracle.com if you need additional information or have any 21 * questions. 22 */ 23 24 #include "precompiled.hpp" 25 #include "classfile/classLoaderDataGraph.hpp" 26 #include "gc/z/zBarrier.inline.hpp" 27 #include "gc/z/zMark.inline.hpp" 28 #include "gc/z/zMarkCache.inline.hpp" 29 #include "gc/z/zMarkStack.inline.hpp" 30 #include "gc/z/zMarkTerminate.inline.hpp" 31 #include "gc/z/zOopClosures.inline.hpp" 32 #include "gc/z/zPage.hpp" 33 #include "gc/z/zPageTable.inline.hpp" 34 #include "gc/z/zRootsIterator.hpp" 35 #include "gc/z/zStat.hpp" 36 #include "gc/z/zTask.hpp" 37 #include "gc/z/zThread.inline.hpp" 38 #include "gc/z/zThreadLocalAllocBuffer.hpp" 39 #include "gc/z/zUtils.inline.hpp" 40 #include "gc/z/zWorkers.inline.hpp" 41 #include "logging/log.hpp" 42 #include "memory/iterator.inline.hpp" 43 #include "oops/objArrayOop.inline.hpp" 44 #include "oops/oop.inline.hpp" 45 #include "runtime/atomic.hpp" 46 #include "runtime/handshake.hpp" 47 #include "runtime/prefetch.inline.hpp" 48 #include "runtime/thread.hpp" 49 #include "utilities/align.hpp" 50 #include "utilities/globalDefinitions.hpp" 51 #include "utilities/ticks.hpp" 52 53 static const ZStatSubPhase ZSubPhaseConcurrentMark("Concurrent Mark"); 54 static const ZStatSubPhase ZSubPhaseConcurrentMarkTryFlush("Concurrent Mark Try Flush"); 55 static const ZStatSubPhase ZSubPhaseConcurrentMarkIdle("Concurrent Mark Idle"); 56 static const ZStatSubPhase ZSubPhaseConcurrentMarkTryTerminate("Concurrent Mark Try Terminate"); 57 static const ZStatSubPhase ZSubPhaseMarkTryComplete("Pause Mark Try Complete"); 58 59 ZMark::ZMark(ZWorkers* workers, ZPageTable* page_table) : 60 _workers(workers), 61 _page_table(page_table), 62 _allocator(), 63 _stripes(), 64 _terminate(), 65 _work_terminateflush(true), 66 _work_nproactiveflush(0), 67 _work_nterminateflush(0), 68 _nproactiveflush(0), 69 _nterminateflush(0), 70 _ntrycomplete(0), 71 _ncontinue(0), 72 _nworkers(0) {} 73 74 bool ZMark::is_initialized() const { 75 return _allocator.is_initialized(); 76 } 77 78 size_t ZMark::calculate_nstripes(uint nworkers) const { 79 // Calculate the number of stripes from the number of workers we use, 80 // where the number of stripes must be a power of two and we want to 81 // have at least one worker per stripe. 82 const size_t nstripes = ZUtils::round_down_power_of_2(nworkers); 83 return MIN2(nstripes, ZMarkStripesMax); 84 } 85 86 void ZMark::prepare_mark() { 87 // Increment global sequence number to invalidate 88 // marking information for all pages. 89 ZGlobalSeqNum++; 90 91 // Reset flush/continue counters 92 _nproactiveflush = 0; 93 _nterminateflush = 0; 94 _ntrycomplete = 0; 95 _ncontinue = 0; 96 97 // Set number of workers to use 98 _nworkers = _workers->nconcurrent(); 99 100 // Set number of mark stripes to use, based on number 101 // of workers we will use in the concurrent mark phase. 102 const size_t nstripes = calculate_nstripes(_nworkers); 103 _stripes.set_nstripes(nstripes); 104 105 // Update statistics 106 ZStatMark::set_at_mark_start(nstripes); 107 108 // Print worker/stripe distribution 109 LogTarget(Debug, gc, marking) log; 110 if (log.is_enabled()) { 111 log.print("Mark Worker/Stripe Distribution"); 112 for (uint worker_id = 0; worker_id < _nworkers; worker_id++) { 113 const ZMarkStripe* const stripe = _stripes.stripe_for_worker(_nworkers, worker_id); 114 const size_t stripe_id = _stripes.stripe_id(stripe); 115 log.print(" Worker %u(%u) -> Stripe " SIZE_FORMAT "(" SIZE_FORMAT ")", 116 worker_id, _nworkers, stripe_id, nstripes); 117 } 118 } 119 } 120 121 class ZMarkRootsIteratorClosure : public ZRootsIteratorClosure { 122 public: 123 ZMarkRootsIteratorClosure() { 124 ZThreadLocalAllocBuffer::reset_statistics(); 125 } 126 127 ~ZMarkRootsIteratorClosure() { 128 ZThreadLocalAllocBuffer::publish_statistics(); 129 } 130 131 virtual void do_thread(Thread* thread) { 132 // Update thread local address bad mask 133 ZThreadLocalData::set_address_bad_mask(thread, ZAddressBadMask); 134 135 // Mark invisible root 136 ZThreadLocalData::do_invisible_root(thread, ZBarrier::mark_barrier_on_invisible_root_oop_field); 137 138 // Retire TLAB 139 ZThreadLocalAllocBuffer::retire(thread); 140 } 141 142 virtual void do_oop(oop* p) { 143 ZBarrier::mark_barrier_on_root_oop_field(p); 144 } 145 146 virtual void do_oop(narrowOop* p) { 147 ShouldNotReachHere(); 148 } 149 }; 150 151 class ZMarkRootsTask : public ZTask { 152 private: 153 ZMark* const _mark; 154 ZRootsIterator _roots; 155 ZMarkRootsIteratorClosure _cl; 156 157 public: 158 ZMarkRootsTask(ZMark* mark) : 159 ZTask("ZMarkRootsTask"), 160 _mark(mark), 161 _roots(false /* visit_jvmti_weak_export */) {} 162 163 virtual void work() { 164 _roots.oops_do(&_cl); 165 166 // Flush and free worker stacks. Needed here since 167 // the set of workers executing during root scanning 168 // can be different from the set of workers executing 169 // during mark. 170 _mark->flush_and_free(); 171 } 172 }; 173 174 void ZMark::start() { 175 // Verification 176 if (ZVerifyMarking) { 177 verify_all_stacks_empty(); 178 } 179 180 // Prepare for concurrent mark 181 prepare_mark(); 182 183 // Mark roots 184 ZMarkRootsTask task(this); 185 _workers->run_parallel(&task); 186 } 187 188 void ZMark::prepare_work() { 189 assert(_nworkers == _workers->nconcurrent(), "Invalid number of workers"); 190 191 // Set number of active workers 192 _terminate.reset(_nworkers); 193 194 // Reset flush counters 195 _work_nproactiveflush = _work_nterminateflush = 0; 196 _work_terminateflush = true; 197 } 198 199 void ZMark::finish_work() { 200 // Accumulate proactive/terminate flush counters 201 _nproactiveflush += _work_nproactiveflush; 202 _nterminateflush += _work_nterminateflush; 203 } 204 205 bool ZMark::is_array(uintptr_t addr) const { 206 return ZOop::from_address(addr)->is_objArray(); 207 } 208 209 void ZMark::push_partial_array(uintptr_t addr, size_t size, bool finalizable) { 210 assert(is_aligned(addr, ZMarkPartialArrayMinSize), "Address misaligned"); 211 ZMarkThreadLocalStacks* const stacks = ZThreadLocalData::stacks(Thread::current()); 212 ZMarkStripe* const stripe = _stripes.stripe_for_addr(addr); 213 const uintptr_t offset = ZAddress::offset(addr) >> ZMarkPartialArrayMinSizeShift; 214 const uintptr_t length = size / oopSize; 215 const ZMarkStackEntry entry(offset, length, finalizable); 216 217 log_develop_trace(gc, marking)("Array push partial: " PTR_FORMAT " (" SIZE_FORMAT "), stripe: " SIZE_FORMAT, 218 addr, size, _stripes.stripe_id(stripe)); 219 220 stacks->push(&_allocator, &_stripes, stripe, entry, false /* publish */); 221 } 222 223 void ZMark::follow_small_array(uintptr_t addr, size_t size, bool finalizable) { 224 assert(size <= ZMarkPartialArrayMinSize, "Too large, should be split"); 225 const size_t length = size / oopSize; 226 227 log_develop_trace(gc, marking)("Array follow small: " PTR_FORMAT " (" SIZE_FORMAT ")", addr, size); 228 229 ZBarrier::mark_barrier_on_oop_array((oop*)addr, length, finalizable); 230 } 231 232 void ZMark::follow_large_array(uintptr_t addr, size_t size, bool finalizable) { 233 assert(size <= (size_t)arrayOopDesc::max_array_length(T_OBJECT) * oopSize, "Too large"); 234 assert(size > ZMarkPartialArrayMinSize, "Too small, should not be split"); 235 const uintptr_t start = addr; 236 const uintptr_t end = start + size; 237 238 // Calculate the aligned middle start/end/size, where the middle start 239 // should always be greater than the start (hence the +1 below) to make 240 // sure we always do some follow work, not just split the array into pieces. 241 const uintptr_t middle_start = align_up(start + 1, ZMarkPartialArrayMinSize); 242 const size_t middle_size = align_down(end - middle_start, ZMarkPartialArrayMinSize); 243 const uintptr_t middle_end = middle_start + middle_size; 244 245 log_develop_trace(gc, marking)("Array follow large: " PTR_FORMAT "-" PTR_FORMAT" (" SIZE_FORMAT "), " 246 "middle: " PTR_FORMAT "-" PTR_FORMAT " (" SIZE_FORMAT ")", 247 start, end, size, middle_start, middle_end, middle_size); 248 249 // Push unaligned trailing part 250 if (end > middle_end) { 251 const uintptr_t trailing_addr = middle_end; 252 const size_t trailing_size = end - middle_end; 253 push_partial_array(trailing_addr, trailing_size, finalizable); 254 } 255 256 // Push aligned middle part(s) 257 uintptr_t partial_addr = middle_end; 258 while (partial_addr > middle_start) { 259 const size_t parts = 2; 260 const size_t partial_size = align_up((partial_addr - middle_start) / parts, ZMarkPartialArrayMinSize); 261 partial_addr -= partial_size; 262 push_partial_array(partial_addr, partial_size, finalizable); 263 } 264 265 // Follow leading part 266 assert(start < middle_start, "Miscalculated middle start"); 267 const uintptr_t leading_addr = start; 268 const size_t leading_size = middle_start - start; 269 follow_small_array(leading_addr, leading_size, finalizable); 270 } 271 272 void ZMark::follow_array(uintptr_t addr, size_t size, bool finalizable) { 273 if (size <= ZMarkPartialArrayMinSize) { 274 follow_small_array(addr, size, finalizable); 275 } else { 276 follow_large_array(addr, size, finalizable); 277 } 278 } 279 280 void ZMark::follow_partial_array(ZMarkStackEntry entry, bool finalizable) { 281 const uintptr_t addr = ZAddress::good(entry.partial_array_offset() << ZMarkPartialArrayMinSizeShift); 282 const size_t size = entry.partial_array_length() * oopSize; 283 284 follow_array(addr, size, finalizable); 285 } 286 287 void ZMark::follow_array_object(objArrayOop obj, bool finalizable) { 288 if (finalizable) { 289 ZMarkBarrierOopClosure<true /* finalizable */> cl; 290 cl.do_klass(obj->klass()); 291 } else { 292 ZMarkBarrierOopClosure<false /* finalizable */> cl; 293 cl.do_klass(obj->klass()); 294 } 295 296 const uintptr_t addr = (uintptr_t)obj->base(); 297 const size_t size = (size_t)obj->length() * oopSize; 298 299 follow_array(addr, size, finalizable); 300 } 301 302 void ZMark::follow_object(oop obj, bool finalizable) { 303 if (finalizable) { 304 ZMarkBarrierOopClosure<true /* finalizable */> cl; 305 obj->oop_iterate(&cl); 306 } else { 307 ZMarkBarrierOopClosure<false /* finalizable */> cl; 308 obj->oop_iterate(&cl); 309 } 310 } 311 312 bool ZMark::try_mark_object(ZMarkCache* cache, uintptr_t addr, bool finalizable) { 313 ZPage* const page = _page_table->get(addr); 314 if (page->is_allocating()) { 315 // Newly allocated objects are implicitly marked 316 return false; 317 } 318 319 // Try mark object 320 bool inc_live = false; 321 const bool success = page->mark_object(addr, finalizable, inc_live); 322 if (inc_live) { 323 // Update live objects/bytes for page. We use the aligned object 324 // size since that is the actual number of bytes used on the page 325 // and alignment paddings can never be reclaimed. 326 const size_t size = ZUtils::object_size(addr); 327 const size_t aligned_size = align_up(size, page->object_alignment()); 328 cache->inc_live(page, aligned_size); 329 } 330 331 return success; 332 } 333 334 void ZMark::mark_and_follow(ZMarkCache* cache, ZMarkStackEntry entry) { 335 // Decode flags 336 const bool finalizable = entry.finalizable(); 337 const bool partial_array = entry.partial_array(); 338 339 if (partial_array) { 340 follow_partial_array(entry, finalizable); 341 return; 342 } 343 344 // Decode object address and follow flag 345 const uintptr_t addr = entry.object_address(); 346 347 if (!try_mark_object(cache, addr, finalizable)) { 348 // Already marked 349 return; 350 } 351 352 if (is_array(addr)) { 353 // Decode follow flag 354 const bool follow = entry.follow(); 355 356 // The follow flag is currently only relevant for object arrays 357 if (follow) { 358 follow_array_object(objArrayOop(ZOop::from_address(addr)), finalizable); 359 } 360 } else { 361 follow_object(ZOop::from_address(addr), finalizable); 362 } 363 } 364 365 template <typename T> 366 bool ZMark::drain(ZMarkStripe* stripe, ZMarkThreadLocalStacks* stacks, ZMarkCache* cache, T* timeout) { 367 ZMarkStackEntry entry; 368 369 // Drain stripe stacks 370 while (stacks->pop(&_allocator, &_stripes, stripe, entry)) { 371 mark_and_follow(cache, entry); 372 373 // Check timeout 374 if (timeout->has_expired()) { 375 // Timeout 376 return false; 377 } 378 } 379 380 // Success 381 return true; 382 } 383 384 template <typename T> 385 bool ZMark::drain_and_flush(ZMarkStripe* stripe, ZMarkThreadLocalStacks* stacks, ZMarkCache* cache, T* timeout) { 386 const bool success = drain(stripe, stacks, cache, timeout); 387 388 // Flush and publish worker stacks 389 stacks->flush(&_allocator, &_stripes); 390 391 return success; 392 } 393 394 bool ZMark::try_steal(ZMarkStripe* stripe, ZMarkThreadLocalStacks* stacks) { 395 // Try to steal a stack from another stripe 396 for (ZMarkStripe* victim_stripe = _stripes.stripe_next(stripe); 397 victim_stripe != stripe; 398 victim_stripe = _stripes.stripe_next(victim_stripe)) { 399 ZMarkStack* const stack = victim_stripe->steal_stack(); 400 if (stack != NULL) { 401 // Success, install the stolen stack 402 stacks->install(&_stripes, stripe, stack); 403 return true; 404 } 405 } 406 407 // Nothing to steal 408 return false; 409 } 410 411 void ZMark::idle() const { 412 ZStatTimer timer(ZSubPhaseConcurrentMarkIdle); 413 os::naked_short_sleep(1); 414 } 415 416 class ZMarkFlushAndFreeStacksClosure : public ThreadClosure { 417 private: 418 ZMark* const _mark; 419 bool _flushed; 420 421 public: 422 ZMarkFlushAndFreeStacksClosure(ZMark* mark) : 423 _mark(mark), 424 _flushed(false) {} 425 426 void do_thread(Thread* thread) { 427 if (_mark->flush_and_free(thread)) { 428 _flushed = true; 429 } 430 } 431 432 bool flushed() const { 433 return _flushed; 434 } 435 }; 436 437 bool ZMark::flush(bool at_safepoint) { 438 ZMarkFlushAndFreeStacksClosure cl(this); 439 if (at_safepoint) { 440 Threads::threads_do(&cl); 441 } else { 442 Handshake::execute(&cl); 443 } 444 445 // Returns true if more work is available 446 return cl.flushed() || !_stripes.is_empty(); 447 } 448 449 bool ZMark::try_flush(volatile size_t* nflush) { 450 // Only flush if handshakes are enabled 451 if (!ThreadLocalHandshakes) { 452 return false; 453 } 454 455 Atomic::inc(nflush); 456 457 ZStatTimer timer(ZSubPhaseConcurrentMarkTryFlush); 458 return flush(false /* at_safepoint */); 459 } 460 461 bool ZMark::try_proactive_flush() { 462 // Only do proactive flushes from worker 0 463 if (ZThread::worker_id() != 0) { 464 return false; 465 } 466 467 if (Atomic::load(&_work_nproactiveflush) == ZMarkProactiveFlushMax || 468 Atomic::load(&_work_nterminateflush) != 0) { 469 // Limit reached or we're trying to terminate 470 return false; 471 } 472 473 return try_flush(&_work_nproactiveflush); 474 } 475 476 bool ZMark::try_terminate() { 477 ZStatTimer timer(ZSubPhaseConcurrentMarkTryTerminate); 478 479 if (_terminate.enter_stage0()) { 480 // Last thread entered stage 0, flush 481 if (Atomic::load(&_work_terminateflush) && 482 Atomic::load(&_work_nterminateflush) != ZMarkTerminateFlushMax) { 483 // Exit stage 0 to allow other threads to continue marking 484 _terminate.exit_stage0(); 485 486 // Flush before termination 487 if (!try_flush(&_work_nterminateflush)) { 488 // No more work available, skip further flush attempts 489 Atomic::store(&_work_terminateflush, false); 490 } 491 492 // Don't terminate, regardless of whether we successfully 493 // flushed out more work or not. We've already exited 494 // termination stage 0, to allow other threads to continue 495 // marking, so this thread has to return false and also 496 // make another round of attempted marking. 497 return false; 498 } 499 } 500 501 for (;;) { 502 if (_terminate.enter_stage1()) { 503 // Last thread entered stage 1, terminate 504 return true; 505 } 506 507 // Idle to give the other threads 508 // a chance to enter termination. 509 idle(); 510 511 if (!_terminate.try_exit_stage1()) { 512 // All workers in stage 1, terminate 513 return true; 514 } 515 516 if (_terminate.try_exit_stage0()) { 517 // More work available, don't terminate 518 return false; 519 } 520 } 521 } 522 523 class ZMarkNoTimeout : public StackObj { 524 public: 525 bool has_expired() { 526 return false; 527 } 528 }; 529 530 void ZMark::work_without_timeout(ZMarkCache* cache, ZMarkStripe* stripe, ZMarkThreadLocalStacks* stacks) { 531 ZStatTimer timer(ZSubPhaseConcurrentMark); 532 ZMarkNoTimeout no_timeout; 533 534 for (;;) { 535 drain_and_flush(stripe, stacks, cache, &no_timeout); 536 537 if (try_steal(stripe, stacks)) { 538 // Stole work 539 continue; 540 } 541 542 if (try_proactive_flush()) { 543 // Work available 544 continue; 545 } 546 547 if (try_terminate()) { 548 // Terminate 549 break; 550 } 551 } 552 } 553 554 class ZMarkTimeout : public StackObj { 555 private: 556 const Ticks _start; 557 const uint64_t _timeout; 558 const uint64_t _check_interval; 559 uint64_t _check_at; 560 uint64_t _check_count; 561 bool _expired; 562 563 public: 564 ZMarkTimeout(uint64_t timeout_in_millis) : 565 _start(Ticks::now()), 566 _timeout(_start.value() + TimeHelper::millis_to_counter(timeout_in_millis)), 567 _check_interval(200), 568 _check_at(_check_interval), 569 _check_count(0), 570 _expired(false) {} 571 572 ~ZMarkTimeout() { 573 const Tickspan duration = Ticks::now() - _start; 574 log_debug(gc, marking)("Mark With Timeout (%s): %s, " UINT64_FORMAT " oops, %.3fms", 575 ZThread::name(), _expired ? "Expired" : "Completed", 576 _check_count, TimeHelper::counter_to_millis(duration.value())); 577 } 578 579 bool has_expired() { 580 if (++_check_count == _check_at) { 581 _check_at += _check_interval; 582 if ((uint64_t)Ticks::now().value() >= _timeout) { 583 // Timeout 584 _expired = true; 585 } 586 } 587 588 return _expired; 589 } 590 }; 591 592 void ZMark::work_with_timeout(ZMarkCache* cache, ZMarkStripe* stripe, ZMarkThreadLocalStacks* stacks, uint64_t timeout_in_millis) { 593 ZStatTimer timer(ZSubPhaseMarkTryComplete); 594 ZMarkTimeout timeout(timeout_in_millis); 595 596 for (;;) { 597 if (!drain_and_flush(stripe, stacks, cache, &timeout)) { 598 // Timed out 599 break; 600 } 601 602 if (try_steal(stripe, stacks)) { 603 // Stole work 604 continue; 605 } 606 607 // Terminate 608 break; 609 } 610 } 611 612 void ZMark::work(uint64_t timeout_in_millis) { 613 ZMarkCache cache(_stripes.nstripes()); 614 ZMarkStripe* const stripe = _stripes.stripe_for_worker(_nworkers, ZThread::worker_id()); 615 ZMarkThreadLocalStacks* const stacks = ZThreadLocalData::stacks(Thread::current()); 616 617 if (timeout_in_millis == 0) { 618 work_without_timeout(&cache, stripe, stacks); 619 } else { 620 work_with_timeout(&cache, stripe, stacks, timeout_in_millis); 621 } 622 623 // Make sure stacks have been flushed 624 assert(stacks->is_empty(&_stripes), "Should be empty"); 625 626 // Free remaining stacks 627 stacks->free(&_allocator); 628 } 629 630 class ZMarkConcurrentRootsIteratorClosure : public ZRootsIteratorClosure { 631 public: 632 virtual void do_oop(oop* p) { 633 ZBarrier::mark_barrier_on_oop_field(p, false /* finalizable */); 634 } 635 636 virtual void do_oop(narrowOop* p) { 637 ShouldNotReachHere(); 638 } 639 }; 640 641 642 class ZMarkConcurrentRootsTask : public ZTask { 643 private: 644 SuspendibleThreadSetJoiner _sts_joiner; 645 ZConcurrentRootsIteratorClaimStrong _roots; 646 ZMarkConcurrentRootsIteratorClosure _cl; 647 648 public: 649 ZMarkConcurrentRootsTask(ZMark* mark) : 650 ZTask("ZMarkConcurrentRootsTask"), 651 _sts_joiner(), 652 _roots(), 653 _cl() { 654 ClassLoaderDataGraph_lock->lock(); 655 } 656 657 ~ZMarkConcurrentRootsTask() { 658 ClassLoaderDataGraph_lock->unlock(); 659 } 660 661 virtual void work() { 662 _roots.oops_do(&_cl); 663 } 664 }; 665 666 class ZMarkTask : public ZTask { 667 private: 668 ZMark* const _mark; 669 const uint64_t _timeout_in_millis; 670 671 public: 672 ZMarkTask(ZMark* mark, uint64_t timeout_in_millis = 0) : 673 ZTask("ZMarkTask"), 674 _mark(mark), 675 _timeout_in_millis(timeout_in_millis) { 676 _mark->prepare_work(); 677 } 678 679 ~ZMarkTask() { 680 _mark->finish_work(); 681 } 682 683 virtual void work() { 684 _mark->work(_timeout_in_millis); 685 } 686 }; 687 688 void ZMark::mark(bool initial) { 689 if (initial) { 690 ZMarkConcurrentRootsTask task(this); 691 _workers->run_concurrent(&task); 692 } 693 694 ZMarkTask task(this); 695 _workers->run_concurrent(&task); 696 } 697 698 bool ZMark::try_complete() { 699 _ntrycomplete++; 700 701 // Use nconcurrent number of worker threads to maintain the 702 // worker/stripe distribution used during concurrent mark. 703 ZMarkTask task(this, ZMarkCompleteTimeout); 704 _workers->run_concurrent(&task); 705 706 // Successful if all stripes are empty 707 return _stripes.is_empty(); 708 } 709 710 bool ZMark::try_end() { 711 // Flush all mark stacks 712 if (!flush(true /* at_safepoint */)) { 713 // Mark completed 714 return true; 715 } 716 717 // Try complete marking by doing a limited 718 // amount of mark work in this phase. 719 return try_complete(); 720 } 721 722 bool ZMark::end() { 723 // Try end marking 724 if (!try_end()) { 725 // Mark not completed 726 _ncontinue++; 727 return false; 728 } 729 730 // Verification 731 if (ZVerifyMarking) { 732 verify_all_stacks_empty(); 733 } 734 735 // Update statistics 736 ZStatMark::set_at_mark_end(_nproactiveflush, _nterminateflush, _ntrycomplete, _ncontinue); 737 738 // Mark completed 739 return true; 740 } 741 742 void ZMark::flush_and_free() { 743 Thread* const thread = Thread::current(); 744 flush_and_free(thread); 745 } 746 747 bool ZMark::flush_and_free(Thread* thread) { 748 ZMarkThreadLocalStacks* const stacks = ZThreadLocalData::stacks(thread); 749 const bool flushed = stacks->flush(&_allocator, &_stripes); 750 stacks->free(&_allocator); 751 return flushed; 752 } 753 754 class ZVerifyMarkStacksEmptyClosure : public ThreadClosure { 755 private: 756 const ZMarkStripeSet* const _stripes; 757 758 public: 759 ZVerifyMarkStacksEmptyClosure(const ZMarkStripeSet* stripes) : 760 _stripes(stripes) {} 761 762 void do_thread(Thread* thread) { 763 ZMarkThreadLocalStacks* const stacks = ZThreadLocalData::stacks(thread); 764 guarantee(stacks->is_empty(_stripes), "Should be empty"); 765 } 766 }; 767 768 void ZMark::verify_all_stacks_empty() const { 769 // Verify thread stacks 770 ZVerifyMarkStacksEmptyClosure cl(&_stripes); 771 Threads::threads_do(&cl); 772 773 // Verify stripe stacks 774 guarantee(_stripes.is_empty(), "Should be empty"); 775 }