1 /*
2 * Copyright (c) 2015, 2018, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 */
23
24 #include "precompiled.hpp"
25 #include "gc/z/zBarrier.inline.hpp"
26 #include "gc/z/zMark.inline.hpp"
27 #include "gc/z/zMarkCache.inline.hpp"
28 #include "gc/z/zMarkStack.inline.hpp"
29 #include "gc/z/zMarkTerminate.inline.hpp"
30 #include "gc/z/zOopClosures.inline.hpp"
31 #include "gc/z/zPage.hpp"
32 #include "gc/z/zPageTable.inline.hpp"
33 #include "gc/z/zRootsIterator.hpp"
34 #include "gc/z/zStat.hpp"
35 #include "gc/z/zStatTLAB.hpp"
36 #include "gc/z/zTask.hpp"
37 #include "gc/z/zThread.hpp"
38 #include "gc/z/zUtils.inline.hpp"
39 #include "gc/z/zWorkers.inline.hpp"
40 #include "logging/log.hpp"
41 #include "memory/iterator.inline.hpp"
42 #include "oops/objArrayOop.inline.hpp"
43 #include "oops/oop.inline.hpp"
44 #include "runtime/atomic.hpp"
45 #include "runtime/handshake.hpp"
46 #include "runtime/orderAccess.hpp"
47 #include "runtime/prefetch.inline.hpp"
48 #include "runtime/thread.hpp"
49 #include "utilities/align.hpp"
50 #include "utilities/globalDefinitions.hpp"
51 #include "utilities/ticks.hpp"
52
53 static const ZStatSubPhase ZSubPhaseConcurrentMark("Concurrent Mark");
54 static const ZStatSubPhase ZSubPhaseConcurrentMarkTryFlush("Concurrent Mark Try Flush");
55 static const ZStatSubPhase ZSubPhaseConcurrentMarkIdle("Concurrent Mark Idle");
56 static const ZStatSubPhase ZSubPhaseConcurrentMarkTryTerminate("Concurrent Mark Try Terminate");
57 static const ZStatSubPhase ZSubPhaseMarkTryComplete("Pause Mark Try Complete");
58
59 ZMark::ZMark(ZWorkers* workers, ZPageTable* pagetable) :
60 _workers(workers),
61 _pagetable(pagetable),
62 _allocator(),
63 _stripes(),
64 _terminate(),
65 _work_terminateflush(true),
66 _work_nproactiveflush(0),
67 _work_nterminateflush(0),
68 _nproactiveflush(0),
69 _nterminateflush(0),
70 _ntrycomplete(0),
71 _ncontinue(0),
72 _nworkers(0) {}
73
74 bool ZMark::is_initialized() const {
75 return _allocator.is_initialized();
76 }
77
78 size_t ZMark::calculate_nstripes(uint nworkers) const {
79 // Calculate the number of stripes from the number of workers we use,
80 // where the number of stripes must be a power of two and we want to
81 // have at least one worker per stripe.
82 const size_t nstripes = ZUtils::round_down_power_of_2(nworkers);
83 return MIN2(nstripes, ZMarkStripesMax);
84 }
85
86 void ZMark::prepare_mark() {
87 // Increment global sequence number to invalidate
88 // marking information for all pages.
89 ZGlobalSeqNum++;
90
91 // Reset flush/continue counters
92 _nproactiveflush = 0;
93 _nterminateflush = 0;
94 _ntrycomplete = 0;
95 _ncontinue = 0;
96
97 // Set number of workers to use
98 _nworkers = _workers->nconcurrent();
99
100 // Set number of mark stripes to use, based on number
101 // of workers we will use in the concurrent mark phase.
102 const size_t nstripes = calculate_nstripes(_nworkers);
103 _stripes.set_nstripes(nstripes);
104
105 // Update statistics
106 ZStatMark::set_at_mark_start(nstripes);
107
108 // Print worker/stripe distribution
109 LogTarget(Debug, gc, marking) log;
110 if (log.is_enabled()) {
111 log.print("Mark Worker/Stripe Distribution");
112 for (uint worker_id = 0; worker_id < _nworkers; worker_id++) {
113 const ZMarkStripe* const stripe = _stripes.stripe_for_worker(_nworkers, worker_id);
114 const size_t stripe_id = _stripes.stripe_id(stripe);
115 log.print(" Worker %u(%u) -> Stripe " SIZE_FORMAT "(" SIZE_FORMAT ")",
116 worker_id, _nworkers, stripe_id, nstripes);
117 }
118 }
119 }
120
121 class ZMarkRootsIteratorClosure : public ZRootsIteratorClosure {
122 public:
123 ZMarkRootsIteratorClosure() {
124 ZStatTLAB::reset();
125 }
126
127 ~ZMarkRootsIteratorClosure() {
128 ZStatTLAB::publish();
129 }
130
131 virtual void do_thread(Thread* thread) {
132 ZRootsIteratorClosure::do_thread(thread);
133
134 // Update thread local address bad mask
135 ZThreadLocalData::set_address_bad_mask(thread, ZAddressBadMask);
136
137 // Retire TLAB
138 if (UseTLAB && thread->is_Java_thread()) {
139 thread->tlab().retire(ZStatTLAB::get());
140 thread->tlab().resize();
141 }
142 }
143
144 virtual void do_oop(oop* p) {
145 ZBarrier::mark_barrier_on_root_oop_field(p);
146 }
147
148 virtual void do_oop(narrowOop* p) {
149 ShouldNotReachHere();
150 }
151 };
152
153 class ZMarkRootsTask : public ZTask {
154 private:
155 ZMark* const _mark;
156 ZRootsIterator _roots;
157 ZMarkRootsIteratorClosure _cl;
158
159 public:
160 ZMarkRootsTask(ZMark* mark) :
161 ZTask("ZMarkRootsTask"),
162 _mark(mark),
163 _roots() {}
164
165 virtual void work() {
166 _roots.oops_do(&_cl);
167
168 // Flush and free worker stacks. Needed here since
169 // the set of workers executing during root scanning
170 // can be different from the set of workers executing
171 // during mark.
172 _mark->flush_and_free();
173 }
174 };
175
176 void ZMark::start() {
177 // Verification
178 if (ZVerifyMarking) {
179 verify_all_stacks_empty();
180 }
181
182 // Prepare for concurrent mark
183 prepare_mark();
184
185 // Mark roots
186 ZMarkRootsTask task(this);
187 _workers->run_parallel(&task);
188 }
189
190 void ZMark::prepare_work() {
191 assert(_nworkers == _workers->nconcurrent(), "Invalid number of workers");
192
193 // Set number of active workers
194 _terminate.reset(_nworkers);
195
196 // Reset flush counters
197 _work_nproactiveflush = _work_nterminateflush = 0;
198 _work_terminateflush = true;
199 }
200
201 void ZMark::finish_work() {
202 // Accumulate proactive/terminate flush counters
203 _nproactiveflush += _work_nproactiveflush;
204 _nterminateflush += _work_nterminateflush;
205 }
206
207 bool ZMark::is_array(uintptr_t addr) const {
208 return ZOop::to_oop(addr)->is_objArray();
209 }
210
211 void ZMark::push_partial_array(uintptr_t addr, size_t size, bool finalizable) {
212 assert(is_aligned(addr, ZMarkPartialArrayMinSize), "Address misaligned");
213 ZMarkThreadLocalStacks* const stacks = ZThreadLocalData::stacks(Thread::current());
214 ZMarkStripe* const stripe = _stripes.stripe_for_addr(addr);
215 const uintptr_t offset = ZAddress::offset(addr) >> ZMarkPartialArrayMinSizeShift;
216 const uintptr_t length = size / oopSize;
217 const ZMarkStackEntry entry(offset, length, finalizable);
218
219 log_develop_trace(gc, marking)("Array push partial: " PTR_FORMAT " (" SIZE_FORMAT "), stripe: " SIZE_FORMAT,
220 addr, size, _stripes.stripe_id(stripe));
221
222 stacks->push(&_allocator, &_stripes, stripe, entry, false /* publish */);
223 }
224
225 void ZMark::follow_small_array(uintptr_t addr, size_t size, bool finalizable) {
226 assert(size <= ZMarkPartialArrayMinSize, "Too large, should be split");
227 const size_t length = size / oopSize;
228
229 log_develop_trace(gc, marking)("Array follow small: " PTR_FORMAT " (" SIZE_FORMAT ")", addr, size);
230
231 ZBarrier::mark_barrier_on_oop_array((oop*)addr, length, finalizable);
232 }
233
234 void ZMark::follow_large_array(uintptr_t addr, size_t size, bool finalizable) {
235 assert(size <= (size_t)arrayOopDesc::max_array_length(T_OBJECT) * oopSize, "Too large");
236 assert(size > ZMarkPartialArrayMinSize, "Too small, should not be split");
237 const uintptr_t start = addr;
238 const uintptr_t end = start + size;
239
240 // Calculate the aligned middle start/end/size, where the middle start
241 // should always be greater than the start (hence the +1 below) to make
242 // sure we always do some follow work, not just split the array into pieces.
243 const uintptr_t middle_start = align_up(start + 1, ZMarkPartialArrayMinSize);
244 const size_t middle_size = align_down(end - middle_start, ZMarkPartialArrayMinSize);
245 const uintptr_t middle_end = middle_start + middle_size;
246
247 log_develop_trace(gc, marking)("Array follow large: " PTR_FORMAT "-" PTR_FORMAT" (" SIZE_FORMAT "), "
248 "middle: " PTR_FORMAT "-" PTR_FORMAT " (" SIZE_FORMAT ")",
249 start, end, size, middle_start, middle_end, middle_size);
250
251 // Push unaligned trailing part
252 if (end > middle_end) {
253 const uintptr_t trailing_addr = middle_end;
254 const size_t trailing_size = end - middle_end;
255 push_partial_array(trailing_addr, trailing_size, finalizable);
256 }
257
258 // Push aligned middle part(s)
259 uintptr_t partial_addr = middle_end;
260 while (partial_addr > middle_start) {
261 const size_t parts = 2;
262 const size_t partial_size = align_up((partial_addr - middle_start) / parts, ZMarkPartialArrayMinSize);
263 partial_addr -= partial_size;
264 push_partial_array(partial_addr, partial_size, finalizable);
265 }
266
267 // Follow leading part
268 assert(start < middle_start, "Miscalculated middle start");
269 const uintptr_t leading_addr = start;
270 const size_t leading_size = middle_start - start;
271 follow_small_array(leading_addr, leading_size, finalizable);
272 }
273
274 void ZMark::follow_array(uintptr_t addr, size_t size, bool finalizable) {
275 if (size <= ZMarkPartialArrayMinSize) {
276 follow_small_array(addr, size, finalizable);
277 } else {
278 follow_large_array(addr, size, finalizable);
279 }
280 }
281
282 void ZMark::follow_partial_array(ZMarkStackEntry entry, bool finalizable) {
283 const uintptr_t addr = ZAddress::good(entry.partial_array_offset() << ZMarkPartialArrayMinSizeShift);
284 const size_t size = entry.partial_array_length() * oopSize;
285
286 follow_array(addr, size, finalizable);
287 }
288
289 void ZMark::follow_array_object(objArrayOop obj, bool finalizable) {
290 if (finalizable) {
291 ZMarkBarrierOopClosure<true /* finalizable */> cl;
292 cl.do_klass(obj->klass());
293 } else {
294 ZMarkBarrierOopClosure<false /* finalizable */> cl;
295 cl.do_klass(obj->klass());
296 }
297
298 const uintptr_t addr = (uintptr_t)obj->base();
299 const size_t size = (size_t)obj->length() * oopSize;
300
301 follow_array(addr, size, finalizable);
302 }
303
304 void ZMark::follow_object(oop obj, bool finalizable) {
305 if (finalizable) {
306 ZMarkBarrierOopClosure<true /* finalizable */> cl;
307 obj->oop_iterate(&cl);
308 } else {
309 ZMarkBarrierOopClosure<false /* finalizable */> cl;
310 obj->oop_iterate(&cl);
311 }
312 }
313
314 bool ZMark::try_mark_object(ZMarkCache* cache, uintptr_t addr, bool finalizable) {
315 ZPage* const page = _pagetable->get(addr);
316 if (page->is_allocating()) {
317 // Newly allocated objects are implicitly marked
318 return false;
319 }
320
321 // Try mark object
322 bool inc_live = false;
323 const bool success = page->mark_object(addr, finalizable, inc_live);
324 if (inc_live) {
325 // Update live objects/bytes for page. We use the aligned object
326 // size since that is the actual number of bytes used on the page
327 // and alignment paddings can never be reclaimed.
328 const size_t size = ZUtils::object_size(addr);
329 const size_t aligned_size = align_up(size, page->object_alignment());
330 cache->inc_live(page, aligned_size);
331 }
332
333 return success;
334 }
335
336 void ZMark::mark_and_follow(ZMarkCache* cache, ZMarkStackEntry entry) {
337 // Decode flags
338 const bool finalizable = entry.finalizable();
339 const bool partial_array = entry.partial_array();
340
341 if (partial_array) {
342 follow_partial_array(entry, finalizable);
343 return;
344 }
345
346 // Decode object address
347 const uintptr_t addr = entry.object_address();
348
349 if (!try_mark_object(cache, addr, finalizable)) {
350 // Already marked
351 return;
352 }
353
354 if (is_array(addr)) {
355 follow_array_object(objArrayOop(ZOop::to_oop(addr)), finalizable);
356 } else {
357 follow_object(ZOop::to_oop(addr), finalizable);
358 }
359 }
360
361 template <typename T>
362 bool ZMark::drain(ZMarkStripe* stripe, ZMarkThreadLocalStacks* stacks, ZMarkCache* cache, T* timeout) {
363 ZMarkStackEntry entry;
364
365 // Drain stripe stacks
366 while (stacks->pop(&_allocator, &_stripes, stripe, entry)) {
367 mark_and_follow(cache, entry);
368
369 // Check timeout
370 if (timeout->has_expired()) {
371 // Timeout
372 return false;
373 }
374 }
375
376 // Success
377 return true;
378 }
379
380 template <typename T>
381 bool ZMark::drain_and_flush(ZMarkStripe* stripe, ZMarkThreadLocalStacks* stacks, ZMarkCache* cache, T* timeout) {
382 const bool success = drain(stripe, stacks, cache, timeout);
383
384 // Flush and publish worker stacks
385 stacks->flush(&_allocator, &_stripes);
386
387 return success;
388 }
389
390 bool ZMark::try_steal(ZMarkStripe* stripe, ZMarkThreadLocalStacks* stacks) {
391 // Try to steal a stack from another stripe
392 for (ZMarkStripe* victim_stripe = _stripes.stripe_next(stripe);
393 victim_stripe != stripe;
394 victim_stripe = _stripes.stripe_next(victim_stripe)) {
395 ZMarkStack* const stack = victim_stripe->steal_stack();
396 if (stack != NULL) {
397 // Success, install the stolen stack
398 stacks->install(&_stripes, stripe, stack);
399 return true;
400 }
401 }
402
403 // Nothing to steal
404 return false;
405 }
406
407 void ZMark::idle() const {
408 ZStatTimer timer(ZSubPhaseConcurrentMarkIdle);
409 os::naked_short_sleep(1);
410 }
411
412 class ZMarkFlushAndFreeStacksClosure : public ThreadClosure {
413 private:
414 ZMark* const _mark;
415 bool _flushed;
416
417 public:
418 ZMarkFlushAndFreeStacksClosure(ZMark* mark) :
419 _mark(mark),
420 _flushed(false) {}
421
422 void do_thread(Thread* thread) {
423 if (_mark->flush_and_free(thread)) {
424 _flushed = true;
425 }
426 }
427
428 bool flushed() const {
429 return _flushed;
430 }
431 };
432
433 bool ZMark::flush(bool at_safepoint) {
434 ZMarkFlushAndFreeStacksClosure cl(this);
435 if (at_safepoint) {
436 Threads::threads_do(&cl);
437 } else {
438 Handshake::execute(&cl);
439 }
440
441 // Returns true if more work is available
442 return cl.flushed() || !_stripes.is_empty();
443 }
444
445 bool ZMark::try_flush(volatile size_t* nflush) {
446 // Only flush if handshakes are enabled
447 if (!ThreadLocalHandshakes) {
448 return false;
449 }
450
451 Atomic::inc(nflush);
452
453 ZStatTimer timer(ZSubPhaseConcurrentMarkTryFlush);
454 return flush(false /* at_safepoint */);
455 }
456
457 bool ZMark::try_proactive_flush() {
458 // Only do proactive flushes from worker 0
459 if (ZThread::worker_id() != 0) {
460 return false;
461 }
462
463 if (Atomic::load(&_work_nproactiveflush) == ZMarkProactiveFlushMax ||
464 Atomic::load(&_work_nterminateflush) != 0) {
465 // Limit reached or we're trying to terminate
466 return false;
467 }
468
469 return try_flush(&_work_nproactiveflush);
470 }
471
472 bool ZMark::try_terminate() {
473 ZStatTimer timer(ZSubPhaseConcurrentMarkTryTerminate);
474
475 if (_terminate.enter_stage0()) {
476 // Last thread entered stage 0, flush
477 if (Atomic::load(&_work_terminateflush) &&
478 Atomic::load(&_work_nterminateflush) != ZMarkTerminateFlushMax) {
479 // Exit stage 0 to allow other threads to continue marking
480 _terminate.exit_stage0();
481
482 // Flush before termination
483 if (!try_flush(&_work_nterminateflush)) {
484 // No more work available, skip further flush attempts
485 Atomic::store(false, &_work_terminateflush);
486 }
487
488 // Don't terminate, regardless of whether we successfully
489 // flushed out more work or not. We've already exited
490 // termination stage 0, to allow other threads to continue
491 // marking, so this thread has to return false and also
492 // make another round of attempted marking.
493 return false;
494 }
495 }
496
497 for (;;) {
498 if (_terminate.enter_stage1()) {
499 // Last thread entered stage 1, terminate
500 return true;
501 }
502
503 // Idle to give the other threads
504 // a chance to enter termination.
505 idle();
506
507 if (!_terminate.try_exit_stage1()) {
508 // All workers in stage 1, terminate
509 return true;
510 }
511
512 if (_terminate.try_exit_stage0()) {
513 // More work available, don't terminate
514 return false;
515 }
516 }
517 }
518
519 class ZMarkNoTimeout : public StackObj {
520 public:
521 bool has_expired() {
522 return false;
523 }
524 };
525
526 void ZMark::work_without_timeout(ZMarkCache* cache, ZMarkStripe* stripe, ZMarkThreadLocalStacks* stacks) {
527 ZStatTimer timer(ZSubPhaseConcurrentMark);
528 ZMarkNoTimeout no_timeout;
529
530 for (;;) {
531 drain_and_flush(stripe, stacks, cache, &no_timeout);
532
533 if (try_steal(stripe, stacks)) {
534 // Stole work
535 continue;
536 }
537
538 if (try_proactive_flush()) {
539 // Work available
540 continue;
541 }
542
543 if (try_terminate()) {
544 // Terminate
545 break;
546 }
547 }
548 }
549
550 class ZMarkTimeout : public StackObj {
551 private:
552 const Ticks _start;
553 const uint64_t _timeout;
554 const uint64_t _check_interval;
555 uint64_t _check_at;
556 uint64_t _check_count;
557 bool _expired;
558
559 public:
560 ZMarkTimeout(uint64_t timeout_in_millis) :
561 _start(Ticks::now()),
562 _timeout(_start.value() + TimeHelper::millis_to_counter(timeout_in_millis)),
563 _check_interval(200),
564 _check_at(_check_interval),
565 _check_count(0),
566 _expired(false) {}
567
568 ~ZMarkTimeout() {
569 const Tickspan duration = Ticks::now() - _start;
570 log_debug(gc, marking)("Mark With Timeout (%s): %s, " UINT64_FORMAT " oops, %.3fms",
571 ZThread::name(), _expired ? "Expired" : "Completed",
572 _check_count, TimeHelper::counter_to_millis(duration.value()));
573 }
574
575 bool has_expired() {
576 if (++_check_count == _check_at) {
577 _check_at += _check_interval;
578 if ((uint64_t)Ticks::now().value() >= _timeout) {
579 // Timeout
580 _expired = true;
581 }
582 }
583
584 return _expired;
585 }
586 };
587
588 void ZMark::work_with_timeout(ZMarkCache* cache, ZMarkStripe* stripe, ZMarkThreadLocalStacks* stacks, uint64_t timeout_in_millis) {
589 ZStatTimer timer(ZSubPhaseMarkTryComplete);
590 ZMarkTimeout timeout(timeout_in_millis);
591
592 for (;;) {
593 if (!drain_and_flush(stripe, stacks, cache, &timeout)) {
594 // Timed out
595 break;
596 }
597
598 if (try_steal(stripe, stacks)) {
599 // Stole work
600 continue;
601 }
602
603 // Terminate
604 break;
605 }
606 }
607
608 void ZMark::work(uint64_t timeout_in_millis) {
609 ZMarkCache cache(_stripes.nstripes());
610 ZMarkStripe* const stripe = _stripes.stripe_for_worker(_nworkers, ZThread::worker_id());
611 ZMarkThreadLocalStacks* const stacks = ZThreadLocalData::stacks(Thread::current());
612
613 if (timeout_in_millis == 0) {
614 work_without_timeout(&cache, stripe, stacks);
615 } else {
616 work_with_timeout(&cache, stripe, stacks, timeout_in_millis);
617 }
618
619 // Make sure stacks have been flushed
620 assert(stacks->is_empty(&_stripes), "Should be empty");
621
622 // Free remaining stacks
623 stacks->free(&_allocator);
624 }
625
626 class ZMarkConcurrentRootsIteratorClosure : public ZRootsIteratorClosure {
627 public:
628 virtual void do_oop(oop* p) {
629 ZBarrier::mark_barrier_on_oop_field(p, false /* finalizable */);
630 }
631
632 virtual void do_oop(narrowOop* p) {
633 ShouldNotReachHere();
634 }
635 };
636
637
638 class ZMarkConcurrentRootsTask : public ZTask {
639 private:
640 ZConcurrentRootsIterator _roots;
641 ZMarkConcurrentRootsIteratorClosure _cl;
642
643 public:
644 ZMarkConcurrentRootsTask(ZMark* mark) :
645 ZTask("ZMarkConcurrentRootsTask"),
646 _roots(true /* marking */),
647 _cl() {}
648
649 virtual void work() {
650 _roots.oops_do(&_cl);
651 }
652 };
653
654 class ZMarkTask : public ZTask {
655 private:
656 ZMark* const _mark;
657 const uint64_t _timeout_in_millis;
658
659 public:
660 ZMarkTask(ZMark* mark, uint64_t timeout_in_millis = 0) :
661 ZTask("ZMarkTask"),
662 _mark(mark),
663 _timeout_in_millis(timeout_in_millis) {
664 _mark->prepare_work();
665 }
666
667 ~ZMarkTask() {
668 _mark->finish_work();
669 }
670
671 virtual void work() {
672 _mark->work(_timeout_in_millis);
673 }
674 };
675
676 void ZMark::mark(bool initial) {
677 if (initial) {
678 ZMarkConcurrentRootsTask task(this);
679 _workers->run_concurrent(&task);
680 }
681
682 ZMarkTask task(this);
683 _workers->run_concurrent(&task);
684 }
685
686 bool ZMark::try_complete() {
687 _ntrycomplete++;
688
689 // Use nconcurrent number of worker threads to maintain the
690 // worker/stripe distribution used during concurrent mark.
691 ZMarkTask task(this, ZMarkCompleteTimeout);
692 _workers->run_concurrent(&task);
693
694 // Successful if all stripes are empty
695 return _stripes.is_empty();
696 }
697
698 bool ZMark::try_end() {
699 // Flush all mark stacks
700 if (!flush(true /* at_safepoint */)) {
701 // Mark completed
702 return true;
703 }
704
705 // Try complete marking by doing a limited
706 // amount of mark work in this phase.
707 return try_complete();
708 }
709
710 bool ZMark::end() {
711 // Try end marking
712 if (!try_end()) {
713 // Mark not completed
714 _ncontinue++;
715 return false;
716 }
717
718 // Verification
719 if (ZVerifyMarking) {
720 verify_all_stacks_empty();
721 }
722
723 // Update statistics
724 ZStatMark::set_at_mark_end(_nproactiveflush, _nterminateflush, _ntrycomplete, _ncontinue);
725
726 // Mark completed
727 return true;
728 }
729
730 void ZMark::flush_and_free() {
731 Thread* const thread = Thread::current();
732 flush_and_free(thread);
733 }
734
735 bool ZMark::flush_and_free(Thread* thread) {
736 ZMarkThreadLocalStacks* const stacks = ZThreadLocalData::stacks(thread);
737 const bool flushed = stacks->flush(&_allocator, &_stripes);
738 stacks->free(&_allocator);
739 return flushed;
740 }
741
742 class ZVerifyMarkStacksEmptyClosure : public ThreadClosure {
743 private:
744 const ZMarkStripeSet* const _stripes;
745
746 public:
747 ZVerifyMarkStacksEmptyClosure(const ZMarkStripeSet* stripes) :
748 _stripes(stripes) {}
749
750 void do_thread(Thread* thread) {
751 ZMarkThreadLocalStacks* const stacks = ZThreadLocalData::stacks(thread);
752 guarantee(stacks->is_empty(_stripes), "Should be empty");
753 }
754 };
755
756 void ZMark::verify_all_stacks_empty() const {
757 // Verify thread stacks
758 ZVerifyMarkStacksEmptyClosure cl(&_stripes);
759 Threads::threads_do(&cl);
760
761 // Verify stripe stacks
762 guarantee(_stripes.is_empty(), "Should be empty");
763 }