1 /*
2 * Copyright (c) 1999, 2018, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "precompiled.hpp"
26 #include "gc/shared/threadLocalAllocBuffer.inline.hpp"
27 #include "logging/log.hpp"
28 #include "memory/resourceArea.hpp"
29 #include "memory/universe.hpp"
30 #include "oops/oop.inline.hpp"
31 #include "runtime/thread.inline.hpp"
32 #include "runtime/threadSMR.hpp"
33 #include "utilities/copy.hpp"
34
35 // Thread-Local Edens support
36
37 // static member initialization
38 size_t ThreadLocalAllocBuffer::_max_size = 0;
39 int ThreadLocalAllocBuffer::_reserve_for_allocation_prefetch = 0;
40 unsigned ThreadLocalAllocBuffer::_target_refills = 0;
41 GlobalTLABStats* ThreadLocalAllocBuffer::_global_stats = NULL;
42
43 void ThreadLocalAllocBuffer::clear_before_allocation() {
44 _slow_refill_waste += (unsigned)remaining();
45 make_parsable(true); // also retire the TLAB
46 }
47
48 size_t ThreadLocalAllocBuffer::remaining() {
49 if (end() == NULL) {
50 return 0;
51 }
52
53 return pointer_delta(hard_end(), top());
54 }
55
56 void ThreadLocalAllocBuffer::accumulate_statistics_before_gc() {
57 global_stats()->initialize();
58
59 for (JavaThreadIteratorWithHandle jtiwh; JavaThread *thread = jtiwh.next(); ) {
60 thread->tlab().accumulate_statistics();
61 thread->tlab().initialize_statistics();
62 }
63
64 // Publish new stats if some allocation occurred.
65 if (global_stats()->allocation() != 0) {
66 global_stats()->publish();
67 global_stats()->print();
68 }
69 }
70
71 void ThreadLocalAllocBuffer::accumulate_statistics() {
72 Thread* thread = myThread();
73 size_t capacity = Universe::heap()->tlab_capacity(thread);
74 size_t used = Universe::heap()->tlab_used(thread);
75
76 _gc_waste += (unsigned)remaining();
77 size_t total_allocated = thread->allocated_bytes();
78 size_t allocated_since_last_gc = total_allocated - _allocated_before_last_gc;
79 _allocated_before_last_gc = total_allocated;
80
81 print_stats("gc");
82
83 if (_number_of_refills > 0) {
84 // Update allocation history if a reasonable amount of eden was allocated.
85 bool update_allocation_history = used > 0.5 * capacity;
86
87 if (update_allocation_history) {
88 // Average the fraction of eden allocated in a tlab by this
89 // thread for use in the next resize operation.
90 // _gc_waste is not subtracted because it's included in
91 // "used".
92 // The result can be larger than 1.0 due to direct to old allocations.
93 // These allocations should ideally not be counted but since it is not possible
94 // to filter them out here we just cap the fraction to be at most 1.0.
95 double alloc_frac = MIN2(1.0, (double) allocated_since_last_gc / used);
96 _allocation_fraction.sample(alloc_frac);
97 }
98 global_stats()->update_allocating_threads();
99 global_stats()->update_number_of_refills(_number_of_refills);
100 global_stats()->update_allocation(_allocated_size);
101 global_stats()->update_gc_waste(_gc_waste);
102 global_stats()->update_slow_refill_waste(_slow_refill_waste);
103 global_stats()->update_fast_refill_waste(_fast_refill_waste);
104
105 } else {
106 assert(_number_of_refills == 0 && _fast_refill_waste == 0 &&
107 _slow_refill_waste == 0 && _gc_waste == 0,
108 "tlab stats == 0");
109 }
110 global_stats()->update_slow_allocations(_slow_allocations);
111 }
112
113 // Fills the current tlab with a dummy filler array to create
114 // an illusion of a contiguous Eden and optionally retires the tlab.
115 // Waste accounting should be done in caller as appropriate; see,
116 // for example, clear_before_allocation().
117 void ThreadLocalAllocBuffer::make_parsable(bool retire, bool zap) {
118 if (end() != NULL) {
119 invariants();
120
121 if (retire) {
122 myThread()->incr_allocated_bytes(used_bytes());
123 }
124
125 Universe::heap()->fill_with_dummy_object(top(), hard_end(), retire && zap);
126
127 if (retire || ZeroTLAB) { // "Reset" the TLAB
128 set_start(NULL);
129 set_top(NULL);
130 set_pf_top(NULL);
131 set_end(NULL);
132 set_allocation_end(NULL);
133 }
134 }
135 assert(!(retire || ZeroTLAB) ||
136 (start() == NULL && end() == NULL && top() == NULL &&
137 _allocation_end == NULL),
138 "TLAB must be reset");
139 }
140
141 void ThreadLocalAllocBuffer::resize_all_tlabs() {
142 if (ResizeTLAB) {
143 for (JavaThreadIteratorWithHandle jtiwh; JavaThread *thread = jtiwh.next(); ) {
144 thread->tlab().resize();
145 }
146 }
147 }
148
149 void ThreadLocalAllocBuffer::resize() {
150 // Compute the next tlab size using expected allocation amount
151 assert(ResizeTLAB, "Should not call this otherwise");
152 size_t alloc = (size_t)(_allocation_fraction.average() *
153 (Universe::heap()->tlab_capacity(myThread()) / HeapWordSize));
154 size_t new_size = alloc / _target_refills;
155
156 new_size = MIN2(MAX2(new_size, min_size()), max_size());
157
158 size_t aligned_new_size = align_object_size(new_size);
159
160 log_trace(gc, tlab)("TLAB new size: thread: " INTPTR_FORMAT " [id: %2d]"
161 " refills %d alloc: %8.6f desired_size: " SIZE_FORMAT " -> " SIZE_FORMAT,
162 p2i(myThread()), myThread()->osthread()->thread_id(),
163 _target_refills, _allocation_fraction.average(), desired_size(), aligned_new_size);
164
165 set_desired_size(aligned_new_size);
166 set_refill_waste_limit(initial_refill_waste_limit());
167 }
168
169 void ThreadLocalAllocBuffer::initialize_statistics() {
170 _number_of_refills = 0;
171 _fast_refill_waste = 0;
172 _slow_refill_waste = 0;
173 _gc_waste = 0;
174 _slow_allocations = 0;
175 _allocated_size = 0;
176 }
177
178 void ThreadLocalAllocBuffer::fill(HeapWord* start,
179 HeapWord* top,
180 size_t new_size) {
181 _number_of_refills++;
182 _allocated_size += new_size;
183 print_stats("fill");
184 assert(top <= start + new_size - alignment_reserve(), "size too small");
185
186 initialize(start, top, start + new_size - alignment_reserve());
187
188 // Reset amount of internal fragmentation
189 set_refill_waste_limit(initial_refill_waste_limit());
190 }
191
192 void ThreadLocalAllocBuffer::initialize(HeapWord* start,
193 HeapWord* top,
194 HeapWord* end) {
195 set_start(start);
196 set_top(top);
197 set_pf_top(top);
198 set_end(end);
199 set_allocation_end(end);
200 invariants();
201 }
202
203 void ThreadLocalAllocBuffer::initialize() {
204 initialize(NULL, // start
205 NULL, // top
206 NULL); // end
207
208 set_desired_size(initial_desired_size());
209
210 // Following check is needed because at startup the main
211 // thread is initialized before the heap is. The initialization for
212 // this thread is redone in startup_initialization below.
213 if (Universe::heap() != NULL) {
214 size_t capacity = Universe::heap()->tlab_capacity(myThread()) / HeapWordSize;
215 double alloc_frac = desired_size() * target_refills() / (double) capacity;
216 _allocation_fraction.sample(alloc_frac);
217 }
218
219 set_refill_waste_limit(initial_refill_waste_limit());
220
221 initialize_statistics();
222 }
223
224 void ThreadLocalAllocBuffer::startup_initialization() {
225
226 // Assuming each thread's active tlab is, on average,
227 // 1/2 full at a GC
228 _target_refills = 100 / (2 * TLABWasteTargetPercent);
229 _target_refills = MAX2(_target_refills, (unsigned)1U);
230
231 _global_stats = new GlobalTLABStats();
232
233 #ifdef COMPILER2
234 // If the C2 compiler is present, extra space is needed at the end of
235 // TLABs, otherwise prefetching instructions generated by the C2
236 // compiler will fault (due to accessing memory outside of heap).
237 // The amount of space is the max of the number of lines to
238 // prefetch for array and for instance allocations. (Extra space must be
239 // reserved to accommodate both types of allocations.)
240 //
241 // Only SPARC-specific BIS instructions are known to fault. (Those
242 // instructions are generated if AllocatePrefetchStyle==3 and
243 // AllocatePrefetchInstr==1). To be on the safe side, however,
244 // extra space is reserved for all combinations of
245 // AllocatePrefetchStyle and AllocatePrefetchInstr.
246 //
247 // If the C2 compiler is not present, no space is reserved.
248
249 // +1 for rounding up to next cache line, +1 to be safe
250 if (is_server_compilation_mode_vm()) {
251 int lines = MAX2(AllocatePrefetchLines, AllocateInstancePrefetchLines) + 2;
252 _reserve_for_allocation_prefetch = (AllocatePrefetchDistance + AllocatePrefetchStepSize * lines) /
253 (int)HeapWordSize;
254 }
255 #endif
256
257 // During jvm startup, the main thread is initialized
258 // before the heap is initialized. So reinitialize it now.
259 guarantee(Thread::current()->is_Java_thread(), "tlab initialization thread not Java thread");
260 Thread::current()->tlab().initialize();
261
262 log_develop_trace(gc, tlab)("TLAB min: " SIZE_FORMAT " initial: " SIZE_FORMAT " max: " SIZE_FORMAT,
263 min_size(), Thread::current()->tlab().initial_desired_size(), max_size());
264 }
265
266 size_t ThreadLocalAllocBuffer::initial_desired_size() {
267 size_t init_sz = 0;
268
269 if (TLABSize > 0) {
270 init_sz = TLABSize / HeapWordSize;
271 } else if (global_stats() != NULL) {
272 // Initial size is a function of the average number of allocating threads.
273 unsigned nof_threads = global_stats()->allocating_threads_avg();
274
275 init_sz = (Universe::heap()->tlab_capacity(myThread()) / HeapWordSize) /
276 (nof_threads * target_refills());
277 init_sz = align_object_size(init_sz);
278 }
279 init_sz = MIN2(MAX2(init_sz, min_size()), max_size());
280 return init_sz;
281 }
282
283 void ThreadLocalAllocBuffer::print_stats(const char* tag) {
284 Log(gc, tlab) log;
285 if (!log.is_trace()) {
286 return;
287 }
288
289 Thread* thrd = myThread();
290 size_t waste = _gc_waste + _slow_refill_waste + _fast_refill_waste;
291 double waste_percent = percent_of(waste, _allocated_size);
292 size_t tlab_used = Universe::heap()->tlab_used(thrd);
293 log.trace("TLAB: %s thread: " INTPTR_FORMAT " [id: %2d]"
294 " desired_size: " SIZE_FORMAT "KB"
295 " slow allocs: %d refill waste: " SIZE_FORMAT "B"
296 " alloc:%8.5f %8.0fKB refills: %d waste %4.1f%% gc: %dB"
297 " slow: %dB fast: %dB",
298 tag, p2i(thrd), thrd->osthread()->thread_id(),
299 _desired_size / (K / HeapWordSize),
300 _slow_allocations, _refill_waste_limit * HeapWordSize,
301 _allocation_fraction.average(),
302 _allocation_fraction.average() * tlab_used / K,
303 _number_of_refills, waste_percent,
304 _gc_waste * HeapWordSize,
305 _slow_refill_waste * HeapWordSize,
306 _fast_refill_waste * HeapWordSize);
307 }
308
309 void ThreadLocalAllocBuffer::verify() {
310 HeapWord* p = start();
311 HeapWord* t = top();
312 HeapWord* prev_p = NULL;
313 while (p < t) {
314 oopDesc::verify(oop(p));
315 prev_p = p;
316 p += oop(p)->size();
317 }
318 guarantee(p == top(), "end of last object must match end of space");
319 }
320
321 void ThreadLocalAllocBuffer::set_sample_end() {
322 size_t heap_words_remaining = pointer_delta(_end, _top);
323 size_t bytes_until_sample = myThread()->heap_sampler().bytes_until_sample();
324 size_t words_until_sample = bytes_until_sample / HeapWordSize;
325
326 if (heap_words_remaining > words_until_sample) {
327 HeapWord* new_end = _top + words_until_sample;
328 set_end(new_end);
329 _bytes_since_last_sample_point = bytes_until_sample;
330 } else {
331 _bytes_since_last_sample_point = heap_words_remaining * HeapWordSize;
332 }
333 }
334
335 Thread* ThreadLocalAllocBuffer::myThread() {
336 return (Thread*)(((char *)this) +
337 in_bytes(start_offset()) -
338 in_bytes(Thread::tlab_start_offset()));
339 }
340
341 void ThreadLocalAllocBuffer::set_back_allocation_end() {
342 _end = _allocation_end;
343 }
344
345 HeapWord* ThreadLocalAllocBuffer::hard_end() {
346 return _allocation_end + alignment_reserve();
347 }
348
349 GlobalTLABStats::GlobalTLABStats() :
350 _allocating_threads_avg(TLABAllocationWeight) {
351
352 initialize();
353
354 _allocating_threads_avg.sample(1); // One allocating thread at startup
355
356 if (UsePerfData) {
357
358 EXCEPTION_MARK;
359 ResourceMark rm;
360
361 char* cname = PerfDataManager::counter_name("tlab", "allocThreads");
362 _perf_allocating_threads =
363 PerfDataManager::create_variable(SUN_GC, cname, PerfData::U_None, CHECK);
364
365 cname = PerfDataManager::counter_name("tlab", "fills");
366 _perf_total_refills =
367 PerfDataManager::create_variable(SUN_GC, cname, PerfData::U_None, CHECK);
368
369 cname = PerfDataManager::counter_name("tlab", "maxFills");
370 _perf_max_refills =
371 PerfDataManager::create_variable(SUN_GC, cname, PerfData::U_None, CHECK);
372
373 cname = PerfDataManager::counter_name("tlab", "alloc");
374 _perf_allocation =
375 PerfDataManager::create_variable(SUN_GC, cname, PerfData::U_Bytes, CHECK);
376
377 cname = PerfDataManager::counter_name("tlab", "gcWaste");
378 _perf_gc_waste =
379 PerfDataManager::create_variable(SUN_GC, cname, PerfData::U_Bytes, CHECK);
380
381 cname = PerfDataManager::counter_name("tlab", "maxGcWaste");
382 _perf_max_gc_waste =
383 PerfDataManager::create_variable(SUN_GC, cname, PerfData::U_Bytes, CHECK);
384
385 cname = PerfDataManager::counter_name("tlab", "slowWaste");
386 _perf_slow_refill_waste =
387 PerfDataManager::create_variable(SUN_GC, cname, PerfData::U_Bytes, CHECK);
388
389 cname = PerfDataManager::counter_name("tlab", "maxSlowWaste");
390 _perf_max_slow_refill_waste =
391 PerfDataManager::create_variable(SUN_GC, cname, PerfData::U_Bytes, CHECK);
392
393 cname = PerfDataManager::counter_name("tlab", "fastWaste");
394 _perf_fast_refill_waste =
395 PerfDataManager::create_variable(SUN_GC, cname, PerfData::U_Bytes, CHECK);
396
397 cname = PerfDataManager::counter_name("tlab", "maxFastWaste");
398 _perf_max_fast_refill_waste =
399 PerfDataManager::create_variable(SUN_GC, cname, PerfData::U_Bytes, CHECK);
400
401 cname = PerfDataManager::counter_name("tlab", "slowAlloc");
402 _perf_slow_allocations =
403 PerfDataManager::create_variable(SUN_GC, cname, PerfData::U_None, CHECK);
404
405 cname = PerfDataManager::counter_name("tlab", "maxSlowAlloc");
406 _perf_max_slow_allocations =
407 PerfDataManager::create_variable(SUN_GC, cname, PerfData::U_None, CHECK);
408 }
409 }
410
411 void GlobalTLABStats::initialize() {
412 // Clear counters summarizing info from all threads
413 _allocating_threads = 0;
414 _total_refills = 0;
415 _max_refills = 0;
416 _total_allocation = 0;
417 _total_gc_waste = 0;
418 _max_gc_waste = 0;
419 _total_slow_refill_waste = 0;
420 _max_slow_refill_waste = 0;
421 _total_fast_refill_waste = 0;
422 _max_fast_refill_waste = 0;
423 _total_slow_allocations = 0;
424 _max_slow_allocations = 0;
425 }
426
427 void GlobalTLABStats::publish() {
428 _allocating_threads_avg.sample(_allocating_threads);
429 if (UsePerfData) {
430 _perf_allocating_threads ->set_value(_allocating_threads);
431 _perf_total_refills ->set_value(_total_refills);
432 _perf_max_refills ->set_value(_max_refills);
433 _perf_allocation ->set_value(_total_allocation);
434 _perf_gc_waste ->set_value(_total_gc_waste);
435 _perf_max_gc_waste ->set_value(_max_gc_waste);
436 _perf_slow_refill_waste ->set_value(_total_slow_refill_waste);
437 _perf_max_slow_refill_waste->set_value(_max_slow_refill_waste);
438 _perf_fast_refill_waste ->set_value(_total_fast_refill_waste);
439 _perf_max_fast_refill_waste->set_value(_max_fast_refill_waste);
440 _perf_slow_allocations ->set_value(_total_slow_allocations);
441 _perf_max_slow_allocations ->set_value(_max_slow_allocations);
442 }
443 }
444
445 void GlobalTLABStats::print() {
446 Log(gc, tlab) log;
447 if (!log.is_debug()) {
448 return;
449 }
450
451 size_t waste = _total_gc_waste + _total_slow_refill_waste + _total_fast_refill_waste;
452 double waste_percent = percent_of(waste, _total_allocation);
453 log.debug("TLAB totals: thrds: %d refills: %d max: %d"
454 " slow allocs: %d max %d waste: %4.1f%%"
455 " gc: " SIZE_FORMAT "B max: " SIZE_FORMAT "B"
456 " slow: " SIZE_FORMAT "B max: " SIZE_FORMAT "B"
457 " fast: " SIZE_FORMAT "B max: " SIZE_FORMAT "B",
458 _allocating_threads,
459 _total_refills, _max_refills,
460 _total_slow_allocations, _max_slow_allocations,
461 waste_percent,
462 _total_gc_waste * HeapWordSize,
463 _max_gc_waste * HeapWordSize,
464 _total_slow_refill_waste * HeapWordSize,
465 _max_slow_refill_waste * HeapWordSize,
466 _total_fast_refill_waste * HeapWordSize,
467 _max_fast_refill_waste * HeapWordSize);
468 }