12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "precompiled.hpp"
26 #include "gc/shared/genCollectedHeap.hpp"
27 #include "gc/shared/threadLocalAllocBuffer.inline.hpp"
28 #include "logging/log.hpp"
29 #include "memory/resourceArea.hpp"
30 #include "memory/universe.inline.hpp"
31 #include "oops/oop.inline.hpp"
32 #include "runtime/thread.inline.hpp"
33 #include "utilities/copy.hpp"
34
35 // Thread-Local Edens support
36
37 // static member initialization
38 size_t ThreadLocalAllocBuffer::_max_size = 0;
39 int ThreadLocalAllocBuffer::_reserve_for_allocation_prefetch = 0;
40 unsigned ThreadLocalAllocBuffer::_target_refills = 0;
41 GlobalTLABStats* ThreadLocalAllocBuffer::_global_stats = NULL;
42
43 void ThreadLocalAllocBuffer::clear_before_allocation() {
44 _slow_refill_waste += (unsigned)remaining();
45 make_parsable(true); // also retire the TLAB
46 }
47
48 void ThreadLocalAllocBuffer::accumulate_statistics_before_gc() {
49 global_stats()->initialize();
50
51 for (JavaThread *thread = Threads::first(); thread != NULL; thread = thread->next()) {
104
105 // Fills the current tlab with a dummy filler array to create
106 // an illusion of a contiguous Eden and optionally retires the tlab.
107 // Waste accounting should be done in caller as appropriate; see,
108 // for example, clear_before_allocation().
109 void ThreadLocalAllocBuffer::make_parsable(bool retire, bool zap) {
110 if (end() != NULL) {
111 invariants();
112
113 if (retire) {
114 myThread()->incr_allocated_bytes(used_bytes());
115 }
116
117 CollectedHeap::fill_with_object(top(), hard_end(), retire && zap);
118
119 if (retire || ZeroTLAB) { // "Reset" the TLAB
120 set_start(NULL);
121 set_top(NULL);
122 set_pf_top(NULL);
123 set_end(NULL);
124 }
125 }
126 assert(!(retire || ZeroTLAB) ||
127 (start() == NULL && end() == NULL && top() == NULL),
128 "TLAB must be reset");
129 }
130
131 void ThreadLocalAllocBuffer::resize_all_tlabs() {
132 if (ResizeTLAB) {
133 for (JavaThread *thread = Threads::first(); thread != NULL; thread = thread->next()) {
134 thread->tlab().resize();
135 }
136 }
137 }
138
139 void ThreadLocalAllocBuffer::resize() {
140 // Compute the next tlab size using expected allocation amount
141 assert(ResizeTLAB, "Should not call this otherwise");
142 size_t alloc = (size_t)(_allocation_fraction.average() *
143 (Universe::heap()->tlab_capacity(myThread()) / HeapWordSize));
144 size_t new_size = alloc / _target_refills;
145
146 new_size = MIN2(MAX2(new_size, min_size()), max_size());
147
153 _target_refills, _allocation_fraction.average(), desired_size(), aligned_new_size);
154
155 set_desired_size(aligned_new_size);
156 set_refill_waste_limit(initial_refill_waste_limit());
157 }
158
159 void ThreadLocalAllocBuffer::initialize_statistics() {
160 _number_of_refills = 0;
161 _fast_refill_waste = 0;
162 _slow_refill_waste = 0;
163 _gc_waste = 0;
164 _slow_allocations = 0;
165 }
166
167 void ThreadLocalAllocBuffer::fill(HeapWord* start,
168 HeapWord* top,
169 size_t new_size) {
170 _number_of_refills++;
171 print_stats("fill");
172 assert(top <= start + new_size - alignment_reserve(), "size too small");
173 initialize(start, top, start + new_size - alignment_reserve());
174
175 // Reset amount of internal fragmentation
176 set_refill_waste_limit(initial_refill_waste_limit());
177 }
178
179 void ThreadLocalAllocBuffer::initialize(HeapWord* start,
180 HeapWord* top,
181 HeapWord* end) {
182 set_start(start);
183 set_top(top);
184 set_pf_top(top);
185 set_end(end);
186 invariants();
187 }
188
189 void ThreadLocalAllocBuffer::initialize() {
190 initialize(NULL, // start
191 NULL, // top
192 NULL); // end
193
194 set_desired_size(initial_desired_size());
195
196 // Following check is needed because at startup the main (primordial)
197 // thread is initialized before the heap is. The initialization for
198 // this thread is redone in startup_initialization below.
199 if (Universe::heap() != NULL) {
200 size_t capacity = Universe::heap()->tlab_capacity(myThread()) / HeapWordSize;
201 double alloc_frac = desired_size() * target_refills() / (double) capacity;
202 _allocation_fraction.sample(alloc_frac);
203 }
204
205 set_refill_waste_limit(initial_refill_waste_limit());
206
289 _allocation_fraction.average(),
290 _allocation_fraction.average() * tlab_used / K,
291 _number_of_refills, waste_percent,
292 _gc_waste * HeapWordSize,
293 _slow_refill_waste * HeapWordSize,
294 _fast_refill_waste * HeapWordSize);
295 }
296
297 void ThreadLocalAllocBuffer::verify() {
298 HeapWord* p = start();
299 HeapWord* t = top();
300 HeapWord* prev_p = NULL;
301 while (p < t) {
302 oop(p)->verify();
303 prev_p = p;
304 p += oop(p)->size();
305 }
306 guarantee(p == top(), "end of last object must match end of space");
307 }
308
309 Thread* ThreadLocalAllocBuffer::myThread() {
310 return (Thread*)(((char *)this) +
311 in_bytes(start_offset()) -
312 in_bytes(Thread::tlab_start_offset()));
313 }
314
315
316 GlobalTLABStats::GlobalTLABStats() :
317 _allocating_threads_avg(TLABAllocationWeight) {
318
319 initialize();
320
321 _allocating_threads_avg.sample(1); // One allocating thread at startup
322
323 if (UsePerfData) {
324
325 EXCEPTION_MARK;
326 ResourceMark rm;
327
328 char* cname = PerfDataManager::counter_name("tlab", "allocThreads");
329 _perf_allocating_threads =
330 PerfDataManager::create_variable(SUN_GC, cname, PerfData::U_None, CHECK);
331
332 cname = PerfDataManager::counter_name("tlab", "fills");
333 _perf_total_refills =
334 PerfDataManager::create_variable(SUN_GC, cname, PerfData::U_None, CHECK);
335
|
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "precompiled.hpp"
26 #include "gc/shared/genCollectedHeap.hpp"
27 #include "gc/shared/threadLocalAllocBuffer.inline.hpp"
28 #include "logging/log.hpp"
29 #include "memory/resourceArea.hpp"
30 #include "memory/universe.inline.hpp"
31 #include "oops/oop.inline.hpp"
32 #include "runtime/heapMonitoring.hpp"
33 #include "runtime/thread.inline.hpp"
34 #include "utilities/copy.hpp"
35
36 // Thread-Local Edens support
37
38 // static member initialization
39 size_t ThreadLocalAllocBuffer::_max_size = 0;
40 int ThreadLocalAllocBuffer::_reserve_for_allocation_prefetch = 0;
41 unsigned ThreadLocalAllocBuffer::_target_refills = 0;
42 GlobalTLABStats* ThreadLocalAllocBuffer::_global_stats = NULL;
43
44 void ThreadLocalAllocBuffer::clear_before_allocation() {
45 _slow_refill_waste += (unsigned)remaining();
46 make_parsable(true); // also retire the TLAB
47 }
48
49 void ThreadLocalAllocBuffer::accumulate_statistics_before_gc() {
50 global_stats()->initialize();
51
52 for (JavaThread *thread = Threads::first(); thread != NULL; thread = thread->next()) {
105
106 // Fills the current tlab with a dummy filler array to create
107 // an illusion of a contiguous Eden and optionally retires the tlab.
108 // Waste accounting should be done in caller as appropriate; see,
109 // for example, clear_before_allocation().
110 void ThreadLocalAllocBuffer::make_parsable(bool retire, bool zap) {
111 if (end() != NULL) {
112 invariants();
113
114 if (retire) {
115 myThread()->incr_allocated_bytes(used_bytes());
116 }
117
118 CollectedHeap::fill_with_object(top(), hard_end(), retire && zap);
119
120 if (retire || ZeroTLAB) { // "Reset" the TLAB
121 set_start(NULL);
122 set_top(NULL);
123 set_pf_top(NULL);
124 set_end(NULL);
125 set_actual_end(NULL);
126 set_slow_path_end(NULL);
127 }
128 }
129 assert(!(retire || ZeroTLAB) ||
130 (start() == NULL && end() == NULL && top() == NULL &&
131 actual_end() == NULL && slow_path_end() == NULL),
132 "TLAB must be reset");
133 }
134
135 void ThreadLocalAllocBuffer::resize_all_tlabs() {
136 if (ResizeTLAB) {
137 for (JavaThread *thread = Threads::first(); thread != NULL; thread = thread->next()) {
138 thread->tlab().resize();
139 }
140 }
141 }
142
143 void ThreadLocalAllocBuffer::resize() {
144 // Compute the next tlab size using expected allocation amount
145 assert(ResizeTLAB, "Should not call this otherwise");
146 size_t alloc = (size_t)(_allocation_fraction.average() *
147 (Universe::heap()->tlab_capacity(myThread()) / HeapWordSize));
148 size_t new_size = alloc / _target_refills;
149
150 new_size = MIN2(MAX2(new_size, min_size()), max_size());
151
157 _target_refills, _allocation_fraction.average(), desired_size(), aligned_new_size);
158
159 set_desired_size(aligned_new_size);
160 set_refill_waste_limit(initial_refill_waste_limit());
161 }
162
163 void ThreadLocalAllocBuffer::initialize_statistics() {
164 _number_of_refills = 0;
165 _fast_refill_waste = 0;
166 _slow_refill_waste = 0;
167 _gc_waste = 0;
168 _slow_allocations = 0;
169 }
170
171 void ThreadLocalAllocBuffer::fill(HeapWord* start,
172 HeapWord* top,
173 size_t new_size) {
174 _number_of_refills++;
175 print_stats("fill");
176 assert(top <= start + new_size - alignment_reserve(), "size too small");
177
178 // Remember old bytes until sample for the next tlab only if this is our first
179 // actual refill.
180 size_t old_bytes_until_sample = 0;
181 if (_number_of_refills > 1) {
182 old_bytes_until_sample = bytes_until_sample();
183 }
184
185 initialize(start, top, start + new_size - alignment_reserve());
186
187 if (old_bytes_until_sample > 0) {
188 set_bytes_until_sample(old_bytes_until_sample);
189 set_sample_end();
190 }
191
192 // Reset amount of internal fragmentation
193 set_refill_waste_limit(initial_refill_waste_limit());
194 }
195
196 void ThreadLocalAllocBuffer::initialize(HeapWord* start,
197 HeapWord* top,
198 HeapWord* end) {
199 set_start(start);
200 set_top(top);
201 set_pf_top(top);
202 set_end(end);
203 set_actual_end(end);
204 set_slow_path_end(end);
205 invariants();
206 _bytes_until_sample = 0;
207 }
208
209 void ThreadLocalAllocBuffer::initialize() {
210 initialize(NULL, // start
211 NULL, // top
212 NULL); // end
213
214 set_desired_size(initial_desired_size());
215
216 // Following check is needed because at startup the main (primordial)
217 // thread is initialized before the heap is. The initialization for
218 // this thread is redone in startup_initialization below.
219 if (Universe::heap() != NULL) {
220 size_t capacity = Universe::heap()->tlab_capacity(myThread()) / HeapWordSize;
221 double alloc_frac = desired_size() * target_refills() / (double) capacity;
222 _allocation_fraction.sample(alloc_frac);
223 }
224
225 set_refill_waste_limit(initial_refill_waste_limit());
226
309 _allocation_fraction.average(),
310 _allocation_fraction.average() * tlab_used / K,
311 _number_of_refills, waste_percent,
312 _gc_waste * HeapWordSize,
313 _slow_refill_waste * HeapWordSize,
314 _fast_refill_waste * HeapWordSize);
315 }
316
317 void ThreadLocalAllocBuffer::verify() {
318 HeapWord* p = start();
319 HeapWord* t = top();
320 HeapWord* prev_p = NULL;
321 while (p < t) {
322 oop(p)->verify();
323 prev_p = p;
324 p += oop(p)->size();
325 }
326 guarantee(p == top(), "end of last object must match end of space");
327 }
328
329 void ThreadLocalAllocBuffer::set_sample_end() {
330 size_t heap_words_remaining = _end - _top;
331 size_t bytes_left = bytes_until_sample();
332 size_t words_until_sample = bytes_left / HeapWordSize;
333
334 if (heap_words_remaining > words_until_sample) {
335 HeapWord* new_end = _top + words_until_sample;
336 set_end(new_end);
337 set_slow_path_end(new_end);
338 set_bytes_until_sample(0);
339 } else {
340 bytes_left -= heap_words_remaining * HeapWordSize;
341 set_bytes_until_sample(bytes_left);
342 }
343 }
344
345 void ThreadLocalAllocBuffer::pick_next_sample(size_t diff) {
346 if (!HeapMonitoring::enabled()) {
347 return;
348 }
349
350 if (bytes_until_sample() == 0) {
351 HeapMonitoring::pick_next_sample(bytes_until_sample_addr());
352 }
353
354 if (diff > 0) {
355 // Try to correct sample size by removing extra space from last allocation.
356 if (bytes_until_sample() > diff * HeapWordSize) {
357 set_bytes_until_sample(bytes_until_sample() - diff * HeapWordSize);
358 }
359 }
360
361 set_sample_end();
362
363 log_trace(gc, tlab)("TLAB picked next sample: thread: " INTPTR_FORMAT " [id: %2d]"
364 " start: %p top: %p end: %p actual_end: %p slow_path_end: %p",
365 p2i(myThread()), myThread()->osthread()->thread_id(),
366 start(), top(), end(),
367 actual_end(), slow_path_end());
368 }
369
370 Thread* ThreadLocalAllocBuffer::myThread() {
371 return (Thread*)(((char *)this) +
372 in_bytes(start_offset()) -
373 in_bytes(Thread::tlab_start_offset()));
374 }
375
376 void ThreadLocalAllocBuffer::set_back_actual_end() {
377 // Did a fast TLAB refill occur?
378 if (_slow_path_end != _end) {
379 // Fix up the actual end to be now the end of this TLAB.
380 _slow_path_end = _end;
381 _actual_end = _end;
382 } else {
383 _end = _actual_end;
384 }
385 }
386
387 void ThreadLocalAllocBuffer::handle_sample(Thread* thread, HeapWord* result,
388 size_t size) {
389 if (!HeapMonitoring::enabled()) {
390 return;
391 }
392
393 size_t size_in_bytes = size * HeapWordSize;
394 if (bytes_until_sample() > size_in_bytes) {
395 set_bytes_until_sample(bytes_until_sample() - size_in_bytes);
396 } else {
397 // Technically this is not exactly right, we probably should remember how many bytes are
398 // negative probably to then reduce our next sample size.
399 set_bytes_until_sample(0);
400 }
401
402 // Should we sample now?
403 if (should_sample()) {
404 HeapMonitoring::object_alloc_do_sample(thread,
405 reinterpret_cast<oopDesc*>(result),
406 size_in_bytes);
407 set_back_actual_end();
408 pick_next_sample();
409 }
410 }
411
412 HeapWord* ThreadLocalAllocBuffer::hard_end() {
413 // Did a fast TLAB refill occur?
414 if (_slow_path_end != _end) {
415 // Fix up the actual end to be now the end of this TLAB.
416 _slow_path_end = _end;
417 _actual_end = _end;
418 }
419
420 return _actual_end + alignment_reserve();
421 }
422 GlobalTLABStats::GlobalTLABStats() :
423 _allocating_threads_avg(TLABAllocationWeight) {
424
425 initialize();
426
427 _allocating_threads_avg.sample(1); // One allocating thread at startup
428
429 if (UsePerfData) {
430
431 EXCEPTION_MARK;
432 ResourceMark rm;
433
434 char* cname = PerfDataManager::counter_name("tlab", "allocThreads");
435 _perf_allocating_threads =
436 PerfDataManager::create_variable(SUN_GC, cname, PerfData::U_None, CHECK);
437
438 cname = PerfDataManager::counter_name("tlab", "fills");
439 _perf_total_refills =
440 PerfDataManager::create_variable(SUN_GC, cname, PerfData::U_None, CHECK);
441
|