44 HeapWord* _start; // address of TLAB
45 HeapWord* _top; // address after last allocation
46 HeapWord* _pf_top; // allocation prefetch watermark
47 HeapWord* _end; // allocation end (excluding alignment_reserve)
48 size_t _desired_size; // desired size (including alignment_reserve)
49 size_t _refill_waste_limit; // hold onto tlab if free() is larger than this
50 size_t _allocated_before_last_gc; // total bytes allocated up until the last gc
51
52 static size_t _max_size; // maximum size of any TLAB
53 static int _reserve_for_allocation_prefetch; // Reserve at the end of the TLAB
54 static unsigned _target_refills; // expected number of refills between GCs
55
56 unsigned _number_of_refills;
57 unsigned _fast_refill_waste;
58 unsigned _slow_refill_waste;
59 unsigned _gc_waste;
60 unsigned _slow_allocations;
61
62 AdaptiveWeightedAverage _allocation_fraction; // fraction of eden allocated in tlabs
63
64 bool _gclab;
65 bool _initialized;
66
67 void set_start(HeapWord* start) { _start = start; }
68 void set_end(HeapWord* end) { _end = end; }
69 void set_top(HeapWord* top) { _top = top; }
70 void set_pf_top(HeapWord* pf_top) { _pf_top = pf_top; }
71 void set_desired_size(size_t desired_size) { _desired_size = desired_size; }
72 void set_refill_waste_limit(size_t waste) { _refill_waste_limit = waste; }
73
74 size_t initial_refill_waste_limit() { return desired_size() / TLABRefillWasteFraction; }
75
76 static int target_refills() { return _target_refills; }
77 size_t initial_desired_size();
78
79 size_t remaining() const { return end() == NULL ? 0 : pointer_delta(hard_end(), top()); }
80
81 // Make parsable and release it.
82 void reset();
83
84 void invariants() const { assert(top() >= start() && top() <= end(), "invalid tlab"); }
85
86 void initialize(HeapWord* start, HeapWord* top, HeapWord* end);
87
88 void print_stats(const char* tag);
89
90 Thread* myThread();
91
92 // statistics
93
94 int number_of_refills() const { return _number_of_refills; }
95 int fast_refill_waste() const { return _fast_refill_waste; }
96 int slow_refill_waste() const { return _slow_refill_waste; }
97 int gc_waste() const { return _gc_waste; }
98 int slow_allocations() const { return _slow_allocations; }
99
100 static GlobalTLABStats* _global_stats;
101 static GlobalTLABStats* global_stats() { return _global_stats; }
102
103 public:
104 ThreadLocalAllocBuffer() : _allocation_fraction(TLABAllocationWeight), _allocated_before_last_gc(0), _initialized(false) {
105 // do nothing. tlabs must be inited by initialize() calls
106 }
107
108 bool is_initialized() const { return _initialized; };
109
110 static size_t min_size() { return align_object_size(MinTLABSize / HeapWordSize) + alignment_reserve(); }
111 static size_t max_size() { assert(_max_size != 0, "max_size not set up"); return _max_size; }
112 static size_t max_size_in_bytes() { return max_size() * BytesPerWord; }
113 static void set_max_size(size_t max_size) { _max_size = max_size; }
114
115 HeapWord* start() const { return _start; }
116 HeapWord* end() const { return _end; }
117 HeapWord* hard_end() const { return _end + alignment_reserve(); }
118 HeapWord* top() const { return _top; }
119 HeapWord* pf_top() const { return _pf_top; }
120 size_t desired_size() const { return _desired_size; }
121 size_t used() const { return pointer_delta(top(), start()); }
122 size_t used_bytes() const { return pointer_delta(top(), start(), 1); }
123 size_t free() const { return pointer_delta(end(), top()); }
124 // Don't discard tlab if remaining space is larger than this.
125 size_t refill_waste_limit() const { return _refill_waste_limit; }
126
127 // Allocate size HeapWords. The memory is NOT initialized to zero.
128 inline HeapWord* allocate(size_t size);
129
130 // Resize based on amount of allocation, etc.
131 void resize();
132
133 void accumulate_statistics();
134 void initialize_statistics();
135
136 // Rolls back a single allocation of the given size.
137 void rollback(size_t size);
138
139 // Reserve space at the end of TLAB
140 static size_t end_reserve();
141 static size_t alignment_reserve() { return align_object_size(end_reserve()); }
142 static size_t alignment_reserve_in_bytes() { return alignment_reserve() * HeapWordSize; }
143
144 // Return tlab size or remaining space in eden such that the
145 // space is large enough to hold obj_size and necessary fill space.
146 // Otherwise return 0;
147 inline size_t compute_size(size_t obj_size);
148
149 // Record slow allocation
150 inline void record_slow_allocation(size_t obj_size);
151
152 // Initialization at startup
153 static void startup_initialization();
154
155 // Make an in-use tlab parsable, optionally retiring and/or zapping it.
156 void make_parsable(bool retire, bool zap = true);
157
158 // Retire in-use tlab before allocation of a new tlab
159 void clear_before_allocation();
160
161 // Accumulate statistics across all tlabs before gc
162 static void accumulate_statistics_before_gc();
163
164 // Resize tlabs for all threads
165 static void resize_all_tlabs();
166
167 void fill(HeapWord* start, HeapWord* top, size_t new_size);
168 void initialize(bool gclab);
169
170 static size_t refill_waste_limit_increment() { return TLABWasteIncrement; }
171
172 // Code generation support
173 static ByteSize start_offset() { return byte_offset_of(ThreadLocalAllocBuffer, _start); }
174 static ByteSize end_offset() { return byte_offset_of(ThreadLocalAllocBuffer, _end ); }
175 static ByteSize top_offset() { return byte_offset_of(ThreadLocalAllocBuffer, _top ); }
176 static ByteSize pf_top_offset() { return byte_offset_of(ThreadLocalAllocBuffer, _pf_top ); }
177 static ByteSize size_offset() { return byte_offset_of(ThreadLocalAllocBuffer, _desired_size ); }
178 static ByteSize refill_waste_limit_offset() { return byte_offset_of(ThreadLocalAllocBuffer, _refill_waste_limit ); }
179
180 static ByteSize number_of_refills_offset() { return byte_offset_of(ThreadLocalAllocBuffer, _number_of_refills ); }
181 static ByteSize fast_refill_waste_offset() { return byte_offset_of(ThreadLocalAllocBuffer, _fast_refill_waste ); }
182 static ByteSize slow_allocations_offset() { return byte_offset_of(ThreadLocalAllocBuffer, _slow_allocations ); }
183
184 void verify();
185 };
186
187 class GlobalTLABStats: public CHeapObj<mtThread> {
188 private:
|
44 HeapWord* _start; // address of TLAB
45 HeapWord* _top; // address after last allocation
46 HeapWord* _pf_top; // allocation prefetch watermark
47 HeapWord* _end; // allocation end (excluding alignment_reserve)
48 size_t _desired_size; // desired size (including alignment_reserve)
49 size_t _refill_waste_limit; // hold onto tlab if free() is larger than this
50 size_t _allocated_before_last_gc; // total bytes allocated up until the last gc
51
52 static size_t _max_size; // maximum size of any TLAB
53 static int _reserve_for_allocation_prefetch; // Reserve at the end of the TLAB
54 static unsigned _target_refills; // expected number of refills between GCs
55
56 unsigned _number_of_refills;
57 unsigned _fast_refill_waste;
58 unsigned _slow_refill_waste;
59 unsigned _gc_waste;
60 unsigned _slow_allocations;
61
62 AdaptiveWeightedAverage _allocation_fraction; // fraction of eden allocated in tlabs
63
64 void accumulate_statistics();
65 void initialize_statistics();
66
67 void set_start(HeapWord* start) { _start = start; }
68 void set_end(HeapWord* end) { _end = end; }
69 void set_top(HeapWord* top) { _top = top; }
70 void set_pf_top(HeapWord* pf_top) { _pf_top = pf_top; }
71 void set_desired_size(size_t desired_size) { _desired_size = desired_size; }
72 void set_refill_waste_limit(size_t waste) { _refill_waste_limit = waste; }
73
74 size_t initial_refill_waste_limit() { return desired_size() / TLABRefillWasteFraction; }
75
76 static int target_refills() { return _target_refills; }
77 size_t initial_desired_size();
78
79 size_t remaining() const { return end() == NULL ? 0 : pointer_delta(hard_end(), top()); }
80
81 // Make parsable and release it.
82 void reset();
83
84 // Resize based on amount of allocation, etc.
85 void resize();
86
87 void invariants() const { assert(top() >= start() && top() <= end(), "invalid tlab"); }
88
89 void initialize(HeapWord* start, HeapWord* top, HeapWord* end);
90
91 void print_stats(const char* tag);
92
93 Thread* myThread();
94
95 // statistics
96
97 int number_of_refills() const { return _number_of_refills; }
98 int fast_refill_waste() const { return _fast_refill_waste; }
99 int slow_refill_waste() const { return _slow_refill_waste; }
100 int gc_waste() const { return _gc_waste; }
101 int slow_allocations() const { return _slow_allocations; }
102
103 static GlobalTLABStats* _global_stats;
104 static GlobalTLABStats* global_stats() { return _global_stats; }
105
106 public:
107 ThreadLocalAllocBuffer() : _allocation_fraction(TLABAllocationWeight), _allocated_before_last_gc(0) {
108 // do nothing. tlabs must be inited by initialize() calls
109 }
110
111 static size_t min_size() { return align_object_size(MinTLABSize / HeapWordSize) + alignment_reserve(); }
112 static size_t max_size() { assert(_max_size != 0, "max_size not set up"); return _max_size; }
113 static size_t max_size_in_bytes() { return max_size() * BytesPerWord; }
114 static void set_max_size(size_t max_size) { _max_size = max_size; }
115
116 HeapWord* start() const { return _start; }
117 HeapWord* end() const { return _end; }
118 HeapWord* hard_end() const { return _end + alignment_reserve(); }
119 HeapWord* top() const { return _top; }
120 HeapWord* pf_top() const { return _pf_top; }
121 size_t desired_size() const { return _desired_size; }
122 size_t used() const { return pointer_delta(top(), start()); }
123 size_t used_bytes() const { return pointer_delta(top(), start(), 1); }
124 size_t free() const { return pointer_delta(end(), top()); }
125 // Don't discard tlab if remaining space is larger than this.
126 size_t refill_waste_limit() const { return _refill_waste_limit; }
127
128 // Allocate size HeapWords. The memory is NOT initialized to zero.
129 inline HeapWord* allocate(size_t size);
130
131 // Reserve space at the end of TLAB
132 static size_t end_reserve() {
133 int reserve_size = typeArrayOopDesc::header_size(T_INT);
134 return MAX2(reserve_size, _reserve_for_allocation_prefetch);
135 }
136 static size_t alignment_reserve() { return align_object_size(end_reserve()); }
137 static size_t alignment_reserve_in_bytes() { return alignment_reserve() * HeapWordSize; }
138
139 // Return tlab size or remaining space in eden such that the
140 // space is large enough to hold obj_size and necessary fill space.
141 // Otherwise return 0;
142 inline size_t compute_size(size_t obj_size);
143
144 // Record slow allocation
145 inline void record_slow_allocation(size_t obj_size);
146
147 // Initialization at startup
148 static void startup_initialization();
149
150 // Make an in-use tlab parsable, optionally retiring and/or zapping it.
151 void make_parsable(bool retire, bool zap = true);
152
153 // Retire in-use tlab before allocation of a new tlab
154 void clear_before_allocation();
155
156 // Accumulate statistics across all tlabs before gc
157 static void accumulate_statistics_before_gc();
158
159 // Resize tlabs for all threads
160 static void resize_all_tlabs();
161
162 void fill(HeapWord* start, HeapWord* top, size_t new_size);
163 void initialize();
164
165 static size_t refill_waste_limit_increment() { return TLABWasteIncrement; }
166
167 // Code generation support
168 static ByteSize start_offset() { return byte_offset_of(ThreadLocalAllocBuffer, _start); }
169 static ByteSize end_offset() { return byte_offset_of(ThreadLocalAllocBuffer, _end ); }
170 static ByteSize top_offset() { return byte_offset_of(ThreadLocalAllocBuffer, _top ); }
171 static ByteSize pf_top_offset() { return byte_offset_of(ThreadLocalAllocBuffer, _pf_top ); }
172 static ByteSize size_offset() { return byte_offset_of(ThreadLocalAllocBuffer, _desired_size ); }
173 static ByteSize refill_waste_limit_offset() { return byte_offset_of(ThreadLocalAllocBuffer, _refill_waste_limit ); }
174
175 static ByteSize number_of_refills_offset() { return byte_offset_of(ThreadLocalAllocBuffer, _number_of_refills ); }
176 static ByteSize fast_refill_waste_offset() { return byte_offset_of(ThreadLocalAllocBuffer, _fast_refill_waste ); }
177 static ByteSize slow_allocations_offset() { return byte_offset_of(ThreadLocalAllocBuffer, _slow_allocations ); }
178
179 void verify();
180 };
181
182 class GlobalTLABStats: public CHeapObj<mtThread> {
183 private:
|