1 /*
2 * Copyright (c) 2020, Oracle and/or its affiliates. All rights reserved.
3 * Copyright (c) 2020 SAP SE. All rights reserved.
4 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
5 *
6 * This code is free software; you can redistribute it and/or modify it
7 * under the terms of the GNU General Public License version 2 only, as
8 * published by the Free Software Foundation.
9 *
10 * This code is distributed in the hope that it will be useful, but WITHOUT
11 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
13 * version 2 for more details (a copy is included in the LICENSE file that
14 * accompanied this code).
15 *
16 * You should have received a copy of the GNU General Public License version
17 * 2 along with this work; if not, write to the Free Software Foundation,
18 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
19 *
20 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
21 * or visit www.oracle.com if you need additional information or have any
22 * questions.
23 *
24 */
25
26 #include "precompiled.hpp"
27
28 #include "logging/log.hpp"
29 #include "logging/logStream.hpp"
30 #include "memory/metaspace/allocationGuard.hpp"
31 #include "memory/metaspace/arenaGrowthPolicy.hpp"
32 #include "memory/metaspace/freeBlocks.hpp"
33 #include "memory/metaspace/chunkManager.hpp"
34 #include "memory/metaspace/internStat.hpp"
35 #include "memory/metaspace/metachunk.hpp"
36 #include "memory/metaspace/metaspaceArena.hpp"
37 #include "memory/metaspace/metaspaceCommon.hpp"
38 #include "memory/metaspace/metaspaceStatistics.hpp"
39 #include "memory/metaspace/virtualSpaceList.hpp"
40 #include "runtime/atomic.hpp"
41 #include "runtime/init.hpp"
42 #include "services/memoryService.hpp"
43 #include "utilities/align.hpp"
44 #include "utilities/debug.hpp"
45 #include "utilities/globalDefinitions.hpp"
46
47 namespace metaspace {
48
49 #define LOGFMT "Arena @" PTR_FORMAT " (%s)"
50 #define LOGFMT_ARGS p2i(this), this->_name
51
52 // Given a net allocation word size, return the raw word size we actually allocate.
53 // Note: externally visible for gtests.
54 //static
55 size_t get_raw_allocation_word_size(size_t net_word_size) {
56
57 size_t byte_size = net_word_size * BytesPerWord;
58
59 // Deallocated metablocks are kept in a binlist which limits their minimal
60 // size to at least the size of a binlist item (2 words).
61 byte_size = MAX2(byte_size, FreeBlocks::minimal_word_size * BytesPerWord);
62
63 // Metaspace allocations are aligned to word size.
64 byte_size = align_up(byte_size, allocation_alignment_bytes);
65
66 // If we guard allocations, we need additional space for a prefix.
67 #ifdef ASSERT
68 if (Settings::use_allocation_guard()) {
69 byte_size += align_up(prefix_size(), allocation_alignment_bytes);
70 }
71 #endif
72
73 size_t word_size = byte_size / BytesPerWord;
74
75 assert(word_size * BytesPerWord == byte_size, "Sanity");
76
77 return word_size;
78
79 }
80
81 // Returns the level of the next chunk to be added, acc to growth policy.
82 chunklevel_t MetaspaceArena::next_chunk_level() const {
83 const int growth_step = _chunks.count();
84 return _growth_policy->get_level_at_step(growth_step);
85 }
86
87 // Given a chunk, add its remaining free committed space to the free block list.
88 void MetaspaceArena::salvage_chunk(Metachunk* c) {
89
90 if (Settings::handle_deallocations() == false) {
91 return;
92 }
93
94 assert_lock_strong(lock());
95
96 // If the chunk is completely empty, just return it to the chunk manager.
97 if (c->used_words() == 0) {
98 UL2(trace, "salvage: returning empty chunk " METACHUNK_FORMAT ".", METACHUNK_FORMAT_ARGS(c));
99 _chunk_manager->return_chunk(c);
100 return;
101 }
102
103 size_t remaining_words = c->free_below_committed_words();
104
105 if (remaining_words > FreeBlocks::minimal_word_size) {
106
107 UL2(trace, "salvaging chunk " METACHUNK_FULL_FORMAT ".", METACHUNK_FULL_FORMAT_ARGS(c));
108
109 MetaWord* ptr = c->allocate(remaining_words);
110 assert(ptr != NULL, "Should have worked");
111 _total_used_words_counter->increment_by(remaining_words);
112
113 add_allocation_to_fbl(ptr, remaining_words);
114
115 // After this operation: the chunk should have no free committed space left.
116 assert(c->free_below_committed_words() == 0,
117 "Salvaging chunk failed (chunk " METACHUNK_FULL_FORMAT ").",
118 METACHUNK_FULL_FORMAT_ARGS(c));
119
120 }
121
122 }
123
124 // Allocate a new chunk from the underlying chunk manager able to hold at least
125 // requested word size.
126 Metachunk* MetaspaceArena::allocate_new_chunk(size_t requested_word_size) {
127
128 assert_lock_strong(lock());
129
130 // Should this ever happen, we need to increase the maximum possible chunk size.
131 guarantee(requested_word_size <= chunklevel::MAX_CHUNK_WORD_SIZE,
132 "Requested size too large (" SIZE_FORMAT ") - max allowed size per allocation is " SIZE_FORMAT ".",
133 requested_word_size, chunklevel::MAX_CHUNK_WORD_SIZE);
134
135 const int growth_step = _chunks.count();
136 const chunklevel_t max_level = chunklevel::level_fitting_word_size(requested_word_size);
137 const chunklevel_t preferred_level = MIN2(max_level, next_chunk_level());
138
139 Metachunk* c = _chunk_manager->get_chunk(preferred_level, max_level, requested_word_size);
140 if (c == NULL) {
141 return NULL;
142 }
143
144 assert(c->is_in_use(), "Wrong chunk state.");
145 assert(c->free_below_committed_words() >= requested_word_size, "Chunk not committed");
146
147 return c;
148
149 }
150
151 void MetaspaceArena::add_allocation_to_fbl(MetaWord* p, size_t word_size) {
152 assert(Settings::handle_deallocations(), "Sanity");
153 if (_fbl == NULL) {
154 _fbl = new FreeBlocks(); // Create only on demand
155 }
156 _fbl->add_block(p, word_size);
157 }
158
159 MetaspaceArena::MetaspaceArena(ChunkManager* chunk_manager,
160 const ArenaGrowthPolicy* growth_policy,
161 Mutex* lock,
162 SizeAtomicCounter* total_used_words_counter,
163 const char* name)
164 : _lock(lock),
165 _chunk_manager(chunk_manager),
166 _growth_policy(growth_policy),
167 _chunks(),
168 _fbl(NULL),
169 _total_used_words_counter(total_used_words_counter),
170 _name(name)
171 {
172 UL(debug, ": born.");
173
174 // Update statistics
175 InternalStats::inc_num_arena_births();
176 }
177
178 MetaspaceArena::~MetaspaceArena() {
179
180 DEBUG_ONLY(verify(true);)
181
182 MutexLocker fcl(lock(), Mutex::_no_safepoint_check_flag);
183
184 MemRangeCounter return_counter;
185
186 Metachunk* c = _chunks.first();
187 Metachunk* c2 = NULL;
188
189 while(c) {
190 c2 = c->next();
191 return_counter.add(c->used_words());
192 DEBUG_ONLY(c->set_prev(NULL);)
193 DEBUG_ONLY(c->set_next(NULL);)
194 UL2(debug, "return chunk: " METACHUNK_FORMAT ".", METACHUNK_FORMAT_ARGS(c));
195 _chunk_manager->return_chunk(c);
196 // c may be invalid after return_chunk(c) was called. Don't access anymore.
197 c = c2;
198 }
199
200 UL2(info, "returned %d chunks, total capacity " SIZE_FORMAT " words.",
201 return_counter.count(), return_counter.total_size());
202
203 _total_used_words_counter->decrement_by(return_counter.total_size());
204
205 DEBUG_ONLY(chunk_manager()->verify(true);)
206
207 delete _fbl;
208
209 UL(debug, ": dies.");
210
211 // Update statistics
212 InternalStats::inc_num_arena_deaths();
213
214 }
215
216 // Attempt to enlarge the current chunk to make it large enough to hold at least
217 // requested_word_size additional words.
218 //
219 // On success, true is returned, false otherwise.
220 bool MetaspaceArena::attempt_enlarge_current_chunk(size_t requested_word_size) {
221
222 assert_lock_strong(lock());
223
224 Metachunk* c = current_chunk();
225 assert(c->free_words() < requested_word_size, "Sanity");
226
227 // Not if chunk enlargment is switched off...
228 if (Settings::enlarge_chunks_in_place() == false) {
229 return false;
230 }
231
232 // ... we also disallow it for very large chunks...
233 if (c->word_size() > Settings::enlarge_chunks_in_place_max_word_size()) {
234 return false;
235 }
236
237 // ... nor if we are already a root chunk ...
238 if (c->is_root_chunk()) {
239 return false;
240 }
241
242 // ... nor if the combined size of chunk content and new content would bring us above the size of a root chunk ...
243 if ((c->used_words() + requested_word_size) > metaspace::chunklevel::MAX_CHUNK_WORD_SIZE) {
244 return false;
245 }
246
247 const chunklevel_t new_level =
248 chunklevel::level_fitting_word_size(c->used_words() + requested_word_size);
249 assert(new_level < c->level(), "Sanity");
250
251 // Atm we only enlarge by one level (so, doubling the chunk in size). So, if the requested enlargement
252 // would require the chunk to more than double in size, we bail. But this covers about 99% of all cases,
253 // so this is good enough.
254 if (new_level < c->level() - 1) {
255 return false;
256 }
257
258 // This only works if chunk is the leader of its buddy pair (and also if buddy
259 // is free and unsplit, but that we cannot check outside of metaspace lock).
260 if (!c->is_leader()) {
261 return false;
262 }
263
264 // If the size added to the chunk would be larger than allowed for the next growth step
265 // dont enlarge.
266 if (next_chunk_level() > c->level()) {
267 return false;
268 }
269
270 bool success = _chunk_manager->attempt_enlarge_chunk(c);
271
272 assert(success == false || c->free_words() >= requested_word_size, "Sanity");
273
274 return success;
275
276 }
277
278 // Allocate memory from Metaspace.
279 // 1) Attempt to allocate from the free block list.
280 // 2) Attempt to allocate from the current chunk.
281 // 3) Attempt to enlarge the current chunk in place if it is too small.
282 // 4) Attempt to get a new chunk and allocate from that chunk.
283 // At any point, if we hit a commit limit, we return NULL.
284 MetaWord* MetaspaceArena::allocate(size_t requested_word_size) {
285
286 MutexLocker cl(lock(), Mutex::_no_safepoint_check_flag);
287
288 UL2(trace, "requested " SIZE_FORMAT " words.", requested_word_size);
289
290 MetaWord* p = NULL;
291
292 const size_t raw_word_size = get_raw_word_size_for_requested_word_size(requested_word_size);
293
294 // 1) Attempt to allocate from the free blocks list
295 if (Settings::handle_deallocations() && _fbl != NULL && !_fbl->is_empty()) {
296 p = _fbl->get_block(raw_word_size);
297 if (p != NULL) {
298 DEBUG_ONLY(InternalStats::inc_num_allocs_from_deallocated_blocks();)
299 UL2(trace, "taken from fbl (now: %d, " SIZE_FORMAT ").",
300 _fbl->count(), _fbl->total_size());
301 // Note: Space in the freeblock dictionary counts as already used (see retire_current_chunk()) -
302 // that means that we do not modify any counters and therefore can skip the epilog.
303 return p;
304 }
305 }
306
307 bool current_chunk_too_small = false;
308 bool commit_failure = false;
309
310 if (current_chunk() != NULL) {
311
312 // 2) Attempt to satisfy the allocation from the current chunk.
313
314 // If the current chunk is too small to hold the requested size, attempt to enlarge it.
315 // If that fails, retire the chunk.
316 if (current_chunk()->free_words() < raw_word_size) {
317 if (!attempt_enlarge_current_chunk(raw_word_size)) {
318 current_chunk_too_small = true;
319 } else {
320 DEBUG_ONLY(InternalStats::inc_num_chunks_enlarged();)
321 UL(debug, "enlarged chunk.");
322 }
323 }
324
325 // Commit the chunk far enough to hold the requested word size. If that fails, we
326 // hit a limit (either GC threshold or MaxMetaspaceSize). In that case retire the
327 // chunk.
328 if (!current_chunk_too_small) {
329 if (!current_chunk()->ensure_committed_additional(raw_word_size)) {
330 UL2(info, "commit failure (requested size: " SIZE_FORMAT ")", raw_word_size);
331 commit_failure = true;
332 }
333 }
334
335 // Allocate from the current chunk. This should work now.
336 if (!current_chunk_too_small && !commit_failure) {
337 p = current_chunk()->allocate(raw_word_size);
338 assert(p != NULL, "Allocation from chunk failed.");
339 }
340
341 }
342
343 if (p == NULL) {
344
345 // If we are here, we either had no current chunk to begin with or it was deemed insufficient.
346 assert(current_chunk() == NULL ||
347 current_chunk_too_small || commit_failure, "Sanity");
348
349 Metachunk* new_chunk = allocate_new_chunk(raw_word_size);
350
351 if (new_chunk != NULL) {
352
353 UL2(debug, "allocated new chunk " METACHUNK_FORMAT " for requested word size " SIZE_FORMAT ".",
354 METACHUNK_FORMAT_ARGS(new_chunk), requested_word_size);
355
356 assert(new_chunk->free_below_committed_words() >= raw_word_size, "Sanity");
357
358 // We have a new chunk. Before making it the current chunk, retire the old one.
359 if (current_chunk() != NULL) {
360 salvage_chunk(current_chunk());
361 DEBUG_ONLY(InternalStats::inc_num_chunks_retired();)
362 }
363
364 _chunks.add(new_chunk);
365
366 // Now, allocate from that chunk. That should work.
367 p = current_chunk()->allocate(raw_word_size);
368 assert(p != NULL, "Allocation from chunk failed.");
369
370 } else {
371 UL2(info, "failed to allocate new chunk for requested word size " SIZE_FORMAT ".", requested_word_size);
372 }
373
374 }
375
376 #ifdef ASSERT
377 // When using allocation guards, establish a prefix.
378 if (p != NULL && Settings::use_allocation_guard()) {
379 p = establish_prefix(p, raw_word_size);
380 }
381 #endif
382
383 if (p == NULL) {
384 InternalStats::inc_num_allocs_failed_limit();
385 } else {
386 DEBUG_ONLY(InternalStats::inc_num_allocs();)
387 _total_used_words_counter->increment_by(raw_word_size);
388 }
389
390 SOMETIMES(verify_locked(true);)
391
392 if (p == NULL) {
393 UL(info, "allocation failed, returned NULL.");
394 } else {
395 UL2(trace, "returned " PTR_FORMAT ".", p2i(p));
396 }
397
398 return p;
399
400 }
401
402 // Prematurely returns a metaspace allocation to the _block_freelists
403 // because it is not needed anymore (requires CLD lock to be active).
404 void MetaspaceArena::deallocate_locked(MetaWord* p, size_t word_size) {
405
406 if (Settings::handle_deallocations() == false) {
407 return;
408 }
409
410 assert_lock_strong(lock());
411
412 // At this point a current chunk must exist since we only deallocate if we did allocate before.
413 assert(current_chunk() != NULL, "stray deallocation?");
414
415 assert(is_valid_area(p, word_size),
416 "Pointer range not part of this Arena and cannot be deallocated: (" PTR_FORMAT ".." PTR_FORMAT ").",
417 p2i(p), p2i(p + word_size));
418
419 UL2(trace, "deallocating " PTR_FORMAT ", word size: " SIZE_FORMAT ".",
420 p2i(p), word_size);
421
422 size_t raw_word_size = get_raw_word_size_for_requested_word_size(word_size);
423 add_allocation_to_fbl(p, raw_word_size);
424
425 DEBUG_ONLY(verify_locked(false);)
426
427 }
428
429 // Prematurely returns a metaspace allocation to the _block_freelists because it is not
430 // needed anymore.
431 void MetaspaceArena::deallocate(MetaWord* p, size_t word_size) {
432 MutexLocker cl(lock(), Mutex::_no_safepoint_check_flag);
433 deallocate_locked(p, word_size);
434 }
435
436 // Update statistics. This walks all in-use chunks.
437 void MetaspaceArena::add_to_statistics(arena_stats_t* out) const {
438
439 MutexLocker cl(lock(), Mutex::_no_safepoint_check_flag);
440
441 for (const Metachunk* c = _chunks.first(); c != NULL; c = c->next()) {
442 in_use_chunk_stats_t& ucs = out->stats[c->level()];
443 ucs.num ++;
444 ucs.word_size += c->word_size();
445 ucs.committed_words += c->committed_words();
446 ucs.used_words += c->used_words();
447 // Note: for free and waste, we only count what's committed.
448 if (c == current_chunk()) {
449 ucs.free_words += c->free_below_committed_words();
450 } else {
451 ucs.waste_words += c->free_below_committed_words();
452 }
453 }
454
455 if (_fbl != NULL) {
456 out->free_blocks_num += _fbl->count();
457 out->free_blocks_word_size += _fbl->total_size();
458 }
459
460 SOMETIMES(out->verify();)
461
462 }
463
464 // Convenience method to get the most important usage statistics.
465 // For deeper analysis use add_to_statistics().
466 void MetaspaceArena::usage_numbers(size_t* p_used_words, size_t* p_committed_words, size_t* p_capacity_words) const {
467 MutexLocker cl(lock(), Mutex::_no_safepoint_check_flag);
468 size_t used = 0, comm = 0, cap = 0;
469 for (const Metachunk* c = _chunks.first(); c != NULL; c = c->next()) {
470 used += c->used_words();
471 comm += c->committed_words();
472 cap += c->word_size();
473 }
474 if (p_used_words != NULL) {
475 *p_used_words = used;
476 }
477 if (p_committed_words != NULL) {
478 *p_committed_words = comm;
479 }
480 if (p_capacity_words != NULL) {
481 *p_capacity_words = cap;
482 }
483 }
484
485
486 #ifdef ASSERT
487
488 void MetaspaceArena::verify_locked(bool slow) const {
489
490 assert_lock_strong(lock());
491
492 assert(_growth_policy != NULL && _chunk_manager != NULL, "Sanity");
493
494 _chunks.verify();
495
496 if (_fbl != NULL) {
497 _fbl->verify();
498 }
499
500 // In slow mode, verify guard zones of all allocations
501 if (slow && Settings::use_allocation_guard()) {
502 for (const Metachunk* c = _chunks.first(); c != NULL; c = c->next()) {
503 const MetaWord* p = c->base();
504 while (p < c->top()) {
505 const prefix_t* pp = (const prefix_t*)p;
506 check_prefix(pp);
507 p += pp->word_size;
508 }
509 }
510 }
511
512 }
513
514 void MetaspaceArena::verify(bool slow) const {
515
516 MutexLocker cl(lock(), Mutex::_no_safepoint_check_flag);
517 verify_locked(slow);
518
519 }
520
521 // Returns true if the area indicated by pointer and size have actually been allocated
522 // from this arena.
523 bool MetaspaceArena::is_valid_area(MetaWord* p, size_t word_size) const {
524 assert(p != NULL && word_size > 0, "Sanity");
525 bool found = false;
526 if (!found) {
527 for (const Metachunk* c = _chunks.first(); c != NULL && !found; c = c->next()) {
528 assert(c->is_valid_committed_pointer(p) ==
529 c->is_valid_committed_pointer(p + word_size - 1), "range intersects");
530 found = c->is_valid_committed_pointer(p);
531 }
532 }
533 return found;
534 }
535
536 #endif // ASSERT
537
538 void MetaspaceArena::print_on(outputStream* st) const {
539 MutexLocker fcl(_lock, Mutex::_no_safepoint_check_flag);
540 print_on_locked(st);
541 }
542
543 void MetaspaceArena::print_on_locked(outputStream* st) const {
544 assert_lock_strong(_lock);
545 st->print_cr("sm %s: %d chunks, total word size: " SIZE_FORMAT ", committed word size: " SIZE_FORMAT, _name,
546 _chunks.count(), _chunks.calc_word_size(), _chunks.calc_committed_word_size());
547 _chunks.print_on(st);
548 st->cr();
549 st->print_cr("growth-policy " PTR_FORMAT ", lock " PTR_FORMAT ", cm " PTR_FORMAT ", fbl " PTR_FORMAT,
550 p2i(_growth_policy), p2i(_lock), p2i(_chunk_manager), p2i(_fbl));
551 }
552
553
554
555 } // namespace metaspace
556