46 ZObjectAllocator::ZObjectAllocator() :
47 _use_per_cpu_shared_small_pages(ZHeuristics::use_per_cpu_shared_small_pages()),
48 _used(0),
49 _undone(0),
50 _shared_medium_page(NULL),
51 _shared_small_page(NULL),
52 _worker_small_page(NULL) {}
53
54 ZPage** ZObjectAllocator::shared_small_page_addr() {
55 return _use_per_cpu_shared_small_pages ? _shared_small_page.addr() : _shared_small_page.addr(0);
56 }
57
58 ZPage* const* ZObjectAllocator::shared_small_page_addr() const {
59 return _use_per_cpu_shared_small_pages ? _shared_small_page.addr() : _shared_small_page.addr(0);
60 }
61
62 ZPage* ZObjectAllocator::alloc_page(uint8_t type, size_t size, ZAllocationFlags flags) {
63 ZPage* const page = ZHeap::heap()->alloc_page(type, size, flags);
64 if (page != NULL) {
65 // Increment used bytes
66 Atomic::add(size, _used.addr());
67 }
68
69 return page;
70 }
71
72 void ZObjectAllocator::undo_alloc_page(ZPage* page) {
73 // Increment undone bytes
74 Atomic::add(page->size(), _undone.addr());
75
76 ZHeap::heap()->undo_alloc_page(page);
77 }
78
79 uintptr_t ZObjectAllocator::alloc_object_in_shared_page(ZPage** shared_page,
80 uint8_t page_type,
81 size_t page_size,
82 size_t size,
83 ZAllocationFlags flags) {
84 uintptr_t addr = 0;
85 ZPage* page = Atomic::load_acquire(shared_page);
86
87 if (page != NULL) {
88 addr = page->alloc_object_atomic(size);
89 }
90
91 if (addr == 0) {
92 // Allocate new page
93 ZPage* const new_page = alloc_page(page_type, page_size, flags);
94 if (new_page != NULL) {
95 // Allocate object before installing the new page
96 addr = new_page->alloc_object(size);
97
98 retry:
99 // Install new page
100 ZPage* const prev_page = Atomic::cmpxchg(new_page, shared_page, page);
101 if (prev_page != page) {
102 if (prev_page == NULL) {
103 // Previous page was retired, retry installing the new page
104 page = prev_page;
105 goto retry;
106 }
107
108 // Another page already installed, try allocation there first
109 const uintptr_t prev_addr = prev_page->alloc_object_atomic(size);
110 if (prev_addr == 0) {
111 // Allocation failed, retry installing the new page
112 page = prev_page;
113 goto retry;
114 }
115
116 // Allocation succeeded in already installed page
117 addr = prev_addr;
118
119 // Undo new page allocation
120 undo_alloc_page(new_page);
|
46 ZObjectAllocator::ZObjectAllocator() :
47 _use_per_cpu_shared_small_pages(ZHeuristics::use_per_cpu_shared_small_pages()),
48 _used(0),
49 _undone(0),
50 _shared_medium_page(NULL),
51 _shared_small_page(NULL),
52 _worker_small_page(NULL) {}
53
54 ZPage** ZObjectAllocator::shared_small_page_addr() {
55 return _use_per_cpu_shared_small_pages ? _shared_small_page.addr() : _shared_small_page.addr(0);
56 }
57
58 ZPage* const* ZObjectAllocator::shared_small_page_addr() const {
59 return _use_per_cpu_shared_small_pages ? _shared_small_page.addr() : _shared_small_page.addr(0);
60 }
61
62 ZPage* ZObjectAllocator::alloc_page(uint8_t type, size_t size, ZAllocationFlags flags) {
63 ZPage* const page = ZHeap::heap()->alloc_page(type, size, flags);
64 if (page != NULL) {
65 // Increment used bytes
66 Atomic::add(_used.addr(), size);
67 }
68
69 return page;
70 }
71
72 void ZObjectAllocator::undo_alloc_page(ZPage* page) {
73 // Increment undone bytes
74 Atomic::add(_undone.addr(), page->size());
75
76 ZHeap::heap()->undo_alloc_page(page);
77 }
78
79 uintptr_t ZObjectAllocator::alloc_object_in_shared_page(ZPage** shared_page,
80 uint8_t page_type,
81 size_t page_size,
82 size_t size,
83 ZAllocationFlags flags) {
84 uintptr_t addr = 0;
85 ZPage* page = Atomic::load_acquire(shared_page);
86
87 if (page != NULL) {
88 addr = page->alloc_object_atomic(size);
89 }
90
91 if (addr == 0) {
92 // Allocate new page
93 ZPage* const new_page = alloc_page(page_type, page_size, flags);
94 if (new_page != NULL) {
95 // Allocate object before installing the new page
96 addr = new_page->alloc_object(size);
97
98 retry:
99 // Install new page
100 ZPage* const prev_page = Atomic::cmpxchg(shared_page, page, new_page);
101 if (prev_page != page) {
102 if (prev_page == NULL) {
103 // Previous page was retired, retry installing the new page
104 page = prev_page;
105 goto retry;
106 }
107
108 // Another page already installed, try allocation there first
109 const uintptr_t prev_addr = prev_page->alloc_object_atomic(size);
110 if (prev_addr == 0) {
111 // Allocation failed, retry installing the new page
112 page = prev_page;
113 goto retry;
114 }
115
116 // Allocation succeeded in already installed page
117 addr = prev_addr;
118
119 // Undo new page allocation
120 undo_alloc_page(new_page);
|