133 }
134 }
135
136 size_t BufferNode::Allocator::free_count() const {
137 return Atomic::load(&_free_count);
138 }
139
140 BufferNode* BufferNode::Allocator::allocate() {
141 BufferNode* node;
142 {
143 // Protect against ABA; see release().
144 GlobalCounter::CriticalSection cs(Thread::current());
145 node = _free_list.pop();
146 }
147 if (node == NULL) {
148 node = BufferNode::allocate(_buffer_size);
149 } else {
150 // Decrement count after getting buffer from free list. This, along
151 // with incrementing count before adding to free list, ensures count
152 // never underflows.
153 size_t count = Atomic::sub(1u, &_free_count);
154 assert((count + 1) != 0, "_free_count underflow");
155 }
156 return node;
157 }
158
159 // To solve the ABA problem for lock-free stack pop, allocate does the
160 // pop inside a critical section, and release synchronizes on the
161 // critical sections before adding to the _free_list. But we don't
162 // want to make every release have to do a synchronize. Instead, we
163 // initially place released nodes on the _pending_list, and transfer
164 // them to the _free_list in batches. Only one transfer at a time is
165 // permitted, with a lock bit to control access to that phase. A
166 // transfer takes all the nodes from the _pending_list, synchronizes on
167 // the _free_list pops, and then adds the former pending nodes to the
168 // _free_list. While that's happening, other threads might be adding
169 // other nodes to the _pending_list, to be dealt with by some later
170 // transfer.
171 void BufferNode::Allocator::release(BufferNode* node) {
172 assert(node != NULL, "precondition");
173 assert(node->next() == NULL, "precondition");
174
175 // Desired minimum transfer batch size. There is relatively little
176 // importance to the specific number. It shouldn't be too big, else
177 // we're wasting space when the release rate is low. If the release
178 // rate is high, we might accumulate more than this before being
179 // able to start a new transfer, but that's okay. Also note that
180 // the allocation rate and the release rate are going to be fairly
181 // similar, due to how the buffers are used.
182 const size_t trigger_transfer = 10;
183
184 // Add to pending list. Update count first so no underflow in transfer.
185 size_t pending_count = Atomic::add(1u, &_pending_count);
186 _pending_list.push(*node);
187 if (pending_count > trigger_transfer) {
188 try_transfer_pending();
189 }
190 }
191
192 // Try to transfer nodes from _pending_list to _free_list, with a
193 // synchronization delay for any in-progress pops from the _free_list,
194 // to solve ABA there. Return true if performed a (possibly empty)
195 // transfer, false if blocked from doing so by some other thread's
196 // in-progress transfer.
197 bool BufferNode::Allocator::try_transfer_pending() {
198 // Attempt to claim the lock.
199 if (Atomic::load(&_transfer_lock) || // Skip CAS if likely to fail.
200 Atomic::cmpxchg(true, &_transfer_lock, false)) {
201 return false;
202 }
203 // Have the lock; perform the transfer.
204
205 // Claim all the pending nodes.
206 BufferNode* first = _pending_list.pop_all();
207 if (first != NULL) {
208 // Prepare to add the claimed nodes, and update _pending_count.
209 BufferNode* last = first;
210 size_t count = 1;
211 for (BufferNode* next = first->next(); next != NULL; next = next->next()) {
212 last = next;
213 ++count;
214 }
215 Atomic::sub(count, &_pending_count);
216
217 // Wait for any in-progress pops, to avoid ABA for them.
218 GlobalCounter::write_synchronize();
219
220 // Add synchronized nodes to _free_list.
221 // Update count first so no underflow in allocate().
222 Atomic::add(count, &_free_count);
223 _free_list.prepend(*first, *last);
224 log_trace(gc, ptrqueue, freelist)
225 ("Transferred %s pending to free: " SIZE_FORMAT, name(), count);
226 }
227 Atomic::release_store(&_transfer_lock, false);
228 return true;
229 }
230
231 size_t BufferNode::Allocator::reduce_free_list(size_t remove_goal) {
232 try_transfer_pending();
233 size_t removed = 0;
234 for ( ; removed < remove_goal; ++removed) {
235 BufferNode* node = _free_list.pop();
236 if (node == NULL) break;
237 BufferNode::deallocate(node);
238 }
239 size_t new_count = Atomic::sub(removed, &_free_count);
240 log_debug(gc, ptrqueue, freelist)
241 ("Reduced %s free list by " SIZE_FORMAT " to " SIZE_FORMAT,
242 name(), removed, new_count);
243 return removed;
244 }
245
246 PtrQueueSet::PtrQueueSet(BufferNode::Allocator* allocator) :
247 _allocator(allocator),
248 _all_active(false)
249 {}
250
251 PtrQueueSet::~PtrQueueSet() {}
252
253 void** PtrQueueSet::allocate_buffer() {
254 BufferNode* node = _allocator->allocate();
255 return BufferNode::make_buffer_from_node(node);
256 }
257
258 void PtrQueueSet::deallocate_buffer(BufferNode* node) {
259 _allocator->release(node);
260 }
261
|
133 }
134 }
135
136 size_t BufferNode::Allocator::free_count() const {
137 return Atomic::load(&_free_count);
138 }
139
140 BufferNode* BufferNode::Allocator::allocate() {
141 BufferNode* node;
142 {
143 // Protect against ABA; see release().
144 GlobalCounter::CriticalSection cs(Thread::current());
145 node = _free_list.pop();
146 }
147 if (node == NULL) {
148 node = BufferNode::allocate(_buffer_size);
149 } else {
150 // Decrement count after getting buffer from free list. This, along
151 // with incrementing count before adding to free list, ensures count
152 // never underflows.
153 size_t count = Atomic::sub(&_free_count, 1u);
154 assert((count + 1) != 0, "_free_count underflow");
155 }
156 return node;
157 }
158
159 // To solve the ABA problem for lock-free stack pop, allocate does the
160 // pop inside a critical section, and release synchronizes on the
161 // critical sections before adding to the _free_list. But we don't
162 // want to make every release have to do a synchronize. Instead, we
163 // initially place released nodes on the _pending_list, and transfer
164 // them to the _free_list in batches. Only one transfer at a time is
165 // permitted, with a lock bit to control access to that phase. A
166 // transfer takes all the nodes from the _pending_list, synchronizes on
167 // the _free_list pops, and then adds the former pending nodes to the
168 // _free_list. While that's happening, other threads might be adding
169 // other nodes to the _pending_list, to be dealt with by some later
170 // transfer.
171 void BufferNode::Allocator::release(BufferNode* node) {
172 assert(node != NULL, "precondition");
173 assert(node->next() == NULL, "precondition");
174
175 // Desired minimum transfer batch size. There is relatively little
176 // importance to the specific number. It shouldn't be too big, else
177 // we're wasting space when the release rate is low. If the release
178 // rate is high, we might accumulate more than this before being
179 // able to start a new transfer, but that's okay. Also note that
180 // the allocation rate and the release rate are going to be fairly
181 // similar, due to how the buffers are used.
182 const size_t trigger_transfer = 10;
183
184 // Add to pending list. Update count first so no underflow in transfer.
185 size_t pending_count = Atomic::add(&_pending_count, 1u);
186 _pending_list.push(*node);
187 if (pending_count > trigger_transfer) {
188 try_transfer_pending();
189 }
190 }
191
192 // Try to transfer nodes from _pending_list to _free_list, with a
193 // synchronization delay for any in-progress pops from the _free_list,
194 // to solve ABA there. Return true if performed a (possibly empty)
195 // transfer, false if blocked from doing so by some other thread's
196 // in-progress transfer.
197 bool BufferNode::Allocator::try_transfer_pending() {
198 // Attempt to claim the lock.
199 if (Atomic::load(&_transfer_lock) || // Skip CAS if likely to fail.
200 Atomic::cmpxchg(&_transfer_lock, false, true)) {
201 return false;
202 }
203 // Have the lock; perform the transfer.
204
205 // Claim all the pending nodes.
206 BufferNode* first = _pending_list.pop_all();
207 if (first != NULL) {
208 // Prepare to add the claimed nodes, and update _pending_count.
209 BufferNode* last = first;
210 size_t count = 1;
211 for (BufferNode* next = first->next(); next != NULL; next = next->next()) {
212 last = next;
213 ++count;
214 }
215 Atomic::sub(&_pending_count, count);
216
217 // Wait for any in-progress pops, to avoid ABA for them.
218 GlobalCounter::write_synchronize();
219
220 // Add synchronized nodes to _free_list.
221 // Update count first so no underflow in allocate().
222 Atomic::add(&_free_count, count);
223 _free_list.prepend(*first, *last);
224 log_trace(gc, ptrqueue, freelist)
225 ("Transferred %s pending to free: " SIZE_FORMAT, name(), count);
226 }
227 Atomic::release_store(&_transfer_lock, false);
228 return true;
229 }
230
231 size_t BufferNode::Allocator::reduce_free_list(size_t remove_goal) {
232 try_transfer_pending();
233 size_t removed = 0;
234 for ( ; removed < remove_goal; ++removed) {
235 BufferNode* node = _free_list.pop();
236 if (node == NULL) break;
237 BufferNode::deallocate(node);
238 }
239 size_t new_count = Atomic::sub(&_free_count, removed);
240 log_debug(gc, ptrqueue, freelist)
241 ("Reduced %s free list by " SIZE_FORMAT " to " SIZE_FORMAT,
242 name(), removed, new_count);
243 return removed;
244 }
245
246 PtrQueueSet::PtrQueueSet(BufferNode::Allocator* allocator) :
247 _allocator(allocator),
248 _all_active(false)
249 {}
250
251 PtrQueueSet::~PtrQueueSet() {}
252
253 void** PtrQueueSet::allocate_buffer() {
254 BufferNode* node = _allocator->allocate();
255 return BufferNode::make_buffer_from_node(node);
256 }
257
258 void PtrQueueSet::deallocate_buffer(BufferNode* node) {
259 _allocator->release(node);
260 }
|