1 /*
2 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
3 *
4 * This code is free software; you can redistribute it and/or modify it
5 * under the terms of the GNU General Public License version 2 only, as
6 * published by the Free Software Foundation. Oracle designates this
7 * particular file as subject to the "Classpath" exception as provided
8 * by Oracle in the LICENSE file that accompanied this code.
9 *
10 * This code is distributed in the hope that it will be useful, but WITHOUT
11 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
13 * version 2 for more details (a copy is included in the LICENSE file that
14 * accompanied this code).
15 *
16 * You should have received a copy of the GNU General Public License version
17 * 2 along with this work; if not, write to the Free Software Foundation,
18 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
19 *
20 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
21 * or visit www.oracle.com if you need additional information or have any
22 * questions.
23 */
24
25 /*
26 * This file is available under and governed by the GNU General Public
27 * License version 2 only, as published by the Free Software Foundation.
28 * However, the following notice accompanied the original version of this
29 * file:
30 *
31 * Written by Doug Lea with assistance from members of JCP JSR-166
32 * Expert Group and released to the public domain, as explained at
33 * http://creativecommons.org/publicdomain/zero/1.0/
34 */
35
36 package java.util.concurrent.atomic;
37
38 import java.lang.invoke.MethodHandles;
39 import java.lang.invoke.VarHandle;
40 import java.util.Arrays;
41 import java.util.concurrent.ThreadLocalRandom;
42 import java.util.function.DoubleBinaryOperator;
43 import java.util.function.LongBinaryOperator;
44
45 /**
46 * A package-local class holding common representation and mechanics
47 * for classes supporting dynamic striping on 64bit values. The class
48 * extends Number so that concrete subclasses must publicly do so.
49 */
50 @SuppressWarnings("serial")
51 abstract class Striped64 extends Number {
52 /*
53 * This class maintains a lazily-initialized table of atomically
54 * updated variables, plus an extra "base" field. The table size
55 * is a power of two. Indexing uses masked per-thread hash codes.
56 * Nearly all declarations in this class are package-private,
57 * accessed directly by subclasses.
58 *
59 * Table entries are of class Cell; a variant of AtomicLong padded
60 * (via @Contended) to reduce cache contention. Padding is
61 * overkill for most Atomics because they are usually irregularly
62 * scattered in memory and thus don't interfere much with each
63 * other. But Atomic objects residing in arrays will tend to be
64 * placed adjacent to each other, and so will most often share
65 * cache lines (with a huge negative performance impact) without
66 * this precaution.
67 *
68 * In part because Cells are relatively large, we avoid creating
69 * them until they are needed. When there is no contention, all
70 * updates are made to the base field. Upon first contention (a
71 * failed CAS on base update), the table is initialized to size 2.
72 * The table size is doubled upon further contention until
73 * reaching the nearest power of two greater than or equal to the
74 * number of CPUS. Table slots remain empty (null) until they are
75 * needed.
76 *
77 * A single spinlock ("cellsBusy") is used for initializing and
78 * resizing the table, as well as populating slots with new Cells.
79 * There is no need for a blocking lock; when the lock is not
80 * available, threads try other slots (or the base). During these
81 * retries, there is increased contention and reduced locality,
82 * which is still better than alternatives.
83 *
84 * The Thread probe fields maintained via ThreadLocalRandom serve
85 * as per-thread hash codes. We let them remain uninitialized as
86 * zero (if they come in this way) until they contend at slot
87 * 0. They are then initialized to values that typically do not
88 * often conflict with others. Contention and/or table collisions
89 * are indicated by failed CASes when performing an update
90 * operation. Upon a collision, if the table size is less than
91 * the capacity, it is doubled in size unless some other thread
92 * holds the lock. If a hashed slot is empty, and lock is
93 * available, a new Cell is created. Otherwise, if the slot
94 * exists, a CAS is tried. Retries proceed by "double hashing",
95 * using a secondary hash (Marsaglia XorShift) to try to find a
96 * free slot.
97 *
98 * The table size is capped because, when there are more threads
99 * than CPUs, supposing that each thread were bound to a CPU,
100 * there would exist a perfect hash function mapping threads to
101 * slots that eliminates collisions. When we reach capacity, we
102 * search for this mapping by randomly varying the hash codes of
103 * colliding threads. Because search is random, and collisions
104 * only become known via CAS failures, convergence can be slow,
105 * and because threads are typically not bound to CPUS forever,
106 * may not occur at all. However, despite these limitations,
107 * observed contention rates are typically low in these cases.
108 *
109 * It is possible for a Cell to become unused when threads that
110 * once hashed to it terminate, as well as in the case where
111 * doubling the table causes no thread to hash to it under
112 * expanded mask. We do not try to detect or remove such cells,
113 * under the assumption that for long-running instances, observed
114 * contention levels will recur, so the cells will eventually be
115 * needed again; and for short-lived ones, it does not matter.
116 */
117
118 /**
119 * Padded variant of AtomicLong supporting only raw accesses plus CAS.
120 *
121 * JVM intrinsics note: It would be possible to use a release-only
122 * form of CAS here, if it were provided.
123 */
124 @jdk.internal.vm.annotation.Contended static final class Cell {
125 volatile long value;
126 Cell(long x) { value = x; }
127 final boolean cas(long cmp, long val) {
128 return VALUE.weakCompareAndSetRelease(this, cmp, val);
129 }
130 final void reset() {
131 VALUE.setVolatile(this, 0L);
132 }
133 final void reset(long identity) {
134 VALUE.setVolatile(this, identity);
135 }
136 final long getAndSet(long val) {
137 return (long)VALUE.getAndSet(this, val);
138 }
139
140 // VarHandle mechanics
141 private static final VarHandle VALUE;
142 static {
143 try {
144 MethodHandles.Lookup l = MethodHandles.lookup();
145 VALUE = l.findVarHandle(Cell.class, "value", long.class);
146 } catch (ReflectiveOperationException e) {
147 throw new ExceptionInInitializerError(e);
148 }
149 }
150 }
151
152 /** Number of CPUS, to place bound on table size */
153 static final int NCPU = Runtime.getRuntime().availableProcessors();
154
155 /**
156 * Table of cells. When non-null, size is a power of 2.
157 */
158 transient volatile Cell[] cells;
159
160 /**
161 * Base value, used mainly when there is no contention, but also as
162 * a fallback during table initialization races. Updated via CAS.
163 */
164 transient volatile long base;
165
166 /**
167 * Spinlock (locked via CAS) used when resizing and/or creating Cells.
168 */
169 transient volatile int cellsBusy;
170
171 /**
172 * Package-private default constructor.
173 */
174 Striped64() {
175 }
176
177 /**
178 * CASes the base field.
179 */
180 final boolean casBase(long cmp, long val) {
181 return BASE.weakCompareAndSetRelease(this, cmp, val);
182 }
183
184 final long getAndSetBase(long val) {
185 return (long)BASE.getAndSet(this, val);
186 }
187
188 /**
189 * CASes the cellsBusy field from 0 to 1 to acquire lock.
190 */
191 final boolean casCellsBusy() {
192 return CELLSBUSY.compareAndSet(this, 0, 1);
193 }
194
195 /**
196 * Returns the probe value for the current thread.
197 * Duplicated from ThreadLocalRandom because of packaging restrictions.
198 */
199 static final int getProbe() {
200 return (int) THREAD_PROBE.get(Thread.currentThread());
201 }
202
203 /**
204 * Pseudo-randomly advances and records the given probe value for the
205 * given thread.
206 * Duplicated from ThreadLocalRandom because of packaging restrictions.
207 */
208 static final int advanceProbe(int probe) {
209 probe ^= probe << 13; // xorshift
210 probe ^= probe >>> 17;
211 probe ^= probe << 5;
212 THREAD_PROBE.set(Thread.currentThread(), probe);
213 return probe;
214 }
215
216 /**
217 * Handles cases of updates involving initialization, resizing,
218 * creating new Cells, and/or contention. See above for
219 * explanation. This method suffers the usual non-modularity
220 * problems of optimistic retry code, relying on rechecked sets of
221 * reads.
222 *
223 * @param x the value
224 * @param fn the update function, or null for add (this convention
225 * avoids the need for an extra field or function in LongAdder).
226 * @param wasUncontended false if CAS failed before call
227 * @param index thread index from getProbe
228 */
229 final void longAccumulate(long x, LongBinaryOperator fn,
230 boolean wasUncontended, int index) {
231 if (index == 0) {
232 ThreadLocalRandom.current(); // force initialization
233 index = getProbe();
234 wasUncontended = true;
235 }
236 for (boolean collide = false;;) { // True if last slot nonempty
237 Cell[] cs; Cell c; int n; long v;
238 if ((cs = cells) != null && (n = cs.length) > 0) {
239 if ((c = cs[(n - 1) & index]) == null) {
240 if (cellsBusy == 0) { // Try to attach new Cell
241 Cell r = new Cell(x); // Optimistically create
242 if (cellsBusy == 0 && casCellsBusy()) {
243 try { // Recheck under lock
244 Cell[] rs; int m, j;
245 if ((rs = cells) != null &&
246 (m = rs.length) > 0 &&
247 rs[j = (m - 1) & index] == null) {
248 rs[j] = r;
249 break;
250 }
251 } finally {
252 cellsBusy = 0;
253 }
254 continue; // Slot is now non-empty
255 }
256 }
257 collide = false;
258 }
259 else if (!wasUncontended) // CAS already known to fail
260 wasUncontended = true; // Continue after rehash
261 else if (c.cas(v = c.value,
262 (fn == null) ? v + x : fn.applyAsLong(v, x)))
263 break;
264 else if (n >= NCPU || cells != cs)
265 collide = false; // At max size or stale
266 else if (!collide)
267 collide = true;
268 else if (cellsBusy == 0 && casCellsBusy()) {
269 try {
270 if (cells == cs) // Expand table unless stale
271 cells = Arrays.copyOf(cs, n << 1);
272 } finally {
273 cellsBusy = 0;
274 }
275 collide = false;
276 continue; // Retry with expanded table
277 }
278 index = advanceProbe(index);
279 }
280 else if (cellsBusy == 0 && cells == cs && casCellsBusy()) {
281 try { // Initialize table
282 if (cells == cs) {
283 Cell[] rs = new Cell[2];
284 rs[index & 1] = new Cell(x);
285 cells = rs;
286 break;
287 }
288 } finally {
289 cellsBusy = 0;
290 }
291 }
292 // Fall back on using base
293 else if (casBase(v = base,
294 (fn == null) ? v + x : fn.applyAsLong(v, x)))
295 break;
296 }
297 }
298
299 private static long apply(DoubleBinaryOperator fn, long v, double x) {
300 double d = Double.longBitsToDouble(v);
301 d = (fn == null) ? d + x : fn.applyAsDouble(d, x);
302 return Double.doubleToRawLongBits(d);
303 }
304
305 /**
306 * Same as longAccumulate, but injecting long/double conversions
307 * in too many places to sensibly merge with long version, given
308 * the low-overhead requirements of this class. So must instead be
309 * maintained by copy/paste/adapt.
310 */
311 final void doubleAccumulate(double x, DoubleBinaryOperator fn,
312 boolean wasUncontended, int index) {
313 if (index == 0) {
314 ThreadLocalRandom.current(); // force initialization
315 index = getProbe();
316 wasUncontended = true;
317 }
318 for (boolean collide = false;;) { // True if last slot nonempty
319 Cell[] cs; Cell c; int n; long v;
320 if ((cs = cells) != null && (n = cs.length) > 0) {
321 if ((c = cs[(n - 1) & index]) == null) {
322 if (cellsBusy == 0) { // Try to attach new Cell
323 Cell r = new Cell(Double.doubleToRawLongBits(x));
324 if (cellsBusy == 0 && casCellsBusy()) {
325 try { // Recheck under lock
326 Cell[] rs; int m, j;
327 if ((rs = cells) != null &&
328 (m = rs.length) > 0 &&
329 rs[j = (m - 1) & index] == null) {
330 rs[j] = r;
331 break;
332 }
333 } finally {
334 cellsBusy = 0;
335 }
336 continue; // Slot is now non-empty
337 }
338 }
339 collide = false;
340 }
341 else if (!wasUncontended) // CAS already known to fail
342 wasUncontended = true; // Continue after rehash
343 else if (c.cas(v = c.value, apply(fn, v, x)))
344 break;
345 else if (n >= NCPU || cells != cs)
346 collide = false; // At max size or stale
347 else if (!collide)
348 collide = true;
349 else if (cellsBusy == 0 && casCellsBusy()) {
350 try {
351 if (cells == cs) // Expand table unless stale
352 cells = Arrays.copyOf(cs, n << 1);
353 } finally {
354 cellsBusy = 0;
355 }
356 collide = false;
357 continue; // Retry with expanded table
358 }
359 index = advanceProbe(index);
360 }
361 else if (cellsBusy == 0 && cells == cs && casCellsBusy()) {
362 try { // Initialize table
363 if (cells == cs) {
364 Cell[] rs = new Cell[2];
365 rs[index & 1] = new Cell(Double.doubleToRawLongBits(x));
366 cells = rs;
367 break;
368 }
369 } finally {
370 cellsBusy = 0;
371 }
372 }
373 // Fall back on using base
374 else if (casBase(v = base, apply(fn, v, x)))
375 break;
376 }
377 }
378
379 // VarHandle mechanics
380 private static final VarHandle BASE;
381 private static final VarHandle CELLSBUSY;
382 private static final VarHandle THREAD_PROBE;
383 static {
384 try {
385 MethodHandles.Lookup l = MethodHandles.lookup();
386 BASE = l.findVarHandle(Striped64.class,
387 "base", long.class);
388 CELLSBUSY = l.findVarHandle(Striped64.class,
389 "cellsBusy", int.class);
390 l = java.security.AccessController.doPrivileged(
391 new java.security.PrivilegedAction<>() {
392 public MethodHandles.Lookup run() {
393 try {
394 return MethodHandles.privateLookupIn(Thread.class, MethodHandles.lookup());
395 } catch (ReflectiveOperationException e) {
396 throw new ExceptionInInitializerError(e);
397 }
398 }});
399 THREAD_PROBE = l.findVarHandle(Thread.class,
400 "threadLocalRandomProbe", int.class);
401 } catch (ReflectiveOperationException e) {
402 throw new ExceptionInInitializerError(e);
403 }
404 }
405
406 }