1 /*
   2  * Copyright (c) 2000, 2017, Oracle and/or its affiliates. All rights reserved.
   3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4  *
   5  * This code is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License version 2 only, as
   7  * published by the Free Software Foundation.  Oracle designates this
   8  * particular file as subject to the "Classpath" exception as provided
   9  * by Oracle in the LICENSE file that accompanied this code.
  10  *
  11  * This code is distributed in the hope that it will be useful, but WITHOUT
  12  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  13  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  14  * version 2 for more details (a copy is included in the LICENSE file that
  15  * accompanied this code).
  16  *
  17  * You should have received a copy of the GNU General Public License version
  18  * 2 along with this work; if not, write to the Free Software Foundation,
  19  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  20  *
  21  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  22  * or visit www.oracle.com if you need additional information or have any
  23  * questions.
  24  */
  25 
  26 package jdk.internal.misc;
  27 
  28 import jdk.internal.HotSpotIntrinsicCandidate;
  29 import jdk.internal.vm.annotation.ForceInline;
  30 
  31 import java.lang.reflect.Field;
  32 import java.security.ProtectionDomain;
  33 
  34 
  35 /**
  36  * A collection of methods for performing low-level, unsafe operations.
  37  * Although the class and all methods are public, use of this class is
  38  * limited because only trusted code can obtain instances of it.
  39  *
  40  * <em>Note:</em> It is the resposibility of the caller to make sure
  41  * arguments are checked before methods of this class are
  42  * called. While some rudimentary checks are performed on the input,
  43  * the checks are best effort and when performance is an overriding
  44  * priority, as when methods of this class are optimized by the
  45  * runtime compiler, some or all checks (if any) may be elided. Hence,
  46  * the caller must not rely on the checks and corresponding
  47  * exceptions!
  48  *
  49  * @author John R. Rose
  50  * @see #getUnsafe
  51  */
  52 
  53 public final class Unsafe {
  54 
  55     private static native void registerNatives();
  56     static {
  57         registerNatives();
  58     }
  59 
  60     private Unsafe() {}
  61 
  62     private static final Unsafe theUnsafe = new Unsafe();
  63 
  64     /**
  65      * Provides the caller with the capability of performing unsafe
  66      * operations.
  67      *
  68      * <p>The returned {@code Unsafe} object should be carefully guarded
  69      * by the caller, since it can be used to read and write data at arbitrary
  70      * memory addresses.  It must never be passed to untrusted code.
  71      *
  72      * <p>Most methods in this class are very low-level, and correspond to a
  73      * small number of hardware instructions (on typical machines).  Compilers
  74      * are encouraged to optimize these methods accordingly.
  75      *
  76      * <p>Here is a suggested idiom for using unsafe operations:
  77      *
  78      * <pre> {@code
  79      * class MyTrustedClass {
  80      *   private static final Unsafe unsafe = Unsafe.getUnsafe();
  81      *   ...
  82      *   private long myCountAddress = ...;
  83      *   public int getCount() { return unsafe.getByte(myCountAddress); }
  84      * }}</pre>
  85      *
  86      * (It may assist compilers to make the local variable {@code final}.)
  87      */
  88     public static Unsafe getUnsafe() {
  89         return theUnsafe;
  90     }
  91 
  92     /// peek and poke operations
  93     /// (compilers should optimize these to memory ops)
  94 
  95     // These work on object fields in the Java heap.
  96     // They will not work on elements of packed arrays.
  97 
  98     /**
  99      * Fetches a value from a given Java variable.
 100      * More specifically, fetches a field or array element within the given
 101      * object {@code o} at the given offset, or (if {@code o} is null)
 102      * from the memory address whose numerical value is the given offset.
 103      * <p>
 104      * The results are undefined unless one of the following cases is true:
 105      * <ul>
 106      * <li>The offset was obtained from {@link #objectFieldOffset} on
 107      * the {@link java.lang.reflect.Field} of some Java field and the object
 108      * referred to by {@code o} is of a class compatible with that
 109      * field's class.
 110      *
 111      * <li>The offset and object reference {@code o} (either null or
 112      * non-null) were both obtained via {@link #staticFieldOffset}
 113      * and {@link #staticFieldBase} (respectively) from the
 114      * reflective {@link Field} representation of some Java field.
 115      *
 116      * <li>The object referred to by {@code o} is an array, and the offset
 117      * is an integer of the form {@code B+N*S}, where {@code N} is
 118      * a valid index into the array, and {@code B} and {@code S} are
 119      * the values obtained by {@link #arrayBaseOffset} and {@link
 120      * #arrayIndexScale} (respectively) from the array's class.  The value
 121      * referred to is the {@code N}<em>th</em> element of the array.
 122      *
 123      * </ul>
 124      * <p>
 125      * If one of the above cases is true, the call references a specific Java
 126      * variable (field or array element).  However, the results are undefined
 127      * if that variable is not in fact of the type returned by this method.
 128      * <p>
 129      * This method refers to a variable by means of two parameters, and so
 130      * it provides (in effect) a <em>double-register</em> addressing mode
 131      * for Java variables.  When the object reference is null, this method
 132      * uses its offset as an absolute address.  This is similar in operation
 133      * to methods such as {@link #getInt(long)}, which provide (in effect) a
 134      * <em>single-register</em> addressing mode for non-Java variables.
 135      * However, because Java variables may have a different layout in memory
 136      * from non-Java variables, programmers should not assume that these
 137      * two addressing modes are ever equivalent.  Also, programmers should
 138      * remember that offsets from the double-register addressing mode cannot
 139      * be portably confused with longs used in the single-register addressing
 140      * mode.
 141      *
 142      * @param o Java heap object in which the variable resides, if any, else
 143      *        null
 144      * @param offset indication of where the variable resides in a Java heap
 145      *        object, if any, else a memory address locating the variable
 146      *        statically
 147      * @return the value fetched from the indicated Java variable
 148      * @throws RuntimeException No defined exceptions are thrown, not even
 149      *         {@link NullPointerException}
 150      */
 151     @HotSpotIntrinsicCandidate
 152     public native int getInt(Object o, long offset);
 153 
 154     /**
 155      * Stores a value into a given Java variable.
 156      * <p>
 157      * The first two parameters are interpreted exactly as with
 158      * {@link #getInt(Object, long)} to refer to a specific
 159      * Java variable (field or array element).  The given value
 160      * is stored into that variable.
 161      * <p>
 162      * The variable must be of the same type as the method
 163      * parameter {@code x}.
 164      *
 165      * @param o Java heap object in which the variable resides, if any, else
 166      *        null
 167      * @param offset indication of where the variable resides in a Java heap
 168      *        object, if any, else a memory address locating the variable
 169      *        statically
 170      * @param x the value to store into the indicated Java variable
 171      * @throws RuntimeException No defined exceptions are thrown, not even
 172      *         {@link NullPointerException}
 173      */
 174     @HotSpotIntrinsicCandidate
 175     public native void putInt(Object o, long offset, int x);
 176 
 177     /**
 178      * Fetches a reference value from a given Java variable.
 179      * @see #getInt(Object, long)
 180      */
 181     @HotSpotIntrinsicCandidate
 182     public native Object getObject(Object o, long offset);
 183 
 184     /**
 185      * Stores a reference value into a given Java variable.
 186      * <p>
 187      * Unless the reference {@code x} being stored is either null
 188      * or matches the field type, the results are undefined.
 189      * If the reference {@code o} is non-null, card marks or
 190      * other store barriers for that object (if the VM requires them)
 191      * are updated.
 192      * @see #putInt(Object, long, int)
 193      */
 194     @HotSpotIntrinsicCandidate
 195     public native void putObject(Object o, long offset, Object x);
 196 
 197     /** @see #getInt(Object, long) */
 198     @HotSpotIntrinsicCandidate
 199     public native boolean getBoolean(Object o, long offset);
 200 
 201     /** @see #putInt(Object, long, int) */
 202     @HotSpotIntrinsicCandidate
 203     public native void    putBoolean(Object o, long offset, boolean x);
 204 
 205     /** @see #getInt(Object, long) */
 206     @HotSpotIntrinsicCandidate
 207     public native byte    getByte(Object o, long offset);
 208 
 209     /** @see #putInt(Object, long, int) */
 210     @HotSpotIntrinsicCandidate
 211     public native void    putByte(Object o, long offset, byte x);
 212 
 213     /** @see #getInt(Object, long) */
 214     @HotSpotIntrinsicCandidate
 215     public native short   getShort(Object o, long offset);
 216 
 217     /** @see #putInt(Object, long, int) */
 218     @HotSpotIntrinsicCandidate
 219     public native void    putShort(Object o, long offset, short x);
 220 
 221     /** @see #getInt(Object, long) */
 222     @HotSpotIntrinsicCandidate
 223     public native char    getChar(Object o, long offset);
 224 
 225     /** @see #putInt(Object, long, int) */
 226     @HotSpotIntrinsicCandidate
 227     public native void    putChar(Object o, long offset, char x);
 228 
 229     /** @see #getInt(Object, long) */
 230     @HotSpotIntrinsicCandidate
 231     public native long    getLong(Object o, long offset);
 232 
 233     /** @see #putInt(Object, long, int) */
 234     @HotSpotIntrinsicCandidate
 235     public native void    putLong(Object o, long offset, long x);
 236 
 237     /** @see #getInt(Object, long) */
 238     @HotSpotIntrinsicCandidate
 239     public native float   getFloat(Object o, long offset);
 240 
 241     /** @see #putInt(Object, long, int) */
 242     @HotSpotIntrinsicCandidate
 243     public native void    putFloat(Object o, long offset, float x);
 244 
 245     /** @see #getInt(Object, long) */
 246     @HotSpotIntrinsicCandidate
 247     public native double  getDouble(Object o, long offset);
 248 
 249     /** @see #putInt(Object, long, int) */
 250     @HotSpotIntrinsicCandidate
 251     public native void    putDouble(Object o, long offset, double x);
 252 
 253     /**
 254      * Fetches a native pointer from a given memory address.  If the address is
 255      * zero, or does not point into a block obtained from {@link
 256      * #allocateMemory}, the results are undefined.
 257      *
 258      * <p>If the native pointer is less than 64 bits wide, it is extended as
 259      * an unsigned number to a Java long.  The pointer may be indexed by any
 260      * given byte offset, simply by adding that offset (as a simple integer) to
 261      * the long representing the pointer.  The number of bytes actually read
 262      * from the target address may be determined by consulting {@link
 263      * #addressSize}.
 264      *
 265      * @see #allocateMemory
 266      * @see #getInt(Object, long)
 267      */
 268     @ForceInline
 269     public long getAddress(Object o, long offset) {
 270         if (ADDRESS_SIZE == 4) {
 271             return Integer.toUnsignedLong(getInt(o, offset));
 272         } else {
 273             return getLong(o, offset);
 274         }
 275     }
 276 
 277     /**
 278      * Stores a native pointer into a given memory address.  If the address is
 279      * zero, or does not point into a block obtained from {@link
 280      * #allocateMemory}, the results are undefined.
 281      *
 282      * <p>The number of bytes actually written at the target address may be
 283      * determined by consulting {@link #addressSize}.
 284      *
 285      * @see #allocateMemory
 286      * @see #putInt(Object, long, int)
 287      */
 288     @ForceInline
 289     public void putAddress(Object o, long offset, long x) {
 290         if (ADDRESS_SIZE == 4) {
 291             putInt(o, offset, (int)x);
 292         } else {
 293             putLong(o, offset, x);
 294         }
 295     }
 296 
 297     // These read VM internal data.
 298 
 299     /**
 300      * Fetches an uncompressed reference value from a given native variable
 301      * ignoring the VM's compressed references mode.
 302      *
 303      * @param address a memory address locating the variable
 304      * @return the value fetched from the indicated native variable
 305      */
 306     public native Object getUncompressedObject(long address);
 307 
 308     // These work on values in the C heap.
 309 
 310     /**
 311      * Fetches a value from a given memory address.  If the address is zero, or
 312      * does not point into a block obtained from {@link #allocateMemory}, the
 313      * results are undefined.
 314      *
 315      * @see #allocateMemory
 316      */
 317     @ForceInline
 318     public byte getByte(long address) {
 319         return getByte(null, address);
 320     }
 321 
 322     /**
 323      * Stores a value into a given memory address.  If the address is zero, or
 324      * does not point into a block obtained from {@link #allocateMemory}, the
 325      * results are undefined.
 326      *
 327      * @see #getByte(long)
 328      */
 329     @ForceInline
 330     public void putByte(long address, byte x) {
 331         putByte(null, address, x);
 332     }
 333 
 334     /** @see #getByte(long) */
 335     @ForceInline
 336     public short getShort(long address) {
 337         return getShort(null, address);
 338     }
 339 
 340     /** @see #putByte(long, byte) */
 341     @ForceInline
 342     public void putShort(long address, short x) {
 343         putShort(null, address, x);
 344     }
 345 
 346     /** @see #getByte(long) */
 347     @ForceInline
 348     public char getChar(long address) {
 349         return getChar(null, address);
 350     }
 351 
 352     /** @see #putByte(long, byte) */
 353     @ForceInline
 354     public void putChar(long address, char x) {
 355         putChar(null, address, x);
 356     }
 357 
 358     /** @see #getByte(long) */
 359     @ForceInline
 360     public int getInt(long address) {
 361         return getInt(null, address);
 362     }
 363 
 364     /** @see #putByte(long, byte) */
 365     @ForceInline
 366     public void putInt(long address, int x) {
 367         putInt(null, address, x);
 368     }
 369 
 370     /** @see #getByte(long) */
 371     @ForceInline
 372     public long getLong(long address) {
 373         return getLong(null, address);
 374     }
 375 
 376     /** @see #putByte(long, byte) */
 377     @ForceInline
 378     public void putLong(long address, long x) {
 379         putLong(null, address, x);
 380     }
 381 
 382     /** @see #getByte(long) */
 383     @ForceInline
 384     public float getFloat(long address) {
 385         return getFloat(null, address);
 386     }
 387 
 388     /** @see #putByte(long, byte) */
 389     @ForceInline
 390     public void putFloat(long address, float x) {
 391         putFloat(null, address, x);
 392     }
 393 
 394     /** @see #getByte(long) */
 395     @ForceInline
 396     public double getDouble(long address) {
 397         return getDouble(null, address);
 398     }
 399 
 400     /** @see #putByte(long, byte) */
 401     @ForceInline
 402     public void putDouble(long address, double x) {
 403         putDouble(null, address, x);
 404     }
 405 
 406     /** @see #getAddress(Object, long) */
 407     @ForceInline
 408     public long getAddress(long address) {
 409         return getAddress(null, address);
 410     }
 411 
 412     /** @see #putAddress(Object, long, long) */
 413     @ForceInline
 414     public void putAddress(long address, long x) {
 415         putAddress(null, address, x);
 416     }
 417 
 418 
 419 
 420     /// helper methods for validating various types of objects/values
 421 
 422     /**
 423      * Create an exception reflecting that some of the input was invalid
 424      *
 425      * <em>Note:</em> It is the resposibility of the caller to make
 426      * sure arguments are checked before the methods are called. While
 427      * some rudimentary checks are performed on the input, the checks
 428      * are best effort and when performance is an overriding priority,
 429      * as when methods of this class are optimized by the runtime
 430      * compiler, some or all checks (if any) may be elided. Hence, the
 431      * caller must not rely on the checks and corresponding
 432      * exceptions!
 433      *
 434      * @return an exception object
 435      */
 436     private RuntimeException invalidInput() {
 437         return new IllegalArgumentException();
 438     }
 439 
 440     /**
 441      * Check if a value is 32-bit clean (32 MSB are all zero)
 442      *
 443      * @param value the 64-bit value to check
 444      *
 445      * @return true if the value is 32-bit clean
 446      */
 447     private boolean is32BitClean(long value) {
 448         return value >>> 32 == 0;
 449     }
 450 
 451     /**
 452      * Check the validity of a size (the equivalent of a size_t)
 453      *
 454      * @throws RuntimeException if the size is invalid
 455      *         (<em>Note:</em> after optimization, invalid inputs may
 456      *         go undetected, which will lead to unpredictable
 457      *         behavior)
 458      */
 459     private void checkSize(long size) {
 460         if (ADDRESS_SIZE == 4) {
 461             // Note: this will also check for negative sizes
 462             if (!is32BitClean(size)) {
 463                 throw invalidInput();
 464             }
 465         } else if (size < 0) {
 466             throw invalidInput();
 467         }
 468     }
 469 
 470     /**
 471      * Check the validity of a native address (the equivalent of void*)
 472      *
 473      * @throws RuntimeException if the address is invalid
 474      *         (<em>Note:</em> after optimization, invalid inputs may
 475      *         go undetected, which will lead to unpredictable
 476      *         behavior)
 477      */
 478     private void checkNativeAddress(long address) {
 479         if (ADDRESS_SIZE == 4) {
 480             // Accept both zero and sign extended pointers. A valid
 481             // pointer will, after the +1 below, either have produced
 482             // the value 0x0 or 0x1. Masking off the low bit allows
 483             // for testing against 0.
 484             if ((((address >> 32) + 1) & ~1) != 0) {
 485                 throw invalidInput();
 486             }
 487         }
 488     }
 489 
 490     /**
 491      * Check the validity of an offset, relative to a base object
 492      *
 493      * @param o the base object
 494      * @param offset the offset to check
 495      *
 496      * @throws RuntimeException if the size is invalid
 497      *         (<em>Note:</em> after optimization, invalid inputs may
 498      *         go undetected, which will lead to unpredictable
 499      *         behavior)
 500      */
 501     private void checkOffset(Object o, long offset) {
 502         if (ADDRESS_SIZE == 4) {
 503             // Note: this will also check for negative offsets
 504             if (!is32BitClean(offset)) {
 505                 throw invalidInput();
 506             }
 507         } else if (offset < 0) {
 508             throw invalidInput();
 509         }
 510     }
 511 
 512     /**
 513      * Check the validity of a double-register pointer
 514      *
 515      * Note: This code deliberately does *not* check for NPE for (at
 516      * least) three reasons:
 517      *
 518      * 1) NPE is not just NULL/0 - there is a range of values all
 519      * resulting in an NPE, which is not trivial to check for
 520      *
 521      * 2) It is the responsibility of the callers of Unsafe methods
 522      * to verify the input, so throwing an exception here is not really
 523      * useful - passing in a NULL pointer is a critical error and the
 524      * must not expect an exception to be thrown anyway.
 525      *
 526      * 3) the actual operations will detect NULL pointers anyway by
 527      * means of traps and signals (like SIGSEGV).
 528      *
 529      * @param o Java heap object, or null
 530      * @param offset indication of where the variable resides in a Java heap
 531      *        object, if any, else a memory address locating the variable
 532      *        statically
 533      *
 534      * @throws RuntimeException if the pointer is invalid
 535      *         (<em>Note:</em> after optimization, invalid inputs may
 536      *         go undetected, which will lead to unpredictable
 537      *         behavior)
 538      */
 539     private void checkPointer(Object o, long offset) {
 540         if (o == null) {
 541             checkNativeAddress(offset);
 542         } else {
 543             checkOffset(o, offset);
 544         }
 545     }
 546 
 547     /**
 548      * Check if a type is a primitive array type
 549      *
 550      * @param c the type to check
 551      *
 552      * @return true if the type is a primitive array type
 553      */
 554     private void checkPrimitiveArray(Class<?> c) {
 555         Class<?> componentType = c.getComponentType();
 556         if (componentType == null || !componentType.isPrimitive()) {
 557             throw invalidInput();
 558         }
 559     }
 560 
 561     /**
 562      * Check that a pointer is a valid primitive array type pointer
 563      *
 564      * Note: pointers off-heap are considered to be primitive arrays
 565      *
 566      * @throws RuntimeException if the pointer is invalid
 567      *         (<em>Note:</em> after optimization, invalid inputs may
 568      *         go undetected, which will lead to unpredictable
 569      *         behavior)
 570      */
 571     private void checkPrimitivePointer(Object o, long offset) {
 572         checkPointer(o, offset);
 573 
 574         if (o != null) {
 575             // If on heap, it must be a primitive array
 576             checkPrimitiveArray(o.getClass());
 577         }
 578     }
 579 
 580 
 581     /// wrappers for malloc, realloc, free:
 582 
 583     /**
 584      * Allocates a new block of native memory, of the given size in bytes.  The
 585      * contents of the memory are uninitialized; they will generally be
 586      * garbage.  The resulting native pointer will never be zero, and will be
 587      * aligned for all value types.  Dispose of this memory by calling {@link
 588      * #freeMemory}, or resize it with {@link #reallocateMemory}.
 589      *
 590      * <em>Note:</em> It is the resposibility of the caller to make
 591      * sure arguments are checked before the methods are called. While
 592      * some rudimentary checks are performed on the input, the checks
 593      * are best effort and when performance is an overriding priority,
 594      * as when methods of this class are optimized by the runtime
 595      * compiler, some or all checks (if any) may be elided. Hence, the
 596      * caller must not rely on the checks and corresponding
 597      * exceptions!
 598      *
 599      * @throws RuntimeException if the size is negative or too large
 600      *         for the native size_t type
 601      *
 602      * @throws OutOfMemoryError if the allocation is refused by the system
 603      *
 604      * @see #getByte(long)
 605      * @see #putByte(long, byte)
 606      */
 607     public long allocateMemory(long bytes) {
 608         allocateMemoryChecks(bytes);
 609 
 610         if (bytes == 0) {
 611             return 0;
 612         }
 613 
 614         long p = allocateMemory0(bytes);
 615         if (p == 0) {
 616             throw new OutOfMemoryError();
 617         }
 618 
 619         return p;
 620     }
 621 
 622     /**
 623      * Validate the arguments to allocateMemory
 624      *
 625      * @throws RuntimeException if the arguments are invalid
 626      *         (<em>Note:</em> after optimization, invalid inputs may
 627      *         go undetected, which will lead to unpredictable
 628      *         behavior)
 629      */
 630     private void allocateMemoryChecks(long bytes) {
 631         checkSize(bytes);
 632     }
 633 
 634     /**
 635      * Resizes a new block of native memory, to the given size in bytes.  The
 636      * contents of the new block past the size of the old block are
 637      * uninitialized; they will generally be garbage.  The resulting native
 638      * pointer will be zero if and only if the requested size is zero.  The
 639      * resulting native pointer will be aligned for all value types.  Dispose
 640      * of this memory by calling {@link #freeMemory}, or resize it with {@link
 641      * #reallocateMemory}.  The address passed to this method may be null, in
 642      * which case an allocation will be performed.
 643      *
 644      * <em>Note:</em> It is the resposibility of the caller to make
 645      * sure arguments are checked before the methods are called. While
 646      * some rudimentary checks are performed on the input, the checks
 647      * are best effort and when performance is an overriding priority,
 648      * as when methods of this class are optimized by the runtime
 649      * compiler, some or all checks (if any) may be elided. Hence, the
 650      * caller must not rely on the checks and corresponding
 651      * exceptions!
 652      *
 653      * @throws RuntimeException if the size is negative or too large
 654      *         for the native size_t type
 655      *
 656      * @throws OutOfMemoryError if the allocation is refused by the system
 657      *
 658      * @see #allocateMemory
 659      */
 660     public long reallocateMemory(long address, long bytes) {
 661         reallocateMemoryChecks(address, bytes);
 662 
 663         if (bytes == 0) {
 664             freeMemory(address);
 665             return 0;
 666         }
 667 
 668         long p = (address == 0) ? allocateMemory0(bytes) : reallocateMemory0(address, bytes);
 669         if (p == 0) {
 670             throw new OutOfMemoryError();
 671         }
 672 
 673         return p;
 674     }
 675 
 676     /**
 677      * Validate the arguments to reallocateMemory
 678      *
 679      * @throws RuntimeException if the arguments are invalid
 680      *         (<em>Note:</em> after optimization, invalid inputs may
 681      *         go undetected, which will lead to unpredictable
 682      *         behavior)
 683      */
 684     private void reallocateMemoryChecks(long address, long bytes) {
 685         checkPointer(null, address);
 686         checkSize(bytes);
 687     }
 688 
 689     /**
 690      * Sets all bytes in a given block of memory to a fixed value
 691      * (usually zero).
 692      *
 693      * <p>This method determines a block's base address by means of two parameters,
 694      * and so it provides (in effect) a <em>double-register</em> addressing mode,
 695      * as discussed in {@link #getInt(Object,long)}.  When the object reference is null,
 696      * the offset supplies an absolute base address.
 697      *
 698      * <p>The stores are in coherent (atomic) units of a size determined
 699      * by the address and length parameters.  If the effective address and
 700      * length are all even modulo 8, the stores take place in 'long' units.
 701      * If the effective address and length are (resp.) even modulo 4 or 2,
 702      * the stores take place in units of 'int' or 'short'.
 703      *
 704      * <em>Note:</em> It is the resposibility of the caller to make
 705      * sure arguments are checked before the methods are called. While
 706      * some rudimentary checks are performed on the input, the checks
 707      * are best effort and when performance is an overriding priority,
 708      * as when methods of this class are optimized by the runtime
 709      * compiler, some or all checks (if any) may be elided. Hence, the
 710      * caller must not rely on the checks and corresponding
 711      * exceptions!
 712      *
 713      * @throws RuntimeException if any of the arguments is invalid
 714      *
 715      * @since 1.7
 716      */
 717     public void setMemory(Object o, long offset, long bytes, byte value) {
 718         setMemoryChecks(o, offset, bytes, value);
 719 
 720         if (bytes == 0) {
 721             return;
 722         }
 723 
 724         setMemory0(o, offset, bytes, value);
 725     }
 726 
 727     /**
 728      * Sets all bytes in a given block of memory to a fixed value
 729      * (usually zero).  This provides a <em>single-register</em> addressing mode,
 730      * as discussed in {@link #getInt(Object,long)}.
 731      *
 732      * <p>Equivalent to {@code setMemory(null, address, bytes, value)}.
 733      */
 734     public void setMemory(long address, long bytes, byte value) {
 735         setMemory(null, address, bytes, value);
 736     }
 737 
 738     /**
 739      * Validate the arguments to setMemory
 740      *
 741      * @throws RuntimeException if the arguments are invalid
 742      *         (<em>Note:</em> after optimization, invalid inputs may
 743      *         go undetected, which will lead to unpredictable
 744      *         behavior)
 745      */
 746     private void setMemoryChecks(Object o, long offset, long bytes, byte value) {
 747         checkPrimitivePointer(o, offset);
 748         checkSize(bytes);
 749     }
 750 
 751     /**
 752      * Sets all bytes in a given block of memory to a copy of another
 753      * block.
 754      *
 755      * <p>This method determines each block's base address by means of two parameters,
 756      * and so it provides (in effect) a <em>double-register</em> addressing mode,
 757      * as discussed in {@link #getInt(Object,long)}.  When the object reference is null,
 758      * the offset supplies an absolute base address.
 759      *
 760      * <p>The transfers are in coherent (atomic) units of a size determined
 761      * by the address and length parameters.  If the effective addresses and
 762      * length are all even modulo 8, the transfer takes place in 'long' units.
 763      * If the effective addresses and length are (resp.) even modulo 4 or 2,
 764      * the transfer takes place in units of 'int' or 'short'.
 765      *
 766      * <em>Note:</em> It is the resposibility of the caller to make
 767      * sure arguments are checked before the methods are called. While
 768      * some rudimentary checks are performed on the input, the checks
 769      * are best effort and when performance is an overriding priority,
 770      * as when methods of this class are optimized by the runtime
 771      * compiler, some or all checks (if any) may be elided. Hence, the
 772      * caller must not rely on the checks and corresponding
 773      * exceptions!
 774      *
 775      * @throws RuntimeException if any of the arguments is invalid
 776      *
 777      * @since 1.7
 778      */
 779     public void copyMemory(Object srcBase, long srcOffset,
 780                            Object destBase, long destOffset,
 781                            long bytes) {
 782         copyMemoryChecks(srcBase, srcOffset, destBase, destOffset, bytes);
 783 
 784         if (bytes == 0) {
 785             return;
 786         }
 787 
 788         copyMemory0(srcBase, srcOffset, destBase, destOffset, bytes);
 789     }
 790 
 791     /**
 792      * Sets all bytes in a given block of memory to a copy of another
 793      * block.  This provides a <em>single-register</em> addressing mode,
 794      * as discussed in {@link #getInt(Object,long)}.
 795      *
 796      * Equivalent to {@code copyMemory(null, srcAddress, null, destAddress, bytes)}.
 797      */
 798     public void copyMemory(long srcAddress, long destAddress, long bytes) {
 799         copyMemory(null, srcAddress, null, destAddress, bytes);
 800     }
 801 
 802     /**
 803      * Validate the arguments to copyMemory
 804      *
 805      * @throws RuntimeException if any of the arguments is invalid
 806      *         (<em>Note:</em> after optimization, invalid inputs may
 807      *         go undetected, which will lead to unpredictable
 808      *         behavior)
 809      */
 810     private void copyMemoryChecks(Object srcBase, long srcOffset,
 811                                   Object destBase, long destOffset,
 812                                   long bytes) {
 813         checkSize(bytes);
 814         checkPrimitivePointer(srcBase, srcOffset);
 815         checkPrimitivePointer(destBase, destOffset);
 816     }
 817 
 818     /**
 819      * Copies all elements from one block of memory to another block,
 820      * *unconditionally* byte swapping the elements on the fly.
 821      *
 822      * <p>This method determines each block's base address by means of two parameters,
 823      * and so it provides (in effect) a <em>double-register</em> addressing mode,
 824      * as discussed in {@link #getInt(Object,long)}.  When the object reference is null,
 825      * the offset supplies an absolute base address.
 826      *
 827      * <em>Note:</em> It is the resposibility of the caller to make
 828      * sure arguments are checked before the methods are called. While
 829      * some rudimentary checks are performed on the input, the checks
 830      * are best effort and when performance is an overriding priority,
 831      * as when methods of this class are optimized by the runtime
 832      * compiler, some or all checks (if any) may be elided. Hence, the
 833      * caller must not rely on the checks and corresponding
 834      * exceptions!
 835      *
 836      * @throws RuntimeException if any of the arguments is invalid
 837      *
 838      * @since 9
 839      */
 840     public void copySwapMemory(Object srcBase, long srcOffset,
 841                                Object destBase, long destOffset,
 842                                long bytes, long elemSize) {
 843         copySwapMemoryChecks(srcBase, srcOffset, destBase, destOffset, bytes, elemSize);
 844 
 845         if (bytes == 0) {
 846             return;
 847         }
 848 
 849         copySwapMemory0(srcBase, srcOffset, destBase, destOffset, bytes, elemSize);
 850     }
 851 
 852     private void copySwapMemoryChecks(Object srcBase, long srcOffset,
 853                                       Object destBase, long destOffset,
 854                                       long bytes, long elemSize) {
 855         checkSize(bytes);
 856 
 857         if (elemSize != 2 && elemSize != 4 && elemSize != 8) {
 858             throw invalidInput();
 859         }
 860         if (bytes % elemSize != 0) {
 861             throw invalidInput();
 862         }
 863 
 864         checkPrimitivePointer(srcBase, srcOffset);
 865         checkPrimitivePointer(destBase, destOffset);
 866     }
 867 
 868    /**
 869      * Copies all elements from one block of memory to another block, byte swapping the
 870      * elements on the fly.
 871      *
 872      * This provides a <em>single-register</em> addressing mode, as
 873      * discussed in {@link #getInt(Object,long)}.
 874      *
 875      * Equivalent to {@code copySwapMemory(null, srcAddress, null, destAddress, bytes, elemSize)}.
 876      */
 877     public void copySwapMemory(long srcAddress, long destAddress, long bytes, long elemSize) {
 878         copySwapMemory(null, srcAddress, null, destAddress, bytes, elemSize);
 879     }
 880 
 881     /**
 882      * Disposes of a block of native memory, as obtained from {@link
 883      * #allocateMemory} or {@link #reallocateMemory}.  The address passed to
 884      * this method may be null, in which case no action is taken.
 885      *
 886      * <em>Note:</em> It is the resposibility of the caller to make
 887      * sure arguments are checked before the methods are called. While
 888      * some rudimentary checks are performed on the input, the checks
 889      * are best effort and when performance is an overriding priority,
 890      * as when methods of this class are optimized by the runtime
 891      * compiler, some or all checks (if any) may be elided. Hence, the
 892      * caller must not rely on the checks and corresponding
 893      * exceptions!
 894      *
 895      * @throws RuntimeException if any of the arguments is invalid
 896      *
 897      * @see #allocateMemory
 898      */
 899     public void freeMemory(long address) {
 900         freeMemoryChecks(address);
 901 
 902         if (address == 0) {
 903             return;
 904         }
 905 
 906         freeMemory0(address);
 907     }
 908 
 909     /**
 910      * Validate the arguments to freeMemory
 911      *
 912      * @throws RuntimeException if the arguments are invalid
 913      *         (<em>Note:</em> after optimization, invalid inputs may
 914      *         go undetected, which will lead to unpredictable
 915      *         behavior)
 916      */
 917     private void freeMemoryChecks(long address) {
 918         checkPointer(null, address);
 919     }
 920 
 921     /**
 922      * ensure writeback of a specified virtual memory address range
 923      * from cache to physical memory. all bytes in the address range
 924      * are guaranteed to have been written back to physical memory on
 925      * return from this call i.e. subsequently executed store
 926      * instructions are guaranteed not to be visible before the
 927      * writeback is completed.
 928      *
 929      * @param address
 930      *        the lowest byte address that must be guaranteed written
 931      *        back to memory. bytes at lower addresses may also be
 932      *        written back.
 933      *
 934      * @param length
 935      *        the length in bytes of the region starting at address
 936      *        that must be guaranteed written back to memory.
 937      *        
 938      * @throws RuntimeException if the arguments are invalid
 939      *         (<em>Note:</em> after optimization, invalid inputs may
 940      *         go undetected, which will lead to unpredictable
 941      *         behavior)
 942      *
 943      * @since 12
 944      */
 945     
 946     public void writebackMemory(long address, long length)
 947     {
 948         checkWritebackMemory(address, length);
 949 
 950         // perform any required pre-writeback barrier
 951         writebackPreSync0();
 952 
 953         // write back one cache line at a time
 954         long line = (address & CACHE_LINE_MASK);
 955         long end = address + length;
 956         while (line < end) {
 957             writeback0(line);
 958             line += CACHE_LINE_FLUSH_SIZE;
 959         }
 960 
 961         // perform any required post-writeback barrier
 962         writebackPostSync0();
 963     }
 964 
 965     /**
 966      * Validate the arguments to writebackMemory
 967      *
 968      * @throws RuntimeException if the arguments are invalid
 969      *         (<em>Note:</em> after optimization, invalid inputs may
 970      *         go undetected, which will lead to unpredictable
 971      *         behavior)
 972      */
 973     private void checkWritebackMemory(long address, long length)
 974     {
 975         checkNativeAddress(address);
 976         checkSize(length);
 977     }
 978     
 979     /**
 980      * The size of an L1 data cache line which will be a power of two.
 981      */
 982     private static final long CACHE_LINE_FLUSH_SIZE = (long)theUnsafe.dataCacheLineFlushSize0();
 983     private static final long CACHE_LINE_MASK = ~(CACHE_LINE_FLUSH_SIZE - 1);
 984 
 985     /**
 986      * primitive operation forcing writeback of a single cache line.
 987      *
 988      * @param address
 989      *        the start address of the cache line to be written back
 990      */
 991     // native used to write back an individual cache line starting at
 992     // the supplied address
 993     @HotSpotIntrinsicCandidate
 994     private native void writeback0(long address);
 995     // native used to serialise writeback operations relative to
 996     // preceding memory writes
 997     @HotSpotIntrinsicCandidate
 998     private native void writebackPreSync0();
 999     // native used to serialise writeback operations relative to
1000     // following memory writes
1001     @HotSpotIntrinsicCandidate
1002     private native void writebackPostSync0();
1003     
1004     /// random queries
1005 
1006     /**
1007      * This constant differs from all results that will ever be returned from
1008      * {@link #staticFieldOffset}, {@link #objectFieldOffset},
1009      * or {@link #arrayBaseOffset}.
1010      */
1011     public static final int INVALID_FIELD_OFFSET = -1;
1012 
1013     /**
1014      * Reports the location of a given field in the storage allocation of its
1015      * class.  Do not expect to perform any sort of arithmetic on this offset;
1016      * it is just a cookie which is passed to the unsafe heap memory accessors.
1017      *
1018      * <p>Any given field will always have the same offset and base, and no
1019      * two distinct fields of the same class will ever have the same offset
1020      * and base.
1021      *
1022      * <p>As of 1.4.1, offsets for fields are represented as long values,
1023      * although the Sun JVM does not use the most significant 32 bits.
1024      * However, JVM implementations which store static fields at absolute
1025      * addresses can use long offsets and null base pointers to express
1026      * the field locations in a form usable by {@link #getInt(Object,long)}.
1027      * Therefore, code which will be ported to such JVMs on 64-bit platforms
1028      * must preserve all bits of static field offsets.
1029      * @see #getInt(Object, long)
1030      */
1031     public long objectFieldOffset(Field f) {
1032         if (f == null) {
1033             throw new NullPointerException();
1034         }
1035 
1036         return objectFieldOffset0(f);
1037     }
1038 
1039     /**
1040      * Reports the location of the field with a given name in the storage
1041      * allocation of its class.
1042      *
1043      * @throws NullPointerException if any parameter is {@code null}.
1044      * @throws InternalError if there is no field named {@code name} declared
1045      *         in class {@code c}, i.e., if {@code c.getDeclaredField(name)}
1046      *         would throw {@code java.lang.NoSuchFieldException}.
1047      *
1048      * @see #objectFieldOffset(Field)
1049      */
1050     public long objectFieldOffset(Class<?> c, String name) {
1051         if (c == null || name == null) {
1052             throw new NullPointerException();
1053         }
1054 
1055         return objectFieldOffset1(c, name);
1056     }
1057 
1058     /**
1059      * Reports the location of a given static field, in conjunction with {@link
1060      * #staticFieldBase}.
1061      * <p>Do not expect to perform any sort of arithmetic on this offset;
1062      * it is just a cookie which is passed to the unsafe heap memory accessors.
1063      *
1064      * <p>Any given field will always have the same offset, and no two distinct
1065      * fields of the same class will ever have the same offset.
1066      *
1067      * <p>As of 1.4.1, offsets for fields are represented as long values,
1068      * although the Sun JVM does not use the most significant 32 bits.
1069      * It is hard to imagine a JVM technology which needs more than
1070      * a few bits to encode an offset within a non-array object,
1071      * However, for consistency with other methods in this class,
1072      * this method reports its result as a long value.
1073      * @see #getInt(Object, long)
1074      */
1075     public long staticFieldOffset(Field f) {
1076         if (f == null) {
1077             throw new NullPointerException();
1078         }
1079 
1080         return staticFieldOffset0(f);
1081     }
1082 
1083     /**
1084      * Reports the location of a given static field, in conjunction with {@link
1085      * #staticFieldOffset}.
1086      * <p>Fetch the base "Object", if any, with which static fields of the
1087      * given class can be accessed via methods like {@link #getInt(Object,
1088      * long)}.  This value may be null.  This value may refer to an object
1089      * which is a "cookie", not guaranteed to be a real Object, and it should
1090      * not be used in any way except as argument to the get and put routines in
1091      * this class.
1092      */
1093     public Object staticFieldBase(Field f) {
1094         if (f == null) {
1095             throw new NullPointerException();
1096         }
1097 
1098         return staticFieldBase0(f);
1099     }
1100 
1101     /**
1102      * Detects if the given class may need to be initialized. This is often
1103      * needed in conjunction with obtaining the static field base of a
1104      * class.
1105      * @return false only if a call to {@code ensureClassInitialized} would have no effect
1106      */
1107     public boolean shouldBeInitialized(Class<?> c) {
1108         if (c == null) {
1109             throw new NullPointerException();
1110         }
1111 
1112         return shouldBeInitialized0(c);
1113     }
1114 
1115     /**
1116      * Ensures the given class has been initialized. This is often
1117      * needed in conjunction with obtaining the static field base of a
1118      * class.
1119      */
1120     public void ensureClassInitialized(Class<?> c) {
1121         if (c == null) {
1122             throw new NullPointerException();
1123         }
1124 
1125         ensureClassInitialized0(c);
1126     }
1127 
1128     /**
1129      * Reports the offset of the first element in the storage allocation of a
1130      * given array class.  If {@link #arrayIndexScale} returns a non-zero value
1131      * for the same class, you may use that scale factor, together with this
1132      * base offset, to form new offsets to access elements of arrays of the
1133      * given class.
1134      *
1135      * @see #getInt(Object, long)
1136      * @see #putInt(Object, long, int)
1137      */
1138     public int arrayBaseOffset(Class<?> arrayClass) {
1139         if (arrayClass == null) {
1140             throw new NullPointerException();
1141         }
1142 
1143         return arrayBaseOffset0(arrayClass);
1144     }
1145 
1146 
1147     /** The value of {@code arrayBaseOffset(boolean[].class)} */
1148     public static final int ARRAY_BOOLEAN_BASE_OFFSET
1149             = theUnsafe.arrayBaseOffset(boolean[].class);
1150 
1151     /** The value of {@code arrayBaseOffset(byte[].class)} */
1152     public static final int ARRAY_BYTE_BASE_OFFSET
1153             = theUnsafe.arrayBaseOffset(byte[].class);
1154 
1155     /** The value of {@code arrayBaseOffset(short[].class)} */
1156     public static final int ARRAY_SHORT_BASE_OFFSET
1157             = theUnsafe.arrayBaseOffset(short[].class);
1158 
1159     /** The value of {@code arrayBaseOffset(char[].class)} */
1160     public static final int ARRAY_CHAR_BASE_OFFSET
1161             = theUnsafe.arrayBaseOffset(char[].class);
1162 
1163     /** The value of {@code arrayBaseOffset(int[].class)} */
1164     public static final int ARRAY_INT_BASE_OFFSET
1165             = theUnsafe.arrayBaseOffset(int[].class);
1166 
1167     /** The value of {@code arrayBaseOffset(long[].class)} */
1168     public static final int ARRAY_LONG_BASE_OFFSET
1169             = theUnsafe.arrayBaseOffset(long[].class);
1170 
1171     /** The value of {@code arrayBaseOffset(float[].class)} */
1172     public static final int ARRAY_FLOAT_BASE_OFFSET
1173             = theUnsafe.arrayBaseOffset(float[].class);
1174 
1175     /** The value of {@code arrayBaseOffset(double[].class)} */
1176     public static final int ARRAY_DOUBLE_BASE_OFFSET
1177             = theUnsafe.arrayBaseOffset(double[].class);
1178 
1179     /** The value of {@code arrayBaseOffset(Object[].class)} */
1180     public static final int ARRAY_OBJECT_BASE_OFFSET
1181             = theUnsafe.arrayBaseOffset(Object[].class);
1182 
1183     /**
1184      * Reports the scale factor for addressing elements in the storage
1185      * allocation of a given array class.  However, arrays of "narrow" types
1186      * will generally not work properly with accessors like {@link
1187      * #getByte(Object, long)}, so the scale factor for such classes is reported
1188      * as zero.
1189      *
1190      * @see #arrayBaseOffset
1191      * @see #getInt(Object, long)
1192      * @see #putInt(Object, long, int)
1193      */
1194     public int arrayIndexScale(Class<?> arrayClass) {
1195         if (arrayClass == null) {
1196             throw new NullPointerException();
1197         }
1198 
1199         return arrayIndexScale0(arrayClass);
1200     }
1201 
1202 
1203     /** The value of {@code arrayIndexScale(boolean[].class)} */
1204     public static final int ARRAY_BOOLEAN_INDEX_SCALE
1205             = theUnsafe.arrayIndexScale(boolean[].class);
1206 
1207     /** The value of {@code arrayIndexScale(byte[].class)} */
1208     public static final int ARRAY_BYTE_INDEX_SCALE
1209             = theUnsafe.arrayIndexScale(byte[].class);
1210 
1211     /** The value of {@code arrayIndexScale(short[].class)} */
1212     public static final int ARRAY_SHORT_INDEX_SCALE
1213             = theUnsafe.arrayIndexScale(short[].class);
1214 
1215     /** The value of {@code arrayIndexScale(char[].class)} */
1216     public static final int ARRAY_CHAR_INDEX_SCALE
1217             = theUnsafe.arrayIndexScale(char[].class);
1218 
1219     /** The value of {@code arrayIndexScale(int[].class)} */
1220     public static final int ARRAY_INT_INDEX_SCALE
1221             = theUnsafe.arrayIndexScale(int[].class);
1222 
1223     /** The value of {@code arrayIndexScale(long[].class)} */
1224     public static final int ARRAY_LONG_INDEX_SCALE
1225             = theUnsafe.arrayIndexScale(long[].class);
1226 
1227     /** The value of {@code arrayIndexScale(float[].class)} */
1228     public static final int ARRAY_FLOAT_INDEX_SCALE
1229             = theUnsafe.arrayIndexScale(float[].class);
1230 
1231     /** The value of {@code arrayIndexScale(double[].class)} */
1232     public static final int ARRAY_DOUBLE_INDEX_SCALE
1233             = theUnsafe.arrayIndexScale(double[].class);
1234 
1235     /** The value of {@code arrayIndexScale(Object[].class)} */
1236     public static final int ARRAY_OBJECT_INDEX_SCALE
1237             = theUnsafe.arrayIndexScale(Object[].class);
1238 
1239     /**
1240      * Reports the size in bytes of a native pointer, as stored via {@link
1241      * #putAddress}.  This value will be either 4 or 8.  Note that the sizes of
1242      * other primitive types (as stored in native memory blocks) is determined
1243      * fully by their information content.
1244      */
1245     public int addressSize() {
1246         return ADDRESS_SIZE;
1247     }
1248 
1249     /** The value of {@code addressSize()} */
1250     public static final int ADDRESS_SIZE = theUnsafe.addressSize0();
1251 
1252     /**
1253      * Reports the size in bytes of a native memory page (whatever that is).
1254      * This value will always be a power of two.
1255      */
1256     public native int pageSize();
1257 
1258     /// random trusted operations from JNI:
1259 
1260     /**
1261      * Tells the VM to define a class, without security checks.  By default, the
1262      * class loader and protection domain come from the caller's class.
1263      */
1264     public Class<?> defineClass(String name, byte[] b, int off, int len,
1265                                 ClassLoader loader,
1266                                 ProtectionDomain protectionDomain) {
1267         if (b == null) {
1268             throw new NullPointerException();
1269         }
1270         if (len < 0) {
1271             throw new ArrayIndexOutOfBoundsException();
1272         }
1273 
1274         return defineClass0(name, b, off, len, loader, protectionDomain);
1275     }
1276 
1277     public native Class<?> defineClass0(String name, byte[] b, int off, int len,
1278                                         ClassLoader loader,
1279                                         ProtectionDomain protectionDomain);
1280 
1281     /**
1282      * Defines a class but does not make it known to the class loader or system dictionary.
1283      * <p>
1284      * For each CP entry, the corresponding CP patch must either be null or have
1285      * the a format that matches its tag:
1286      * <ul>
1287      * <li>Integer, Long, Float, Double: the corresponding wrapper object type from java.lang
1288      * <li>Utf8: a string (must have suitable syntax if used as signature or name)
1289      * <li>Class: any java.lang.Class object
1290      * <li>String: any object (not just a java.lang.String)
1291      * <li>InterfaceMethodRef: (NYI) a method handle to invoke on that call site's arguments
1292      * </ul>
1293      * @param hostClass context for linkage, access control, protection domain, and class loader
1294      * @param data      bytes of a class file
1295      * @param cpPatches where non-null entries exist, they replace corresponding CP entries in data
1296      */
1297     public Class<?> defineAnonymousClass(Class<?> hostClass, byte[] data, Object[] cpPatches) {
1298         if (hostClass == null || data == null) {
1299             throw new NullPointerException();
1300         }
1301         if (hostClass.isArray() || hostClass.isPrimitive()) {
1302             throw new IllegalArgumentException();
1303         }
1304 
1305         return defineAnonymousClass0(hostClass, data, cpPatches);
1306     }
1307 
1308     /**
1309      * Allocates an instance but does not run any constructor.
1310      * Initializes the class if it has not yet been.
1311      */
1312     @HotSpotIntrinsicCandidate
1313     public native Object allocateInstance(Class<?> cls)
1314         throws InstantiationException;
1315 
1316     /**
1317      * Allocates an array of a given type, but does not do zeroing.
1318      * <p>
1319      * This method should only be used in the very rare cases where a high-performance code
1320      * overwrites the destination array completely, and compilers cannot assist in zeroing elimination.
1321      * In an overwhelming majority of cases, a normal Java allocation should be used instead.
1322      * <p>
1323      * Users of this method are <b>required</b> to overwrite the initial (garbage) array contents
1324      * before allowing untrusted code, or code in other threads, to observe the reference
1325      * to the newly allocated array. In addition, the publication of the array reference must be
1326      * safe according to the Java Memory Model requirements.
1327      * <p>
1328      * The safest approach to deal with an uninitialized array is to keep the reference to it in local
1329      * variable at least until the initialization is complete, and then publish it <b>once</b>, either
1330      * by writing it to a <em>volatile</em> field, or storing it into a <em>final</em> field in constructor,
1331      * or issuing a {@link #storeFence} before publishing the reference.
1332      * <p>
1333      * @implnote This method can only allocate primitive arrays, to avoid garbage reference
1334      * elements that could break heap integrity.
1335      *
1336      * @param componentType array component type to allocate
1337      * @param length array size to allocate
1338      * @throws IllegalArgumentException if component type is null, or not a primitive class;
1339      *                                  or the length is negative
1340      */
1341     public Object allocateUninitializedArray(Class<?> componentType, int length) {
1342        if (componentType == null) {
1343            throw new IllegalArgumentException("Component type is null");
1344        }
1345        if (!componentType.isPrimitive()) {
1346            throw new IllegalArgumentException("Component type is not primitive");
1347        }
1348        if (length < 0) {
1349            throw new IllegalArgumentException("Negative length");
1350        }
1351        return allocateUninitializedArray0(componentType, length);
1352     }
1353 
1354     @HotSpotIntrinsicCandidate
1355     private Object allocateUninitializedArray0(Class<?> componentType, int length) {
1356        // These fallbacks provide zeroed arrays, but intrinsic is not required to
1357        // return the zeroed arrays.
1358        if (componentType == byte.class)    return new byte[length];
1359        if (componentType == boolean.class) return new boolean[length];
1360        if (componentType == short.class)   return new short[length];
1361        if (componentType == char.class)    return new char[length];
1362        if (componentType == int.class)     return new int[length];
1363        if (componentType == float.class)   return new float[length];
1364        if (componentType == long.class)    return new long[length];
1365        if (componentType == double.class)  return new double[length];
1366        return null;
1367     }
1368 
1369     /** Throws the exception without telling the verifier. */
1370     public native void throwException(Throwable ee);
1371 
1372     /**
1373      * Atomically updates Java variable to {@code x} if it is currently
1374      * holding {@code expected}.
1375      *
1376      * <p>This operation has memory semantics of a {@code volatile} read
1377      * and write.  Corresponds to C11 atomic_compare_exchange_strong.
1378      *
1379      * @return {@code true} if successful
1380      */
1381     @HotSpotIntrinsicCandidate
1382     public final native boolean compareAndSetObject(Object o, long offset,
1383                                                     Object expected,
1384                                                     Object x);
1385 
1386     @HotSpotIntrinsicCandidate
1387     public final native Object compareAndExchangeObject(Object o, long offset,
1388                                                         Object expected,
1389                                                         Object x);
1390 
1391     @HotSpotIntrinsicCandidate
1392     public final Object compareAndExchangeObjectAcquire(Object o, long offset,
1393                                                                Object expected,
1394                                                                Object x) {
1395         return compareAndExchangeObject(o, offset, expected, x);
1396     }
1397 
1398     @HotSpotIntrinsicCandidate
1399     public final Object compareAndExchangeObjectRelease(Object o, long offset,
1400                                                                Object expected,
1401                                                                Object x) {
1402         return compareAndExchangeObject(o, offset, expected, x);
1403     }
1404 
1405     @HotSpotIntrinsicCandidate
1406     public final boolean weakCompareAndSetObjectPlain(Object o, long offset,
1407                                                       Object expected,
1408                                                       Object x) {
1409         return compareAndSetObject(o, offset, expected, x);
1410     }
1411 
1412     @HotSpotIntrinsicCandidate
1413     public final boolean weakCompareAndSetObjectAcquire(Object o, long offset,
1414                                                         Object expected,
1415                                                         Object x) {
1416         return compareAndSetObject(o, offset, expected, x);
1417     }
1418 
1419     @HotSpotIntrinsicCandidate
1420     public final boolean weakCompareAndSetObjectRelease(Object o, long offset,
1421                                                         Object expected,
1422                                                         Object x) {
1423         return compareAndSetObject(o, offset, expected, x);
1424     }
1425 
1426     @HotSpotIntrinsicCandidate
1427     public final boolean weakCompareAndSetObject(Object o, long offset,
1428                                                  Object expected,
1429                                                  Object x) {
1430         return compareAndSetObject(o, offset, expected, x);
1431     }
1432 
1433     /**
1434      * Atomically updates Java variable to {@code x} if it is currently
1435      * holding {@code expected}.
1436      *
1437      * <p>This operation has memory semantics of a {@code volatile} read
1438      * and write.  Corresponds to C11 atomic_compare_exchange_strong.
1439      *
1440      * @return {@code true} if successful
1441      */
1442     @HotSpotIntrinsicCandidate
1443     public final native boolean compareAndSetInt(Object o, long offset,
1444                                                  int expected,
1445                                                  int x);
1446 
1447     @HotSpotIntrinsicCandidate
1448     public final native int compareAndExchangeInt(Object o, long offset,
1449                                                   int expected,
1450                                                   int x);
1451 
1452     @HotSpotIntrinsicCandidate
1453     public final int compareAndExchangeIntAcquire(Object o, long offset,
1454                                                          int expected,
1455                                                          int x) {
1456         return compareAndExchangeInt(o, offset, expected, x);
1457     }
1458 
1459     @HotSpotIntrinsicCandidate
1460     public final int compareAndExchangeIntRelease(Object o, long offset,
1461                                                          int expected,
1462                                                          int x) {
1463         return compareAndExchangeInt(o, offset, expected, x);
1464     }
1465 
1466     @HotSpotIntrinsicCandidate
1467     public final boolean weakCompareAndSetIntPlain(Object o, long offset,
1468                                                    int expected,
1469                                                    int x) {
1470         return compareAndSetInt(o, offset, expected, x);
1471     }
1472 
1473     @HotSpotIntrinsicCandidate
1474     public final boolean weakCompareAndSetIntAcquire(Object o, long offset,
1475                                                      int expected,
1476                                                      int x) {
1477         return compareAndSetInt(o, offset, expected, x);
1478     }
1479 
1480     @HotSpotIntrinsicCandidate
1481     public final boolean weakCompareAndSetIntRelease(Object o, long offset,
1482                                                      int expected,
1483                                                      int x) {
1484         return compareAndSetInt(o, offset, expected, x);
1485     }
1486 
1487     @HotSpotIntrinsicCandidate
1488     public final boolean weakCompareAndSetInt(Object o, long offset,
1489                                               int expected,
1490                                               int x) {
1491         return compareAndSetInt(o, offset, expected, x);
1492     }
1493 
1494     @HotSpotIntrinsicCandidate
1495     public final byte compareAndExchangeByte(Object o, long offset,
1496                                              byte expected,
1497                                              byte x) {
1498         long wordOffset = offset & ~3;
1499         int shift = (int) (offset & 3) << 3;
1500         if (BE) {
1501             shift = 24 - shift;
1502         }
1503         int mask           = 0xFF << shift;
1504         int maskedExpected = (expected & 0xFF) << shift;
1505         int maskedX        = (x & 0xFF) << shift;
1506         int fullWord;
1507         do {
1508             fullWord = getIntVolatile(o, wordOffset);
1509             if ((fullWord & mask) != maskedExpected)
1510                 return (byte) ((fullWord & mask) >> shift);
1511         } while (!weakCompareAndSetInt(o, wordOffset,
1512                                                 fullWord, (fullWord & ~mask) | maskedX));
1513         return expected;
1514     }
1515 
1516     @HotSpotIntrinsicCandidate
1517     public final boolean compareAndSetByte(Object o, long offset,
1518                                            byte expected,
1519                                            byte x) {
1520         return compareAndExchangeByte(o, offset, expected, x) == expected;
1521     }
1522 
1523     @HotSpotIntrinsicCandidate
1524     public final boolean weakCompareAndSetByte(Object o, long offset,
1525                                                byte expected,
1526                                                byte x) {
1527         return compareAndSetByte(o, offset, expected, x);
1528     }
1529 
1530     @HotSpotIntrinsicCandidate
1531     public final boolean weakCompareAndSetByteAcquire(Object o, long offset,
1532                                                       byte expected,
1533                                                       byte x) {
1534         return weakCompareAndSetByte(o, offset, expected, x);
1535     }
1536 
1537     @HotSpotIntrinsicCandidate
1538     public final boolean weakCompareAndSetByteRelease(Object o, long offset,
1539                                                       byte expected,
1540                                                       byte x) {
1541         return weakCompareAndSetByte(o, offset, expected, x);
1542     }
1543 
1544     @HotSpotIntrinsicCandidate
1545     public final boolean weakCompareAndSetBytePlain(Object o, long offset,
1546                                                     byte expected,
1547                                                     byte x) {
1548         return weakCompareAndSetByte(o, offset, expected, x);
1549     }
1550 
1551     @HotSpotIntrinsicCandidate
1552     public final byte compareAndExchangeByteAcquire(Object o, long offset,
1553                                                     byte expected,
1554                                                     byte x) {
1555         return compareAndExchangeByte(o, offset, expected, x);
1556     }
1557 
1558     @HotSpotIntrinsicCandidate
1559     public final byte compareAndExchangeByteRelease(Object o, long offset,
1560                                                     byte expected,
1561                                                     byte x) {
1562         return compareAndExchangeByte(o, offset, expected, x);
1563     }
1564 
1565     @HotSpotIntrinsicCandidate
1566     public final short compareAndExchangeShort(Object o, long offset,
1567                                                short expected,
1568                                                short x) {
1569         if ((offset & 3) == 3) {
1570             throw new IllegalArgumentException("Update spans the word, not supported");
1571         }
1572         long wordOffset = offset & ~3;
1573         int shift = (int) (offset & 3) << 3;
1574         if (BE) {
1575             shift = 16 - shift;
1576         }
1577         int mask           = 0xFFFF << shift;
1578         int maskedExpected = (expected & 0xFFFF) << shift;
1579         int maskedX        = (x & 0xFFFF) << shift;
1580         int fullWord;
1581         do {
1582             fullWord = getIntVolatile(o, wordOffset);
1583             if ((fullWord & mask) != maskedExpected) {
1584                 return (short) ((fullWord & mask) >> shift);
1585             }
1586         } while (!weakCompareAndSetInt(o, wordOffset,
1587                                                 fullWord, (fullWord & ~mask) | maskedX));
1588         return expected;
1589     }
1590 
1591     @HotSpotIntrinsicCandidate
1592     public final boolean compareAndSetShort(Object o, long offset,
1593                                             short expected,
1594                                             short x) {
1595         return compareAndExchangeShort(o, offset, expected, x) == expected;
1596     }
1597 
1598     @HotSpotIntrinsicCandidate
1599     public final boolean weakCompareAndSetShort(Object o, long offset,
1600                                                 short expected,
1601                                                 short x) {
1602         return compareAndSetShort(o, offset, expected, x);
1603     }
1604 
1605     @HotSpotIntrinsicCandidate
1606     public final boolean weakCompareAndSetShortAcquire(Object o, long offset,
1607                                                        short expected,
1608                                                        short x) {
1609         return weakCompareAndSetShort(o, offset, expected, x);
1610     }
1611 
1612     @HotSpotIntrinsicCandidate
1613     public final boolean weakCompareAndSetShortRelease(Object o, long offset,
1614                                                        short expected,
1615                                                        short x) {
1616         return weakCompareAndSetShort(o, offset, expected, x);
1617     }
1618 
1619     @HotSpotIntrinsicCandidate
1620     public final boolean weakCompareAndSetShortPlain(Object o, long offset,
1621                                                      short expected,
1622                                                      short x) {
1623         return weakCompareAndSetShort(o, offset, expected, x);
1624     }
1625 
1626 
1627     @HotSpotIntrinsicCandidate
1628     public final short compareAndExchangeShortAcquire(Object o, long offset,
1629                                                      short expected,
1630                                                      short x) {
1631         return compareAndExchangeShort(o, offset, expected, x);
1632     }
1633 
1634     @HotSpotIntrinsicCandidate
1635     public final short compareAndExchangeShortRelease(Object o, long offset,
1636                                                     short expected,
1637                                                     short x) {
1638         return compareAndExchangeShort(o, offset, expected, x);
1639     }
1640 
1641     @ForceInline
1642     private char s2c(short s) {
1643         return (char) s;
1644     }
1645 
1646     @ForceInline
1647     private short c2s(char s) {
1648         return (short) s;
1649     }
1650 
1651     @ForceInline
1652     public final boolean compareAndSetChar(Object o, long offset,
1653                                            char expected,
1654                                            char x) {
1655         return compareAndSetShort(o, offset, c2s(expected), c2s(x));
1656     }
1657 
1658     @ForceInline
1659     public final char compareAndExchangeChar(Object o, long offset,
1660                                              char expected,
1661                                              char x) {
1662         return s2c(compareAndExchangeShort(o, offset, c2s(expected), c2s(x)));
1663     }
1664 
1665     @ForceInline
1666     public final char compareAndExchangeCharAcquire(Object o, long offset,
1667                                             char expected,
1668                                             char x) {
1669         return s2c(compareAndExchangeShortAcquire(o, offset, c2s(expected), c2s(x)));
1670     }
1671 
1672     @ForceInline
1673     public final char compareAndExchangeCharRelease(Object o, long offset,
1674                                             char expected,
1675                                             char x) {
1676         return s2c(compareAndExchangeShortRelease(o, offset, c2s(expected), c2s(x)));
1677     }
1678 
1679     @ForceInline
1680     public final boolean weakCompareAndSetChar(Object o, long offset,
1681                                                char expected,
1682                                                char x) {
1683         return weakCompareAndSetShort(o, offset, c2s(expected), c2s(x));
1684     }
1685 
1686     @ForceInline
1687     public final boolean weakCompareAndSetCharAcquire(Object o, long offset,
1688                                                       char expected,
1689                                                       char x) {
1690         return weakCompareAndSetShortAcquire(o, offset, c2s(expected), c2s(x));
1691     }
1692 
1693     @ForceInline
1694     public final boolean weakCompareAndSetCharRelease(Object o, long offset,
1695                                                       char expected,
1696                                                       char x) {
1697         return weakCompareAndSetShortRelease(o, offset, c2s(expected), c2s(x));
1698     }
1699 
1700     @ForceInline
1701     public final boolean weakCompareAndSetCharPlain(Object o, long offset,
1702                                                     char expected,
1703                                                     char x) {
1704         return weakCompareAndSetShortPlain(o, offset, c2s(expected), c2s(x));
1705     }
1706 
1707     /**
1708      * The JVM converts integral values to boolean values using two
1709      * different conventions, byte testing against zero and truncation
1710      * to least-significant bit.
1711      *
1712      * <p>The JNI documents specify that, at least for returning
1713      * values from native methods, a Java boolean value is converted
1714      * to the value-set 0..1 by first truncating to a byte (0..255 or
1715      * maybe -128..127) and then testing against zero. Thus, Java
1716      * booleans in non-Java data structures are by convention
1717      * represented as 8-bit containers containing either zero (for
1718      * false) or any non-zero value (for true).
1719      *
1720      * <p>Java booleans in the heap are also stored in bytes, but are
1721      * strongly normalized to the value-set 0..1 (i.e., they are
1722      * truncated to the least-significant bit).
1723      *
1724      * <p>The main reason for having different conventions for
1725      * conversion is performance: Truncation to the least-significant
1726      * bit can be usually implemented with fewer (machine)
1727      * instructions than byte testing against zero.
1728      *
1729      * <p>A number of Unsafe methods load boolean values from the heap
1730      * as bytes. Unsafe converts those values according to the JNI
1731      * rules (i.e, using the "testing against zero" convention). The
1732      * method {@code byte2bool} implements that conversion.
1733      *
1734      * @param b the byte to be converted to boolean
1735      * @return the result of the conversion
1736      */
1737     @ForceInline
1738     private boolean byte2bool(byte b) {
1739         return b != 0;
1740     }
1741 
1742     /**
1743      * Convert a boolean value to a byte. The return value is strongly
1744      * normalized to the value-set 0..1 (i.e., the value is truncated
1745      * to the least-significant bit). See {@link #byte2bool(byte)} for
1746      * more details on conversion conventions.
1747      *
1748      * @param b the boolean to be converted to byte (and then normalized)
1749      * @return the result of the conversion
1750      */
1751     @ForceInline
1752     private byte bool2byte(boolean b) {
1753         return b ? (byte)1 : (byte)0;
1754     }
1755 
1756     @ForceInline
1757     public final boolean compareAndSetBoolean(Object o, long offset,
1758                                               boolean expected,
1759                                               boolean x) {
1760         return compareAndSetByte(o, offset, bool2byte(expected), bool2byte(x));
1761     }
1762 
1763     @ForceInline
1764     public final boolean compareAndExchangeBoolean(Object o, long offset,
1765                                                    boolean expected,
1766                                                    boolean x) {
1767         return byte2bool(compareAndExchangeByte(o, offset, bool2byte(expected), bool2byte(x)));
1768     }
1769 
1770     @ForceInline
1771     public final boolean compareAndExchangeBooleanAcquire(Object o, long offset,
1772                                                     boolean expected,
1773                                                     boolean x) {
1774         return byte2bool(compareAndExchangeByteAcquire(o, offset, bool2byte(expected), bool2byte(x)));
1775     }
1776 
1777     @ForceInline
1778     public final boolean compareAndExchangeBooleanRelease(Object o, long offset,
1779                                                        boolean expected,
1780                                                        boolean x) {
1781         return byte2bool(compareAndExchangeByteRelease(o, offset, bool2byte(expected), bool2byte(x)));
1782     }
1783 
1784     @ForceInline
1785     public final boolean weakCompareAndSetBoolean(Object o, long offset,
1786                                                   boolean expected,
1787                                                   boolean x) {
1788         return weakCompareAndSetByte(o, offset, bool2byte(expected), bool2byte(x));
1789     }
1790 
1791     @ForceInline
1792     public final boolean weakCompareAndSetBooleanAcquire(Object o, long offset,
1793                                                          boolean expected,
1794                                                          boolean x) {
1795         return weakCompareAndSetByteAcquire(o, offset, bool2byte(expected), bool2byte(x));
1796     }
1797 
1798     @ForceInline
1799     public final boolean weakCompareAndSetBooleanRelease(Object o, long offset,
1800                                                          boolean expected,
1801                                                          boolean x) {
1802         return weakCompareAndSetByteRelease(o, offset, bool2byte(expected), bool2byte(x));
1803     }
1804 
1805     @ForceInline
1806     public final boolean weakCompareAndSetBooleanPlain(Object o, long offset,
1807                                                        boolean expected,
1808                                                        boolean x) {
1809         return weakCompareAndSetBytePlain(o, offset, bool2byte(expected), bool2byte(x));
1810     }
1811 
1812     /**
1813      * Atomically updates Java variable to {@code x} if it is currently
1814      * holding {@code expected}.
1815      *
1816      * <p>This operation has memory semantics of a {@code volatile} read
1817      * and write.  Corresponds to C11 atomic_compare_exchange_strong.
1818      *
1819      * @return {@code true} if successful
1820      */
1821     @ForceInline
1822     public final boolean compareAndSetFloat(Object o, long offset,
1823                                             float expected,
1824                                             float x) {
1825         return compareAndSetInt(o, offset,
1826                                  Float.floatToRawIntBits(expected),
1827                                  Float.floatToRawIntBits(x));
1828     }
1829 
1830     @ForceInline
1831     public final float compareAndExchangeFloat(Object o, long offset,
1832                                                float expected,
1833                                                float x) {
1834         int w = compareAndExchangeInt(o, offset,
1835                                       Float.floatToRawIntBits(expected),
1836                                       Float.floatToRawIntBits(x));
1837         return Float.intBitsToFloat(w);
1838     }
1839 
1840     @ForceInline
1841     public final float compareAndExchangeFloatAcquire(Object o, long offset,
1842                                                   float expected,
1843                                                   float x) {
1844         int w = compareAndExchangeIntAcquire(o, offset,
1845                                              Float.floatToRawIntBits(expected),
1846                                              Float.floatToRawIntBits(x));
1847         return Float.intBitsToFloat(w);
1848     }
1849 
1850     @ForceInline
1851     public final float compareAndExchangeFloatRelease(Object o, long offset,
1852                                                   float expected,
1853                                                   float x) {
1854         int w = compareAndExchangeIntRelease(o, offset,
1855                                              Float.floatToRawIntBits(expected),
1856                                              Float.floatToRawIntBits(x));
1857         return Float.intBitsToFloat(w);
1858     }
1859 
1860     @ForceInline
1861     public final boolean weakCompareAndSetFloatPlain(Object o, long offset,
1862                                                      float expected,
1863                                                      float x) {
1864         return weakCompareAndSetIntPlain(o, offset,
1865                                      Float.floatToRawIntBits(expected),
1866                                      Float.floatToRawIntBits(x));
1867     }
1868 
1869     @ForceInline
1870     public final boolean weakCompareAndSetFloatAcquire(Object o, long offset,
1871                                                        float expected,
1872                                                        float x) {
1873         return weakCompareAndSetIntAcquire(o, offset,
1874                                             Float.floatToRawIntBits(expected),
1875                                             Float.floatToRawIntBits(x));
1876     }
1877 
1878     @ForceInline
1879     public final boolean weakCompareAndSetFloatRelease(Object o, long offset,
1880                                                        float expected,
1881                                                        float x) {
1882         return weakCompareAndSetIntRelease(o, offset,
1883                                             Float.floatToRawIntBits(expected),
1884                                             Float.floatToRawIntBits(x));
1885     }
1886 
1887     @ForceInline
1888     public final boolean weakCompareAndSetFloat(Object o, long offset,
1889                                                 float expected,
1890                                                 float x) {
1891         return weakCompareAndSetInt(o, offset,
1892                                              Float.floatToRawIntBits(expected),
1893                                              Float.floatToRawIntBits(x));
1894     }
1895 
1896     /**
1897      * Atomically updates Java variable to {@code x} if it is currently
1898      * holding {@code expected}.
1899      *
1900      * <p>This operation has memory semantics of a {@code volatile} read
1901      * and write.  Corresponds to C11 atomic_compare_exchange_strong.
1902      *
1903      * @return {@code true} if successful
1904      */
1905     @ForceInline
1906     public final boolean compareAndSetDouble(Object o, long offset,
1907                                              double expected,
1908                                              double x) {
1909         return compareAndSetLong(o, offset,
1910                                  Double.doubleToRawLongBits(expected),
1911                                  Double.doubleToRawLongBits(x));
1912     }
1913 
1914     @ForceInline
1915     public final double compareAndExchangeDouble(Object o, long offset,
1916                                                  double expected,
1917                                                  double x) {
1918         long w = compareAndExchangeLong(o, offset,
1919                                         Double.doubleToRawLongBits(expected),
1920                                         Double.doubleToRawLongBits(x));
1921         return Double.longBitsToDouble(w);
1922     }
1923 
1924     @ForceInline
1925     public final double compareAndExchangeDoubleAcquire(Object o, long offset,
1926                                                         double expected,
1927                                                         double x) {
1928         long w = compareAndExchangeLongAcquire(o, offset,
1929                                                Double.doubleToRawLongBits(expected),
1930                                                Double.doubleToRawLongBits(x));
1931         return Double.longBitsToDouble(w);
1932     }
1933 
1934     @ForceInline
1935     public final double compareAndExchangeDoubleRelease(Object o, long offset,
1936                                                         double expected,
1937                                                         double x) {
1938         long w = compareAndExchangeLongRelease(o, offset,
1939                                                Double.doubleToRawLongBits(expected),
1940                                                Double.doubleToRawLongBits(x));
1941         return Double.longBitsToDouble(w);
1942     }
1943 
1944     @ForceInline
1945     public final boolean weakCompareAndSetDoublePlain(Object o, long offset,
1946                                                       double expected,
1947                                                       double x) {
1948         return weakCompareAndSetLongPlain(o, offset,
1949                                      Double.doubleToRawLongBits(expected),
1950                                      Double.doubleToRawLongBits(x));
1951     }
1952 
1953     @ForceInline
1954     public final boolean weakCompareAndSetDoubleAcquire(Object o, long offset,
1955                                                         double expected,
1956                                                         double x) {
1957         return weakCompareAndSetLongAcquire(o, offset,
1958                                              Double.doubleToRawLongBits(expected),
1959                                              Double.doubleToRawLongBits(x));
1960     }
1961 
1962     @ForceInline
1963     public final boolean weakCompareAndSetDoubleRelease(Object o, long offset,
1964                                                         double expected,
1965                                                         double x) {
1966         return weakCompareAndSetLongRelease(o, offset,
1967                                              Double.doubleToRawLongBits(expected),
1968                                              Double.doubleToRawLongBits(x));
1969     }
1970 
1971     @ForceInline
1972     public final boolean weakCompareAndSetDouble(Object o, long offset,
1973                                                  double expected,
1974                                                  double x) {
1975         return weakCompareAndSetLong(o, offset,
1976                                               Double.doubleToRawLongBits(expected),
1977                                               Double.doubleToRawLongBits(x));
1978     }
1979 
1980     /**
1981      * Atomically updates Java variable to {@code x} if it is currently
1982      * holding {@code expected}.
1983      *
1984      * <p>This operation has memory semantics of a {@code volatile} read
1985      * and write.  Corresponds to C11 atomic_compare_exchange_strong.
1986      *
1987      * @return {@code true} if successful
1988      */
1989     @HotSpotIntrinsicCandidate
1990     public final native boolean compareAndSetLong(Object o, long offset,
1991                                                   long expected,
1992                                                   long x);
1993 
1994     @HotSpotIntrinsicCandidate
1995     public final native long compareAndExchangeLong(Object o, long offset,
1996                                                     long expected,
1997                                                     long x);
1998 
1999     @HotSpotIntrinsicCandidate
2000     public final long compareAndExchangeLongAcquire(Object o, long offset,
2001                                                            long expected,
2002                                                            long x) {
2003         return compareAndExchangeLong(o, offset, expected, x);
2004     }
2005 
2006     @HotSpotIntrinsicCandidate
2007     public final long compareAndExchangeLongRelease(Object o, long offset,
2008                                                            long expected,
2009                                                            long x) {
2010         return compareAndExchangeLong(o, offset, expected, x);
2011     }
2012 
2013     @HotSpotIntrinsicCandidate
2014     public final boolean weakCompareAndSetLongPlain(Object o, long offset,
2015                                                     long expected,
2016                                                     long x) {
2017         return compareAndSetLong(o, offset, expected, x);
2018     }
2019 
2020     @HotSpotIntrinsicCandidate
2021     public final boolean weakCompareAndSetLongAcquire(Object o, long offset,
2022                                                       long expected,
2023                                                       long x) {
2024         return compareAndSetLong(o, offset, expected, x);
2025     }
2026 
2027     @HotSpotIntrinsicCandidate
2028     public final boolean weakCompareAndSetLongRelease(Object o, long offset,
2029                                                       long expected,
2030                                                       long x) {
2031         return compareAndSetLong(o, offset, expected, x);
2032     }
2033 
2034     @HotSpotIntrinsicCandidate
2035     public final boolean weakCompareAndSetLong(Object o, long offset,
2036                                                long expected,
2037                                                long x) {
2038         return compareAndSetLong(o, offset, expected, x);
2039     }
2040 
2041     /**
2042      * Fetches a reference value from a given Java variable, with volatile
2043      * load semantics. Otherwise identical to {@link #getObject(Object, long)}
2044      */
2045     @HotSpotIntrinsicCandidate
2046     public native Object getObjectVolatile(Object o, long offset);
2047 
2048     /**
2049      * Stores a reference value into a given Java variable, with
2050      * volatile store semantics. Otherwise identical to {@link #putObject(Object, long, Object)}
2051      */
2052     @HotSpotIntrinsicCandidate
2053     public native void    putObjectVolatile(Object o, long offset, Object x);
2054 
2055     /** Volatile version of {@link #getInt(Object, long)}  */
2056     @HotSpotIntrinsicCandidate
2057     public native int     getIntVolatile(Object o, long offset);
2058 
2059     /** Volatile version of {@link #putInt(Object, long, int)}  */
2060     @HotSpotIntrinsicCandidate
2061     public native void    putIntVolatile(Object o, long offset, int x);
2062 
2063     /** Volatile version of {@link #getBoolean(Object, long)}  */
2064     @HotSpotIntrinsicCandidate
2065     public native boolean getBooleanVolatile(Object o, long offset);
2066 
2067     /** Volatile version of {@link #putBoolean(Object, long, boolean)}  */
2068     @HotSpotIntrinsicCandidate
2069     public native void    putBooleanVolatile(Object o, long offset, boolean x);
2070 
2071     /** Volatile version of {@link #getByte(Object, long)}  */
2072     @HotSpotIntrinsicCandidate
2073     public native byte    getByteVolatile(Object o, long offset);
2074 
2075     /** Volatile version of {@link #putByte(Object, long, byte)}  */
2076     @HotSpotIntrinsicCandidate
2077     public native void    putByteVolatile(Object o, long offset, byte x);
2078 
2079     /** Volatile version of {@link #getShort(Object, long)}  */
2080     @HotSpotIntrinsicCandidate
2081     public native short   getShortVolatile(Object o, long offset);
2082 
2083     /** Volatile version of {@link #putShort(Object, long, short)}  */
2084     @HotSpotIntrinsicCandidate
2085     public native void    putShortVolatile(Object o, long offset, short x);
2086 
2087     /** Volatile version of {@link #getChar(Object, long)}  */
2088     @HotSpotIntrinsicCandidate
2089     public native char    getCharVolatile(Object o, long offset);
2090 
2091     /** Volatile version of {@link #putChar(Object, long, char)}  */
2092     @HotSpotIntrinsicCandidate
2093     public native void    putCharVolatile(Object o, long offset, char x);
2094 
2095     /** Volatile version of {@link #getLong(Object, long)}  */
2096     @HotSpotIntrinsicCandidate
2097     public native long    getLongVolatile(Object o, long offset);
2098 
2099     /** Volatile version of {@link #putLong(Object, long, long)}  */
2100     @HotSpotIntrinsicCandidate
2101     public native void    putLongVolatile(Object o, long offset, long x);
2102 
2103     /** Volatile version of {@link #getFloat(Object, long)}  */
2104     @HotSpotIntrinsicCandidate
2105     public native float   getFloatVolatile(Object o, long offset);
2106 
2107     /** Volatile version of {@link #putFloat(Object, long, float)}  */
2108     @HotSpotIntrinsicCandidate
2109     public native void    putFloatVolatile(Object o, long offset, float x);
2110 
2111     /** Volatile version of {@link #getDouble(Object, long)}  */
2112     @HotSpotIntrinsicCandidate
2113     public native double  getDoubleVolatile(Object o, long offset);
2114 
2115     /** Volatile version of {@link #putDouble(Object, long, double)}  */
2116     @HotSpotIntrinsicCandidate
2117     public native void    putDoubleVolatile(Object o, long offset, double x);
2118 
2119 
2120 
2121     /** Acquire version of {@link #getObjectVolatile(Object, long)} */
2122     @HotSpotIntrinsicCandidate
2123     public final Object getObjectAcquire(Object o, long offset) {
2124         return getObjectVolatile(o, offset);
2125     }
2126 
2127     /** Acquire version of {@link #getBooleanVolatile(Object, long)} */
2128     @HotSpotIntrinsicCandidate
2129     public final boolean getBooleanAcquire(Object o, long offset) {
2130         return getBooleanVolatile(o, offset);
2131     }
2132 
2133     /** Acquire version of {@link #getByteVolatile(Object, long)} */
2134     @HotSpotIntrinsicCandidate
2135     public final byte getByteAcquire(Object o, long offset) {
2136         return getByteVolatile(o, offset);
2137     }
2138 
2139     /** Acquire version of {@link #getShortVolatile(Object, long)} */
2140     @HotSpotIntrinsicCandidate
2141     public final short getShortAcquire(Object o, long offset) {
2142         return getShortVolatile(o, offset);
2143     }
2144 
2145     /** Acquire version of {@link #getCharVolatile(Object, long)} */
2146     @HotSpotIntrinsicCandidate
2147     public final char getCharAcquire(Object o, long offset) {
2148         return getCharVolatile(o, offset);
2149     }
2150 
2151     /** Acquire version of {@link #getIntVolatile(Object, long)} */
2152     @HotSpotIntrinsicCandidate
2153     public final int getIntAcquire(Object o, long offset) {
2154         return getIntVolatile(o, offset);
2155     }
2156 
2157     /** Acquire version of {@link #getFloatVolatile(Object, long)} */
2158     @HotSpotIntrinsicCandidate
2159     public final float getFloatAcquire(Object o, long offset) {
2160         return getFloatVolatile(o, offset);
2161     }
2162 
2163     /** Acquire version of {@link #getLongVolatile(Object, long)} */
2164     @HotSpotIntrinsicCandidate
2165     public final long getLongAcquire(Object o, long offset) {
2166         return getLongVolatile(o, offset);
2167     }
2168 
2169     /** Acquire version of {@link #getDoubleVolatile(Object, long)} */
2170     @HotSpotIntrinsicCandidate
2171     public final double getDoubleAcquire(Object o, long offset) {
2172         return getDoubleVolatile(o, offset);
2173     }
2174 
2175     /*
2176       * Versions of {@link #putObjectVolatile(Object, long, Object)}
2177       * that do not guarantee immediate visibility of the store to
2178       * other threads. This method is generally only useful if the
2179       * underlying field is a Java volatile (or if an array cell, one
2180       * that is otherwise only accessed using volatile accesses).
2181       *
2182       * Corresponds to C11 atomic_store_explicit(..., memory_order_release).
2183       */
2184 
2185     /** Release version of {@link #putObjectVolatile(Object, long, Object)} */
2186     @HotSpotIntrinsicCandidate
2187     public final void putObjectRelease(Object o, long offset, Object x) {
2188         putObjectVolatile(o, offset, x);
2189     }
2190 
2191     /** Release version of {@link #putBooleanVolatile(Object, long, boolean)} */
2192     @HotSpotIntrinsicCandidate
2193     public final void putBooleanRelease(Object o, long offset, boolean x) {
2194         putBooleanVolatile(o, offset, x);
2195     }
2196 
2197     /** Release version of {@link #putByteVolatile(Object, long, byte)} */
2198     @HotSpotIntrinsicCandidate
2199     public final void putByteRelease(Object o, long offset, byte x) {
2200         putByteVolatile(o, offset, x);
2201     }
2202 
2203     /** Release version of {@link #putShortVolatile(Object, long, short)} */
2204     @HotSpotIntrinsicCandidate
2205     public final void putShortRelease(Object o, long offset, short x) {
2206         putShortVolatile(o, offset, x);
2207     }
2208 
2209     /** Release version of {@link #putCharVolatile(Object, long, char)} */
2210     @HotSpotIntrinsicCandidate
2211     public final void putCharRelease(Object o, long offset, char x) {
2212         putCharVolatile(o, offset, x);
2213     }
2214 
2215     /** Release version of {@link #putIntVolatile(Object, long, int)} */
2216     @HotSpotIntrinsicCandidate
2217     public final void putIntRelease(Object o, long offset, int x) {
2218         putIntVolatile(o, offset, x);
2219     }
2220 
2221     /** Release version of {@link #putFloatVolatile(Object, long, float)} */
2222     @HotSpotIntrinsicCandidate
2223     public final void putFloatRelease(Object o, long offset, float x) {
2224         putFloatVolatile(o, offset, x);
2225     }
2226 
2227     /** Release version of {@link #putLongVolatile(Object, long, long)} */
2228     @HotSpotIntrinsicCandidate
2229     public final void putLongRelease(Object o, long offset, long x) {
2230         putLongVolatile(o, offset, x);
2231     }
2232 
2233     /** Release version of {@link #putDoubleVolatile(Object, long, double)} */
2234     @HotSpotIntrinsicCandidate
2235     public final void putDoubleRelease(Object o, long offset, double x) {
2236         putDoubleVolatile(o, offset, x);
2237     }
2238 
2239     // ------------------------------ Opaque --------------------------------------
2240 
2241     /** Opaque version of {@link #getObjectVolatile(Object, long)} */
2242     @HotSpotIntrinsicCandidate
2243     public final Object getObjectOpaque(Object o, long offset) {
2244         return getObjectVolatile(o, offset);
2245     }
2246 
2247     /** Opaque version of {@link #getBooleanVolatile(Object, long)} */
2248     @HotSpotIntrinsicCandidate
2249     public final boolean getBooleanOpaque(Object o, long offset) {
2250         return getBooleanVolatile(o, offset);
2251     }
2252 
2253     /** Opaque version of {@link #getByteVolatile(Object, long)} */
2254     @HotSpotIntrinsicCandidate
2255     public final byte getByteOpaque(Object o, long offset) {
2256         return getByteVolatile(o, offset);
2257     }
2258 
2259     /** Opaque version of {@link #getShortVolatile(Object, long)} */
2260     @HotSpotIntrinsicCandidate
2261     public final short getShortOpaque(Object o, long offset) {
2262         return getShortVolatile(o, offset);
2263     }
2264 
2265     /** Opaque version of {@link #getCharVolatile(Object, long)} */
2266     @HotSpotIntrinsicCandidate
2267     public final char getCharOpaque(Object o, long offset) {
2268         return getCharVolatile(o, offset);
2269     }
2270 
2271     /** Opaque version of {@link #getIntVolatile(Object, long)} */
2272     @HotSpotIntrinsicCandidate
2273     public final int getIntOpaque(Object o, long offset) {
2274         return getIntVolatile(o, offset);
2275     }
2276 
2277     /** Opaque version of {@link #getFloatVolatile(Object, long)} */
2278     @HotSpotIntrinsicCandidate
2279     public final float getFloatOpaque(Object o, long offset) {
2280         return getFloatVolatile(o, offset);
2281     }
2282 
2283     /** Opaque version of {@link #getLongVolatile(Object, long)} */
2284     @HotSpotIntrinsicCandidate
2285     public final long getLongOpaque(Object o, long offset) {
2286         return getLongVolatile(o, offset);
2287     }
2288 
2289     /** Opaque version of {@link #getDoubleVolatile(Object, long)} */
2290     @HotSpotIntrinsicCandidate
2291     public final double getDoubleOpaque(Object o, long offset) {
2292         return getDoubleVolatile(o, offset);
2293     }
2294 
2295     /** Opaque version of {@link #putObjectVolatile(Object, long, Object)} */
2296     @HotSpotIntrinsicCandidate
2297     public final void putObjectOpaque(Object o, long offset, Object x) {
2298         putObjectVolatile(o, offset, x);
2299     }
2300 
2301     /** Opaque version of {@link #putBooleanVolatile(Object, long, boolean)} */
2302     @HotSpotIntrinsicCandidate
2303     public final void putBooleanOpaque(Object o, long offset, boolean x) {
2304         putBooleanVolatile(o, offset, x);
2305     }
2306 
2307     /** Opaque version of {@link #putByteVolatile(Object, long, byte)} */
2308     @HotSpotIntrinsicCandidate
2309     public final void putByteOpaque(Object o, long offset, byte x) {
2310         putByteVolatile(o, offset, x);
2311     }
2312 
2313     /** Opaque version of {@link #putShortVolatile(Object, long, short)} */
2314     @HotSpotIntrinsicCandidate
2315     public final void putShortOpaque(Object o, long offset, short x) {
2316         putShortVolatile(o, offset, x);
2317     }
2318 
2319     /** Opaque version of {@link #putCharVolatile(Object, long, char)} */
2320     @HotSpotIntrinsicCandidate
2321     public final void putCharOpaque(Object o, long offset, char x) {
2322         putCharVolatile(o, offset, x);
2323     }
2324 
2325     /** Opaque version of {@link #putIntVolatile(Object, long, int)} */
2326     @HotSpotIntrinsicCandidate
2327     public final void putIntOpaque(Object o, long offset, int x) {
2328         putIntVolatile(o, offset, x);
2329     }
2330 
2331     /** Opaque version of {@link #putFloatVolatile(Object, long, float)} */
2332     @HotSpotIntrinsicCandidate
2333     public final void putFloatOpaque(Object o, long offset, float x) {
2334         putFloatVolatile(o, offset, x);
2335     }
2336 
2337     /** Opaque version of {@link #putLongVolatile(Object, long, long)} */
2338     @HotSpotIntrinsicCandidate
2339     public final void putLongOpaque(Object o, long offset, long x) {
2340         putLongVolatile(o, offset, x);
2341     }
2342 
2343     /** Opaque version of {@link #putDoubleVolatile(Object, long, double)} */
2344     @HotSpotIntrinsicCandidate
2345     public final void putDoubleOpaque(Object o, long offset, double x) {
2346         putDoubleVolatile(o, offset, x);
2347     }
2348 
2349     /**
2350      * Unblocks the given thread blocked on {@code park}, or, if it is
2351      * not blocked, causes the subsequent call to {@code park} not to
2352      * block.  Note: this operation is "unsafe" solely because the
2353      * caller must somehow ensure that the thread has not been
2354      * destroyed. Nothing special is usually required to ensure this
2355      * when called from Java (in which there will ordinarily be a live
2356      * reference to the thread) but this is not nearly-automatically
2357      * so when calling from native code.
2358      *
2359      * @param thread the thread to unpark.
2360      */
2361     @HotSpotIntrinsicCandidate
2362     public native void unpark(Object thread);
2363 
2364     /**
2365      * Blocks current thread, returning when a balancing
2366      * {@code unpark} occurs, or a balancing {@code unpark} has
2367      * already occurred, or the thread is interrupted, or, if not
2368      * absolute and time is not zero, the given time nanoseconds have
2369      * elapsed, or if absolute, the given deadline in milliseconds
2370      * since Epoch has passed, or spuriously (i.e., returning for no
2371      * "reason"). Note: This operation is in the Unsafe class only
2372      * because {@code unpark} is, so it would be strange to place it
2373      * elsewhere.
2374      */
2375     @HotSpotIntrinsicCandidate
2376     public native void park(boolean isAbsolute, long time);
2377 
2378     /**
2379      * Gets the load average in the system run queue assigned
2380      * to the available processors averaged over various periods of time.
2381      * This method retrieves the given {@code nelem} samples and
2382      * assigns to the elements of the given {@code loadavg} array.
2383      * The system imposes a maximum of 3 samples, representing
2384      * averages over the last 1,  5,  and  15 minutes, respectively.
2385      *
2386      * @param loadavg an array of double of size nelems
2387      * @param nelems the number of samples to be retrieved and
2388      *        must be 1 to 3.
2389      *
2390      * @return the number of samples actually retrieved; or -1
2391      *         if the load average is unobtainable.
2392      */
2393     public int getLoadAverage(double[] loadavg, int nelems) {
2394         if (nelems < 0 || nelems > 3 || nelems > loadavg.length) {
2395             throw new ArrayIndexOutOfBoundsException();
2396         }
2397 
2398         return getLoadAverage0(loadavg, nelems);
2399     }
2400 
2401     // The following contain CAS-based Java implementations used on
2402     // platforms not supporting native instructions
2403 
2404     /**
2405      * Atomically adds the given value to the current value of a field
2406      * or array element within the given object {@code o}
2407      * at the given {@code offset}.
2408      *
2409      * @param o object/array to update the field/element in
2410      * @param offset field/element offset
2411      * @param delta the value to add
2412      * @return the previous value
2413      * @since 1.8
2414      */
2415     @HotSpotIntrinsicCandidate
2416     public final int getAndAddInt(Object o, long offset, int delta) {
2417         int v;
2418         do {
2419             v = getIntVolatile(o, offset);
2420         } while (!weakCompareAndSetInt(o, offset, v, v + delta));
2421         return v;
2422     }
2423 
2424     @ForceInline
2425     public final int getAndAddIntRelease(Object o, long offset, int delta) {
2426         int v;
2427         do {
2428             v = getInt(o, offset);
2429         } while (!weakCompareAndSetIntRelease(o, offset, v, v + delta));
2430         return v;
2431     }
2432 
2433     @ForceInline
2434     public final int getAndAddIntAcquire(Object o, long offset, int delta) {
2435         int v;
2436         do {
2437             v = getIntAcquire(o, offset);
2438         } while (!weakCompareAndSetIntAcquire(o, offset, v, v + delta));
2439         return v;
2440     }
2441 
2442     /**
2443      * Atomically adds the given value to the current value of a field
2444      * or array element within the given object {@code o}
2445      * at the given {@code offset}.
2446      *
2447      * @param o object/array to update the field/element in
2448      * @param offset field/element offset
2449      * @param delta the value to add
2450      * @return the previous value
2451      * @since 1.8
2452      */
2453     @HotSpotIntrinsicCandidate
2454     public final long getAndAddLong(Object o, long offset, long delta) {
2455         long v;
2456         do {
2457             v = getLongVolatile(o, offset);
2458         } while (!weakCompareAndSetLong(o, offset, v, v + delta));
2459         return v;
2460     }
2461 
2462     @ForceInline
2463     public final long getAndAddLongRelease(Object o, long offset, long delta) {
2464         long v;
2465         do {
2466             v = getLong(o, offset);
2467         } while (!weakCompareAndSetLongRelease(o, offset, v, v + delta));
2468         return v;
2469     }
2470 
2471     @ForceInline
2472     public final long getAndAddLongAcquire(Object o, long offset, long delta) {
2473         long v;
2474         do {
2475             v = getLongAcquire(o, offset);
2476         } while (!weakCompareAndSetLongAcquire(o, offset, v, v + delta));
2477         return v;
2478     }
2479 
2480     @HotSpotIntrinsicCandidate
2481     public final byte getAndAddByte(Object o, long offset, byte delta) {
2482         byte v;
2483         do {
2484             v = getByteVolatile(o, offset);
2485         } while (!weakCompareAndSetByte(o, offset, v, (byte) (v + delta)));
2486         return v;
2487     }
2488 
2489     @ForceInline
2490     public final byte getAndAddByteRelease(Object o, long offset, byte delta) {
2491         byte v;
2492         do {
2493             v = getByte(o, offset);
2494         } while (!weakCompareAndSetByteRelease(o, offset, v, (byte) (v + delta)));
2495         return v;
2496     }
2497 
2498     @ForceInline
2499     public final byte getAndAddByteAcquire(Object o, long offset, byte delta) {
2500         byte v;
2501         do {
2502             v = getByteAcquire(o, offset);
2503         } while (!weakCompareAndSetByteAcquire(o, offset, v, (byte) (v + delta)));
2504         return v;
2505     }
2506 
2507     @HotSpotIntrinsicCandidate
2508     public final short getAndAddShort(Object o, long offset, short delta) {
2509         short v;
2510         do {
2511             v = getShortVolatile(o, offset);
2512         } while (!weakCompareAndSetShort(o, offset, v, (short) (v + delta)));
2513         return v;
2514     }
2515 
2516     @ForceInline
2517     public final short getAndAddShortRelease(Object o, long offset, short delta) {
2518         short v;
2519         do {
2520             v = getShort(o, offset);
2521         } while (!weakCompareAndSetShortRelease(o, offset, v, (short) (v + delta)));
2522         return v;
2523     }
2524 
2525     @ForceInline
2526     public final short getAndAddShortAcquire(Object o, long offset, short delta) {
2527         short v;
2528         do {
2529             v = getShortAcquire(o, offset);
2530         } while (!weakCompareAndSetShortAcquire(o, offset, v, (short) (v + delta)));
2531         return v;
2532     }
2533 
2534     @ForceInline
2535     public final char getAndAddChar(Object o, long offset, char delta) {
2536         return (char) getAndAddShort(o, offset, (short) delta);
2537     }
2538 
2539     @ForceInline
2540     public final char getAndAddCharRelease(Object o, long offset, char delta) {
2541         return (char) getAndAddShortRelease(o, offset, (short) delta);
2542     }
2543 
2544     @ForceInline
2545     public final char getAndAddCharAcquire(Object o, long offset, char delta) {
2546         return (char) getAndAddShortAcquire(o, offset, (short) delta);
2547     }
2548 
2549     @ForceInline
2550     public final float getAndAddFloat(Object o, long offset, float delta) {
2551         int expectedBits;
2552         float v;
2553         do {
2554             // Load and CAS with the raw bits to avoid issues with NaNs and
2555             // possible bit conversion from signaling NaNs to quiet NaNs that
2556             // may result in the loop not terminating.
2557             expectedBits = getIntVolatile(o, offset);
2558             v = Float.intBitsToFloat(expectedBits);
2559         } while (!weakCompareAndSetInt(o, offset,
2560                                                 expectedBits, Float.floatToRawIntBits(v + delta)));
2561         return v;
2562     }
2563 
2564     @ForceInline
2565     public final float getAndAddFloatRelease(Object o, long offset, float delta) {
2566         int expectedBits;
2567         float v;
2568         do {
2569             // Load and CAS with the raw bits to avoid issues with NaNs and
2570             // possible bit conversion from signaling NaNs to quiet NaNs that
2571             // may result in the loop not terminating.
2572             expectedBits = getInt(o, offset);
2573             v = Float.intBitsToFloat(expectedBits);
2574         } while (!weakCompareAndSetIntRelease(o, offset,
2575                                                expectedBits, Float.floatToRawIntBits(v + delta)));
2576         return v;
2577     }
2578 
2579     @ForceInline
2580     public final float getAndAddFloatAcquire(Object o, long offset, float delta) {
2581         int expectedBits;
2582         float v;
2583         do {
2584             // Load and CAS with the raw bits to avoid issues with NaNs and
2585             // possible bit conversion from signaling NaNs to quiet NaNs that
2586             // may result in the loop not terminating.
2587             expectedBits = getIntAcquire(o, offset);
2588             v = Float.intBitsToFloat(expectedBits);
2589         } while (!weakCompareAndSetIntAcquire(o, offset,
2590                                                expectedBits, Float.floatToRawIntBits(v + delta)));
2591         return v;
2592     }
2593 
2594     @ForceInline
2595     public final double getAndAddDouble(Object o, long offset, double delta) {
2596         long expectedBits;
2597         double v;
2598         do {
2599             // Load and CAS with the raw bits to avoid issues with NaNs and
2600             // possible bit conversion from signaling NaNs to quiet NaNs that
2601             // may result in the loop not terminating.
2602             expectedBits = getLongVolatile(o, offset);
2603             v = Double.longBitsToDouble(expectedBits);
2604         } while (!weakCompareAndSetLong(o, offset,
2605                                                  expectedBits, Double.doubleToRawLongBits(v + delta)));
2606         return v;
2607     }
2608 
2609     @ForceInline
2610     public final double getAndAddDoubleRelease(Object o, long offset, double delta) {
2611         long expectedBits;
2612         double v;
2613         do {
2614             // Load and CAS with the raw bits to avoid issues with NaNs and
2615             // possible bit conversion from signaling NaNs to quiet NaNs that
2616             // may result in the loop not terminating.
2617             expectedBits = getLong(o, offset);
2618             v = Double.longBitsToDouble(expectedBits);
2619         } while (!weakCompareAndSetLongRelease(o, offset,
2620                                                 expectedBits, Double.doubleToRawLongBits(v + delta)));
2621         return v;
2622     }
2623 
2624     @ForceInline
2625     public final double getAndAddDoubleAcquire(Object o, long offset, double delta) {
2626         long expectedBits;
2627         double v;
2628         do {
2629             // Load and CAS with the raw bits to avoid issues with NaNs and
2630             // possible bit conversion from signaling NaNs to quiet NaNs that
2631             // may result in the loop not terminating.
2632             expectedBits = getLongAcquire(o, offset);
2633             v = Double.longBitsToDouble(expectedBits);
2634         } while (!weakCompareAndSetLongAcquire(o, offset,
2635                                                 expectedBits, Double.doubleToRawLongBits(v + delta)));
2636         return v;
2637     }
2638 
2639     /**
2640      * Atomically exchanges the given value with the current value of
2641      * a field or array element within the given object {@code o}
2642      * at the given {@code offset}.
2643      *
2644      * @param o object/array to update the field/element in
2645      * @param offset field/element offset
2646      * @param newValue new value
2647      * @return the previous value
2648      * @since 1.8
2649      */
2650     @HotSpotIntrinsicCandidate
2651     public final int getAndSetInt(Object o, long offset, int newValue) {
2652         int v;
2653         do {
2654             v = getIntVolatile(o, offset);
2655         } while (!weakCompareAndSetInt(o, offset, v, newValue));
2656         return v;
2657     }
2658 
2659     @ForceInline
2660     public final int getAndSetIntRelease(Object o, long offset, int newValue) {
2661         int v;
2662         do {
2663             v = getInt(o, offset);
2664         } while (!weakCompareAndSetIntRelease(o, offset, v, newValue));
2665         return v;
2666     }
2667 
2668     @ForceInline
2669     public final int getAndSetIntAcquire(Object o, long offset, int newValue) {
2670         int v;
2671         do {
2672             v = getIntAcquire(o, offset);
2673         } while (!weakCompareAndSetIntAcquire(o, offset, v, newValue));
2674         return v;
2675     }
2676 
2677     /**
2678      * Atomically exchanges the given value with the current value of
2679      * a field or array element within the given object {@code o}
2680      * at the given {@code offset}.
2681      *
2682      * @param o object/array to update the field/element in
2683      * @param offset field/element offset
2684      * @param newValue new value
2685      * @return the previous value
2686      * @since 1.8
2687      */
2688     @HotSpotIntrinsicCandidate
2689     public final long getAndSetLong(Object o, long offset, long newValue) {
2690         long v;
2691         do {
2692             v = getLongVolatile(o, offset);
2693         } while (!weakCompareAndSetLong(o, offset, v, newValue));
2694         return v;
2695     }
2696 
2697     @ForceInline
2698     public final long getAndSetLongRelease(Object o, long offset, long newValue) {
2699         long v;
2700         do {
2701             v = getLong(o, offset);
2702         } while (!weakCompareAndSetLongRelease(o, offset, v, newValue));
2703         return v;
2704     }
2705 
2706     @ForceInline
2707     public final long getAndSetLongAcquire(Object o, long offset, long newValue) {
2708         long v;
2709         do {
2710             v = getLongAcquire(o, offset);
2711         } while (!weakCompareAndSetLongAcquire(o, offset, v, newValue));
2712         return v;
2713     }
2714 
2715     /**
2716      * Atomically exchanges the given reference value with the current
2717      * reference value of a field or array element within the given
2718      * object {@code o} at the given {@code offset}.
2719      *
2720      * @param o object/array to update the field/element in
2721      * @param offset field/element offset
2722      * @param newValue new value
2723      * @return the previous value
2724      * @since 1.8
2725      */
2726     @HotSpotIntrinsicCandidate
2727     public final Object getAndSetObject(Object o, long offset, Object newValue) {
2728         Object v;
2729         do {
2730             v = getObjectVolatile(o, offset);
2731         } while (!weakCompareAndSetObject(o, offset, v, newValue));
2732         return v;
2733     }
2734 
2735     @ForceInline
2736     public final Object getAndSetObjectRelease(Object o, long offset, Object newValue) {
2737         Object v;
2738         do {
2739             v = getObject(o, offset);
2740         } while (!weakCompareAndSetObjectRelease(o, offset, v, newValue));
2741         return v;
2742     }
2743 
2744     @ForceInline
2745     public final Object getAndSetObjectAcquire(Object o, long offset, Object newValue) {
2746         Object v;
2747         do {
2748             v = getObjectAcquire(o, offset);
2749         } while (!weakCompareAndSetObjectAcquire(o, offset, v, newValue));
2750         return v;
2751     }
2752 
2753     @HotSpotIntrinsicCandidate
2754     public final byte getAndSetByte(Object o, long offset, byte newValue) {
2755         byte v;
2756         do {
2757             v = getByteVolatile(o, offset);
2758         } while (!weakCompareAndSetByte(o, offset, v, newValue));
2759         return v;
2760     }
2761 
2762     @ForceInline
2763     public final byte getAndSetByteRelease(Object o, long offset, byte newValue) {
2764         byte v;
2765         do {
2766             v = getByte(o, offset);
2767         } while (!weakCompareAndSetByteRelease(o, offset, v, newValue));
2768         return v;
2769     }
2770 
2771     @ForceInline
2772     public final byte getAndSetByteAcquire(Object o, long offset, byte newValue) {
2773         byte v;
2774         do {
2775             v = getByteAcquire(o, offset);
2776         } while (!weakCompareAndSetByteAcquire(o, offset, v, newValue));
2777         return v;
2778     }
2779 
2780     @ForceInline
2781     public final boolean getAndSetBoolean(Object o, long offset, boolean newValue) {
2782         return byte2bool(getAndSetByte(o, offset, bool2byte(newValue)));
2783     }
2784 
2785     @ForceInline
2786     public final boolean getAndSetBooleanRelease(Object o, long offset, boolean newValue) {
2787         return byte2bool(getAndSetByteRelease(o, offset, bool2byte(newValue)));
2788     }
2789 
2790     @ForceInline
2791     public final boolean getAndSetBooleanAcquire(Object o, long offset, boolean newValue) {
2792         return byte2bool(getAndSetByteAcquire(o, offset, bool2byte(newValue)));
2793     }
2794 
2795     @HotSpotIntrinsicCandidate
2796     public final short getAndSetShort(Object o, long offset, short newValue) {
2797         short v;
2798         do {
2799             v = getShortVolatile(o, offset);
2800         } while (!weakCompareAndSetShort(o, offset, v, newValue));
2801         return v;
2802     }
2803 
2804     @ForceInline
2805     public final short getAndSetShortRelease(Object o, long offset, short newValue) {
2806         short v;
2807         do {
2808             v = getShort(o, offset);
2809         } while (!weakCompareAndSetShortRelease(o, offset, v, newValue));
2810         return v;
2811     }
2812 
2813     @ForceInline
2814     public final short getAndSetShortAcquire(Object o, long offset, short newValue) {
2815         short v;
2816         do {
2817             v = getShortAcquire(o, offset);
2818         } while (!weakCompareAndSetShortAcquire(o, offset, v, newValue));
2819         return v;
2820     }
2821 
2822     @ForceInline
2823     public final char getAndSetChar(Object o, long offset, char newValue) {
2824         return s2c(getAndSetShort(o, offset, c2s(newValue)));
2825     }
2826 
2827     @ForceInline
2828     public final char getAndSetCharRelease(Object o, long offset, char newValue) {
2829         return s2c(getAndSetShortRelease(o, offset, c2s(newValue)));
2830     }
2831 
2832     @ForceInline
2833     public final char getAndSetCharAcquire(Object o, long offset, char newValue) {
2834         return s2c(getAndSetShortAcquire(o, offset, c2s(newValue)));
2835     }
2836 
2837     @ForceInline
2838     public final float getAndSetFloat(Object o, long offset, float newValue) {
2839         int v = getAndSetInt(o, offset, Float.floatToRawIntBits(newValue));
2840         return Float.intBitsToFloat(v);
2841     }
2842 
2843     @ForceInline
2844     public final float getAndSetFloatRelease(Object o, long offset, float newValue) {
2845         int v = getAndSetIntRelease(o, offset, Float.floatToRawIntBits(newValue));
2846         return Float.intBitsToFloat(v);
2847     }
2848 
2849     @ForceInline
2850     public final float getAndSetFloatAcquire(Object o, long offset, float newValue) {
2851         int v = getAndSetIntAcquire(o, offset, Float.floatToRawIntBits(newValue));
2852         return Float.intBitsToFloat(v);
2853     }
2854 
2855     @ForceInline
2856     public final double getAndSetDouble(Object o, long offset, double newValue) {
2857         long v = getAndSetLong(o, offset, Double.doubleToRawLongBits(newValue));
2858         return Double.longBitsToDouble(v);
2859     }
2860 
2861     @ForceInline
2862     public final double getAndSetDoubleRelease(Object o, long offset, double newValue) {
2863         long v = getAndSetLongRelease(o, offset, Double.doubleToRawLongBits(newValue));
2864         return Double.longBitsToDouble(v);
2865     }
2866 
2867     @ForceInline
2868     public final double getAndSetDoubleAcquire(Object o, long offset, double newValue) {
2869         long v = getAndSetLongAcquire(o, offset, Double.doubleToRawLongBits(newValue));
2870         return Double.longBitsToDouble(v);
2871     }
2872 
2873 
2874     // The following contain CAS-based Java implementations used on
2875     // platforms not supporting native instructions
2876 
2877     @ForceInline
2878     public final boolean getAndBitwiseOrBoolean(Object o, long offset, boolean mask) {
2879         return byte2bool(getAndBitwiseOrByte(o, offset, bool2byte(mask)));
2880     }
2881 
2882     @ForceInline
2883     public final boolean getAndBitwiseOrBooleanRelease(Object o, long offset, boolean mask) {
2884         return byte2bool(getAndBitwiseOrByteRelease(o, offset, bool2byte(mask)));
2885     }
2886 
2887     @ForceInline
2888     public final boolean getAndBitwiseOrBooleanAcquire(Object o, long offset, boolean mask) {
2889         return byte2bool(getAndBitwiseOrByteAcquire(o, offset, bool2byte(mask)));
2890     }
2891 
2892     @ForceInline
2893     public final boolean getAndBitwiseAndBoolean(Object o, long offset, boolean mask) {
2894         return byte2bool(getAndBitwiseAndByte(o, offset, bool2byte(mask)));
2895     }
2896 
2897     @ForceInline
2898     public final boolean getAndBitwiseAndBooleanRelease(Object o, long offset, boolean mask) {
2899         return byte2bool(getAndBitwiseAndByteRelease(o, offset, bool2byte(mask)));
2900     }
2901 
2902     @ForceInline
2903     public final boolean getAndBitwiseAndBooleanAcquire(Object o, long offset, boolean mask) {
2904         return byte2bool(getAndBitwiseAndByteAcquire(o, offset, bool2byte(mask)));
2905     }
2906 
2907     @ForceInline
2908     public final boolean getAndBitwiseXorBoolean(Object o, long offset, boolean mask) {
2909         return byte2bool(getAndBitwiseXorByte(o, offset, bool2byte(mask)));
2910     }
2911 
2912     @ForceInline
2913     public final boolean getAndBitwiseXorBooleanRelease(Object o, long offset, boolean mask) {
2914         return byte2bool(getAndBitwiseXorByteRelease(o, offset, bool2byte(mask)));
2915     }
2916 
2917     @ForceInline
2918     public final boolean getAndBitwiseXorBooleanAcquire(Object o, long offset, boolean mask) {
2919         return byte2bool(getAndBitwiseXorByteAcquire(o, offset, bool2byte(mask)));
2920     }
2921 
2922 
2923     @ForceInline
2924     public final byte getAndBitwiseOrByte(Object o, long offset, byte mask) {
2925         byte current;
2926         do {
2927             current = getByteVolatile(o, offset);
2928         } while (!weakCompareAndSetByte(o, offset,
2929                                                   current, (byte) (current | mask)));
2930         return current;
2931     }
2932 
2933     @ForceInline
2934     public final byte getAndBitwiseOrByteRelease(Object o, long offset, byte mask) {
2935         byte current;
2936         do {
2937             current = getByte(o, offset);
2938         } while (!weakCompareAndSetByteRelease(o, offset,
2939                                                  current, (byte) (current | mask)));
2940         return current;
2941     }
2942 
2943     @ForceInline
2944     public final byte getAndBitwiseOrByteAcquire(Object o, long offset, byte mask) {
2945         byte current;
2946         do {
2947             // Plain read, the value is a hint, the acquire CAS does the work
2948             current = getByte(o, offset);
2949         } while (!weakCompareAndSetByteAcquire(o, offset,
2950                                                  current, (byte) (current | mask)));
2951         return current;
2952     }
2953 
2954     @ForceInline
2955     public final byte getAndBitwiseAndByte(Object o, long offset, byte mask) {
2956         byte current;
2957         do {
2958             current = getByteVolatile(o, offset);
2959         } while (!weakCompareAndSetByte(o, offset,
2960                                                   current, (byte) (current & mask)));
2961         return current;
2962     }
2963 
2964     @ForceInline
2965     public final byte getAndBitwiseAndByteRelease(Object o, long offset, byte mask) {
2966         byte current;
2967         do {
2968             current = getByte(o, offset);
2969         } while (!weakCompareAndSetByteRelease(o, offset,
2970                                                  current, (byte) (current & mask)));
2971         return current;
2972     }
2973 
2974     @ForceInline
2975     public final byte getAndBitwiseAndByteAcquire(Object o, long offset, byte mask) {
2976         byte current;
2977         do {
2978             // Plain read, the value is a hint, the acquire CAS does the work
2979             current = getByte(o, offset);
2980         } while (!weakCompareAndSetByteAcquire(o, offset,
2981                                                  current, (byte) (current & mask)));
2982         return current;
2983     }
2984 
2985     @ForceInline
2986     public final byte getAndBitwiseXorByte(Object o, long offset, byte mask) {
2987         byte current;
2988         do {
2989             current = getByteVolatile(o, offset);
2990         } while (!weakCompareAndSetByte(o, offset,
2991                                                   current, (byte) (current ^ mask)));
2992         return current;
2993     }
2994 
2995     @ForceInline
2996     public final byte getAndBitwiseXorByteRelease(Object o, long offset, byte mask) {
2997         byte current;
2998         do {
2999             current = getByte(o, offset);
3000         } while (!weakCompareAndSetByteRelease(o, offset,
3001                                                  current, (byte) (current ^ mask)));
3002         return current;
3003     }
3004 
3005     @ForceInline
3006     public final byte getAndBitwiseXorByteAcquire(Object o, long offset, byte mask) {
3007         byte current;
3008         do {
3009             // Plain read, the value is a hint, the acquire CAS does the work
3010             current = getByte(o, offset);
3011         } while (!weakCompareAndSetByteAcquire(o, offset,
3012                                                  current, (byte) (current ^ mask)));
3013         return current;
3014     }
3015 
3016 
3017     @ForceInline
3018     public final char getAndBitwiseOrChar(Object o, long offset, char mask) {
3019         return s2c(getAndBitwiseOrShort(o, offset, c2s(mask)));
3020     }
3021 
3022     @ForceInline
3023     public final char getAndBitwiseOrCharRelease(Object o, long offset, char mask) {
3024         return s2c(getAndBitwiseOrShortRelease(o, offset, c2s(mask)));
3025     }
3026 
3027     @ForceInline
3028     public final char getAndBitwiseOrCharAcquire(Object o, long offset, char mask) {
3029         return s2c(getAndBitwiseOrShortAcquire(o, offset, c2s(mask)));
3030     }
3031 
3032     @ForceInline
3033     public final char getAndBitwiseAndChar(Object o, long offset, char mask) {
3034         return s2c(getAndBitwiseAndShort(o, offset, c2s(mask)));
3035     }
3036 
3037     @ForceInline
3038     public final char getAndBitwiseAndCharRelease(Object o, long offset, char mask) {
3039         return s2c(getAndBitwiseAndShortRelease(o, offset, c2s(mask)));
3040     }
3041 
3042     @ForceInline
3043     public final char getAndBitwiseAndCharAcquire(Object o, long offset, char mask) {
3044         return s2c(getAndBitwiseAndShortAcquire(o, offset, c2s(mask)));
3045     }
3046 
3047     @ForceInline
3048     public final char getAndBitwiseXorChar(Object o, long offset, char mask) {
3049         return s2c(getAndBitwiseXorShort(o, offset, c2s(mask)));
3050     }
3051 
3052     @ForceInline
3053     public final char getAndBitwiseXorCharRelease(Object o, long offset, char mask) {
3054         return s2c(getAndBitwiseXorShortRelease(o, offset, c2s(mask)));
3055     }
3056 
3057     @ForceInline
3058     public final char getAndBitwiseXorCharAcquire(Object o, long offset, char mask) {
3059         return s2c(getAndBitwiseXorShortAcquire(o, offset, c2s(mask)));
3060     }
3061 
3062 
3063     @ForceInline
3064     public final short getAndBitwiseOrShort(Object o, long offset, short mask) {
3065         short current;
3066         do {
3067             current = getShortVolatile(o, offset);
3068         } while (!weakCompareAndSetShort(o, offset,
3069                                                 current, (short) (current | mask)));
3070         return current;
3071     }
3072 
3073     @ForceInline
3074     public final short getAndBitwiseOrShortRelease(Object o, long offset, short mask) {
3075         short current;
3076         do {
3077             current = getShort(o, offset);
3078         } while (!weakCompareAndSetShortRelease(o, offset,
3079                                                current, (short) (current | mask)));
3080         return current;
3081     }
3082 
3083     @ForceInline
3084     public final short getAndBitwiseOrShortAcquire(Object o, long offset, short mask) {
3085         short current;
3086         do {
3087             // Plain read, the value is a hint, the acquire CAS does the work
3088             current = getShort(o, offset);
3089         } while (!weakCompareAndSetShortAcquire(o, offset,
3090                                                current, (short) (current | mask)));
3091         return current;
3092     }
3093 
3094     @ForceInline
3095     public final short getAndBitwiseAndShort(Object o, long offset, short mask) {
3096         short current;
3097         do {
3098             current = getShortVolatile(o, offset);
3099         } while (!weakCompareAndSetShort(o, offset,
3100                                                 current, (short) (current & mask)));
3101         return current;
3102     }
3103 
3104     @ForceInline
3105     public final short getAndBitwiseAndShortRelease(Object o, long offset, short mask) {
3106         short current;
3107         do {
3108             current = getShort(o, offset);
3109         } while (!weakCompareAndSetShortRelease(o, offset,
3110                                                current, (short) (current & mask)));
3111         return current;
3112     }
3113 
3114     @ForceInline
3115     public final short getAndBitwiseAndShortAcquire(Object o, long offset, short mask) {
3116         short current;
3117         do {
3118             // Plain read, the value is a hint, the acquire CAS does the work
3119             current = getShort(o, offset);
3120         } while (!weakCompareAndSetShortAcquire(o, offset,
3121                                                current, (short) (current & mask)));
3122         return current;
3123     }
3124 
3125     @ForceInline
3126     public final short getAndBitwiseXorShort(Object o, long offset, short mask) {
3127         short current;
3128         do {
3129             current = getShortVolatile(o, offset);
3130         } while (!weakCompareAndSetShort(o, offset,
3131                                                 current, (short) (current ^ mask)));
3132         return current;
3133     }
3134 
3135     @ForceInline
3136     public final short getAndBitwiseXorShortRelease(Object o, long offset, short mask) {
3137         short current;
3138         do {
3139             current = getShort(o, offset);
3140         } while (!weakCompareAndSetShortRelease(o, offset,
3141                                                current, (short) (current ^ mask)));
3142         return current;
3143     }
3144 
3145     @ForceInline
3146     public final short getAndBitwiseXorShortAcquire(Object o, long offset, short mask) {
3147         short current;
3148         do {
3149             // Plain read, the value is a hint, the acquire CAS does the work
3150             current = getShort(o, offset);
3151         } while (!weakCompareAndSetShortAcquire(o, offset,
3152                                                current, (short) (current ^ mask)));
3153         return current;
3154     }
3155 
3156 
3157     @ForceInline
3158     public final int getAndBitwiseOrInt(Object o, long offset, int mask) {
3159         int current;
3160         do {
3161             current = getIntVolatile(o, offset);
3162         } while (!weakCompareAndSetInt(o, offset,
3163                                                 current, current | mask));
3164         return current;
3165     }
3166 
3167     @ForceInline
3168     public final int getAndBitwiseOrIntRelease(Object o, long offset, int mask) {
3169         int current;
3170         do {
3171             current = getInt(o, offset);
3172         } while (!weakCompareAndSetIntRelease(o, offset,
3173                                                current, current | mask));
3174         return current;
3175     }
3176 
3177     @ForceInline
3178     public final int getAndBitwiseOrIntAcquire(Object o, long offset, int mask) {
3179         int current;
3180         do {
3181             // Plain read, the value is a hint, the acquire CAS does the work
3182             current = getInt(o, offset);
3183         } while (!weakCompareAndSetIntAcquire(o, offset,
3184                                                current, current | mask));
3185         return current;
3186     }
3187 
3188     /**
3189      * Atomically replaces the current value of a field or array element within
3190      * the given object with the result of bitwise AND between the current value
3191      * and mask.
3192      *
3193      * @param o object/array to update the field/element in
3194      * @param offset field/element offset
3195      * @param mask the mask value
3196      * @return the previous value
3197      * @since 1.9
3198      */
3199     @ForceInline
3200     public final int getAndBitwiseAndInt(Object o, long offset, int mask) {
3201         int current;
3202         do {
3203             current = getIntVolatile(o, offset);
3204         } while (!weakCompareAndSetInt(o, offset,
3205                                                 current, current & mask));
3206         return current;
3207     }
3208 
3209     @ForceInline
3210     public final int getAndBitwiseAndIntRelease(Object o, long offset, int mask) {
3211         int current;
3212         do {
3213             current = getInt(o, offset);
3214         } while (!weakCompareAndSetIntRelease(o, offset,
3215                                                current, current & mask));
3216         return current;
3217     }
3218 
3219     @ForceInline
3220     public final int getAndBitwiseAndIntAcquire(Object o, long offset, int mask) {
3221         int current;
3222         do {
3223             // Plain read, the value is a hint, the acquire CAS does the work
3224             current = getInt(o, offset);
3225         } while (!weakCompareAndSetIntAcquire(o, offset,
3226                                                current, current & mask));
3227         return current;
3228     }
3229 
3230     @ForceInline
3231     public final int getAndBitwiseXorInt(Object o, long offset, int mask) {
3232         int current;
3233         do {
3234             current = getIntVolatile(o, offset);
3235         } while (!weakCompareAndSetInt(o, offset,
3236                                                 current, current ^ mask));
3237         return current;
3238     }
3239 
3240     @ForceInline
3241     public final int getAndBitwiseXorIntRelease(Object o, long offset, int mask) {
3242         int current;
3243         do {
3244             current = getInt(o, offset);
3245         } while (!weakCompareAndSetIntRelease(o, offset,
3246                                                current, current ^ mask));
3247         return current;
3248     }
3249 
3250     @ForceInline
3251     public final int getAndBitwiseXorIntAcquire(Object o, long offset, int mask) {
3252         int current;
3253         do {
3254             // Plain read, the value is a hint, the acquire CAS does the work
3255             current = getInt(o, offset);
3256         } while (!weakCompareAndSetIntAcquire(o, offset,
3257                                                current, current ^ mask));
3258         return current;
3259     }
3260 
3261 
3262     @ForceInline
3263     public final long getAndBitwiseOrLong(Object o, long offset, long mask) {
3264         long current;
3265         do {
3266             current = getLongVolatile(o, offset);
3267         } while (!weakCompareAndSetLong(o, offset,
3268                                                 current, current | mask));
3269         return current;
3270     }
3271 
3272     @ForceInline
3273     public final long getAndBitwiseOrLongRelease(Object o, long offset, long mask) {
3274         long current;
3275         do {
3276             current = getLong(o, offset);
3277         } while (!weakCompareAndSetLongRelease(o, offset,
3278                                                current, current | mask));
3279         return current;
3280     }
3281 
3282     @ForceInline
3283     public final long getAndBitwiseOrLongAcquire(Object o, long offset, long mask) {
3284         long current;
3285         do {
3286             // Plain read, the value is a hint, the acquire CAS does the work
3287             current = getLong(o, offset);
3288         } while (!weakCompareAndSetLongAcquire(o, offset,
3289                                                current, current | mask));
3290         return current;
3291     }
3292 
3293     @ForceInline
3294     public final long getAndBitwiseAndLong(Object o, long offset, long mask) {
3295         long current;
3296         do {
3297             current = getLongVolatile(o, offset);
3298         } while (!weakCompareAndSetLong(o, offset,
3299                                                 current, current & mask));
3300         return current;
3301     }
3302 
3303     @ForceInline
3304     public final long getAndBitwiseAndLongRelease(Object o, long offset, long mask) {
3305         long current;
3306         do {
3307             current = getLong(o, offset);
3308         } while (!weakCompareAndSetLongRelease(o, offset,
3309                                                current, current & mask));
3310         return current;
3311     }
3312 
3313     @ForceInline
3314     public final long getAndBitwiseAndLongAcquire(Object o, long offset, long mask) {
3315         long current;
3316         do {
3317             // Plain read, the value is a hint, the acquire CAS does the work
3318             current = getLong(o, offset);
3319         } while (!weakCompareAndSetLongAcquire(o, offset,
3320                                                current, current & mask));
3321         return current;
3322     }
3323 
3324     @ForceInline
3325     public final long getAndBitwiseXorLong(Object o, long offset, long mask) {
3326         long current;
3327         do {
3328             current = getLongVolatile(o, offset);
3329         } while (!weakCompareAndSetLong(o, offset,
3330                                                 current, current ^ mask));
3331         return current;
3332     }
3333 
3334     @ForceInline
3335     public final long getAndBitwiseXorLongRelease(Object o, long offset, long mask) {
3336         long current;
3337         do {
3338             current = getLong(o, offset);
3339         } while (!weakCompareAndSetLongRelease(o, offset,
3340                                                current, current ^ mask));
3341         return current;
3342     }
3343 
3344     @ForceInline
3345     public final long getAndBitwiseXorLongAcquire(Object o, long offset, long mask) {
3346         long current;
3347         do {
3348             // Plain read, the value is a hint, the acquire CAS does the work
3349             current = getLong(o, offset);
3350         } while (!weakCompareAndSetLongAcquire(o, offset,
3351                                                current, current ^ mask));
3352         return current;
3353     }
3354 
3355 
3356 
3357     /**
3358      * Ensures that loads before the fence will not be reordered with loads and
3359      * stores after the fence; a "LoadLoad plus LoadStore barrier".
3360      *
3361      * Corresponds to C11 atomic_thread_fence(memory_order_acquire)
3362      * (an "acquire fence").
3363      *
3364      * A pure LoadLoad fence is not provided, since the addition of LoadStore
3365      * is almost always desired, and most current hardware instructions that
3366      * provide a LoadLoad barrier also provide a LoadStore barrier for free.
3367      * @since 1.8
3368      */
3369     @HotSpotIntrinsicCandidate
3370     public native void loadFence();
3371 
3372     /**
3373      * Ensures that loads and stores before the fence will not be reordered with
3374      * stores after the fence; a "StoreStore plus LoadStore barrier".
3375      *
3376      * Corresponds to C11 atomic_thread_fence(memory_order_release)
3377      * (a "release fence").
3378      *
3379      * A pure StoreStore fence is not provided, since the addition of LoadStore
3380      * is almost always desired, and most current hardware instructions that
3381      * provide a StoreStore barrier also provide a LoadStore barrier for free.
3382      * @since 1.8
3383      */
3384     @HotSpotIntrinsicCandidate
3385     public native void storeFence();
3386 
3387     /**
3388      * Ensures that loads and stores before the fence will not be reordered
3389      * with loads and stores after the fence.  Implies the effects of both
3390      * loadFence() and storeFence(), and in addition, the effect of a StoreLoad
3391      * barrier.
3392      *
3393      * Corresponds to C11 atomic_thread_fence(memory_order_seq_cst).
3394      * @since 1.8
3395      */
3396     @HotSpotIntrinsicCandidate
3397     public native void fullFence();
3398 
3399     /**
3400      * Ensures that loads before the fence will not be reordered with
3401      * loads after the fence.
3402      */
3403     public final void loadLoadFence() {
3404         loadFence();
3405     }
3406 
3407     /**
3408      * Ensures that stores before the fence will not be reordered with
3409      * stores after the fence.
3410      */
3411     public final void storeStoreFence() {
3412         storeFence();
3413     }
3414 
3415 
3416     /**
3417      * Throws IllegalAccessError; for use by the VM for access control
3418      * error support.
3419      * @since 1.8
3420      */
3421     private static void throwIllegalAccessError() {
3422         throw new IllegalAccessError();
3423     }
3424 
3425     /**
3426      * @return Returns true if the native byte ordering of this
3427      * platform is big-endian, false if it is little-endian.
3428      */
3429     public final boolean isBigEndian() { return BE; }
3430 
3431     /**
3432      * @return Returns true if this platform is capable of performing
3433      * accesses at addresses which are not aligned for the type of the
3434      * primitive type being accessed, false otherwise.
3435      */
3436     public final boolean unalignedAccess() { return unalignedAccess; }
3437 
3438     /**
3439      * Fetches a value at some byte offset into a given Java object.
3440      * More specifically, fetches a value within the given object
3441      * <code>o</code> at the given offset, or (if <code>o</code> is
3442      * null) from the memory address whose numerical value is the
3443      * given offset.  <p>
3444      *
3445      * The specification of this method is the same as {@link
3446      * #getLong(Object, long)} except that the offset does not need to
3447      * have been obtained from {@link #objectFieldOffset} on the
3448      * {@link java.lang.reflect.Field} of some Java field.  The value
3449      * in memory is raw data, and need not correspond to any Java
3450      * variable.  Unless <code>o</code> is null, the value accessed
3451      * must be entirely within the allocated object.  The endianness
3452      * of the value in memory is the endianness of the native platform.
3453      *
3454      * <p> The read will be atomic with respect to the largest power
3455      * of two that divides the GCD of the offset and the storage size.
3456      * For example, getLongUnaligned will make atomic reads of 2-, 4-,
3457      * or 8-byte storage units if the offset is zero mod 2, 4, or 8,
3458      * respectively.  There are no other guarantees of atomicity.
3459      * <p>
3460      * 8-byte atomicity is only guaranteed on platforms on which
3461      * support atomic accesses to longs.
3462      *
3463      * @param o Java heap object in which the value resides, if any, else
3464      *        null
3465      * @param offset The offset in bytes from the start of the object
3466      * @return the value fetched from the indicated object
3467      * @throws RuntimeException No defined exceptions are thrown, not even
3468      *         {@link NullPointerException}
3469      * @since 9
3470      */
3471     @HotSpotIntrinsicCandidate
3472     public final long getLongUnaligned(Object o, long offset) {
3473         if ((offset & 7) == 0) {
3474             return getLong(o, offset);
3475         } else if ((offset & 3) == 0) {
3476             return makeLong(getInt(o, offset),
3477                             getInt(o, offset + 4));
3478         } else if ((offset & 1) == 0) {
3479             return makeLong(getShort(o, offset),
3480                             getShort(o, offset + 2),
3481                             getShort(o, offset + 4),
3482                             getShort(o, offset + 6));
3483         } else {
3484             return makeLong(getByte(o, offset),
3485                             getByte(o, offset + 1),
3486                             getByte(o, offset + 2),
3487                             getByte(o, offset + 3),
3488                             getByte(o, offset + 4),
3489                             getByte(o, offset + 5),
3490                             getByte(o, offset + 6),
3491                             getByte(o, offset + 7));
3492         }
3493     }
3494     /**
3495      * As {@link #getLongUnaligned(Object, long)} but with an
3496      * additional argument which specifies the endianness of the value
3497      * as stored in memory.
3498      *
3499      * @param o Java heap object in which the variable resides
3500      * @param offset The offset in bytes from the start of the object
3501      * @param bigEndian The endianness of the value
3502      * @return the value fetched from the indicated object
3503      * @since 9
3504      */
3505     public final long getLongUnaligned(Object o, long offset, boolean bigEndian) {
3506         return convEndian(bigEndian, getLongUnaligned(o, offset));
3507     }
3508 
3509     /** @see #getLongUnaligned(Object, long) */
3510     @HotSpotIntrinsicCandidate
3511     public final int getIntUnaligned(Object o, long offset) {
3512         if ((offset & 3) == 0) {
3513             return getInt(o, offset);
3514         } else if ((offset & 1) == 0) {
3515             return makeInt(getShort(o, offset),
3516                            getShort(o, offset + 2));
3517         } else {
3518             return makeInt(getByte(o, offset),
3519                            getByte(o, offset + 1),
3520                            getByte(o, offset + 2),
3521                            getByte(o, offset + 3));
3522         }
3523     }
3524     /** @see #getLongUnaligned(Object, long, boolean) */
3525     public final int getIntUnaligned(Object o, long offset, boolean bigEndian) {
3526         return convEndian(bigEndian, getIntUnaligned(o, offset));
3527     }
3528 
3529     /** @see #getLongUnaligned(Object, long) */
3530     @HotSpotIntrinsicCandidate
3531     public final short getShortUnaligned(Object o, long offset) {
3532         if ((offset & 1) == 0) {
3533             return getShort(o, offset);
3534         } else {
3535             return makeShort(getByte(o, offset),
3536                              getByte(o, offset + 1));
3537         }
3538     }
3539     /** @see #getLongUnaligned(Object, long, boolean) */
3540     public final short getShortUnaligned(Object o, long offset, boolean bigEndian) {
3541         return convEndian(bigEndian, getShortUnaligned(o, offset));
3542     }
3543 
3544     /** @see #getLongUnaligned(Object, long) */
3545     @HotSpotIntrinsicCandidate
3546     public final char getCharUnaligned(Object o, long offset) {
3547         if ((offset & 1) == 0) {
3548             return getChar(o, offset);
3549         } else {
3550             return (char)makeShort(getByte(o, offset),
3551                                    getByte(o, offset + 1));
3552         }
3553     }
3554 
3555     /** @see #getLongUnaligned(Object, long, boolean) */
3556     public final char getCharUnaligned(Object o, long offset, boolean bigEndian) {
3557         return convEndian(bigEndian, getCharUnaligned(o, offset));
3558     }
3559 
3560     /**
3561      * Stores a value at some byte offset into a given Java object.
3562      * <p>
3563      * The specification of this method is the same as {@link
3564      * #getLong(Object, long)} except that the offset does not need to
3565      * have been obtained from {@link #objectFieldOffset} on the
3566      * {@link java.lang.reflect.Field} of some Java field.  The value
3567      * in memory is raw data, and need not correspond to any Java
3568      * variable.  The endianness of the value in memory is the
3569      * endianness of the native platform.
3570      * <p>
3571      * The write will be atomic with respect to the largest power of
3572      * two that divides the GCD of the offset and the storage size.
3573      * For example, putLongUnaligned will make atomic writes of 2-, 4-,
3574      * or 8-byte storage units if the offset is zero mod 2, 4, or 8,
3575      * respectively.  There are no other guarantees of atomicity.
3576      * <p>
3577      * 8-byte atomicity is only guaranteed on platforms on which
3578      * support atomic accesses to longs.
3579      *
3580      * @param o Java heap object in which the value resides, if any, else
3581      *        null
3582      * @param offset The offset in bytes from the start of the object
3583      * @param x the value to store
3584      * @throws RuntimeException No defined exceptions are thrown, not even
3585      *         {@link NullPointerException}
3586      * @since 9
3587      */
3588     @HotSpotIntrinsicCandidate
3589     public final void putLongUnaligned(Object o, long offset, long x) {
3590         if ((offset & 7) == 0) {
3591             putLong(o, offset, x);
3592         } else if ((offset & 3) == 0) {
3593             putLongParts(o, offset,
3594                          (int)(x >> 0),
3595                          (int)(x >>> 32));
3596         } else if ((offset & 1) == 0) {
3597             putLongParts(o, offset,
3598                          (short)(x >>> 0),
3599                          (short)(x >>> 16),
3600                          (short)(x >>> 32),
3601                          (short)(x >>> 48));
3602         } else {
3603             putLongParts(o, offset,
3604                          (byte)(x >>> 0),
3605                          (byte)(x >>> 8),
3606                          (byte)(x >>> 16),
3607                          (byte)(x >>> 24),
3608                          (byte)(x >>> 32),
3609                          (byte)(x >>> 40),
3610                          (byte)(x >>> 48),
3611                          (byte)(x >>> 56));
3612         }
3613     }
3614 
3615     /**
3616      * As {@link #putLongUnaligned(Object, long, long)} but with an additional
3617      * argument which specifies the endianness of the value as stored in memory.
3618      * @param o Java heap object in which the value resides
3619      * @param offset The offset in bytes from the start of the object
3620      * @param x the value to store
3621      * @param bigEndian The endianness of the value
3622      * @throws RuntimeException No defined exceptions are thrown, not even
3623      *         {@link NullPointerException}
3624      * @since 9
3625      */
3626     public final void putLongUnaligned(Object o, long offset, long x, boolean bigEndian) {
3627         putLongUnaligned(o, offset, convEndian(bigEndian, x));
3628     }
3629 
3630     /** @see #putLongUnaligned(Object, long, long) */
3631     @HotSpotIntrinsicCandidate
3632     public final void putIntUnaligned(Object o, long offset, int x) {
3633         if ((offset & 3) == 0) {
3634             putInt(o, offset, x);
3635         } else if ((offset & 1) == 0) {
3636             putIntParts(o, offset,
3637                         (short)(x >> 0),
3638                         (short)(x >>> 16));
3639         } else {
3640             putIntParts(o, offset,
3641                         (byte)(x >>> 0),
3642                         (byte)(x >>> 8),
3643                         (byte)(x >>> 16),
3644                         (byte)(x >>> 24));
3645         }
3646     }
3647     /** @see #putLongUnaligned(Object, long, long, boolean) */
3648     public final void putIntUnaligned(Object o, long offset, int x, boolean bigEndian) {
3649         putIntUnaligned(o, offset, convEndian(bigEndian, x));
3650     }
3651 
3652     /** @see #putLongUnaligned(Object, long, long) */
3653     @HotSpotIntrinsicCandidate
3654     public final void putShortUnaligned(Object o, long offset, short x) {
3655         if ((offset & 1) == 0) {
3656             putShort(o, offset, x);
3657         } else {
3658             putShortParts(o, offset,
3659                           (byte)(x >>> 0),
3660                           (byte)(x >>> 8));
3661         }
3662     }
3663     /** @see #putLongUnaligned(Object, long, long, boolean) */
3664     public final void putShortUnaligned(Object o, long offset, short x, boolean bigEndian) {
3665         putShortUnaligned(o, offset, convEndian(bigEndian, x));
3666     }
3667 
3668     /** @see #putLongUnaligned(Object, long, long) */
3669     @HotSpotIntrinsicCandidate
3670     public final void putCharUnaligned(Object o, long offset, char x) {
3671         putShortUnaligned(o, offset, (short)x);
3672     }
3673     /** @see #putLongUnaligned(Object, long, long, boolean) */
3674     public final void putCharUnaligned(Object o, long offset, char x, boolean bigEndian) {
3675         putCharUnaligned(o, offset, convEndian(bigEndian, x));
3676     }
3677 
3678     // JVM interface methods
3679     // BE is true iff the native endianness of this platform is big.
3680     private static final boolean BE = theUnsafe.isBigEndian0();
3681 
3682     // unalignedAccess is true iff this platform can perform unaligned accesses.
3683     private static final boolean unalignedAccess = theUnsafe.unalignedAccess0();
3684 
3685     private static int pickPos(int top, int pos) { return BE ? top - pos : pos; }
3686 
3687     // These methods construct integers from bytes.  The byte ordering
3688     // is the native endianness of this platform.
3689     private static long makeLong(byte i0, byte i1, byte i2, byte i3, byte i4, byte i5, byte i6, byte i7) {
3690         return ((toUnsignedLong(i0) << pickPos(56, 0))
3691               | (toUnsignedLong(i1) << pickPos(56, 8))
3692               | (toUnsignedLong(i2) << pickPos(56, 16))
3693               | (toUnsignedLong(i3) << pickPos(56, 24))
3694               | (toUnsignedLong(i4) << pickPos(56, 32))
3695               | (toUnsignedLong(i5) << pickPos(56, 40))
3696               | (toUnsignedLong(i6) << pickPos(56, 48))
3697               | (toUnsignedLong(i7) << pickPos(56, 56)));
3698     }
3699     private static long makeLong(short i0, short i1, short i2, short i3) {
3700         return ((toUnsignedLong(i0) << pickPos(48, 0))
3701               | (toUnsignedLong(i1) << pickPos(48, 16))
3702               | (toUnsignedLong(i2) << pickPos(48, 32))
3703               | (toUnsignedLong(i3) << pickPos(48, 48)));
3704     }
3705     private static long makeLong(int i0, int i1) {
3706         return (toUnsignedLong(i0) << pickPos(32, 0))
3707              | (toUnsignedLong(i1) << pickPos(32, 32));
3708     }
3709     private static int makeInt(short i0, short i1) {
3710         return (toUnsignedInt(i0) << pickPos(16, 0))
3711              | (toUnsignedInt(i1) << pickPos(16, 16));
3712     }
3713     private static int makeInt(byte i0, byte i1, byte i2, byte i3) {
3714         return ((toUnsignedInt(i0) << pickPos(24, 0))
3715               | (toUnsignedInt(i1) << pickPos(24, 8))
3716               | (toUnsignedInt(i2) << pickPos(24, 16))
3717               | (toUnsignedInt(i3) << pickPos(24, 24)));
3718     }
3719     private static short makeShort(byte i0, byte i1) {
3720         return (short)((toUnsignedInt(i0) << pickPos(8, 0))
3721                      | (toUnsignedInt(i1) << pickPos(8, 8)));
3722     }
3723 
3724     private static byte  pick(byte  le, byte  be) { return BE ? be : le; }
3725     private static short pick(short le, short be) { return BE ? be : le; }
3726     private static int   pick(int   le, int   be) { return BE ? be : le; }
3727 
3728     // These methods write integers to memory from smaller parts
3729     // provided by their caller.  The ordering in which these parts
3730     // are written is the native endianness of this platform.
3731     private void putLongParts(Object o, long offset, byte i0, byte i1, byte i2, byte i3, byte i4, byte i5, byte i6, byte i7) {
3732         putByte(o, offset + 0, pick(i0, i7));
3733         putByte(o, offset + 1, pick(i1, i6));
3734         putByte(o, offset + 2, pick(i2, i5));
3735         putByte(o, offset + 3, pick(i3, i4));
3736         putByte(o, offset + 4, pick(i4, i3));
3737         putByte(o, offset + 5, pick(i5, i2));
3738         putByte(o, offset + 6, pick(i6, i1));
3739         putByte(o, offset + 7, pick(i7, i0));
3740     }
3741     private void putLongParts(Object o, long offset, short i0, short i1, short i2, short i3) {
3742         putShort(o, offset + 0, pick(i0, i3));
3743         putShort(o, offset + 2, pick(i1, i2));
3744         putShort(o, offset + 4, pick(i2, i1));
3745         putShort(o, offset + 6, pick(i3, i0));
3746     }
3747     private void putLongParts(Object o, long offset, int i0, int i1) {
3748         putInt(o, offset + 0, pick(i0, i1));
3749         putInt(o, offset + 4, pick(i1, i0));
3750     }
3751     private void putIntParts(Object o, long offset, short i0, short i1) {
3752         putShort(o, offset + 0, pick(i0, i1));
3753         putShort(o, offset + 2, pick(i1, i0));
3754     }
3755     private void putIntParts(Object o, long offset, byte i0, byte i1, byte i2, byte i3) {
3756         putByte(o, offset + 0, pick(i0, i3));
3757         putByte(o, offset + 1, pick(i1, i2));
3758         putByte(o, offset + 2, pick(i2, i1));
3759         putByte(o, offset + 3, pick(i3, i0));
3760     }
3761     private void putShortParts(Object o, long offset, byte i0, byte i1) {
3762         putByte(o, offset + 0, pick(i0, i1));
3763         putByte(o, offset + 1, pick(i1, i0));
3764     }
3765 
3766     // Zero-extend an integer
3767     private static int toUnsignedInt(byte n)    { return n & 0xff; }
3768     private static int toUnsignedInt(short n)   { return n & 0xffff; }
3769     private static long toUnsignedLong(byte n)  { return n & 0xffl; }
3770     private static long toUnsignedLong(short n) { return n & 0xffffl; }
3771     private static long toUnsignedLong(int n)   { return n & 0xffffffffl; }
3772 
3773     // Maybe byte-reverse an integer
3774     private static char convEndian(boolean big, char n)   { return big == BE ? n : Character.reverseBytes(n); }
3775     private static short convEndian(boolean big, short n) { return big == BE ? n : Short.reverseBytes(n)    ; }
3776     private static int convEndian(boolean big, int n)     { return big == BE ? n : Integer.reverseBytes(n)  ; }
3777     private static long convEndian(boolean big, long n)   { return big == BE ? n : Long.reverseBytes(n)     ; }
3778 
3779 
3780 
3781     private native long allocateMemory0(long bytes);
3782     private native long reallocateMemory0(long address, long bytes);
3783     private native void freeMemory0(long address);
3784     private native void setMemory0(Object o, long offset, long bytes, byte value);
3785     @HotSpotIntrinsicCandidate
3786     private native void copyMemory0(Object srcBase, long srcOffset, Object destBase, long destOffset, long bytes);
3787     private native void copySwapMemory0(Object srcBase, long srcOffset, Object destBase, long destOffset, long bytes, long elemSize);
3788     private native long objectFieldOffset0(Field f);
3789     private native long objectFieldOffset1(Class<?> c, String name);
3790     private native long staticFieldOffset0(Field f);
3791     private native Object staticFieldBase0(Field f);
3792     private native boolean shouldBeInitialized0(Class<?> c);
3793     private native void ensureClassInitialized0(Class<?> c);
3794     private native int arrayBaseOffset0(Class<?> arrayClass);
3795     private native int arrayIndexScale0(Class<?> arrayClass);
3796     private native int addressSize0();
3797     private native int dataCacheLineFlushSize0();
3798     private native Class<?> defineAnonymousClass0(Class<?> hostClass, byte[] data, Object[] cpPatches);
3799     private native int getLoadAverage0(double[] loadavg, int nelems);
3800     private native boolean unalignedAccess0();
3801     private native boolean isBigEndian0();
3802 }