1 /*
   2  * Copyright (c) 2000, 2017, Oracle and/or its affiliates. All rights reserved.
   3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4  *
   5  * This code is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License version 2 only, as
   7  * published by the Free Software Foundation.  Oracle designates this
   8  * particular file as subject to the "Classpath" exception as provided
   9  * by Oracle in the LICENSE file that accompanied this code.
  10  *
  11  * This code is distributed in the hope that it will be useful, but WITHOUT
  12  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  13  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  14  * version 2 for more details (a copy is included in the LICENSE file that
  15  * accompanied this code).
  16  *
  17  * You should have received a copy of the GNU General Public License version
  18  * 2 along with this work; if not, write to the Free Software Foundation,
  19  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  20  *
  21  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  22  * or visit www.oracle.com if you need additional information or have any
  23  * questions.
  24  */
  25 
  26 package jdk.internal.misc;
  27 
  28 import jdk.internal.HotSpotIntrinsicCandidate;
  29 import jdk.internal.vm.annotation.ForceInline;
  30 
  31 import java.lang.reflect.Field;
  32 import java.security.ProtectionDomain;
  33 
  34 
  35 /**
  36  * A collection of methods for performing low-level, unsafe operations.
  37  * Although the class and all methods are public, use of this class is
  38  * limited because only trusted code can obtain instances of it.
  39  *
  40  * <em>Note:</em> It is the resposibility of the caller to make sure
  41  * arguments are checked before methods of this class are
  42  * called. While some rudimentary checks are performed on the input,
  43  * the checks are best effort and when performance is an overriding
  44  * priority, as when methods of this class are optimized by the
  45  * runtime compiler, some or all checks (if any) may be elided. Hence,
  46  * the caller must not rely on the checks and corresponding
  47  * exceptions!
  48  *
  49  * @author John R. Rose
  50  * @see #getUnsafe
  51  */
  52 
  53 public final class Unsafe {
  54 
  55     private static native void registerNatives();
  56     static {
  57         registerNatives();
  58     }
  59 
  60     private Unsafe() {}
  61 
  62     private static final Unsafe theUnsafe = new Unsafe();
  63 
  64     /**
  65      * Provides the caller with the capability of performing unsafe
  66      * operations.
  67      *
  68      * <p>The returned {@code Unsafe} object should be carefully guarded
  69      * by the caller, since it can be used to read and write data at arbitrary
  70      * memory addresses.  It must never be passed to untrusted code.
  71      *
  72      * <p>Most methods in this class are very low-level, and correspond to a
  73      * small number of hardware instructions (on typical machines).  Compilers
  74      * are encouraged to optimize these methods accordingly.
  75      *
  76      * <p>Here is a suggested idiom for using unsafe operations:
  77      *
  78      * <pre> {@code
  79      * class MyTrustedClass {
  80      *   private static final Unsafe unsafe = Unsafe.getUnsafe();
  81      *   ...
  82      *   private long myCountAddress = ...;
  83      *   public int getCount() { return unsafe.getByte(myCountAddress); }
  84      * }}</pre>
  85      *
  86      * (It may assist compilers to make the local variable {@code final}.)
  87      */
  88     public static Unsafe getUnsafe() {
  89         return theUnsafe;
  90     }
  91 
  92     /// peek and poke operations
  93     /// (compilers should optimize these to memory ops)
  94 
  95     // These work on object fields in the Java heap.
  96     // They will not work on elements of packed arrays.
  97 
  98     /**
  99      * Fetches a value from a given Java variable.
 100      * More specifically, fetches a field or array element within the given
 101      * object {@code o} at the given offset, or (if {@code o} is null)
 102      * from the memory address whose numerical value is the given offset.
 103      * <p>
 104      * The results are undefined unless one of the following cases is true:
 105      * <ul>
 106      * <li>The offset was obtained from {@link #objectFieldOffset} on
 107      * the {@link java.lang.reflect.Field} of some Java field and the object
 108      * referred to by {@code o} is of a class compatible with that
 109      * field's class.
 110      *
 111      * <li>The offset and object reference {@code o} (either null or
 112      * non-null) were both obtained via {@link #staticFieldOffset}
 113      * and {@link #staticFieldBase} (respectively) from the
 114      * reflective {@link Field} representation of some Java field.
 115      *
 116      * <li>The object referred to by {@code o} is an array, and the offset
 117      * is an integer of the form {@code B+N*S}, where {@code N} is
 118      * a valid index into the array, and {@code B} and {@code S} are
 119      * the values obtained by {@link #arrayBaseOffset} and {@link
 120      * #arrayIndexScale} (respectively) from the array's class.  The value
 121      * referred to is the {@code N}<em>th</em> element of the array.
 122      *
 123      * </ul>
 124      * <p>
 125      * If one of the above cases is true, the call references a specific Java
 126      * variable (field or array element).  However, the results are undefined
 127      * if that variable is not in fact of the type returned by this method.
 128      * <p>
 129      * This method refers to a variable by means of two parameters, and so
 130      * it provides (in effect) a <em>double-register</em> addressing mode
 131      * for Java variables.  When the object reference is null, this method
 132      * uses its offset as an absolute address.  This is similar in operation
 133      * to methods such as {@link #getInt(long)}, which provide (in effect) a
 134      * <em>single-register</em> addressing mode for non-Java variables.
 135      * However, because Java variables may have a different layout in memory
 136      * from non-Java variables, programmers should not assume that these
 137      * two addressing modes are ever equivalent.  Also, programmers should
 138      * remember that offsets from the double-register addressing mode cannot
 139      * be portably confused with longs used in the single-register addressing
 140      * mode.
 141      *
 142      * @param o Java heap object in which the variable resides, if any, else
 143      *        null
 144      * @param offset indication of where the variable resides in a Java heap
 145      *        object, if any, else a memory address locating the variable
 146      *        statically
 147      * @return the value fetched from the indicated Java variable
 148      * @throws RuntimeException No defined exceptions are thrown, not even
 149      *         {@link NullPointerException}
 150      */
 151     @HotSpotIntrinsicCandidate
 152     public native int getInt(Object o, long offset);
 153 
 154     /**
 155      * Stores a value into a given Java variable.
 156      * <p>
 157      * The first two parameters are interpreted exactly as with
 158      * {@link #getInt(Object, long)} to refer to a specific
 159      * Java variable (field or array element).  The given value
 160      * is stored into that variable.
 161      * <p>
 162      * The variable must be of the same type as the method
 163      * parameter {@code x}.
 164      *
 165      * @param o Java heap object in which the variable resides, if any, else
 166      *        null
 167      * @param offset indication of where the variable resides in a Java heap
 168      *        object, if any, else a memory address locating the variable
 169      *        statically
 170      * @param x the value to store into the indicated Java variable
 171      * @throws RuntimeException No defined exceptions are thrown, not even
 172      *         {@link NullPointerException}
 173      */
 174     @HotSpotIntrinsicCandidate
 175     public native void putInt(Object o, long offset, int x);
 176 
 177     /**
 178      * Fetches a reference value from a given Java variable.
 179      * @see #getInt(Object, long)
 180      */
 181     @HotSpotIntrinsicCandidate
 182     public native Object getReference(Object o, long offset);
 183 
 184     /**
 185      * Stores a reference value into a given Java variable.
 186      * <p>
 187      * Unless the reference {@code x} being stored is either null
 188      * or matches the field type, the results are undefined.
 189      * If the reference {@code o} is non-null, card marks or
 190      * other store barriers for that object (if the VM requires them)
 191      * are updated.
 192      * @see #putInt(Object, long, int)
 193      */
 194     @HotSpotIntrinsicCandidate
 195     public native void putReference(Object o, long offset, Object x);
 196 
 197     /** @see #getInt(Object, long) */
 198     @HotSpotIntrinsicCandidate
 199     public native boolean getBoolean(Object o, long offset);
 200 
 201     /** @see #putInt(Object, long, int) */
 202     @HotSpotIntrinsicCandidate
 203     public native void    putBoolean(Object o, long offset, boolean x);
 204 
 205     /** @see #getInt(Object, long) */
 206     @HotSpotIntrinsicCandidate
 207     public native byte    getByte(Object o, long offset);
 208 
 209     /** @see #putInt(Object, long, int) */
 210     @HotSpotIntrinsicCandidate
 211     public native void    putByte(Object o, long offset, byte x);
 212 
 213     /** @see #getInt(Object, long) */
 214     @HotSpotIntrinsicCandidate
 215     public native short   getShort(Object o, long offset);
 216 
 217     /** @see #putInt(Object, long, int) */
 218     @HotSpotIntrinsicCandidate
 219     public native void    putShort(Object o, long offset, short x);
 220 
 221     /** @see #getInt(Object, long) */
 222     @HotSpotIntrinsicCandidate
 223     public native char    getChar(Object o, long offset);
 224 
 225     /** @see #putInt(Object, long, int) */
 226     @HotSpotIntrinsicCandidate
 227     public native void    putChar(Object o, long offset, char x);
 228 
 229     /** @see #getInt(Object, long) */
 230     @HotSpotIntrinsicCandidate
 231     public native long    getLong(Object o, long offset);
 232 
 233     /** @see #putInt(Object, long, int) */
 234     @HotSpotIntrinsicCandidate
 235     public native void    putLong(Object o, long offset, long x);
 236 
 237     /** @see #getInt(Object, long) */
 238     @HotSpotIntrinsicCandidate
 239     public native float   getFloat(Object o, long offset);
 240 
 241     /** @see #putInt(Object, long, int) */
 242     @HotSpotIntrinsicCandidate
 243     public native void    putFloat(Object o, long offset, float x);
 244 
 245     /** @see #getInt(Object, long) */
 246     @HotSpotIntrinsicCandidate
 247     public native double  getDouble(Object o, long offset);
 248 
 249     /** @see #putInt(Object, long, int) */
 250     @HotSpotIntrinsicCandidate
 251     public native void    putDouble(Object o, long offset, double x);
 252 
 253     /**
 254      * Fetches a native pointer from a given memory address.  If the address is
 255      * zero, or does not point into a block obtained from {@link
 256      * #allocateMemory}, the results are undefined.
 257      *
 258      * <p>If the native pointer is less than 64 bits wide, it is extended as
 259      * an unsigned number to a Java long.  The pointer may be indexed by any
 260      * given byte offset, simply by adding that offset (as a simple integer) to
 261      * the long representing the pointer.  The number of bytes actually read
 262      * from the target address may be determined by consulting {@link
 263      * #addressSize}.
 264      *
 265      * @see #allocateMemory
 266      * @see #getInt(Object, long)
 267      */
 268     @ForceInline
 269     public long getAddress(Object o, long offset) {
 270         if (ADDRESS_SIZE == 4) {
 271             return Integer.toUnsignedLong(getInt(o, offset));
 272         } else {
 273             return getLong(o, offset);
 274         }
 275     }
 276 
 277     /**
 278      * Stores a native pointer into a given memory address.  If the address is
 279      * zero, or does not point into a block obtained from {@link
 280      * #allocateMemory}, the results are undefined.
 281      *
 282      * <p>The number of bytes actually written at the target address may be
 283      * determined by consulting {@link #addressSize}.
 284      *
 285      * @see #allocateMemory
 286      * @see #putInt(Object, long, int)
 287      */
 288     @ForceInline
 289     public void putAddress(Object o, long offset, long x) {
 290         if (ADDRESS_SIZE == 4) {
 291             putInt(o, offset, (int)x);
 292         } else {
 293             putLong(o, offset, x);
 294         }
 295     }
 296 
 297     // These read VM internal data.
 298 
 299     /**
 300      * Fetches an uncompressed reference value from a given native variable
 301      * ignoring the VM's compressed references mode.
 302      *
 303      * @param address a memory address locating the variable
 304      * @return the value fetched from the indicated native variable
 305      */
 306     public native Object getUncompressedObject(long address);
 307 
 308     // These work on values in the C heap.
 309 
 310     /**
 311      * Fetches a value from a given memory address.  If the address is zero, or
 312      * does not point into a block obtained from {@link #allocateMemory}, the
 313      * results are undefined.
 314      *
 315      * @see #allocateMemory
 316      */
 317     @ForceInline
 318     public byte getByte(long address) {
 319         return getByte(null, address);
 320     }
 321 
 322     /**
 323      * Stores a value into a given memory address.  If the address is zero, or
 324      * does not point into a block obtained from {@link #allocateMemory}, the
 325      * results are undefined.
 326      *
 327      * @see #getByte(long)
 328      */
 329     @ForceInline
 330     public void putByte(long address, byte x) {
 331         putByte(null, address, x);
 332     }
 333 
 334     /** @see #getByte(long) */
 335     @ForceInline
 336     public short getShort(long address) {
 337         return getShort(null, address);
 338     }
 339 
 340     /** @see #putByte(long, byte) */
 341     @ForceInline
 342     public void putShort(long address, short x) {
 343         putShort(null, address, x);
 344     }
 345 
 346     /** @see #getByte(long) */
 347     @ForceInline
 348     public char getChar(long address) {
 349         return getChar(null, address);
 350     }
 351 
 352     /** @see #putByte(long, byte) */
 353     @ForceInline
 354     public void putChar(long address, char x) {
 355         putChar(null, address, x);
 356     }
 357 
 358     /** @see #getByte(long) */
 359     @ForceInline
 360     public int getInt(long address) {
 361         return getInt(null, address);
 362     }
 363 
 364     /** @see #putByte(long, byte) */
 365     @ForceInline
 366     public void putInt(long address, int x) {
 367         putInt(null, address, x);
 368     }
 369 
 370     /** @see #getByte(long) */
 371     @ForceInline
 372     public long getLong(long address) {
 373         return getLong(null, address);
 374     }
 375 
 376     /** @see #putByte(long, byte) */
 377     @ForceInline
 378     public void putLong(long address, long x) {
 379         putLong(null, address, x);
 380     }
 381 
 382     /** @see #getByte(long) */
 383     @ForceInline
 384     public float getFloat(long address) {
 385         return getFloat(null, address);
 386     }
 387 
 388     /** @see #putByte(long, byte) */
 389     @ForceInline
 390     public void putFloat(long address, float x) {
 391         putFloat(null, address, x);
 392     }
 393 
 394     /** @see #getByte(long) */
 395     @ForceInline
 396     public double getDouble(long address) {
 397         return getDouble(null, address);
 398     }
 399 
 400     /** @see #putByte(long, byte) */
 401     @ForceInline
 402     public void putDouble(long address, double x) {
 403         putDouble(null, address, x);
 404     }
 405 
 406     /** @see #getAddress(Object, long) */
 407     @ForceInline
 408     public long getAddress(long address) {
 409         return getAddress(null, address);
 410     }
 411 
 412     /** @see #putAddress(Object, long, long) */
 413     @ForceInline
 414     public void putAddress(long address, long x) {
 415         putAddress(null, address, x);
 416     }
 417 
 418 
 419 
 420     /// helper methods for validating various types of objects/values
 421 
 422     /**
 423      * Create an exception reflecting that some of the input was invalid
 424      *
 425      * <em>Note:</em> It is the resposibility of the caller to make
 426      * sure arguments are checked before the methods are called. While
 427      * some rudimentary checks are performed on the input, the checks
 428      * are best effort and when performance is an overriding priority,
 429      * as when methods of this class are optimized by the runtime
 430      * compiler, some or all checks (if any) may be elided. Hence, the
 431      * caller must not rely on the checks and corresponding
 432      * exceptions!
 433      *
 434      * @return an exception object
 435      */
 436     private RuntimeException invalidInput() {
 437         return new IllegalArgumentException();
 438     }
 439 
 440     /**
 441      * Check if a value is 32-bit clean (32 MSB are all zero)
 442      *
 443      * @param value the 64-bit value to check
 444      *
 445      * @return true if the value is 32-bit clean
 446      */
 447     private boolean is32BitClean(long value) {
 448         return value >>> 32 == 0;
 449     }
 450 
 451     /**
 452      * Check the validity of a size (the equivalent of a size_t)
 453      *
 454      * @throws RuntimeException if the size is invalid
 455      *         (<em>Note:</em> after optimization, invalid inputs may
 456      *         go undetected, which will lead to unpredictable
 457      *         behavior)
 458      */
 459     private void checkSize(long size) {
 460         if (ADDRESS_SIZE == 4) {
 461             // Note: this will also check for negative sizes
 462             if (!is32BitClean(size)) {
 463                 throw invalidInput();
 464             }
 465         } else if (size < 0) {
 466             throw invalidInput();
 467         }
 468     }
 469 
 470     /**
 471      * Check the validity of a native address (the equivalent of void*)
 472      *
 473      * @throws RuntimeException if the address is invalid
 474      *         (<em>Note:</em> after optimization, invalid inputs may
 475      *         go undetected, which will lead to unpredictable
 476      *         behavior)
 477      */
 478     private void checkNativeAddress(long address) {
 479         if (ADDRESS_SIZE == 4) {
 480             // Accept both zero and sign extended pointers. A valid
 481             // pointer will, after the +1 below, either have produced
 482             // the value 0x0 or 0x1. Masking off the low bit allows
 483             // for testing against 0.
 484             if ((((address >> 32) + 1) & ~1) != 0) {
 485                 throw invalidInput();
 486             }
 487         }
 488     }
 489 
 490     /**
 491      * Check the validity of an offset, relative to a base object
 492      *
 493      * @param o the base object
 494      * @param offset the offset to check
 495      *
 496      * @throws RuntimeException if the size is invalid
 497      *         (<em>Note:</em> after optimization, invalid inputs may
 498      *         go undetected, which will lead to unpredictable
 499      *         behavior)
 500      */
 501     private void checkOffset(Object o, long offset) {
 502         if (ADDRESS_SIZE == 4) {
 503             // Note: this will also check for negative offsets
 504             if (!is32BitClean(offset)) {
 505                 throw invalidInput();
 506             }
 507         } else if (offset < 0) {
 508             throw invalidInput();
 509         }
 510     }
 511 
 512     /**
 513      * Check the validity of a double-register pointer
 514      *
 515      * Note: This code deliberately does *not* check for NPE for (at
 516      * least) three reasons:
 517      *
 518      * 1) NPE is not just NULL/0 - there is a range of values all
 519      * resulting in an NPE, which is not trivial to check for
 520      *
 521      * 2) It is the responsibility of the callers of Unsafe methods
 522      * to verify the input, so throwing an exception here is not really
 523      * useful - passing in a NULL pointer is a critical error and the
 524      * must not expect an exception to be thrown anyway.
 525      *
 526      * 3) the actual operations will detect NULL pointers anyway by
 527      * means of traps and signals (like SIGSEGV).
 528      *
 529      * @param o Java heap object, or null
 530      * @param offset indication of where the variable resides in a Java heap
 531      *        object, if any, else a memory address locating the variable
 532      *        statically
 533      *
 534      * @throws RuntimeException if the pointer is invalid
 535      *         (<em>Note:</em> after optimization, invalid inputs may
 536      *         go undetected, which will lead to unpredictable
 537      *         behavior)
 538      */
 539     private void checkPointer(Object o, long offset) {
 540         if (o == null) {
 541             checkNativeAddress(offset);
 542         } else {
 543             checkOffset(o, offset);
 544         }
 545     }
 546 
 547     /**
 548      * Check if a type is a primitive array type
 549      *
 550      * @param c the type to check
 551      *
 552      * @return true if the type is a primitive array type
 553      */
 554     private void checkPrimitiveArray(Class<?> c) {
 555         Class<?> componentType = c.getComponentType();
 556         if (componentType == null || !componentType.isPrimitive()) {
 557             throw invalidInput();
 558         }
 559     }
 560 
 561     /**
 562      * Check that a pointer is a valid primitive array type pointer
 563      *
 564      * Note: pointers off-heap are considered to be primitive arrays
 565      *
 566      * @throws RuntimeException if the pointer is invalid
 567      *         (<em>Note:</em> after optimization, invalid inputs may
 568      *         go undetected, which will lead to unpredictable
 569      *         behavior)
 570      */
 571     private void checkPrimitivePointer(Object o, long offset) {
 572         checkPointer(o, offset);
 573 
 574         if (o != null) {
 575             // If on heap, it must be a primitive array
 576             checkPrimitiveArray(o.getClass());
 577         }
 578     }
 579 
 580 
 581     /// wrappers for malloc, realloc, free:
 582 
 583     /**
 584      * Allocates a new block of native memory, of the given size in bytes.  The
 585      * contents of the memory are uninitialized; they will generally be
 586      * garbage.  The resulting native pointer will never be zero, and will be
 587      * aligned for all value types.  Dispose of this memory by calling {@link
 588      * #freeMemory}, or resize it with {@link #reallocateMemory}.
 589      *
 590      * <em>Note:</em> It is the resposibility of the caller to make
 591      * sure arguments are checked before the methods are called. While
 592      * some rudimentary checks are performed on the input, the checks
 593      * are best effort and when performance is an overriding priority,
 594      * as when methods of this class are optimized by the runtime
 595      * compiler, some or all checks (if any) may be elided. Hence, the
 596      * caller must not rely on the checks and corresponding
 597      * exceptions!
 598      *
 599      * @throws RuntimeException if the size is negative or too large
 600      *         for the native size_t type
 601      *
 602      * @throws OutOfMemoryError if the allocation is refused by the system
 603      *
 604      * @see #getByte(long)
 605      * @see #putByte(long, byte)
 606      */
 607     public long allocateMemory(long bytes) {
 608         allocateMemoryChecks(bytes);
 609 
 610         if (bytes == 0) {
 611             return 0;
 612         }
 613 
 614         long p = allocateMemory0(bytes);
 615         if (p == 0) {
 616             throw new OutOfMemoryError();
 617         }
 618 
 619         return p;
 620     }
 621 
 622     /**
 623      * Validate the arguments to allocateMemory
 624      *
 625      * @throws RuntimeException if the arguments are invalid
 626      *         (<em>Note:</em> after optimization, invalid inputs may
 627      *         go undetected, which will lead to unpredictable
 628      *         behavior)
 629      */
 630     private void allocateMemoryChecks(long bytes) {
 631         checkSize(bytes);
 632     }
 633 
 634     /**
 635      * Resizes a new block of native memory, to the given size in bytes.  The
 636      * contents of the new block past the size of the old block are
 637      * uninitialized; they will generally be garbage.  The resulting native
 638      * pointer will be zero if and only if the requested size is zero.  The
 639      * resulting native pointer will be aligned for all value types.  Dispose
 640      * of this memory by calling {@link #freeMemory}, or resize it with {@link
 641      * #reallocateMemory}.  The address passed to this method may be null, in
 642      * which case an allocation will be performed.
 643      *
 644      * <em>Note:</em> It is the resposibility of the caller to make
 645      * sure arguments are checked before the methods are called. While
 646      * some rudimentary checks are performed on the input, the checks
 647      * are best effort and when performance is an overriding priority,
 648      * as when methods of this class are optimized by the runtime
 649      * compiler, some or all checks (if any) may be elided. Hence, the
 650      * caller must not rely on the checks and corresponding
 651      * exceptions!
 652      *
 653      * @throws RuntimeException if the size is negative or too large
 654      *         for the native size_t type
 655      *
 656      * @throws OutOfMemoryError if the allocation is refused by the system
 657      *
 658      * @see #allocateMemory
 659      */
 660     public long reallocateMemory(long address, long bytes) {
 661         reallocateMemoryChecks(address, bytes);
 662 
 663         if (bytes == 0) {
 664             freeMemory(address);
 665             return 0;
 666         }
 667 
 668         long p = (address == 0) ? allocateMemory0(bytes) : reallocateMemory0(address, bytes);
 669         if (p == 0) {
 670             throw new OutOfMemoryError();
 671         }
 672 
 673         return p;
 674     }
 675 
 676     /**
 677      * Validate the arguments to reallocateMemory
 678      *
 679      * @throws RuntimeException if the arguments are invalid
 680      *         (<em>Note:</em> after optimization, invalid inputs may
 681      *         go undetected, which will lead to unpredictable
 682      *         behavior)
 683      */
 684     private void reallocateMemoryChecks(long address, long bytes) {
 685         checkPointer(null, address);
 686         checkSize(bytes);
 687     }
 688 
 689     /**
 690      * Sets all bytes in a given block of memory to a fixed value
 691      * (usually zero).
 692      *
 693      * <p>This method determines a block's base address by means of two parameters,
 694      * and so it provides (in effect) a <em>double-register</em> addressing mode,
 695      * as discussed in {@link #getInt(Object,long)}.  When the object reference is null,
 696      * the offset supplies an absolute base address.
 697      *
 698      * <p>The stores are in coherent (atomic) units of a size determined
 699      * by the address and length parameters.  If the effective address and
 700      * length are all even modulo 8, the stores take place in 'long' units.
 701      * If the effective address and length are (resp.) even modulo 4 or 2,
 702      * the stores take place in units of 'int' or 'short'.
 703      *
 704      * <em>Note:</em> It is the resposibility of the caller to make
 705      * sure arguments are checked before the methods are called. While
 706      * some rudimentary checks are performed on the input, the checks
 707      * are best effort and when performance is an overriding priority,
 708      * as when methods of this class are optimized by the runtime
 709      * compiler, some or all checks (if any) may be elided. Hence, the
 710      * caller must not rely on the checks and corresponding
 711      * exceptions!
 712      *
 713      * @throws RuntimeException if any of the arguments is invalid
 714      *
 715      * @since 1.7
 716      */
 717     public void setMemory(Object o, long offset, long bytes, byte value) {
 718         setMemoryChecks(o, offset, bytes, value);
 719 
 720         if (bytes == 0) {
 721             return;
 722         }
 723 
 724         setMemory0(o, offset, bytes, value);
 725     }
 726 
 727     /**
 728      * Sets all bytes in a given block of memory to a fixed value
 729      * (usually zero).  This provides a <em>single-register</em> addressing mode,
 730      * as discussed in {@link #getInt(Object,long)}.
 731      *
 732      * <p>Equivalent to {@code setMemory(null, address, bytes, value)}.
 733      */
 734     public void setMemory(long address, long bytes, byte value) {
 735         setMemory(null, address, bytes, value);
 736     }
 737 
 738     /**
 739      * Validate the arguments to setMemory
 740      *
 741      * @throws RuntimeException if the arguments are invalid
 742      *         (<em>Note:</em> after optimization, invalid inputs may
 743      *         go undetected, which will lead to unpredictable
 744      *         behavior)
 745      */
 746     private void setMemoryChecks(Object o, long offset, long bytes, byte value) {
 747         checkPrimitivePointer(o, offset);
 748         checkSize(bytes);
 749     }
 750 
 751     /**
 752      * Sets all bytes in a given block of memory to a copy of another
 753      * block.
 754      *
 755      * <p>This method determines each block's base address by means of two parameters,
 756      * and so it provides (in effect) a <em>double-register</em> addressing mode,
 757      * as discussed in {@link #getInt(Object,long)}.  When the object reference is null,
 758      * the offset supplies an absolute base address.
 759      *
 760      * <p>The transfers are in coherent (atomic) units of a size determined
 761      * by the address and length parameters.  If the effective addresses and
 762      * length are all even modulo 8, the transfer takes place in 'long' units.
 763      * If the effective addresses and length are (resp.) even modulo 4 or 2,
 764      * the transfer takes place in units of 'int' or 'short'.
 765      *
 766      * <em>Note:</em> It is the resposibility of the caller to make
 767      * sure arguments are checked before the methods are called. While
 768      * some rudimentary checks are performed on the input, the checks
 769      * are best effort and when performance is an overriding priority,
 770      * as when methods of this class are optimized by the runtime
 771      * compiler, some or all checks (if any) may be elided. Hence, the
 772      * caller must not rely on the checks and corresponding
 773      * exceptions!
 774      *
 775      * @throws RuntimeException if any of the arguments is invalid
 776      *
 777      * @since 1.7
 778      */
 779     public void copyMemory(Object srcBase, long srcOffset,
 780                            Object destBase, long destOffset,
 781                            long bytes) {
 782         copyMemoryChecks(srcBase, srcOffset, destBase, destOffset, bytes);
 783 
 784         if (bytes == 0) {
 785             return;
 786         }
 787 
 788         copyMemory0(srcBase, srcOffset, destBase, destOffset, bytes);
 789     }
 790 
 791     /**
 792      * Sets all bytes in a given block of memory to a copy of another
 793      * block.  This provides a <em>single-register</em> addressing mode,
 794      * as discussed in {@link #getInt(Object,long)}.
 795      *
 796      * Equivalent to {@code copyMemory(null, srcAddress, null, destAddress, bytes)}.
 797      */
 798     public void copyMemory(long srcAddress, long destAddress, long bytes) {
 799         copyMemory(null, srcAddress, null, destAddress, bytes);
 800     }
 801 
 802     /**
 803      * Validate the arguments to copyMemory
 804      *
 805      * @throws RuntimeException if any of the arguments is invalid
 806      *         (<em>Note:</em> after optimization, invalid inputs may
 807      *         go undetected, which will lead to unpredictable
 808      *         behavior)
 809      */
 810     private void copyMemoryChecks(Object srcBase, long srcOffset,
 811                                   Object destBase, long destOffset,
 812                                   long bytes) {
 813         checkSize(bytes);
 814         checkPrimitivePointer(srcBase, srcOffset);
 815         checkPrimitivePointer(destBase, destOffset);
 816     }
 817 
 818     /**
 819      * Copies all elements from one block of memory to another block,
 820      * *unconditionally* byte swapping the elements on the fly.
 821      *
 822      * <p>This method determines each block's base address by means of two parameters,
 823      * and so it provides (in effect) a <em>double-register</em> addressing mode,
 824      * as discussed in {@link #getInt(Object,long)}.  When the object reference is null,
 825      * the offset supplies an absolute base address.
 826      *
 827      * <em>Note:</em> It is the resposibility of the caller to make
 828      * sure arguments are checked before the methods are called. While
 829      * some rudimentary checks are performed on the input, the checks
 830      * are best effort and when performance is an overriding priority,
 831      * as when methods of this class are optimized by the runtime
 832      * compiler, some or all checks (if any) may be elided. Hence, the
 833      * caller must not rely on the checks and corresponding
 834      * exceptions!
 835      *
 836      * @throws RuntimeException if any of the arguments is invalid
 837      *
 838      * @since 9
 839      */
 840     public void copySwapMemory(Object srcBase, long srcOffset,
 841                                Object destBase, long destOffset,
 842                                long bytes, long elemSize) {
 843         copySwapMemoryChecks(srcBase, srcOffset, destBase, destOffset, bytes, elemSize);
 844 
 845         if (bytes == 0) {
 846             return;
 847         }
 848 
 849         copySwapMemory0(srcBase, srcOffset, destBase, destOffset, bytes, elemSize);
 850     }
 851 
 852     private void copySwapMemoryChecks(Object srcBase, long srcOffset,
 853                                       Object destBase, long destOffset,
 854                                       long bytes, long elemSize) {
 855         checkSize(bytes);
 856 
 857         if (elemSize != 2 && elemSize != 4 && elemSize != 8) {
 858             throw invalidInput();
 859         }
 860         if (bytes % elemSize != 0) {
 861             throw invalidInput();
 862         }
 863 
 864         checkPrimitivePointer(srcBase, srcOffset);
 865         checkPrimitivePointer(destBase, destOffset);
 866     }
 867 
 868    /**
 869      * Copies all elements from one block of memory to another block, byte swapping the
 870      * elements on the fly.
 871      *
 872      * This provides a <em>single-register</em> addressing mode, as
 873      * discussed in {@link #getInt(Object,long)}.
 874      *
 875      * Equivalent to {@code copySwapMemory(null, srcAddress, null, destAddress, bytes, elemSize)}.
 876      */
 877     public void copySwapMemory(long srcAddress, long destAddress, long bytes, long elemSize) {
 878         copySwapMemory(null, srcAddress, null, destAddress, bytes, elemSize);
 879     }
 880 
 881     /**
 882      * Disposes of a block of native memory, as obtained from {@link
 883      * #allocateMemory} or {@link #reallocateMemory}.  The address passed to
 884      * this method may be null, in which case no action is taken.
 885      *
 886      * <em>Note:</em> It is the resposibility of the caller to make
 887      * sure arguments are checked before the methods are called. While
 888      * some rudimentary checks are performed on the input, the checks
 889      * are best effort and when performance is an overriding priority,
 890      * as when methods of this class are optimized by the runtime
 891      * compiler, some or all checks (if any) may be elided. Hence, the
 892      * caller must not rely on the checks and corresponding
 893      * exceptions!
 894      *
 895      * @throws RuntimeException if any of the arguments is invalid
 896      *
 897      * @see #allocateMemory
 898      */
 899     public void freeMemory(long address) {
 900         freeMemoryChecks(address);
 901 
 902         if (address == 0) {
 903             return;
 904         }
 905 
 906         freeMemory0(address);
 907     }
 908 
 909     /**
 910      * Validate the arguments to freeMemory
 911      *
 912      * @throws RuntimeException if the arguments are invalid
 913      *         (<em>Note:</em> after optimization, invalid inputs may
 914      *         go undetected, which will lead to unpredictable
 915      *         behavior)
 916      */
 917     private void freeMemoryChecks(long address) {
 918         checkPointer(null, address);
 919     }
 920 
 921     /// random queries
 922 
 923     /**
 924      * This constant differs from all results that will ever be returned from
 925      * {@link #staticFieldOffset}, {@link #objectFieldOffset},
 926      * or {@link #arrayBaseOffset}.
 927      */
 928     public static final int INVALID_FIELD_OFFSET = -1;
 929 
 930     /**
 931      * Reports the location of a given field in the storage allocation of its
 932      * class.  Do not expect to perform any sort of arithmetic on this offset;
 933      * it is just a cookie which is passed to the unsafe heap memory accessors.
 934      *
 935      * <p>Any given field will always have the same offset and base, and no
 936      * two distinct fields of the same class will ever have the same offset
 937      * and base.
 938      *
 939      * <p>As of 1.4.1, offsets for fields are represented as long values,
 940      * although the Sun JVM does not use the most significant 32 bits.
 941      * However, JVM implementations which store static fields at absolute
 942      * addresses can use long offsets and null base pointers to express
 943      * the field locations in a form usable by {@link #getInt(Object,long)}.
 944      * Therefore, code which will be ported to such JVMs on 64-bit platforms
 945      * must preserve all bits of static field offsets.
 946      * @see #getInt(Object, long)
 947      */
 948     public long objectFieldOffset(Field f) {
 949         if (f == null) {
 950             throw new NullPointerException();
 951         }
 952 
 953         return objectFieldOffset0(f);
 954     }
 955 
 956     /**
 957      * Reports the location of the field with a given name in the storage
 958      * allocation of its class.
 959      *
 960      * @throws NullPointerException if any parameter is {@code null}.
 961      * @throws InternalError if there is no field named {@code name} declared
 962      *         in class {@code c}, i.e., if {@code c.getDeclaredField(name)}
 963      *         would throw {@code java.lang.NoSuchFieldException}.
 964      *
 965      * @see #objectFieldOffset(Field)
 966      */
 967     public long objectFieldOffset(Class<?> c, String name) {
 968         if (c == null || name == null) {
 969             throw new NullPointerException();
 970         }
 971 
 972         return objectFieldOffset1(c, name);
 973     }
 974 
 975     /**
 976      * Reports the location of a given static field, in conjunction with {@link
 977      * #staticFieldBase}.
 978      * <p>Do not expect to perform any sort of arithmetic on this offset;
 979      * it is just a cookie which is passed to the unsafe heap memory accessors.
 980      *
 981      * <p>Any given field will always have the same offset, and no two distinct
 982      * fields of the same class will ever have the same offset.
 983      *
 984      * <p>As of 1.4.1, offsets for fields are represented as long values,
 985      * although the Sun JVM does not use the most significant 32 bits.
 986      * It is hard to imagine a JVM technology which needs more than
 987      * a few bits to encode an offset within a non-array object,
 988      * However, for consistency with other methods in this class,
 989      * this method reports its result as a long value.
 990      * @see #getInt(Object, long)
 991      */
 992     public long staticFieldOffset(Field f) {
 993         if (f == null) {
 994             throw new NullPointerException();
 995         }
 996 
 997         return staticFieldOffset0(f);
 998     }
 999 
1000     /**
1001      * Reports the location of a given static field, in conjunction with {@link
1002      * #staticFieldOffset}.
1003      * <p>Fetch the base "Object", if any, with which static fields of the
1004      * given class can be accessed via methods like {@link #getInt(Object,
1005      * long)}.  This value may be null.  This value may refer to an object
1006      * which is a "cookie", not guaranteed to be a real Object, and it should
1007      * not be used in any way except as argument to the get and put routines in
1008      * this class.
1009      */
1010     public Object staticFieldBase(Field f) {
1011         if (f == null) {
1012             throw new NullPointerException();
1013         }
1014 
1015         return staticFieldBase0(f);
1016     }
1017 
1018     /**
1019      * Detects if the given class may need to be initialized. This is often
1020      * needed in conjunction with obtaining the static field base of a
1021      * class.
1022      * @return false only if a call to {@code ensureClassInitialized} would have no effect
1023      */
1024     public boolean shouldBeInitialized(Class<?> c) {
1025         if (c == null) {
1026             throw new NullPointerException();
1027         }
1028 
1029         return shouldBeInitialized0(c);
1030     }
1031 
1032     /**
1033      * Ensures the given class has been initialized. This is often
1034      * needed in conjunction with obtaining the static field base of a
1035      * class.
1036      */
1037     public void ensureClassInitialized(Class<?> c) {
1038         if (c == null) {
1039             throw new NullPointerException();
1040         }
1041 
1042         ensureClassInitialized0(c);
1043     }
1044 
1045     /**
1046      * Reports the offset of the first element in the storage allocation of a
1047      * given array class.  If {@link #arrayIndexScale} returns a non-zero value
1048      * for the same class, you may use that scale factor, together with this
1049      * base offset, to form new offsets to access elements of arrays of the
1050      * given class.
1051      *
1052      * @see #getInt(Object, long)
1053      * @see #putInt(Object, long, int)
1054      */
1055     public int arrayBaseOffset(Class<?> arrayClass) {
1056         if (arrayClass == null) {
1057             throw new NullPointerException();
1058         }
1059 
1060         return arrayBaseOffset0(arrayClass);
1061     }
1062 
1063 
1064     /** The value of {@code arrayBaseOffset(boolean[].class)} */
1065     public static final int ARRAY_BOOLEAN_BASE_OFFSET
1066             = theUnsafe.arrayBaseOffset(boolean[].class);
1067 
1068     /** The value of {@code arrayBaseOffset(byte[].class)} */
1069     public static final int ARRAY_BYTE_BASE_OFFSET
1070             = theUnsafe.arrayBaseOffset(byte[].class);
1071 
1072     /** The value of {@code arrayBaseOffset(short[].class)} */
1073     public static final int ARRAY_SHORT_BASE_OFFSET
1074             = theUnsafe.arrayBaseOffset(short[].class);
1075 
1076     /** The value of {@code arrayBaseOffset(char[].class)} */
1077     public static final int ARRAY_CHAR_BASE_OFFSET
1078             = theUnsafe.arrayBaseOffset(char[].class);
1079 
1080     /** The value of {@code arrayBaseOffset(int[].class)} */
1081     public static final int ARRAY_INT_BASE_OFFSET
1082             = theUnsafe.arrayBaseOffset(int[].class);
1083 
1084     /** The value of {@code arrayBaseOffset(long[].class)} */
1085     public static final int ARRAY_LONG_BASE_OFFSET
1086             = theUnsafe.arrayBaseOffset(long[].class);
1087 
1088     /** The value of {@code arrayBaseOffset(float[].class)} */
1089     public static final int ARRAY_FLOAT_BASE_OFFSET
1090             = theUnsafe.arrayBaseOffset(float[].class);
1091 
1092     /** The value of {@code arrayBaseOffset(double[].class)} */
1093     public static final int ARRAY_DOUBLE_BASE_OFFSET
1094             = theUnsafe.arrayBaseOffset(double[].class);
1095 
1096     /** The value of {@code arrayBaseOffset(Object[].class)} */
1097     public static final int ARRAY_OBJECT_BASE_OFFSET
1098             = theUnsafe.arrayBaseOffset(Object[].class);
1099 
1100     /**
1101      * Reports the scale factor for addressing elements in the storage
1102      * allocation of a given array class.  However, arrays of "narrow" types
1103      * will generally not work properly with accessors like {@link
1104      * #getByte(Object, long)}, so the scale factor for such classes is reported
1105      * as zero.
1106      *
1107      * @see #arrayBaseOffset
1108      * @see #getInt(Object, long)
1109      * @see #putInt(Object, long, int)
1110      */
1111     public int arrayIndexScale(Class<?> arrayClass) {
1112         if (arrayClass == null) {
1113             throw new NullPointerException();
1114         }
1115 
1116         return arrayIndexScale0(arrayClass);
1117     }
1118 
1119 
1120     /** The value of {@code arrayIndexScale(boolean[].class)} */
1121     public static final int ARRAY_BOOLEAN_INDEX_SCALE
1122             = theUnsafe.arrayIndexScale(boolean[].class);
1123 
1124     /** The value of {@code arrayIndexScale(byte[].class)} */
1125     public static final int ARRAY_BYTE_INDEX_SCALE
1126             = theUnsafe.arrayIndexScale(byte[].class);
1127 
1128     /** The value of {@code arrayIndexScale(short[].class)} */
1129     public static final int ARRAY_SHORT_INDEX_SCALE
1130             = theUnsafe.arrayIndexScale(short[].class);
1131 
1132     /** The value of {@code arrayIndexScale(char[].class)} */
1133     public static final int ARRAY_CHAR_INDEX_SCALE
1134             = theUnsafe.arrayIndexScale(char[].class);
1135 
1136     /** The value of {@code arrayIndexScale(int[].class)} */
1137     public static final int ARRAY_INT_INDEX_SCALE
1138             = theUnsafe.arrayIndexScale(int[].class);
1139 
1140     /** The value of {@code arrayIndexScale(long[].class)} */
1141     public static final int ARRAY_LONG_INDEX_SCALE
1142             = theUnsafe.arrayIndexScale(long[].class);
1143 
1144     /** The value of {@code arrayIndexScale(float[].class)} */
1145     public static final int ARRAY_FLOAT_INDEX_SCALE
1146             = theUnsafe.arrayIndexScale(float[].class);
1147 
1148     /** The value of {@code arrayIndexScale(double[].class)} */
1149     public static final int ARRAY_DOUBLE_INDEX_SCALE
1150             = theUnsafe.arrayIndexScale(double[].class);
1151 
1152     /** The value of {@code arrayIndexScale(Object[].class)} */
1153     public static final int ARRAY_OBJECT_INDEX_SCALE
1154             = theUnsafe.arrayIndexScale(Object[].class);
1155 
1156     /**
1157      * Reports the size in bytes of a native pointer, as stored via {@link
1158      * #putAddress}.  This value will be either 4 or 8.  Note that the sizes of
1159      * other primitive types (as stored in native memory blocks) is determined
1160      * fully by their information content.
1161      */
1162     public int addressSize() {
1163         return ADDRESS_SIZE;
1164     }
1165 
1166     /** The value of {@code addressSize()} */
1167     public static final int ADDRESS_SIZE = theUnsafe.addressSize0();
1168 
1169     /**
1170      * Reports the size in bytes of a native memory page (whatever that is).
1171      * This value will always be a power of two.
1172      */
1173     public native int pageSize();
1174 
1175 
1176     /// random trusted operations from JNI:
1177 
1178     /**
1179      * Tells the VM to define a class, without security checks.  By default, the
1180      * class loader and protection domain come from the caller's class.
1181      */
1182     public Class<?> defineClass(String name, byte[] b, int off, int len,
1183                                 ClassLoader loader,
1184                                 ProtectionDomain protectionDomain) {
1185         if (b == null) {
1186             throw new NullPointerException();
1187         }
1188         if (len < 0) {
1189             throw new ArrayIndexOutOfBoundsException();
1190         }
1191 
1192         return defineClass0(name, b, off, len, loader, protectionDomain);
1193     }
1194 
1195     public native Class<?> defineClass0(String name, byte[] b, int off, int len,
1196                                         ClassLoader loader,
1197                                         ProtectionDomain protectionDomain);
1198 
1199     /**
1200      * Defines a class but does not make it known to the class loader or system dictionary.
1201      * <p>
1202      * For each CP entry, the corresponding CP patch must either be null or have
1203      * the a format that matches its tag:
1204      * <ul>
1205      * <li>Integer, Long, Float, Double: the corresponding wrapper object type from java.lang
1206      * <li>Utf8: a string (must have suitable syntax if used as signature or name)
1207      * <li>Class: any java.lang.Class object
1208      * <li>String: any object (not just a java.lang.String)
1209      * <li>InterfaceMethodRef: (NYI) a method handle to invoke on that call site's arguments
1210      * </ul>
1211      * @param hostClass context for linkage, access control, protection domain, and class loader
1212      * @param data      bytes of a class file
1213      * @param cpPatches where non-null entries exist, they replace corresponding CP entries in data
1214      */
1215     public Class<?> defineAnonymousClass(Class<?> hostClass, byte[] data, Object[] cpPatches) {
1216         if (hostClass == null || data == null) {
1217             throw new NullPointerException();
1218         }
1219         if (hostClass.isArray() || hostClass.isPrimitive()) {
1220             throw new IllegalArgumentException();
1221         }
1222 
1223         return defineAnonymousClass0(hostClass, data, cpPatches);
1224     }
1225 
1226     /**
1227      * Allocates an instance but does not run any constructor.
1228      * Initializes the class if it has not yet been.
1229      */
1230     @HotSpotIntrinsicCandidate
1231     public native Object allocateInstance(Class<?> cls)
1232         throws InstantiationException;
1233 
1234     /**
1235      * Allocates an array of a given type, but does not do zeroing.
1236      * <p>
1237      * This method should only be used in the very rare cases where a high-performance code
1238      * overwrites the destination array completely, and compilers cannot assist in zeroing elimination.
1239      * In an overwhelming majority of cases, a normal Java allocation should be used instead.
1240      * <p>
1241      * Users of this method are <b>required</b> to overwrite the initial (garbage) array contents
1242      * before allowing untrusted code, or code in other threads, to observe the reference
1243      * to the newly allocated array. In addition, the publication of the array reference must be
1244      * safe according to the Java Memory Model requirements.
1245      * <p>
1246      * The safest approach to deal with an uninitialized array is to keep the reference to it in local
1247      * variable at least until the initialization is complete, and then publish it <b>once</b>, either
1248      * by writing it to a <em>volatile</em> field, or storing it into a <em>final</em> field in constructor,
1249      * or issuing a {@link #storeFence} before publishing the reference.
1250      * <p>
1251      * @implnote This method can only allocate primitive arrays, to avoid garbage reference
1252      * elements that could break heap integrity.
1253      *
1254      * @param componentType array component type to allocate
1255      * @param length array size to allocate
1256      * @throws IllegalArgumentException if component type is null, or not a primitive class;
1257      *                                  or the length is negative
1258      */
1259     public Object allocateUninitializedArray(Class<?> componentType, int length) {
1260        if (componentType == null) {
1261            throw new IllegalArgumentException("Component type is null");
1262        }
1263        if (!componentType.isPrimitive()) {
1264            throw new IllegalArgumentException("Component type is not primitive");
1265        }
1266        if (length < 0) {
1267            throw new IllegalArgumentException("Negative length");
1268        }
1269        return allocateUninitializedArray0(componentType, length);
1270     }
1271 
1272     @HotSpotIntrinsicCandidate
1273     private Object allocateUninitializedArray0(Class<?> componentType, int length) {
1274        // These fallbacks provide zeroed arrays, but intrinsic is not required to
1275        // return the zeroed arrays.
1276        if (componentType == byte.class)    return new byte[length];
1277        if (componentType == boolean.class) return new boolean[length];
1278        if (componentType == short.class)   return new short[length];
1279        if (componentType == char.class)    return new char[length];
1280        if (componentType == int.class)     return new int[length];
1281        if (componentType == float.class)   return new float[length];
1282        if (componentType == long.class)    return new long[length];
1283        if (componentType == double.class)  return new double[length];
1284        return null;
1285     }
1286 
1287     /** Throws the exception without telling the verifier. */
1288     public native void throwException(Throwable ee);
1289 
1290     /**
1291      * Atomically updates Java variable to {@code x} if it is currently
1292      * holding {@code expected}.
1293      *
1294      * <p>This operation has memory semantics of a {@code volatile} read
1295      * and write.  Corresponds to C11 atomic_compare_exchange_strong.
1296      *
1297      * @return {@code true} if successful
1298      */
1299     @HotSpotIntrinsicCandidate
1300     public final native boolean compareAndSetReference(Object o, long offset,
1301                                                        Object expected,
1302                                                        Object x);
1303 
1304     @HotSpotIntrinsicCandidate
1305     public final native Object compareAndExchangeReference(Object o, long offset,
1306                                                            Object expected,
1307                                                            Object x);
1308 
1309     @HotSpotIntrinsicCandidate
1310     public final Object compareAndExchangeReferenceAcquire(Object o, long offset,
1311                                                            Object expected,
1312                                                            Object x) {
1313         return compareAndExchangeReference(o, offset, expected, x);
1314     }
1315 
1316     @HotSpotIntrinsicCandidate
1317     public final Object compareAndExchangeReferenceRelease(Object o, long offset,
1318                                                            Object expected,
1319                                                            Object x) {
1320         return compareAndExchangeReference(o, offset, expected, x);
1321     }
1322 
1323     @HotSpotIntrinsicCandidate
1324     public final boolean weakCompareAndSetReferencePlain(Object o, long offset,
1325                                                          Object expected,
1326                                                          Object x) {
1327         return compareAndSetReference(o, offset, expected, x);
1328     }
1329 
1330     @HotSpotIntrinsicCandidate
1331     public final boolean weakCompareAndSetReferenceAcquire(Object o, long offset,
1332                                                            Object expected,
1333                                                            Object x) {
1334         return compareAndSetReference(o, offset, expected, x);
1335     }
1336 
1337     @HotSpotIntrinsicCandidate
1338     public final boolean weakCompareAndSetReferenceRelease(Object o, long offset,
1339                                                            Object expected,
1340                                                            Object x) {
1341         return compareAndSetReference(o, offset, expected, x);
1342     }
1343 
1344     @HotSpotIntrinsicCandidate
1345     public final boolean weakCompareAndSetReference(Object o, long offset,
1346                                                     Object expected,
1347                                                     Object x) {
1348         return compareAndSetReference(o, offset, expected, x);
1349     }
1350 
1351     /**
1352      * Atomically updates Java variable to {@code x} if it is currently
1353      * holding {@code expected}.
1354      *
1355      * <p>This operation has memory semantics of a {@code volatile} read
1356      * and write.  Corresponds to C11 atomic_compare_exchange_strong.
1357      *
1358      * @return {@code true} if successful
1359      */
1360     @HotSpotIntrinsicCandidate
1361     public final native boolean compareAndSetInt(Object o, long offset,
1362                                                  int expected,
1363                                                  int x);
1364 
1365     @HotSpotIntrinsicCandidate
1366     public final native int compareAndExchangeInt(Object o, long offset,
1367                                                   int expected,
1368                                                   int x);
1369 
1370     @HotSpotIntrinsicCandidate
1371     public final int compareAndExchangeIntAcquire(Object o, long offset,
1372                                                          int expected,
1373                                                          int x) {
1374         return compareAndExchangeInt(o, offset, expected, x);
1375     }
1376 
1377     @HotSpotIntrinsicCandidate
1378     public final int compareAndExchangeIntRelease(Object o, long offset,
1379                                                          int expected,
1380                                                          int x) {
1381         return compareAndExchangeInt(o, offset, expected, x);
1382     }
1383 
1384     @HotSpotIntrinsicCandidate
1385     public final boolean weakCompareAndSetIntPlain(Object o, long offset,
1386                                                    int expected,
1387                                                    int x) {
1388         return compareAndSetInt(o, offset, expected, x);
1389     }
1390 
1391     @HotSpotIntrinsicCandidate
1392     public final boolean weakCompareAndSetIntAcquire(Object o, long offset,
1393                                                      int expected,
1394                                                      int x) {
1395         return compareAndSetInt(o, offset, expected, x);
1396     }
1397 
1398     @HotSpotIntrinsicCandidate
1399     public final boolean weakCompareAndSetIntRelease(Object o, long offset,
1400                                                      int expected,
1401                                                      int x) {
1402         return compareAndSetInt(o, offset, expected, x);
1403     }
1404 
1405     @HotSpotIntrinsicCandidate
1406     public final boolean weakCompareAndSetInt(Object o, long offset,
1407                                               int expected,
1408                                               int x) {
1409         return compareAndSetInt(o, offset, expected, x);
1410     }
1411 
1412     @HotSpotIntrinsicCandidate
1413     public final byte compareAndExchangeByte(Object o, long offset,
1414                                              byte expected,
1415                                              byte x) {
1416         long wordOffset = offset & ~3;
1417         int shift = (int) (offset & 3) << 3;
1418         if (BE) {
1419             shift = 24 - shift;
1420         }
1421         int mask           = 0xFF << shift;
1422         int maskedExpected = (expected & 0xFF) << shift;
1423         int maskedX        = (x & 0xFF) << shift;
1424         int fullWord;
1425         do {
1426             fullWord = getIntVolatile(o, wordOffset);
1427             if ((fullWord & mask) != maskedExpected)
1428                 return (byte) ((fullWord & mask) >> shift);
1429         } while (!weakCompareAndSetInt(o, wordOffset,
1430                                                 fullWord, (fullWord & ~mask) | maskedX));
1431         return expected;
1432     }
1433 
1434     @HotSpotIntrinsicCandidate
1435     public final boolean compareAndSetByte(Object o, long offset,
1436                                            byte expected,
1437                                            byte x) {
1438         return compareAndExchangeByte(o, offset, expected, x) == expected;
1439     }
1440 
1441     @HotSpotIntrinsicCandidate
1442     public final boolean weakCompareAndSetByte(Object o, long offset,
1443                                                byte expected,
1444                                                byte x) {
1445         return compareAndSetByte(o, offset, expected, x);
1446     }
1447 
1448     @HotSpotIntrinsicCandidate
1449     public final boolean weakCompareAndSetByteAcquire(Object o, long offset,
1450                                                       byte expected,
1451                                                       byte x) {
1452         return weakCompareAndSetByte(o, offset, expected, x);
1453     }
1454 
1455     @HotSpotIntrinsicCandidate
1456     public final boolean weakCompareAndSetByteRelease(Object o, long offset,
1457                                                       byte expected,
1458                                                       byte x) {
1459         return weakCompareAndSetByte(o, offset, expected, x);
1460     }
1461 
1462     @HotSpotIntrinsicCandidate
1463     public final boolean weakCompareAndSetBytePlain(Object o, long offset,
1464                                                     byte expected,
1465                                                     byte x) {
1466         return weakCompareAndSetByte(o, offset, expected, x);
1467     }
1468 
1469     @HotSpotIntrinsicCandidate
1470     public final byte compareAndExchangeByteAcquire(Object o, long offset,
1471                                                     byte expected,
1472                                                     byte x) {
1473         return compareAndExchangeByte(o, offset, expected, x);
1474     }
1475 
1476     @HotSpotIntrinsicCandidate
1477     public final byte compareAndExchangeByteRelease(Object o, long offset,
1478                                                     byte expected,
1479                                                     byte x) {
1480         return compareAndExchangeByte(o, offset, expected, x);
1481     }
1482 
1483     @HotSpotIntrinsicCandidate
1484     public final short compareAndExchangeShort(Object o, long offset,
1485                                                short expected,
1486                                                short x) {
1487         if ((offset & 3) == 3) {
1488             throw new IllegalArgumentException("Update spans the word, not supported");
1489         }
1490         long wordOffset = offset & ~3;
1491         int shift = (int) (offset & 3) << 3;
1492         if (BE) {
1493             shift = 16 - shift;
1494         }
1495         int mask           = 0xFFFF << shift;
1496         int maskedExpected = (expected & 0xFFFF) << shift;
1497         int maskedX        = (x & 0xFFFF) << shift;
1498         int fullWord;
1499         do {
1500             fullWord = getIntVolatile(o, wordOffset);
1501             if ((fullWord & mask) != maskedExpected) {
1502                 return (short) ((fullWord & mask) >> shift);
1503             }
1504         } while (!weakCompareAndSetInt(o, wordOffset,
1505                                                 fullWord, (fullWord & ~mask) | maskedX));
1506         return expected;
1507     }
1508 
1509     @HotSpotIntrinsicCandidate
1510     public final boolean compareAndSetShort(Object o, long offset,
1511                                             short expected,
1512                                             short x) {
1513         return compareAndExchangeShort(o, offset, expected, x) == expected;
1514     }
1515 
1516     @HotSpotIntrinsicCandidate
1517     public final boolean weakCompareAndSetShort(Object o, long offset,
1518                                                 short expected,
1519                                                 short x) {
1520         return compareAndSetShort(o, offset, expected, x);
1521     }
1522 
1523     @HotSpotIntrinsicCandidate
1524     public final boolean weakCompareAndSetShortAcquire(Object o, long offset,
1525                                                        short expected,
1526                                                        short x) {
1527         return weakCompareAndSetShort(o, offset, expected, x);
1528     }
1529 
1530     @HotSpotIntrinsicCandidate
1531     public final boolean weakCompareAndSetShortRelease(Object o, long offset,
1532                                                        short expected,
1533                                                        short x) {
1534         return weakCompareAndSetShort(o, offset, expected, x);
1535     }
1536 
1537     @HotSpotIntrinsicCandidate
1538     public final boolean weakCompareAndSetShortPlain(Object o, long offset,
1539                                                      short expected,
1540                                                      short x) {
1541         return weakCompareAndSetShort(o, offset, expected, x);
1542     }
1543 
1544 
1545     @HotSpotIntrinsicCandidate
1546     public final short compareAndExchangeShortAcquire(Object o, long offset,
1547                                                      short expected,
1548                                                      short x) {
1549         return compareAndExchangeShort(o, offset, expected, x);
1550     }
1551 
1552     @HotSpotIntrinsicCandidate
1553     public final short compareAndExchangeShortRelease(Object o, long offset,
1554                                                     short expected,
1555                                                     short x) {
1556         return compareAndExchangeShort(o, offset, expected, x);
1557     }
1558 
1559     @ForceInline
1560     private char s2c(short s) {
1561         return (char) s;
1562     }
1563 
1564     @ForceInline
1565     private short c2s(char s) {
1566         return (short) s;
1567     }
1568 
1569     @ForceInline
1570     public final boolean compareAndSetChar(Object o, long offset,
1571                                            char expected,
1572                                            char x) {
1573         return compareAndSetShort(o, offset, c2s(expected), c2s(x));
1574     }
1575 
1576     @ForceInline
1577     public final char compareAndExchangeChar(Object o, long offset,
1578                                              char expected,
1579                                              char x) {
1580         return s2c(compareAndExchangeShort(o, offset, c2s(expected), c2s(x)));
1581     }
1582 
1583     @ForceInline
1584     public final char compareAndExchangeCharAcquire(Object o, long offset,
1585                                             char expected,
1586                                             char x) {
1587         return s2c(compareAndExchangeShortAcquire(o, offset, c2s(expected), c2s(x)));
1588     }
1589 
1590     @ForceInline
1591     public final char compareAndExchangeCharRelease(Object o, long offset,
1592                                             char expected,
1593                                             char x) {
1594         return s2c(compareAndExchangeShortRelease(o, offset, c2s(expected), c2s(x)));
1595     }
1596 
1597     @ForceInline
1598     public final boolean weakCompareAndSetChar(Object o, long offset,
1599                                                char expected,
1600                                                char x) {
1601         return weakCompareAndSetShort(o, offset, c2s(expected), c2s(x));
1602     }
1603 
1604     @ForceInline
1605     public final boolean weakCompareAndSetCharAcquire(Object o, long offset,
1606                                                       char expected,
1607                                                       char x) {
1608         return weakCompareAndSetShortAcquire(o, offset, c2s(expected), c2s(x));
1609     }
1610 
1611     @ForceInline
1612     public final boolean weakCompareAndSetCharRelease(Object o, long offset,
1613                                                       char expected,
1614                                                       char x) {
1615         return weakCompareAndSetShortRelease(o, offset, c2s(expected), c2s(x));
1616     }
1617 
1618     @ForceInline
1619     public final boolean weakCompareAndSetCharPlain(Object o, long offset,
1620                                                     char expected,
1621                                                     char x) {
1622         return weakCompareAndSetShortPlain(o, offset, c2s(expected), c2s(x));
1623     }
1624 
1625     /**
1626      * The JVM converts integral values to boolean values using two
1627      * different conventions, byte testing against zero and truncation
1628      * to least-significant bit.
1629      *
1630      * <p>The JNI documents specify that, at least for returning
1631      * values from native methods, a Java boolean value is converted
1632      * to the value-set 0..1 by first truncating to a byte (0..255 or
1633      * maybe -128..127) and then testing against zero. Thus, Java
1634      * booleans in non-Java data structures are by convention
1635      * represented as 8-bit containers containing either zero (for
1636      * false) or any non-zero value (for true).
1637      *
1638      * <p>Java booleans in the heap are also stored in bytes, but are
1639      * strongly normalized to the value-set 0..1 (i.e., they are
1640      * truncated to the least-significant bit).
1641      *
1642      * <p>The main reason for having different conventions for
1643      * conversion is performance: Truncation to the least-significant
1644      * bit can be usually implemented with fewer (machine)
1645      * instructions than byte testing against zero.
1646      *
1647      * <p>A number of Unsafe methods load boolean values from the heap
1648      * as bytes. Unsafe converts those values according to the JNI
1649      * rules (i.e, using the "testing against zero" convention). The
1650      * method {@code byte2bool} implements that conversion.
1651      *
1652      * @param b the byte to be converted to boolean
1653      * @return the result of the conversion
1654      */
1655     @ForceInline
1656     private boolean byte2bool(byte b) {
1657         return b != 0;
1658     }
1659 
1660     /**
1661      * Convert a boolean value to a byte. The return value is strongly
1662      * normalized to the value-set 0..1 (i.e., the value is truncated
1663      * to the least-significant bit). See {@link #byte2bool(byte)} for
1664      * more details on conversion conventions.
1665      *
1666      * @param b the boolean to be converted to byte (and then normalized)
1667      * @return the result of the conversion
1668      */
1669     @ForceInline
1670     private byte bool2byte(boolean b) {
1671         return b ? (byte)1 : (byte)0;
1672     }
1673 
1674     @ForceInline
1675     public final boolean compareAndSetBoolean(Object o, long offset,
1676                                               boolean expected,
1677                                               boolean x) {
1678         return compareAndSetByte(o, offset, bool2byte(expected), bool2byte(x));
1679     }
1680 
1681     @ForceInline
1682     public final boolean compareAndExchangeBoolean(Object o, long offset,
1683                                                    boolean expected,
1684                                                    boolean x) {
1685         return byte2bool(compareAndExchangeByte(o, offset, bool2byte(expected), bool2byte(x)));
1686     }
1687 
1688     @ForceInline
1689     public final boolean compareAndExchangeBooleanAcquire(Object o, long offset,
1690                                                     boolean expected,
1691                                                     boolean x) {
1692         return byte2bool(compareAndExchangeByteAcquire(o, offset, bool2byte(expected), bool2byte(x)));
1693     }
1694 
1695     @ForceInline
1696     public final boolean compareAndExchangeBooleanRelease(Object o, long offset,
1697                                                        boolean expected,
1698                                                        boolean x) {
1699         return byte2bool(compareAndExchangeByteRelease(o, offset, bool2byte(expected), bool2byte(x)));
1700     }
1701 
1702     @ForceInline
1703     public final boolean weakCompareAndSetBoolean(Object o, long offset,
1704                                                   boolean expected,
1705                                                   boolean x) {
1706         return weakCompareAndSetByte(o, offset, bool2byte(expected), bool2byte(x));
1707     }
1708 
1709     @ForceInline
1710     public final boolean weakCompareAndSetBooleanAcquire(Object o, long offset,
1711                                                          boolean expected,
1712                                                          boolean x) {
1713         return weakCompareAndSetByteAcquire(o, offset, bool2byte(expected), bool2byte(x));
1714     }
1715 
1716     @ForceInline
1717     public final boolean weakCompareAndSetBooleanRelease(Object o, long offset,
1718                                                          boolean expected,
1719                                                          boolean x) {
1720         return weakCompareAndSetByteRelease(o, offset, bool2byte(expected), bool2byte(x));
1721     }
1722 
1723     @ForceInline
1724     public final boolean weakCompareAndSetBooleanPlain(Object o, long offset,
1725                                                        boolean expected,
1726                                                        boolean x) {
1727         return weakCompareAndSetBytePlain(o, offset, bool2byte(expected), bool2byte(x));
1728     }
1729 
1730     /**
1731      * Atomically updates Java variable to {@code x} if it is currently
1732      * holding {@code expected}.
1733      *
1734      * <p>This operation has memory semantics of a {@code volatile} read
1735      * and write.  Corresponds to C11 atomic_compare_exchange_strong.
1736      *
1737      * @return {@code true} if successful
1738      */
1739     @ForceInline
1740     public final boolean compareAndSetFloat(Object o, long offset,
1741                                             float expected,
1742                                             float x) {
1743         return compareAndSetInt(o, offset,
1744                                  Float.floatToRawIntBits(expected),
1745                                  Float.floatToRawIntBits(x));
1746     }
1747 
1748     @ForceInline
1749     public final float compareAndExchangeFloat(Object o, long offset,
1750                                                float expected,
1751                                                float x) {
1752         int w = compareAndExchangeInt(o, offset,
1753                                       Float.floatToRawIntBits(expected),
1754                                       Float.floatToRawIntBits(x));
1755         return Float.intBitsToFloat(w);
1756     }
1757 
1758     @ForceInline
1759     public final float compareAndExchangeFloatAcquire(Object o, long offset,
1760                                                   float expected,
1761                                                   float x) {
1762         int w = compareAndExchangeIntAcquire(o, offset,
1763                                              Float.floatToRawIntBits(expected),
1764                                              Float.floatToRawIntBits(x));
1765         return Float.intBitsToFloat(w);
1766     }
1767 
1768     @ForceInline
1769     public final float compareAndExchangeFloatRelease(Object o, long offset,
1770                                                   float expected,
1771                                                   float x) {
1772         int w = compareAndExchangeIntRelease(o, offset,
1773                                              Float.floatToRawIntBits(expected),
1774                                              Float.floatToRawIntBits(x));
1775         return Float.intBitsToFloat(w);
1776     }
1777 
1778     @ForceInline
1779     public final boolean weakCompareAndSetFloatPlain(Object o, long offset,
1780                                                      float expected,
1781                                                      float x) {
1782         return weakCompareAndSetIntPlain(o, offset,
1783                                      Float.floatToRawIntBits(expected),
1784                                      Float.floatToRawIntBits(x));
1785     }
1786 
1787     @ForceInline
1788     public final boolean weakCompareAndSetFloatAcquire(Object o, long offset,
1789                                                        float expected,
1790                                                        float x) {
1791         return weakCompareAndSetIntAcquire(o, offset,
1792                                             Float.floatToRawIntBits(expected),
1793                                             Float.floatToRawIntBits(x));
1794     }
1795 
1796     @ForceInline
1797     public final boolean weakCompareAndSetFloatRelease(Object o, long offset,
1798                                                        float expected,
1799                                                        float x) {
1800         return weakCompareAndSetIntRelease(o, offset,
1801                                             Float.floatToRawIntBits(expected),
1802                                             Float.floatToRawIntBits(x));
1803     }
1804 
1805     @ForceInline
1806     public final boolean weakCompareAndSetFloat(Object o, long offset,
1807                                                 float expected,
1808                                                 float x) {
1809         return weakCompareAndSetInt(o, offset,
1810                                              Float.floatToRawIntBits(expected),
1811                                              Float.floatToRawIntBits(x));
1812     }
1813 
1814     /**
1815      * Atomically updates Java variable to {@code x} if it is currently
1816      * holding {@code expected}.
1817      *
1818      * <p>This operation has memory semantics of a {@code volatile} read
1819      * and write.  Corresponds to C11 atomic_compare_exchange_strong.
1820      *
1821      * @return {@code true} if successful
1822      */
1823     @ForceInline
1824     public final boolean compareAndSetDouble(Object o, long offset,
1825                                              double expected,
1826                                              double x) {
1827         return compareAndSetLong(o, offset,
1828                                  Double.doubleToRawLongBits(expected),
1829                                  Double.doubleToRawLongBits(x));
1830     }
1831 
1832     @ForceInline
1833     public final double compareAndExchangeDouble(Object o, long offset,
1834                                                  double expected,
1835                                                  double x) {
1836         long w = compareAndExchangeLong(o, offset,
1837                                         Double.doubleToRawLongBits(expected),
1838                                         Double.doubleToRawLongBits(x));
1839         return Double.longBitsToDouble(w);
1840     }
1841 
1842     @ForceInline
1843     public final double compareAndExchangeDoubleAcquire(Object o, long offset,
1844                                                         double expected,
1845                                                         double x) {
1846         long w = compareAndExchangeLongAcquire(o, offset,
1847                                                Double.doubleToRawLongBits(expected),
1848                                                Double.doubleToRawLongBits(x));
1849         return Double.longBitsToDouble(w);
1850     }
1851 
1852     @ForceInline
1853     public final double compareAndExchangeDoubleRelease(Object o, long offset,
1854                                                         double expected,
1855                                                         double x) {
1856         long w = compareAndExchangeLongRelease(o, offset,
1857                                                Double.doubleToRawLongBits(expected),
1858                                                Double.doubleToRawLongBits(x));
1859         return Double.longBitsToDouble(w);
1860     }
1861 
1862     @ForceInline
1863     public final boolean weakCompareAndSetDoublePlain(Object o, long offset,
1864                                                       double expected,
1865                                                       double x) {
1866         return weakCompareAndSetLongPlain(o, offset,
1867                                      Double.doubleToRawLongBits(expected),
1868                                      Double.doubleToRawLongBits(x));
1869     }
1870 
1871     @ForceInline
1872     public final boolean weakCompareAndSetDoubleAcquire(Object o, long offset,
1873                                                         double expected,
1874                                                         double x) {
1875         return weakCompareAndSetLongAcquire(o, offset,
1876                                              Double.doubleToRawLongBits(expected),
1877                                              Double.doubleToRawLongBits(x));
1878     }
1879 
1880     @ForceInline
1881     public final boolean weakCompareAndSetDoubleRelease(Object o, long offset,
1882                                                         double expected,
1883                                                         double x) {
1884         return weakCompareAndSetLongRelease(o, offset,
1885                                              Double.doubleToRawLongBits(expected),
1886                                              Double.doubleToRawLongBits(x));
1887     }
1888 
1889     @ForceInline
1890     public final boolean weakCompareAndSetDouble(Object o, long offset,
1891                                                  double expected,
1892                                                  double x) {
1893         return weakCompareAndSetLong(o, offset,
1894                                               Double.doubleToRawLongBits(expected),
1895                                               Double.doubleToRawLongBits(x));
1896     }
1897 
1898     /**
1899      * Atomically updates Java variable to {@code x} if it is currently
1900      * holding {@code expected}.
1901      *
1902      * <p>This operation has memory semantics of a {@code volatile} read
1903      * and write.  Corresponds to C11 atomic_compare_exchange_strong.
1904      *
1905      * @return {@code true} if successful
1906      */
1907     @HotSpotIntrinsicCandidate
1908     public final native boolean compareAndSetLong(Object o, long offset,
1909                                                   long expected,
1910                                                   long x);
1911 
1912     @HotSpotIntrinsicCandidate
1913     public final native long compareAndExchangeLong(Object o, long offset,
1914                                                     long expected,
1915                                                     long x);
1916 
1917     @HotSpotIntrinsicCandidate
1918     public final long compareAndExchangeLongAcquire(Object o, long offset,
1919                                                            long expected,
1920                                                            long x) {
1921         return compareAndExchangeLong(o, offset, expected, x);
1922     }
1923 
1924     @HotSpotIntrinsicCandidate
1925     public final long compareAndExchangeLongRelease(Object o, long offset,
1926                                                            long expected,
1927                                                            long x) {
1928         return compareAndExchangeLong(o, offset, expected, x);
1929     }
1930 
1931     @HotSpotIntrinsicCandidate
1932     public final boolean weakCompareAndSetLongPlain(Object o, long offset,
1933                                                     long expected,
1934                                                     long x) {
1935         return compareAndSetLong(o, offset, expected, x);
1936     }
1937 
1938     @HotSpotIntrinsicCandidate
1939     public final boolean weakCompareAndSetLongAcquire(Object o, long offset,
1940                                                       long expected,
1941                                                       long x) {
1942         return compareAndSetLong(o, offset, expected, x);
1943     }
1944 
1945     @HotSpotIntrinsicCandidate
1946     public final boolean weakCompareAndSetLongRelease(Object o, long offset,
1947                                                       long expected,
1948                                                       long x) {
1949         return compareAndSetLong(o, offset, expected, x);
1950     }
1951 
1952     @HotSpotIntrinsicCandidate
1953     public final boolean weakCompareAndSetLong(Object o, long offset,
1954                                                long expected,
1955                                                long x) {
1956         return compareAndSetLong(o, offset, expected, x);
1957     }
1958 
1959     /**
1960      * Fetches a reference value from a given Java variable, with volatile
1961      * load semantics. Otherwise identical to {@link #getReference(Object, long)}
1962      */
1963     @HotSpotIntrinsicCandidate
1964     public native Object getReferenceVolatile(Object o, long offset);
1965 
1966     /**
1967      * Stores a reference value into a given Java variable, with
1968      * volatile store semantics. Otherwise identical to {@link #putReference(Object, long, Object)}
1969      */
1970     @HotSpotIntrinsicCandidate
1971     public native void putReferenceVolatile(Object o, long offset, Object x);
1972 
1973     /** Volatile version of {@link #getInt(Object, long)}  */
1974     @HotSpotIntrinsicCandidate
1975     public native int     getIntVolatile(Object o, long offset);
1976 
1977     /** Volatile version of {@link #putInt(Object, long, int)}  */
1978     @HotSpotIntrinsicCandidate
1979     public native void    putIntVolatile(Object o, long offset, int x);
1980 
1981     /** Volatile version of {@link #getBoolean(Object, long)}  */
1982     @HotSpotIntrinsicCandidate
1983     public native boolean getBooleanVolatile(Object o, long offset);
1984 
1985     /** Volatile version of {@link #putBoolean(Object, long, boolean)}  */
1986     @HotSpotIntrinsicCandidate
1987     public native void    putBooleanVolatile(Object o, long offset, boolean x);
1988 
1989     /** Volatile version of {@link #getByte(Object, long)}  */
1990     @HotSpotIntrinsicCandidate
1991     public native byte    getByteVolatile(Object o, long offset);
1992 
1993     /** Volatile version of {@link #putByte(Object, long, byte)}  */
1994     @HotSpotIntrinsicCandidate
1995     public native void    putByteVolatile(Object o, long offset, byte x);
1996 
1997     /** Volatile version of {@link #getShort(Object, long)}  */
1998     @HotSpotIntrinsicCandidate
1999     public native short   getShortVolatile(Object o, long offset);
2000 
2001     /** Volatile version of {@link #putShort(Object, long, short)}  */
2002     @HotSpotIntrinsicCandidate
2003     public native void    putShortVolatile(Object o, long offset, short x);
2004 
2005     /** Volatile version of {@link #getChar(Object, long)}  */
2006     @HotSpotIntrinsicCandidate
2007     public native char    getCharVolatile(Object o, long offset);
2008 
2009     /** Volatile version of {@link #putChar(Object, long, char)}  */
2010     @HotSpotIntrinsicCandidate
2011     public native void    putCharVolatile(Object o, long offset, char x);
2012 
2013     /** Volatile version of {@link #getLong(Object, long)}  */
2014     @HotSpotIntrinsicCandidate
2015     public native long    getLongVolatile(Object o, long offset);
2016 
2017     /** Volatile version of {@link #putLong(Object, long, long)}  */
2018     @HotSpotIntrinsicCandidate
2019     public native void    putLongVolatile(Object o, long offset, long x);
2020 
2021     /** Volatile version of {@link #getFloat(Object, long)}  */
2022     @HotSpotIntrinsicCandidate
2023     public native float   getFloatVolatile(Object o, long offset);
2024 
2025     /** Volatile version of {@link #putFloat(Object, long, float)}  */
2026     @HotSpotIntrinsicCandidate
2027     public native void    putFloatVolatile(Object o, long offset, float x);
2028 
2029     /** Volatile version of {@link #getDouble(Object, long)}  */
2030     @HotSpotIntrinsicCandidate
2031     public native double  getDoubleVolatile(Object o, long offset);
2032 
2033     /** Volatile version of {@link #putDouble(Object, long, double)}  */
2034     @HotSpotIntrinsicCandidate
2035     public native void    putDoubleVolatile(Object o, long offset, double x);
2036 
2037 
2038 
2039     /** Acquire version of {@link #getReferenceVolatile(Object, long)} */
2040     @HotSpotIntrinsicCandidate
2041     public final Object getReferenceAcquire(Object o, long offset) {
2042         return getReferenceVolatile(o, offset);
2043     }
2044 
2045     /** Acquire version of {@link #getBooleanVolatile(Object, long)} */
2046     @HotSpotIntrinsicCandidate
2047     public final boolean getBooleanAcquire(Object o, long offset) {
2048         return getBooleanVolatile(o, offset);
2049     }
2050 
2051     /** Acquire version of {@link #getByteVolatile(Object, long)} */
2052     @HotSpotIntrinsicCandidate
2053     public final byte getByteAcquire(Object o, long offset) {
2054         return getByteVolatile(o, offset);
2055     }
2056 
2057     /** Acquire version of {@link #getShortVolatile(Object, long)} */
2058     @HotSpotIntrinsicCandidate
2059     public final short getShortAcquire(Object o, long offset) {
2060         return getShortVolatile(o, offset);
2061     }
2062 
2063     /** Acquire version of {@link #getCharVolatile(Object, long)} */
2064     @HotSpotIntrinsicCandidate
2065     public final char getCharAcquire(Object o, long offset) {
2066         return getCharVolatile(o, offset);
2067     }
2068 
2069     /** Acquire version of {@link #getIntVolatile(Object, long)} */
2070     @HotSpotIntrinsicCandidate
2071     public final int getIntAcquire(Object o, long offset) {
2072         return getIntVolatile(o, offset);
2073     }
2074 
2075     /** Acquire version of {@link #getFloatVolatile(Object, long)} */
2076     @HotSpotIntrinsicCandidate
2077     public final float getFloatAcquire(Object o, long offset) {
2078         return getFloatVolatile(o, offset);
2079     }
2080 
2081     /** Acquire version of {@link #getLongVolatile(Object, long)} */
2082     @HotSpotIntrinsicCandidate
2083     public final long getLongAcquire(Object o, long offset) {
2084         return getLongVolatile(o, offset);
2085     }
2086 
2087     /** Acquire version of {@link #getDoubleVolatile(Object, long)} */
2088     @HotSpotIntrinsicCandidate
2089     public final double getDoubleAcquire(Object o, long offset) {
2090         return getDoubleVolatile(o, offset);
2091     }
2092 
2093     /*
2094       * Versions of {@link #putReferenceVolatile(Object, long, Object)}
2095       * that do not guarantee immediate visibility of the store to
2096       * other threads. This method is generally only useful if the
2097       * underlying field is a Java volatile (or if an array cell, one
2098       * that is otherwise only accessed using volatile accesses).
2099       *
2100       * Corresponds to C11 atomic_store_explicit(..., memory_order_release).
2101       */
2102 
2103     /** Release version of {@link #putReferenceVolatile(Object, long, Object)} */
2104     @HotSpotIntrinsicCandidate
2105     public final void putReferenceRelease(Object o, long offset, Object x) {
2106         putReferenceVolatile(o, offset, x);
2107     }
2108 
2109     /** Release version of {@link #putBooleanVolatile(Object, long, boolean)} */
2110     @HotSpotIntrinsicCandidate
2111     public final void putBooleanRelease(Object o, long offset, boolean x) {
2112         putBooleanVolatile(o, offset, x);
2113     }
2114 
2115     /** Release version of {@link #putByteVolatile(Object, long, byte)} */
2116     @HotSpotIntrinsicCandidate
2117     public final void putByteRelease(Object o, long offset, byte x) {
2118         putByteVolatile(o, offset, x);
2119     }
2120 
2121     /** Release version of {@link #putShortVolatile(Object, long, short)} */
2122     @HotSpotIntrinsicCandidate
2123     public final void putShortRelease(Object o, long offset, short x) {
2124         putShortVolatile(o, offset, x);
2125     }
2126 
2127     /** Release version of {@link #putCharVolatile(Object, long, char)} */
2128     @HotSpotIntrinsicCandidate
2129     public final void putCharRelease(Object o, long offset, char x) {
2130         putCharVolatile(o, offset, x);
2131     }
2132 
2133     /** Release version of {@link #putIntVolatile(Object, long, int)} */
2134     @HotSpotIntrinsicCandidate
2135     public final void putIntRelease(Object o, long offset, int x) {
2136         putIntVolatile(o, offset, x);
2137     }
2138 
2139     /** Release version of {@link #putFloatVolatile(Object, long, float)} */
2140     @HotSpotIntrinsicCandidate
2141     public final void putFloatRelease(Object o, long offset, float x) {
2142         putFloatVolatile(o, offset, x);
2143     }
2144 
2145     /** Release version of {@link #putLongVolatile(Object, long, long)} */
2146     @HotSpotIntrinsicCandidate
2147     public final void putLongRelease(Object o, long offset, long x) {
2148         putLongVolatile(o, offset, x);
2149     }
2150 
2151     /** Release version of {@link #putDoubleVolatile(Object, long, double)} */
2152     @HotSpotIntrinsicCandidate
2153     public final void putDoubleRelease(Object o, long offset, double x) {
2154         putDoubleVolatile(o, offset, x);
2155     }
2156 
2157     // ------------------------------ Opaque --------------------------------------
2158 
2159     /** Opaque version of {@link #getReferenceVolatile(Object, long)} */
2160     @HotSpotIntrinsicCandidate
2161     public final Object getReferenceOpaque(Object o, long offset) {
2162         return getReferenceVolatile(o, offset);
2163     }
2164 
2165     /** Opaque version of {@link #getBooleanVolatile(Object, long)} */
2166     @HotSpotIntrinsicCandidate
2167     public final boolean getBooleanOpaque(Object o, long offset) {
2168         return getBooleanVolatile(o, offset);
2169     }
2170 
2171     /** Opaque version of {@link #getByteVolatile(Object, long)} */
2172     @HotSpotIntrinsicCandidate
2173     public final byte getByteOpaque(Object o, long offset) {
2174         return getByteVolatile(o, offset);
2175     }
2176 
2177     /** Opaque version of {@link #getShortVolatile(Object, long)} */
2178     @HotSpotIntrinsicCandidate
2179     public final short getShortOpaque(Object o, long offset) {
2180         return getShortVolatile(o, offset);
2181     }
2182 
2183     /** Opaque version of {@link #getCharVolatile(Object, long)} */
2184     @HotSpotIntrinsicCandidate
2185     public final char getCharOpaque(Object o, long offset) {
2186         return getCharVolatile(o, offset);
2187     }
2188 
2189     /** Opaque version of {@link #getIntVolatile(Object, long)} */
2190     @HotSpotIntrinsicCandidate
2191     public final int getIntOpaque(Object o, long offset) {
2192         return getIntVolatile(o, offset);
2193     }
2194 
2195     /** Opaque version of {@link #getFloatVolatile(Object, long)} */
2196     @HotSpotIntrinsicCandidate
2197     public final float getFloatOpaque(Object o, long offset) {
2198         return getFloatVolatile(o, offset);
2199     }
2200 
2201     /** Opaque version of {@link #getLongVolatile(Object, long)} */
2202     @HotSpotIntrinsicCandidate
2203     public final long getLongOpaque(Object o, long offset) {
2204         return getLongVolatile(o, offset);
2205     }
2206 
2207     /** Opaque version of {@link #getDoubleVolatile(Object, long)} */
2208     @HotSpotIntrinsicCandidate
2209     public final double getDoubleOpaque(Object o, long offset) {
2210         return getDoubleVolatile(o, offset);
2211     }
2212 
2213     /** Opaque version of {@link #putReferenceVolatile(Object, long, Object)} */
2214     @HotSpotIntrinsicCandidate
2215     public final void putReferenceOpaque(Object o, long offset, Object x) {
2216         putReferenceVolatile(o, offset, x);
2217     }
2218 
2219     /** Opaque version of {@link #putBooleanVolatile(Object, long, boolean)} */
2220     @HotSpotIntrinsicCandidate
2221     public final void putBooleanOpaque(Object o, long offset, boolean x) {
2222         putBooleanVolatile(o, offset, x);
2223     }
2224 
2225     /** Opaque version of {@link #putByteVolatile(Object, long, byte)} */
2226     @HotSpotIntrinsicCandidate
2227     public final void putByteOpaque(Object o, long offset, byte x) {
2228         putByteVolatile(o, offset, x);
2229     }
2230 
2231     /** Opaque version of {@link #putShortVolatile(Object, long, short)} */
2232     @HotSpotIntrinsicCandidate
2233     public final void putShortOpaque(Object o, long offset, short x) {
2234         putShortVolatile(o, offset, x);
2235     }
2236 
2237     /** Opaque version of {@link #putCharVolatile(Object, long, char)} */
2238     @HotSpotIntrinsicCandidate
2239     public final void putCharOpaque(Object o, long offset, char x) {
2240         putCharVolatile(o, offset, x);
2241     }
2242 
2243     /** Opaque version of {@link #putIntVolatile(Object, long, int)} */
2244     @HotSpotIntrinsicCandidate
2245     public final void putIntOpaque(Object o, long offset, int x) {
2246         putIntVolatile(o, offset, x);
2247     }
2248 
2249     /** Opaque version of {@link #putFloatVolatile(Object, long, float)} */
2250     @HotSpotIntrinsicCandidate
2251     public final void putFloatOpaque(Object o, long offset, float x) {
2252         putFloatVolatile(o, offset, x);
2253     }
2254 
2255     /** Opaque version of {@link #putLongVolatile(Object, long, long)} */
2256     @HotSpotIntrinsicCandidate
2257     public final void putLongOpaque(Object o, long offset, long x) {
2258         putLongVolatile(o, offset, x);
2259     }
2260 
2261     /** Opaque version of {@link #putDoubleVolatile(Object, long, double)} */
2262     @HotSpotIntrinsicCandidate
2263     public final void putDoubleOpaque(Object o, long offset, double x) {
2264         putDoubleVolatile(o, offset, x);
2265     }
2266 
2267     /**
2268      * Unblocks the given thread blocked on {@code park}, or, if it is
2269      * not blocked, causes the subsequent call to {@code park} not to
2270      * block.  Note: this operation is "unsafe" solely because the
2271      * caller must somehow ensure that the thread has not been
2272      * destroyed. Nothing special is usually required to ensure this
2273      * when called from Java (in which there will ordinarily be a live
2274      * reference to the thread) but this is not nearly-automatically
2275      * so when calling from native code.
2276      *
2277      * @param thread the thread to unpark.
2278      */
2279     @HotSpotIntrinsicCandidate
2280     public native void unpark(Object thread);
2281 
2282     /**
2283      * Blocks current thread, returning when a balancing
2284      * {@code unpark} occurs, or a balancing {@code unpark} has
2285      * already occurred, or the thread is interrupted, or, if not
2286      * absolute and time is not zero, the given time nanoseconds have
2287      * elapsed, or if absolute, the given deadline in milliseconds
2288      * since Epoch has passed, or spuriously (i.e., returning for no
2289      * "reason"). Note: This operation is in the Unsafe class only
2290      * because {@code unpark} is, so it would be strange to place it
2291      * elsewhere.
2292      */
2293     @HotSpotIntrinsicCandidate
2294     public native void park(boolean isAbsolute, long time);
2295 
2296     /**
2297      * Gets the load average in the system run queue assigned
2298      * to the available processors averaged over various periods of time.
2299      * This method retrieves the given {@code nelem} samples and
2300      * assigns to the elements of the given {@code loadavg} array.
2301      * The system imposes a maximum of 3 samples, representing
2302      * averages over the last 1,  5,  and  15 minutes, respectively.
2303      *
2304      * @param loadavg an array of double of size nelems
2305      * @param nelems the number of samples to be retrieved and
2306      *        must be 1 to 3.
2307      *
2308      * @return the number of samples actually retrieved; or -1
2309      *         if the load average is unobtainable.
2310      */
2311     public int getLoadAverage(double[] loadavg, int nelems) {
2312         if (nelems < 0 || nelems > 3 || nelems > loadavg.length) {
2313             throw new ArrayIndexOutOfBoundsException();
2314         }
2315 
2316         return getLoadAverage0(loadavg, nelems);
2317     }
2318 
2319     // The following contain CAS-based Java implementations used on
2320     // platforms not supporting native instructions
2321 
2322     /**
2323      * Atomically adds the given value to the current value of a field
2324      * or array element within the given object {@code o}
2325      * at the given {@code offset}.
2326      *
2327      * @param o object/array to update the field/element in
2328      * @param offset field/element offset
2329      * @param delta the value to add
2330      * @return the previous value
2331      * @since 1.8
2332      */
2333     @HotSpotIntrinsicCandidate
2334     public final int getAndAddInt(Object o, long offset, int delta) {
2335         int v;
2336         do {
2337             v = getIntVolatile(o, offset);
2338         } while (!weakCompareAndSetInt(o, offset, v, v + delta));
2339         return v;
2340     }
2341 
2342     @ForceInline
2343     public final int getAndAddIntRelease(Object o, long offset, int delta) {
2344         int v;
2345         do {
2346             v = getInt(o, offset);
2347         } while (!weakCompareAndSetIntRelease(o, offset, v, v + delta));
2348         return v;
2349     }
2350 
2351     @ForceInline
2352     public final int getAndAddIntAcquire(Object o, long offset, int delta) {
2353         int v;
2354         do {
2355             v = getIntAcquire(o, offset);
2356         } while (!weakCompareAndSetIntAcquire(o, offset, v, v + delta));
2357         return v;
2358     }
2359 
2360     /**
2361      * Atomically adds the given value to the current value of a field
2362      * or array element within the given object {@code o}
2363      * at the given {@code offset}.
2364      *
2365      * @param o object/array to update the field/element in
2366      * @param offset field/element offset
2367      * @param delta the value to add
2368      * @return the previous value
2369      * @since 1.8
2370      */
2371     @HotSpotIntrinsicCandidate
2372     public final long getAndAddLong(Object o, long offset, long delta) {
2373         long v;
2374         do {
2375             v = getLongVolatile(o, offset);
2376         } while (!weakCompareAndSetLong(o, offset, v, v + delta));
2377         return v;
2378     }
2379 
2380     @ForceInline
2381     public final long getAndAddLongRelease(Object o, long offset, long delta) {
2382         long v;
2383         do {
2384             v = getLong(o, offset);
2385         } while (!weakCompareAndSetLongRelease(o, offset, v, v + delta));
2386         return v;
2387     }
2388 
2389     @ForceInline
2390     public final long getAndAddLongAcquire(Object o, long offset, long delta) {
2391         long v;
2392         do {
2393             v = getLongAcquire(o, offset);
2394         } while (!weakCompareAndSetLongAcquire(o, offset, v, v + delta));
2395         return v;
2396     }
2397 
2398     @HotSpotIntrinsicCandidate
2399     public final byte getAndAddByte(Object o, long offset, byte delta) {
2400         byte v;
2401         do {
2402             v = getByteVolatile(o, offset);
2403         } while (!weakCompareAndSetByte(o, offset, v, (byte) (v + delta)));
2404         return v;
2405     }
2406 
2407     @ForceInline
2408     public final byte getAndAddByteRelease(Object o, long offset, byte delta) {
2409         byte v;
2410         do {
2411             v = getByte(o, offset);
2412         } while (!weakCompareAndSetByteRelease(o, offset, v, (byte) (v + delta)));
2413         return v;
2414     }
2415 
2416     @ForceInline
2417     public final byte getAndAddByteAcquire(Object o, long offset, byte delta) {
2418         byte v;
2419         do {
2420             v = getByteAcquire(o, offset);
2421         } while (!weakCompareAndSetByteAcquire(o, offset, v, (byte) (v + delta)));
2422         return v;
2423     }
2424 
2425     @HotSpotIntrinsicCandidate
2426     public final short getAndAddShort(Object o, long offset, short delta) {
2427         short v;
2428         do {
2429             v = getShortVolatile(o, offset);
2430         } while (!weakCompareAndSetShort(o, offset, v, (short) (v + delta)));
2431         return v;
2432     }
2433 
2434     @ForceInline
2435     public final short getAndAddShortRelease(Object o, long offset, short delta) {
2436         short v;
2437         do {
2438             v = getShort(o, offset);
2439         } while (!weakCompareAndSetShortRelease(o, offset, v, (short) (v + delta)));
2440         return v;
2441     }
2442 
2443     @ForceInline
2444     public final short getAndAddShortAcquire(Object o, long offset, short delta) {
2445         short v;
2446         do {
2447             v = getShortAcquire(o, offset);
2448         } while (!weakCompareAndSetShortAcquire(o, offset, v, (short) (v + delta)));
2449         return v;
2450     }
2451 
2452     @ForceInline
2453     public final char getAndAddChar(Object o, long offset, char delta) {
2454         return (char) getAndAddShort(o, offset, (short) delta);
2455     }
2456 
2457     @ForceInline
2458     public final char getAndAddCharRelease(Object o, long offset, char delta) {
2459         return (char) getAndAddShortRelease(o, offset, (short) delta);
2460     }
2461 
2462     @ForceInline
2463     public final char getAndAddCharAcquire(Object o, long offset, char delta) {
2464         return (char) getAndAddShortAcquire(o, offset, (short) delta);
2465     }
2466 
2467     @ForceInline
2468     public final float getAndAddFloat(Object o, long offset, float delta) {
2469         int expectedBits;
2470         float v;
2471         do {
2472             // Load and CAS with the raw bits to avoid issues with NaNs and
2473             // possible bit conversion from signaling NaNs to quiet NaNs that
2474             // may result in the loop not terminating.
2475             expectedBits = getIntVolatile(o, offset);
2476             v = Float.intBitsToFloat(expectedBits);
2477         } while (!weakCompareAndSetInt(o, offset,
2478                                                 expectedBits, Float.floatToRawIntBits(v + delta)));
2479         return v;
2480     }
2481 
2482     @ForceInline
2483     public final float getAndAddFloatRelease(Object o, long offset, float delta) {
2484         int expectedBits;
2485         float v;
2486         do {
2487             // Load and CAS with the raw bits to avoid issues with NaNs and
2488             // possible bit conversion from signaling NaNs to quiet NaNs that
2489             // may result in the loop not terminating.
2490             expectedBits = getInt(o, offset);
2491             v = Float.intBitsToFloat(expectedBits);
2492         } while (!weakCompareAndSetIntRelease(o, offset,
2493                                                expectedBits, Float.floatToRawIntBits(v + delta)));
2494         return v;
2495     }
2496 
2497     @ForceInline
2498     public final float getAndAddFloatAcquire(Object o, long offset, float delta) {
2499         int expectedBits;
2500         float v;
2501         do {
2502             // Load and CAS with the raw bits to avoid issues with NaNs and
2503             // possible bit conversion from signaling NaNs to quiet NaNs that
2504             // may result in the loop not terminating.
2505             expectedBits = getIntAcquire(o, offset);
2506             v = Float.intBitsToFloat(expectedBits);
2507         } while (!weakCompareAndSetIntAcquire(o, offset,
2508                                                expectedBits, Float.floatToRawIntBits(v + delta)));
2509         return v;
2510     }
2511 
2512     @ForceInline
2513     public final double getAndAddDouble(Object o, long offset, double delta) {
2514         long expectedBits;
2515         double v;
2516         do {
2517             // Load and CAS with the raw bits to avoid issues with NaNs and
2518             // possible bit conversion from signaling NaNs to quiet NaNs that
2519             // may result in the loop not terminating.
2520             expectedBits = getLongVolatile(o, offset);
2521             v = Double.longBitsToDouble(expectedBits);
2522         } while (!weakCompareAndSetLong(o, offset,
2523                                                  expectedBits, Double.doubleToRawLongBits(v + delta)));
2524         return v;
2525     }
2526 
2527     @ForceInline
2528     public final double getAndAddDoubleRelease(Object o, long offset, double delta) {
2529         long expectedBits;
2530         double v;
2531         do {
2532             // Load and CAS with the raw bits to avoid issues with NaNs and
2533             // possible bit conversion from signaling NaNs to quiet NaNs that
2534             // may result in the loop not terminating.
2535             expectedBits = getLong(o, offset);
2536             v = Double.longBitsToDouble(expectedBits);
2537         } while (!weakCompareAndSetLongRelease(o, offset,
2538                                                 expectedBits, Double.doubleToRawLongBits(v + delta)));
2539         return v;
2540     }
2541 
2542     @ForceInline
2543     public final double getAndAddDoubleAcquire(Object o, long offset, double delta) {
2544         long expectedBits;
2545         double v;
2546         do {
2547             // Load and CAS with the raw bits to avoid issues with NaNs and
2548             // possible bit conversion from signaling NaNs to quiet NaNs that
2549             // may result in the loop not terminating.
2550             expectedBits = getLongAcquire(o, offset);
2551             v = Double.longBitsToDouble(expectedBits);
2552         } while (!weakCompareAndSetLongAcquire(o, offset,
2553                                                 expectedBits, Double.doubleToRawLongBits(v + delta)));
2554         return v;
2555     }
2556 
2557     /**
2558      * Atomically exchanges the given value with the current value of
2559      * a field or array element within the given object {@code o}
2560      * at the given {@code offset}.
2561      *
2562      * @param o object/array to update the field/element in
2563      * @param offset field/element offset
2564      * @param newValue new value
2565      * @return the previous value
2566      * @since 1.8
2567      */
2568     @HotSpotIntrinsicCandidate
2569     public final int getAndSetInt(Object o, long offset, int newValue) {
2570         int v;
2571         do {
2572             v = getIntVolatile(o, offset);
2573         } while (!weakCompareAndSetInt(o, offset, v, newValue));
2574         return v;
2575     }
2576 
2577     @ForceInline
2578     public final int getAndSetIntRelease(Object o, long offset, int newValue) {
2579         int v;
2580         do {
2581             v = getInt(o, offset);
2582         } while (!weakCompareAndSetIntRelease(o, offset, v, newValue));
2583         return v;
2584     }
2585 
2586     @ForceInline
2587     public final int getAndSetIntAcquire(Object o, long offset, int newValue) {
2588         int v;
2589         do {
2590             v = getIntAcquire(o, offset);
2591         } while (!weakCompareAndSetIntAcquire(o, offset, v, newValue));
2592         return v;
2593     }
2594 
2595     /**
2596      * Atomically exchanges the given value with the current value of
2597      * a field or array element within the given object {@code o}
2598      * at the given {@code offset}.
2599      *
2600      * @param o object/array to update the field/element in
2601      * @param offset field/element offset
2602      * @param newValue new value
2603      * @return the previous value
2604      * @since 1.8
2605      */
2606     @HotSpotIntrinsicCandidate
2607     public final long getAndSetLong(Object o, long offset, long newValue) {
2608         long v;
2609         do {
2610             v = getLongVolatile(o, offset);
2611         } while (!weakCompareAndSetLong(o, offset, v, newValue));
2612         return v;
2613     }
2614 
2615     @ForceInline
2616     public final long getAndSetLongRelease(Object o, long offset, long newValue) {
2617         long v;
2618         do {
2619             v = getLong(o, offset);
2620         } while (!weakCompareAndSetLongRelease(o, offset, v, newValue));
2621         return v;
2622     }
2623 
2624     @ForceInline
2625     public final long getAndSetLongAcquire(Object o, long offset, long newValue) {
2626         long v;
2627         do {
2628             v = getLongAcquire(o, offset);
2629         } while (!weakCompareAndSetLongAcquire(o, offset, v, newValue));
2630         return v;
2631     }
2632 
2633     /**
2634      * Atomically exchanges the given reference value with the current
2635      * reference value of a field or array element within the given
2636      * object {@code o} at the given {@code offset}.
2637      *
2638      * @param o object/array to update the field/element in
2639      * @param offset field/element offset
2640      * @param newValue new value
2641      * @return the previous value
2642      * @since 1.8
2643      */
2644     @HotSpotIntrinsicCandidate
2645     public final Object getAndSetReference(Object o, long offset, Object newValue) {
2646         Object v;
2647         do {
2648             v = getReferenceVolatile(o, offset);
2649         } while (!weakCompareAndSetReference(o, offset, v, newValue));
2650         return v;
2651     }
2652 
2653     @ForceInline
2654     public final Object getAndSetReferenceRelease(Object o, long offset, Object newValue) {
2655         Object v;
2656         do {
2657             v = getReference(o, offset);
2658         } while (!weakCompareAndSetReferenceRelease(o, offset, v, newValue));
2659         return v;
2660     }
2661 
2662     @ForceInline
2663     public final Object getAndSetReferenceAcquire(Object o, long offset, Object newValue) {
2664         Object v;
2665         do {
2666             v = getReferenceAcquire(o, offset);
2667         } while (!weakCompareAndSetReferenceAcquire(o, offset, v, newValue));
2668         return v;
2669     }
2670 
2671     @HotSpotIntrinsicCandidate
2672     public final byte getAndSetByte(Object o, long offset, byte newValue) {
2673         byte v;
2674         do {
2675             v = getByteVolatile(o, offset);
2676         } while (!weakCompareAndSetByte(o, offset, v, newValue));
2677         return v;
2678     }
2679 
2680     @ForceInline
2681     public final byte getAndSetByteRelease(Object o, long offset, byte newValue) {
2682         byte v;
2683         do {
2684             v = getByte(o, offset);
2685         } while (!weakCompareAndSetByteRelease(o, offset, v, newValue));
2686         return v;
2687     }
2688 
2689     @ForceInline
2690     public final byte getAndSetByteAcquire(Object o, long offset, byte newValue) {
2691         byte v;
2692         do {
2693             v = getByteAcquire(o, offset);
2694         } while (!weakCompareAndSetByteAcquire(o, offset, v, newValue));
2695         return v;
2696     }
2697 
2698     @ForceInline
2699     public final boolean getAndSetBoolean(Object o, long offset, boolean newValue) {
2700         return byte2bool(getAndSetByte(o, offset, bool2byte(newValue)));
2701     }
2702 
2703     @ForceInline
2704     public final boolean getAndSetBooleanRelease(Object o, long offset, boolean newValue) {
2705         return byte2bool(getAndSetByteRelease(o, offset, bool2byte(newValue)));
2706     }
2707 
2708     @ForceInline
2709     public final boolean getAndSetBooleanAcquire(Object o, long offset, boolean newValue) {
2710         return byte2bool(getAndSetByteAcquire(o, offset, bool2byte(newValue)));
2711     }
2712 
2713     @HotSpotIntrinsicCandidate
2714     public final short getAndSetShort(Object o, long offset, short newValue) {
2715         short v;
2716         do {
2717             v = getShortVolatile(o, offset);
2718         } while (!weakCompareAndSetShort(o, offset, v, newValue));
2719         return v;
2720     }
2721 
2722     @ForceInline
2723     public final short getAndSetShortRelease(Object o, long offset, short newValue) {
2724         short v;
2725         do {
2726             v = getShort(o, offset);
2727         } while (!weakCompareAndSetShortRelease(o, offset, v, newValue));
2728         return v;
2729     }
2730 
2731     @ForceInline
2732     public final short getAndSetShortAcquire(Object o, long offset, short newValue) {
2733         short v;
2734         do {
2735             v = getShortAcquire(o, offset);
2736         } while (!weakCompareAndSetShortAcquire(o, offset, v, newValue));
2737         return v;
2738     }
2739 
2740     @ForceInline
2741     public final char getAndSetChar(Object o, long offset, char newValue) {
2742         return s2c(getAndSetShort(o, offset, c2s(newValue)));
2743     }
2744 
2745     @ForceInline
2746     public final char getAndSetCharRelease(Object o, long offset, char newValue) {
2747         return s2c(getAndSetShortRelease(o, offset, c2s(newValue)));
2748     }
2749 
2750     @ForceInline
2751     public final char getAndSetCharAcquire(Object o, long offset, char newValue) {
2752         return s2c(getAndSetShortAcquire(o, offset, c2s(newValue)));
2753     }
2754 
2755     @ForceInline
2756     public final float getAndSetFloat(Object o, long offset, float newValue) {
2757         int v = getAndSetInt(o, offset, Float.floatToRawIntBits(newValue));
2758         return Float.intBitsToFloat(v);
2759     }
2760 
2761     @ForceInline
2762     public final float getAndSetFloatRelease(Object o, long offset, float newValue) {
2763         int v = getAndSetIntRelease(o, offset, Float.floatToRawIntBits(newValue));
2764         return Float.intBitsToFloat(v);
2765     }
2766 
2767     @ForceInline
2768     public final float getAndSetFloatAcquire(Object o, long offset, float newValue) {
2769         int v = getAndSetIntAcquire(o, offset, Float.floatToRawIntBits(newValue));
2770         return Float.intBitsToFloat(v);
2771     }
2772 
2773     @ForceInline
2774     public final double getAndSetDouble(Object o, long offset, double newValue) {
2775         long v = getAndSetLong(o, offset, Double.doubleToRawLongBits(newValue));
2776         return Double.longBitsToDouble(v);
2777     }
2778 
2779     @ForceInline
2780     public final double getAndSetDoubleRelease(Object o, long offset, double newValue) {
2781         long v = getAndSetLongRelease(o, offset, Double.doubleToRawLongBits(newValue));
2782         return Double.longBitsToDouble(v);
2783     }
2784 
2785     @ForceInline
2786     public final double getAndSetDoubleAcquire(Object o, long offset, double newValue) {
2787         long v = getAndSetLongAcquire(o, offset, Double.doubleToRawLongBits(newValue));
2788         return Double.longBitsToDouble(v);
2789     }
2790 
2791 
2792     // The following contain CAS-based Java implementations used on
2793     // platforms not supporting native instructions
2794 
2795     @ForceInline
2796     public final boolean getAndBitwiseOrBoolean(Object o, long offset, boolean mask) {
2797         return byte2bool(getAndBitwiseOrByte(o, offset, bool2byte(mask)));
2798     }
2799 
2800     @ForceInline
2801     public final boolean getAndBitwiseOrBooleanRelease(Object o, long offset, boolean mask) {
2802         return byte2bool(getAndBitwiseOrByteRelease(o, offset, bool2byte(mask)));
2803     }
2804 
2805     @ForceInline
2806     public final boolean getAndBitwiseOrBooleanAcquire(Object o, long offset, boolean mask) {
2807         return byte2bool(getAndBitwiseOrByteAcquire(o, offset, bool2byte(mask)));
2808     }
2809 
2810     @ForceInline
2811     public final boolean getAndBitwiseAndBoolean(Object o, long offset, boolean mask) {
2812         return byte2bool(getAndBitwiseAndByte(o, offset, bool2byte(mask)));
2813     }
2814 
2815     @ForceInline
2816     public final boolean getAndBitwiseAndBooleanRelease(Object o, long offset, boolean mask) {
2817         return byte2bool(getAndBitwiseAndByteRelease(o, offset, bool2byte(mask)));
2818     }
2819 
2820     @ForceInline
2821     public final boolean getAndBitwiseAndBooleanAcquire(Object o, long offset, boolean mask) {
2822         return byte2bool(getAndBitwiseAndByteAcquire(o, offset, bool2byte(mask)));
2823     }
2824 
2825     @ForceInline
2826     public final boolean getAndBitwiseXorBoolean(Object o, long offset, boolean mask) {
2827         return byte2bool(getAndBitwiseXorByte(o, offset, bool2byte(mask)));
2828     }
2829 
2830     @ForceInline
2831     public final boolean getAndBitwiseXorBooleanRelease(Object o, long offset, boolean mask) {
2832         return byte2bool(getAndBitwiseXorByteRelease(o, offset, bool2byte(mask)));
2833     }
2834 
2835     @ForceInline
2836     public final boolean getAndBitwiseXorBooleanAcquire(Object o, long offset, boolean mask) {
2837         return byte2bool(getAndBitwiseXorByteAcquire(o, offset, bool2byte(mask)));
2838     }
2839 
2840 
2841     @ForceInline
2842     public final byte getAndBitwiseOrByte(Object o, long offset, byte mask) {
2843         byte current;
2844         do {
2845             current = getByteVolatile(o, offset);
2846         } while (!weakCompareAndSetByte(o, offset,
2847                                                   current, (byte) (current | mask)));
2848         return current;
2849     }
2850 
2851     @ForceInline
2852     public final byte getAndBitwiseOrByteRelease(Object o, long offset, byte mask) {
2853         byte current;
2854         do {
2855             current = getByte(o, offset);
2856         } while (!weakCompareAndSetByteRelease(o, offset,
2857                                                  current, (byte) (current | mask)));
2858         return current;
2859     }
2860 
2861     @ForceInline
2862     public final byte getAndBitwiseOrByteAcquire(Object o, long offset, byte mask) {
2863         byte current;
2864         do {
2865             // Plain read, the value is a hint, the acquire CAS does the work
2866             current = getByte(o, offset);
2867         } while (!weakCompareAndSetByteAcquire(o, offset,
2868                                                  current, (byte) (current | mask)));
2869         return current;
2870     }
2871 
2872     @ForceInline
2873     public final byte getAndBitwiseAndByte(Object o, long offset, byte mask) {
2874         byte current;
2875         do {
2876             current = getByteVolatile(o, offset);
2877         } while (!weakCompareAndSetByte(o, offset,
2878                                                   current, (byte) (current & mask)));
2879         return current;
2880     }
2881 
2882     @ForceInline
2883     public final byte getAndBitwiseAndByteRelease(Object o, long offset, byte mask) {
2884         byte current;
2885         do {
2886             current = getByte(o, offset);
2887         } while (!weakCompareAndSetByteRelease(o, offset,
2888                                                  current, (byte) (current & mask)));
2889         return current;
2890     }
2891 
2892     @ForceInline
2893     public final byte getAndBitwiseAndByteAcquire(Object o, long offset, byte mask) {
2894         byte current;
2895         do {
2896             // Plain read, the value is a hint, the acquire CAS does the work
2897             current = getByte(o, offset);
2898         } while (!weakCompareAndSetByteAcquire(o, offset,
2899                                                  current, (byte) (current & mask)));
2900         return current;
2901     }
2902 
2903     @ForceInline
2904     public final byte getAndBitwiseXorByte(Object o, long offset, byte mask) {
2905         byte current;
2906         do {
2907             current = getByteVolatile(o, offset);
2908         } while (!weakCompareAndSetByte(o, offset,
2909                                                   current, (byte) (current ^ mask)));
2910         return current;
2911     }
2912 
2913     @ForceInline
2914     public final byte getAndBitwiseXorByteRelease(Object o, long offset, byte mask) {
2915         byte current;
2916         do {
2917             current = getByte(o, offset);
2918         } while (!weakCompareAndSetByteRelease(o, offset,
2919                                                  current, (byte) (current ^ mask)));
2920         return current;
2921     }
2922 
2923     @ForceInline
2924     public final byte getAndBitwiseXorByteAcquire(Object o, long offset, byte mask) {
2925         byte current;
2926         do {
2927             // Plain read, the value is a hint, the acquire CAS does the work
2928             current = getByte(o, offset);
2929         } while (!weakCompareAndSetByteAcquire(o, offset,
2930                                                  current, (byte) (current ^ mask)));
2931         return current;
2932     }
2933 
2934 
2935     @ForceInline
2936     public final char getAndBitwiseOrChar(Object o, long offset, char mask) {
2937         return s2c(getAndBitwiseOrShort(o, offset, c2s(mask)));
2938     }
2939 
2940     @ForceInline
2941     public final char getAndBitwiseOrCharRelease(Object o, long offset, char mask) {
2942         return s2c(getAndBitwiseOrShortRelease(o, offset, c2s(mask)));
2943     }
2944 
2945     @ForceInline
2946     public final char getAndBitwiseOrCharAcquire(Object o, long offset, char mask) {
2947         return s2c(getAndBitwiseOrShortAcquire(o, offset, c2s(mask)));
2948     }
2949 
2950     @ForceInline
2951     public final char getAndBitwiseAndChar(Object o, long offset, char mask) {
2952         return s2c(getAndBitwiseAndShort(o, offset, c2s(mask)));
2953     }
2954 
2955     @ForceInline
2956     public final char getAndBitwiseAndCharRelease(Object o, long offset, char mask) {
2957         return s2c(getAndBitwiseAndShortRelease(o, offset, c2s(mask)));
2958     }
2959 
2960     @ForceInline
2961     public final char getAndBitwiseAndCharAcquire(Object o, long offset, char mask) {
2962         return s2c(getAndBitwiseAndShortAcquire(o, offset, c2s(mask)));
2963     }
2964 
2965     @ForceInline
2966     public final char getAndBitwiseXorChar(Object o, long offset, char mask) {
2967         return s2c(getAndBitwiseXorShort(o, offset, c2s(mask)));
2968     }
2969 
2970     @ForceInline
2971     public final char getAndBitwiseXorCharRelease(Object o, long offset, char mask) {
2972         return s2c(getAndBitwiseXorShortRelease(o, offset, c2s(mask)));
2973     }
2974 
2975     @ForceInline
2976     public final char getAndBitwiseXorCharAcquire(Object o, long offset, char mask) {
2977         return s2c(getAndBitwiseXorShortAcquire(o, offset, c2s(mask)));
2978     }
2979 
2980 
2981     @ForceInline
2982     public final short getAndBitwiseOrShort(Object o, long offset, short mask) {
2983         short current;
2984         do {
2985             current = getShortVolatile(o, offset);
2986         } while (!weakCompareAndSetShort(o, offset,
2987                                                 current, (short) (current | mask)));
2988         return current;
2989     }
2990 
2991     @ForceInline
2992     public final short getAndBitwiseOrShortRelease(Object o, long offset, short mask) {
2993         short current;
2994         do {
2995             current = getShort(o, offset);
2996         } while (!weakCompareAndSetShortRelease(o, offset,
2997                                                current, (short) (current | mask)));
2998         return current;
2999     }
3000 
3001     @ForceInline
3002     public final short getAndBitwiseOrShortAcquire(Object o, long offset, short mask) {
3003         short current;
3004         do {
3005             // Plain read, the value is a hint, the acquire CAS does the work
3006             current = getShort(o, offset);
3007         } while (!weakCompareAndSetShortAcquire(o, offset,
3008                                                current, (short) (current | mask)));
3009         return current;
3010     }
3011 
3012     @ForceInline
3013     public final short getAndBitwiseAndShort(Object o, long offset, short mask) {
3014         short current;
3015         do {
3016             current = getShortVolatile(o, offset);
3017         } while (!weakCompareAndSetShort(o, offset,
3018                                                 current, (short) (current & mask)));
3019         return current;
3020     }
3021 
3022     @ForceInline
3023     public final short getAndBitwiseAndShortRelease(Object o, long offset, short mask) {
3024         short current;
3025         do {
3026             current = getShort(o, offset);
3027         } while (!weakCompareAndSetShortRelease(o, offset,
3028                                                current, (short) (current & mask)));
3029         return current;
3030     }
3031 
3032     @ForceInline
3033     public final short getAndBitwiseAndShortAcquire(Object o, long offset, short mask) {
3034         short current;
3035         do {
3036             // Plain read, the value is a hint, the acquire CAS does the work
3037             current = getShort(o, offset);
3038         } while (!weakCompareAndSetShortAcquire(o, offset,
3039                                                current, (short) (current & mask)));
3040         return current;
3041     }
3042 
3043     @ForceInline
3044     public final short getAndBitwiseXorShort(Object o, long offset, short mask) {
3045         short current;
3046         do {
3047             current = getShortVolatile(o, offset);
3048         } while (!weakCompareAndSetShort(o, offset,
3049                                                 current, (short) (current ^ mask)));
3050         return current;
3051     }
3052 
3053     @ForceInline
3054     public final short getAndBitwiseXorShortRelease(Object o, long offset, short mask) {
3055         short current;
3056         do {
3057             current = getShort(o, offset);
3058         } while (!weakCompareAndSetShortRelease(o, offset,
3059                                                current, (short) (current ^ mask)));
3060         return current;
3061     }
3062 
3063     @ForceInline
3064     public final short getAndBitwiseXorShortAcquire(Object o, long offset, short mask) {
3065         short current;
3066         do {
3067             // Plain read, the value is a hint, the acquire CAS does the work
3068             current = getShort(o, offset);
3069         } while (!weakCompareAndSetShortAcquire(o, offset,
3070                                                current, (short) (current ^ mask)));
3071         return current;
3072     }
3073 
3074 
3075     @ForceInline
3076     public final int getAndBitwiseOrInt(Object o, long offset, int mask) {
3077         int current;
3078         do {
3079             current = getIntVolatile(o, offset);
3080         } while (!weakCompareAndSetInt(o, offset,
3081                                                 current, current | mask));
3082         return current;
3083     }
3084 
3085     @ForceInline
3086     public final int getAndBitwiseOrIntRelease(Object o, long offset, int mask) {
3087         int current;
3088         do {
3089             current = getInt(o, offset);
3090         } while (!weakCompareAndSetIntRelease(o, offset,
3091                                                current, current | mask));
3092         return current;
3093     }
3094 
3095     @ForceInline
3096     public final int getAndBitwiseOrIntAcquire(Object o, long offset, int mask) {
3097         int current;
3098         do {
3099             // Plain read, the value is a hint, the acquire CAS does the work
3100             current = getInt(o, offset);
3101         } while (!weakCompareAndSetIntAcquire(o, offset,
3102                                                current, current | mask));
3103         return current;
3104     }
3105 
3106     /**
3107      * Atomically replaces the current value of a field or array element within
3108      * the given object with the result of bitwise AND between the current value
3109      * and mask.
3110      *
3111      * @param o object/array to update the field/element in
3112      * @param offset field/element offset
3113      * @param mask the mask value
3114      * @return the previous value
3115      * @since 1.9
3116      */
3117     @ForceInline
3118     public final int getAndBitwiseAndInt(Object o, long offset, int mask) {
3119         int current;
3120         do {
3121             current = getIntVolatile(o, offset);
3122         } while (!weakCompareAndSetInt(o, offset,
3123                                                 current, current & mask));
3124         return current;
3125     }
3126 
3127     @ForceInline
3128     public final int getAndBitwiseAndIntRelease(Object o, long offset, int mask) {
3129         int current;
3130         do {
3131             current = getInt(o, offset);
3132         } while (!weakCompareAndSetIntRelease(o, offset,
3133                                                current, current & mask));
3134         return current;
3135     }
3136 
3137     @ForceInline
3138     public final int getAndBitwiseAndIntAcquire(Object o, long offset, int mask) {
3139         int current;
3140         do {
3141             // Plain read, the value is a hint, the acquire CAS does the work
3142             current = getInt(o, offset);
3143         } while (!weakCompareAndSetIntAcquire(o, offset,
3144                                                current, current & mask));
3145         return current;
3146     }
3147 
3148     @ForceInline
3149     public final int getAndBitwiseXorInt(Object o, long offset, int mask) {
3150         int current;
3151         do {
3152             current = getIntVolatile(o, offset);
3153         } while (!weakCompareAndSetInt(o, offset,
3154                                                 current, current ^ mask));
3155         return current;
3156     }
3157 
3158     @ForceInline
3159     public final int getAndBitwiseXorIntRelease(Object o, long offset, int mask) {
3160         int current;
3161         do {
3162             current = getInt(o, offset);
3163         } while (!weakCompareAndSetIntRelease(o, offset,
3164                                                current, current ^ mask));
3165         return current;
3166     }
3167 
3168     @ForceInline
3169     public final int getAndBitwiseXorIntAcquire(Object o, long offset, int mask) {
3170         int current;
3171         do {
3172             // Plain read, the value is a hint, the acquire CAS does the work
3173             current = getInt(o, offset);
3174         } while (!weakCompareAndSetIntAcquire(o, offset,
3175                                                current, current ^ mask));
3176         return current;
3177     }
3178 
3179 
3180     @ForceInline
3181     public final long getAndBitwiseOrLong(Object o, long offset, long mask) {
3182         long current;
3183         do {
3184             current = getLongVolatile(o, offset);
3185         } while (!weakCompareAndSetLong(o, offset,
3186                                                 current, current | mask));
3187         return current;
3188     }
3189 
3190     @ForceInline
3191     public final long getAndBitwiseOrLongRelease(Object o, long offset, long mask) {
3192         long current;
3193         do {
3194             current = getLong(o, offset);
3195         } while (!weakCompareAndSetLongRelease(o, offset,
3196                                                current, current | mask));
3197         return current;
3198     }
3199 
3200     @ForceInline
3201     public final long getAndBitwiseOrLongAcquire(Object o, long offset, long mask) {
3202         long current;
3203         do {
3204             // Plain read, the value is a hint, the acquire CAS does the work
3205             current = getLong(o, offset);
3206         } while (!weakCompareAndSetLongAcquire(o, offset,
3207                                                current, current | mask));
3208         return current;
3209     }
3210 
3211     @ForceInline
3212     public final long getAndBitwiseAndLong(Object o, long offset, long mask) {
3213         long current;
3214         do {
3215             current = getLongVolatile(o, offset);
3216         } while (!weakCompareAndSetLong(o, offset,
3217                                                 current, current & mask));
3218         return current;
3219     }
3220 
3221     @ForceInline
3222     public final long getAndBitwiseAndLongRelease(Object o, long offset, long mask) {
3223         long current;
3224         do {
3225             current = getLong(o, offset);
3226         } while (!weakCompareAndSetLongRelease(o, offset,
3227                                                current, current & mask));
3228         return current;
3229     }
3230 
3231     @ForceInline
3232     public final long getAndBitwiseAndLongAcquire(Object o, long offset, long mask) {
3233         long current;
3234         do {
3235             // Plain read, the value is a hint, the acquire CAS does the work
3236             current = getLong(o, offset);
3237         } while (!weakCompareAndSetLongAcquire(o, offset,
3238                                                current, current & mask));
3239         return current;
3240     }
3241 
3242     @ForceInline
3243     public final long getAndBitwiseXorLong(Object o, long offset, long mask) {
3244         long current;
3245         do {
3246             current = getLongVolatile(o, offset);
3247         } while (!weakCompareAndSetLong(o, offset,
3248                                                 current, current ^ mask));
3249         return current;
3250     }
3251 
3252     @ForceInline
3253     public final long getAndBitwiseXorLongRelease(Object o, long offset, long mask) {
3254         long current;
3255         do {
3256             current = getLong(o, offset);
3257         } while (!weakCompareAndSetLongRelease(o, offset,
3258                                                current, current ^ mask));
3259         return current;
3260     }
3261 
3262     @ForceInline
3263     public final long getAndBitwiseXorLongAcquire(Object o, long offset, long mask) {
3264         long current;
3265         do {
3266             // Plain read, the value is a hint, the acquire CAS does the work
3267             current = getLong(o, offset);
3268         } while (!weakCompareAndSetLongAcquire(o, offset,
3269                                                current, current ^ mask));
3270         return current;
3271     }
3272 
3273 
3274 
3275     /**
3276      * Ensures that loads before the fence will not be reordered with loads and
3277      * stores after the fence; a "LoadLoad plus LoadStore barrier".
3278      *
3279      * Corresponds to C11 atomic_thread_fence(memory_order_acquire)
3280      * (an "acquire fence").
3281      *
3282      * A pure LoadLoad fence is not provided, since the addition of LoadStore
3283      * is almost always desired, and most current hardware instructions that
3284      * provide a LoadLoad barrier also provide a LoadStore barrier for free.
3285      * @since 1.8
3286      */
3287     @HotSpotIntrinsicCandidate
3288     public native void loadFence();
3289 
3290     /**
3291      * Ensures that loads and stores before the fence will not be reordered with
3292      * stores after the fence; a "StoreStore plus LoadStore barrier".
3293      *
3294      * Corresponds to C11 atomic_thread_fence(memory_order_release)
3295      * (a "release fence").
3296      *
3297      * A pure StoreStore fence is not provided, since the addition of LoadStore
3298      * is almost always desired, and most current hardware instructions that
3299      * provide a StoreStore barrier also provide a LoadStore barrier for free.
3300      * @since 1.8
3301      */
3302     @HotSpotIntrinsicCandidate
3303     public native void storeFence();
3304 
3305     /**
3306      * Ensures that loads and stores before the fence will not be reordered
3307      * with loads and stores after the fence.  Implies the effects of both
3308      * loadFence() and storeFence(), and in addition, the effect of a StoreLoad
3309      * barrier.
3310      *
3311      * Corresponds to C11 atomic_thread_fence(memory_order_seq_cst).
3312      * @since 1.8
3313      */
3314     @HotSpotIntrinsicCandidate
3315     public native void fullFence();
3316 
3317     /**
3318      * Ensures that loads before the fence will not be reordered with
3319      * loads after the fence.
3320      */
3321     public final void loadLoadFence() {
3322         loadFence();
3323     }
3324 
3325     /**
3326      * Ensures that stores before the fence will not be reordered with
3327      * stores after the fence.
3328      */
3329     public final void storeStoreFence() {
3330         storeFence();
3331     }
3332 
3333 
3334     /**
3335      * Throws IllegalAccessError; for use by the VM for access control
3336      * error support.
3337      * @since 1.8
3338      */
3339     private static void throwIllegalAccessError() {
3340         throw new IllegalAccessError();
3341     }
3342 
3343     /**
3344      * @return Returns true if the native byte ordering of this
3345      * platform is big-endian, false if it is little-endian.
3346      */
3347     public final boolean isBigEndian() { return BE; }
3348 
3349     /**
3350      * @return Returns true if this platform is capable of performing
3351      * accesses at addresses which are not aligned for the type of the
3352      * primitive type being accessed, false otherwise.
3353      */
3354     public final boolean unalignedAccess() { return unalignedAccess; }
3355 
3356     /**
3357      * Fetches a value at some byte offset into a given Java object.
3358      * More specifically, fetches a value within the given object
3359      * <code>o</code> at the given offset, or (if <code>o</code> is
3360      * null) from the memory address whose numerical value is the
3361      * given offset.  <p>
3362      *
3363      * The specification of this method is the same as {@link
3364      * #getLong(Object, long)} except that the offset does not need to
3365      * have been obtained from {@link #objectFieldOffset} on the
3366      * {@link java.lang.reflect.Field} of some Java field.  The value
3367      * in memory is raw data, and need not correspond to any Java
3368      * variable.  Unless <code>o</code> is null, the value accessed
3369      * must be entirely within the allocated object.  The endianness
3370      * of the value in memory is the endianness of the native platform.
3371      *
3372      * <p> The read will be atomic with respect to the largest power
3373      * of two that divides the GCD of the offset and the storage size.
3374      * For example, getLongUnaligned will make atomic reads of 2-, 4-,
3375      * or 8-byte storage units if the offset is zero mod 2, 4, or 8,
3376      * respectively.  There are no other guarantees of atomicity.
3377      * <p>
3378      * 8-byte atomicity is only guaranteed on platforms on which
3379      * support atomic accesses to longs.
3380      *
3381      * @param o Java heap object in which the value resides, if any, else
3382      *        null
3383      * @param offset The offset in bytes from the start of the object
3384      * @return the value fetched from the indicated object
3385      * @throws RuntimeException No defined exceptions are thrown, not even
3386      *         {@link NullPointerException}
3387      * @since 9
3388      */
3389     @HotSpotIntrinsicCandidate
3390     public final long getLongUnaligned(Object o, long offset) {
3391         if ((offset & 7) == 0) {
3392             return getLong(o, offset);
3393         } else if ((offset & 3) == 0) {
3394             return makeLong(getInt(o, offset),
3395                             getInt(o, offset + 4));
3396         } else if ((offset & 1) == 0) {
3397             return makeLong(getShort(o, offset),
3398                             getShort(o, offset + 2),
3399                             getShort(o, offset + 4),
3400                             getShort(o, offset + 6));
3401         } else {
3402             return makeLong(getByte(o, offset),
3403                             getByte(o, offset + 1),
3404                             getByte(o, offset + 2),
3405                             getByte(o, offset + 3),
3406                             getByte(o, offset + 4),
3407                             getByte(o, offset + 5),
3408                             getByte(o, offset + 6),
3409                             getByte(o, offset + 7));
3410         }
3411     }
3412     /**
3413      * As {@link #getLongUnaligned(Object, long)} but with an
3414      * additional argument which specifies the endianness of the value
3415      * as stored in memory.
3416      *
3417      * @param o Java heap object in which the variable resides
3418      * @param offset The offset in bytes from the start of the object
3419      * @param bigEndian The endianness of the value
3420      * @return the value fetched from the indicated object
3421      * @since 9
3422      */
3423     public final long getLongUnaligned(Object o, long offset, boolean bigEndian) {
3424         return convEndian(bigEndian, getLongUnaligned(o, offset));
3425     }
3426 
3427     /** @see #getLongUnaligned(Object, long) */
3428     @HotSpotIntrinsicCandidate
3429     public final int getIntUnaligned(Object o, long offset) {
3430         if ((offset & 3) == 0) {
3431             return getInt(o, offset);
3432         } else if ((offset & 1) == 0) {
3433             return makeInt(getShort(o, offset),
3434                            getShort(o, offset + 2));
3435         } else {
3436             return makeInt(getByte(o, offset),
3437                            getByte(o, offset + 1),
3438                            getByte(o, offset + 2),
3439                            getByte(o, offset + 3));
3440         }
3441     }
3442     /** @see #getLongUnaligned(Object, long, boolean) */
3443     public final int getIntUnaligned(Object o, long offset, boolean bigEndian) {
3444         return convEndian(bigEndian, getIntUnaligned(o, offset));
3445     }
3446 
3447     /** @see #getLongUnaligned(Object, long) */
3448     @HotSpotIntrinsicCandidate
3449     public final short getShortUnaligned(Object o, long offset) {
3450         if ((offset & 1) == 0) {
3451             return getShort(o, offset);
3452         } else {
3453             return makeShort(getByte(o, offset),
3454                              getByte(o, offset + 1));
3455         }
3456     }
3457     /** @see #getLongUnaligned(Object, long, boolean) */
3458     public final short getShortUnaligned(Object o, long offset, boolean bigEndian) {
3459         return convEndian(bigEndian, getShortUnaligned(o, offset));
3460     }
3461 
3462     /** @see #getLongUnaligned(Object, long) */
3463     @HotSpotIntrinsicCandidate
3464     public final char getCharUnaligned(Object o, long offset) {
3465         if ((offset & 1) == 0) {
3466             return getChar(o, offset);
3467         } else {
3468             return (char)makeShort(getByte(o, offset),
3469                                    getByte(o, offset + 1));
3470         }
3471     }
3472 
3473     /** @see #getLongUnaligned(Object, long, boolean) */
3474     public final char getCharUnaligned(Object o, long offset, boolean bigEndian) {
3475         return convEndian(bigEndian, getCharUnaligned(o, offset));
3476     }
3477 
3478     /**
3479      * Stores a value at some byte offset into a given Java object.
3480      * <p>
3481      * The specification of this method is the same as {@link
3482      * #getLong(Object, long)} except that the offset does not need to
3483      * have been obtained from {@link #objectFieldOffset} on the
3484      * {@link java.lang.reflect.Field} of some Java field.  The value
3485      * in memory is raw data, and need not correspond to any Java
3486      * variable.  The endianness of the value in memory is the
3487      * endianness of the native platform.
3488      * <p>
3489      * The write will be atomic with respect to the largest power of
3490      * two that divides the GCD of the offset and the storage size.
3491      * For example, putLongUnaligned will make atomic writes of 2-, 4-,
3492      * or 8-byte storage units if the offset is zero mod 2, 4, or 8,
3493      * respectively.  There are no other guarantees of atomicity.
3494      * <p>
3495      * 8-byte atomicity is only guaranteed on platforms on which
3496      * support atomic accesses to longs.
3497      *
3498      * @param o Java heap object in which the value resides, if any, else
3499      *        null
3500      * @param offset The offset in bytes from the start of the object
3501      * @param x the value to store
3502      * @throws RuntimeException No defined exceptions are thrown, not even
3503      *         {@link NullPointerException}
3504      * @since 9
3505      */
3506     @HotSpotIntrinsicCandidate
3507     public final void putLongUnaligned(Object o, long offset, long x) {
3508         if ((offset & 7) == 0) {
3509             putLong(o, offset, x);
3510         } else if ((offset & 3) == 0) {
3511             putLongParts(o, offset,
3512                          (int)(x >> 0),
3513                          (int)(x >>> 32));
3514         } else if ((offset & 1) == 0) {
3515             putLongParts(o, offset,
3516                          (short)(x >>> 0),
3517                          (short)(x >>> 16),
3518                          (short)(x >>> 32),
3519                          (short)(x >>> 48));
3520         } else {
3521             putLongParts(o, offset,
3522                          (byte)(x >>> 0),
3523                          (byte)(x >>> 8),
3524                          (byte)(x >>> 16),
3525                          (byte)(x >>> 24),
3526                          (byte)(x >>> 32),
3527                          (byte)(x >>> 40),
3528                          (byte)(x >>> 48),
3529                          (byte)(x >>> 56));
3530         }
3531     }
3532 
3533     /**
3534      * As {@link #putLongUnaligned(Object, long, long)} but with an additional
3535      * argument which specifies the endianness of the value as stored in memory.
3536      * @param o Java heap object in which the value resides
3537      * @param offset The offset in bytes from the start of the object
3538      * @param x the value to store
3539      * @param bigEndian The endianness of the value
3540      * @throws RuntimeException No defined exceptions are thrown, not even
3541      *         {@link NullPointerException}
3542      * @since 9
3543      */
3544     public final void putLongUnaligned(Object o, long offset, long x, boolean bigEndian) {
3545         putLongUnaligned(o, offset, convEndian(bigEndian, x));
3546     }
3547 
3548     /** @see #putLongUnaligned(Object, long, long) */
3549     @HotSpotIntrinsicCandidate
3550     public final void putIntUnaligned(Object o, long offset, int x) {
3551         if ((offset & 3) == 0) {
3552             putInt(o, offset, x);
3553         } else if ((offset & 1) == 0) {
3554             putIntParts(o, offset,
3555                         (short)(x >> 0),
3556                         (short)(x >>> 16));
3557         } else {
3558             putIntParts(o, offset,
3559                         (byte)(x >>> 0),
3560                         (byte)(x >>> 8),
3561                         (byte)(x >>> 16),
3562                         (byte)(x >>> 24));
3563         }
3564     }
3565     /** @see #putLongUnaligned(Object, long, long, boolean) */
3566     public final void putIntUnaligned(Object o, long offset, int x, boolean bigEndian) {
3567         putIntUnaligned(o, offset, convEndian(bigEndian, x));
3568     }
3569 
3570     /** @see #putLongUnaligned(Object, long, long) */
3571     @HotSpotIntrinsicCandidate
3572     public final void putShortUnaligned(Object o, long offset, short x) {
3573         if ((offset & 1) == 0) {
3574             putShort(o, offset, x);
3575         } else {
3576             putShortParts(o, offset,
3577                           (byte)(x >>> 0),
3578                           (byte)(x >>> 8));
3579         }
3580     }
3581     /** @see #putLongUnaligned(Object, long, long, boolean) */
3582     public final void putShortUnaligned(Object o, long offset, short x, boolean bigEndian) {
3583         putShortUnaligned(o, offset, convEndian(bigEndian, x));
3584     }
3585 
3586     /** @see #putLongUnaligned(Object, long, long) */
3587     @HotSpotIntrinsicCandidate
3588     public final void putCharUnaligned(Object o, long offset, char x) {
3589         putShortUnaligned(o, offset, (short)x);
3590     }
3591     /** @see #putLongUnaligned(Object, long, long, boolean) */
3592     public final void putCharUnaligned(Object o, long offset, char x, boolean bigEndian) {
3593         putCharUnaligned(o, offset, convEndian(bigEndian, x));
3594     }
3595 
3596     // JVM interface methods
3597     // BE is true iff the native endianness of this platform is big.
3598     private static final boolean BE = theUnsafe.isBigEndian0();
3599 
3600     // unalignedAccess is true iff this platform can perform unaligned accesses.
3601     private static final boolean unalignedAccess = theUnsafe.unalignedAccess0();
3602 
3603     private static int pickPos(int top, int pos) { return BE ? top - pos : pos; }
3604 
3605     // These methods construct integers from bytes.  The byte ordering
3606     // is the native endianness of this platform.
3607     private static long makeLong(byte i0, byte i1, byte i2, byte i3, byte i4, byte i5, byte i6, byte i7) {
3608         return ((toUnsignedLong(i0) << pickPos(56, 0))
3609               | (toUnsignedLong(i1) << pickPos(56, 8))
3610               | (toUnsignedLong(i2) << pickPos(56, 16))
3611               | (toUnsignedLong(i3) << pickPos(56, 24))
3612               | (toUnsignedLong(i4) << pickPos(56, 32))
3613               | (toUnsignedLong(i5) << pickPos(56, 40))
3614               | (toUnsignedLong(i6) << pickPos(56, 48))
3615               | (toUnsignedLong(i7) << pickPos(56, 56)));
3616     }
3617     private static long makeLong(short i0, short i1, short i2, short i3) {
3618         return ((toUnsignedLong(i0) << pickPos(48, 0))
3619               | (toUnsignedLong(i1) << pickPos(48, 16))
3620               | (toUnsignedLong(i2) << pickPos(48, 32))
3621               | (toUnsignedLong(i3) << pickPos(48, 48)));
3622     }
3623     private static long makeLong(int i0, int i1) {
3624         return (toUnsignedLong(i0) << pickPos(32, 0))
3625              | (toUnsignedLong(i1) << pickPos(32, 32));
3626     }
3627     private static int makeInt(short i0, short i1) {
3628         return (toUnsignedInt(i0) << pickPos(16, 0))
3629              | (toUnsignedInt(i1) << pickPos(16, 16));
3630     }
3631     private static int makeInt(byte i0, byte i1, byte i2, byte i3) {
3632         return ((toUnsignedInt(i0) << pickPos(24, 0))
3633               | (toUnsignedInt(i1) << pickPos(24, 8))
3634               | (toUnsignedInt(i2) << pickPos(24, 16))
3635               | (toUnsignedInt(i3) << pickPos(24, 24)));
3636     }
3637     private static short makeShort(byte i0, byte i1) {
3638         return (short)((toUnsignedInt(i0) << pickPos(8, 0))
3639                      | (toUnsignedInt(i1) << pickPos(8, 8)));
3640     }
3641 
3642     private static byte  pick(byte  le, byte  be) { return BE ? be : le; }
3643     private static short pick(short le, short be) { return BE ? be : le; }
3644     private static int   pick(int   le, int   be) { return BE ? be : le; }
3645 
3646     // These methods write integers to memory from smaller parts
3647     // provided by their caller.  The ordering in which these parts
3648     // are written is the native endianness of this platform.
3649     private void putLongParts(Object o, long offset, byte i0, byte i1, byte i2, byte i3, byte i4, byte i5, byte i6, byte i7) {
3650         putByte(o, offset + 0, pick(i0, i7));
3651         putByte(o, offset + 1, pick(i1, i6));
3652         putByte(o, offset + 2, pick(i2, i5));
3653         putByte(o, offset + 3, pick(i3, i4));
3654         putByte(o, offset + 4, pick(i4, i3));
3655         putByte(o, offset + 5, pick(i5, i2));
3656         putByte(o, offset + 6, pick(i6, i1));
3657         putByte(o, offset + 7, pick(i7, i0));
3658     }
3659     private void putLongParts(Object o, long offset, short i0, short i1, short i2, short i3) {
3660         putShort(o, offset + 0, pick(i0, i3));
3661         putShort(o, offset + 2, pick(i1, i2));
3662         putShort(o, offset + 4, pick(i2, i1));
3663         putShort(o, offset + 6, pick(i3, i0));
3664     }
3665     private void putLongParts(Object o, long offset, int i0, int i1) {
3666         putInt(o, offset + 0, pick(i0, i1));
3667         putInt(o, offset + 4, pick(i1, i0));
3668     }
3669     private void putIntParts(Object o, long offset, short i0, short i1) {
3670         putShort(o, offset + 0, pick(i0, i1));
3671         putShort(o, offset + 2, pick(i1, i0));
3672     }
3673     private void putIntParts(Object o, long offset, byte i0, byte i1, byte i2, byte i3) {
3674         putByte(o, offset + 0, pick(i0, i3));
3675         putByte(o, offset + 1, pick(i1, i2));
3676         putByte(o, offset + 2, pick(i2, i1));
3677         putByte(o, offset + 3, pick(i3, i0));
3678     }
3679     private void putShortParts(Object o, long offset, byte i0, byte i1) {
3680         putByte(o, offset + 0, pick(i0, i1));
3681         putByte(o, offset + 1, pick(i1, i0));
3682     }
3683 
3684     // Zero-extend an integer
3685     private static int toUnsignedInt(byte n)    { return n & 0xff; }
3686     private static int toUnsignedInt(short n)   { return n & 0xffff; }
3687     private static long toUnsignedLong(byte n)  { return n & 0xffl; }
3688     private static long toUnsignedLong(short n) { return n & 0xffffl; }
3689     private static long toUnsignedLong(int n)   { return n & 0xffffffffl; }
3690 
3691     // Maybe byte-reverse an integer
3692     private static char convEndian(boolean big, char n)   { return big == BE ? n : Character.reverseBytes(n); }
3693     private static short convEndian(boolean big, short n) { return big == BE ? n : Short.reverseBytes(n)    ; }
3694     private static int convEndian(boolean big, int n)     { return big == BE ? n : Integer.reverseBytes(n)  ; }
3695     private static long convEndian(boolean big, long n)   { return big == BE ? n : Long.reverseBytes(n)     ; }
3696 
3697 
3698 
3699     private native long allocateMemory0(long bytes);
3700     private native long reallocateMemory0(long address, long bytes);
3701     private native void freeMemory0(long address);
3702     private native void setMemory0(Object o, long offset, long bytes, byte value);
3703     @HotSpotIntrinsicCandidate
3704     private native void copyMemory0(Object srcBase, long srcOffset, Object destBase, long destOffset, long bytes);
3705     private native void copySwapMemory0(Object srcBase, long srcOffset, Object destBase, long destOffset, long bytes, long elemSize);
3706     private native long objectFieldOffset0(Field f);
3707     private native long objectFieldOffset1(Class<?> c, String name);
3708     private native long staticFieldOffset0(Field f);
3709     private native Object staticFieldBase0(Field f);
3710     private native boolean shouldBeInitialized0(Class<?> c);
3711     private native void ensureClassInitialized0(Class<?> c);
3712     private native int arrayBaseOffset0(Class<?> arrayClass);
3713     private native int arrayIndexScale0(Class<?> arrayClass);
3714     private native int addressSize0();
3715     private native Class<?> defineAnonymousClass0(Class<?> hostClass, byte[] data, Object[] cpPatches);
3716     private native int getLoadAverage0(double[] loadavg, int nelems);
3717     private native boolean unalignedAccess0();
3718     private native boolean isBigEndian0();
3719 }