1 /* 2 * Copyright (c) 2003, 2016, Oracle and/or its affiliates. All rights reserved. 3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 4 * 5 * This code is free software; you can redistribute it and/or modify it 6 * under the terms of the GNU General Public License version 2 only, as 7 * published by the Free Software Foundation. 8 * 9 * This code is distributed in the hope that it will be useful, but WITHOUT 10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 12 * version 2 for more details (a copy is included in the LICENSE file that 13 * accompanied this code). 14 * 15 * You should have received a copy of the GNU General Public License version 16 * 2 along with this work; if not, write to the Free Software Foundation, 17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. 18 * 19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 20 * or visit www.oracle.com if you need additional information or have any 21 * questions. 22 * 23 */ 24 25 #ifndef SHARE_VM_UTILITIES_COPY_HPP 26 #define SHARE_VM_UTILITIES_COPY_HPP 27 28 #include "runtime/stubRoutines.hpp" 29 #include "utilities/align.hpp" 30 #include "utilities/macros.hpp" 31 32 // Assembly code for platforms that need it. 33 extern "C" { 34 void _Copy_conjoint_words(HeapWord* from, HeapWord* to, size_t count); 35 void _Copy_disjoint_words(HeapWord* from, HeapWord* to, size_t count); 36 37 void _Copy_conjoint_words_atomic(HeapWord* from, HeapWord* to, size_t count); 38 void _Copy_disjoint_words_atomic(HeapWord* from, HeapWord* to, size_t count); 39 40 void _Copy_aligned_conjoint_words(HeapWord* from, HeapWord* to, size_t count); 41 void _Copy_aligned_disjoint_words(HeapWord* from, HeapWord* to, size_t count); 42 43 void _Copy_conjoint_bytes(void* from, void* to, size_t count); 44 45 void _Copy_conjoint_bytes_atomic (void* from, void* to, size_t count); 46 void _Copy_conjoint_jshorts_atomic(jshort* from, jshort* to, size_t count); 47 void _Copy_conjoint_jints_atomic (jint* from, jint* to, size_t count); 48 void _Copy_conjoint_jlongs_atomic (jlong* from, jlong* to, size_t count); 49 void _Copy_conjoint_oops_atomic (oop* from, oop* to, size_t count); 50 51 void _Copy_arrayof_conjoint_bytes (HeapWord* from, HeapWord* to, size_t count); 52 void _Copy_arrayof_conjoint_jshorts(HeapWord* from, HeapWord* to, size_t count); 53 void _Copy_arrayof_conjoint_jints (HeapWord* from, HeapWord* to, size_t count); 54 void _Copy_arrayof_conjoint_jlongs (HeapWord* from, HeapWord* to, size_t count); 55 void _Copy_arrayof_conjoint_oops (HeapWord* from, HeapWord* to, size_t count); 56 } 57 58 class Copy : AllStatic { 59 public: 60 // Block copy methods have four attributes. We don't define all possibilities. 61 // alignment: aligned to BytesPerLong 62 // arrayof: arraycopy operation with both operands aligned on the same 63 // boundary as the first element of an array of the copy unit. 64 // This is currently a HeapWord boundary on all platforms, except 65 // for long and double arrays, which are aligned on an 8-byte 66 // boundary on all platforms. 67 // arraycopy operations are implicitly atomic on each array element. 68 // overlap: disjoint or conjoint. 69 // copy unit: bytes or words (i.e., HeapWords) or oops (i.e., pointers). 70 // atomicity: atomic or non-atomic on the copy unit. 71 // 72 // Names are constructed thusly: 73 // 74 // [ 'aligned_' | 'arrayof_' ] 75 // ('conjoint_' | 'disjoint_') 76 // ('words' | 'bytes' | 'jshorts' | 'jints' | 'jlongs' | 'oops') 77 // [ '_atomic' ] 78 // 79 // Except in the arrayof case, whatever the alignment is, we assume we can copy 80 // whole alignment units. E.g., if BytesPerLong is 2x word alignment, an odd 81 // count may copy an extra word. In the arrayof case, we are allowed to copy 82 // only the number of copy units specified. 83 // 84 // All callees check count for 0. 85 // 86 87 // HeapWords 88 89 // Word-aligned words, conjoint, not atomic on each word 90 static void conjoint_words(HeapWord* from, HeapWord* to, size_t count) { 91 assert_params_ok(from, to, LogHeapWordSize); 92 pd_conjoint_words(from, to, count); 93 } 94 95 // Word-aligned words, disjoint, not atomic on each word 96 static void disjoint_words(HeapWord* from, HeapWord* to, size_t count) { 97 assert_params_ok(from, to, LogHeapWordSize); 98 assert_disjoint(from, to, count); 99 pd_disjoint_words(from, to, count); 100 } 101 102 // Word-aligned words, disjoint, atomic on each word 103 static void disjoint_words_atomic(HeapWord* from, HeapWord* to, size_t count) { 104 assert_params_ok(from, to, LogHeapWordSize); 105 assert_disjoint(from, to, count); 106 pd_disjoint_words_atomic(from, to, count); 107 } 108 109 // Object-aligned words, conjoint, not atomic on each word 110 static void aligned_conjoint_words(HeapWord* from, HeapWord* to, size_t count) { 111 assert_params_aligned(from, to); 112 pd_aligned_conjoint_words(from, to, count); 113 } 114 115 // Object-aligned words, disjoint, not atomic on each word 116 static void aligned_disjoint_words(HeapWord* from, HeapWord* to, size_t count) { 117 assert_params_aligned(from, to); 118 assert_disjoint(from, to, count); 119 pd_aligned_disjoint_words(from, to, count); 120 } 121 122 // bytes, jshorts, jints, jlongs, oops 123 124 // bytes, conjoint, not atomic on each byte (not that it matters) 125 static void conjoint_jbytes(void* from, void* to, size_t count) { 126 pd_conjoint_bytes(from, to, count); 127 } 128 129 // bytes, conjoint, atomic on each byte (not that it matters) 130 static void conjoint_jbytes_atomic(void* from, void* to, size_t count) { 131 pd_conjoint_bytes(from, to, count); 132 } 133 134 // jshorts, conjoint, atomic on each jshort 135 static void conjoint_jshorts_atomic(jshort* from, jshort* to, size_t count) { 136 assert_params_ok(from, to, LogBytesPerShort); 137 pd_conjoint_jshorts_atomic(from, to, count); 138 } 139 140 // jints, conjoint, atomic on each jint 141 static void conjoint_jints_atomic(jint* from, jint* to, size_t count) { 142 assert_params_ok(from, to, LogBytesPerInt); 143 pd_conjoint_jints_atomic(from, to, count); 144 } 145 146 // jlongs, conjoint, atomic on each jlong 147 static void conjoint_jlongs_atomic(jlong* from, jlong* to, size_t count) { 148 assert_params_ok(from, to, LogBytesPerLong); 149 pd_conjoint_jlongs_atomic(from, to, count); 150 } 151 152 // oops, conjoint, atomic on each oop 153 static void conjoint_oops_atomic(oop* from, oop* to, size_t count) { 154 assert_params_ok(from, to, LogBytesPerHeapOop); 155 pd_conjoint_oops_atomic(from, to, count); 156 } 157 158 // overloaded for UseCompressedOops 159 static void conjoint_oops_atomic(narrowOop* from, narrowOop* to, size_t count) { 160 assert(sizeof(narrowOop) == sizeof(jint), "this cast is wrong"); 161 assert_params_ok(from, to, LogBytesPerInt); 162 pd_conjoint_jints_atomic((jint*)from, (jint*)to, count); 163 } 164 165 // Copy a span of memory. If the span is an integral number of aligned 166 // longs, words, or ints, copy those units atomically. 167 // The largest atomic transfer unit is 8 bytes, or the largest power 168 // of two which divides all of from, to, and size, whichever is smaller. 169 static void conjoint_memory_atomic(void* from, void* to, size_t size); 170 171 // bytes, conjoint array, atomic on each byte (not that it matters) 172 static void arrayof_conjoint_jbytes(HeapWord* from, HeapWord* to, size_t count) { 173 pd_arrayof_conjoint_bytes(from, to, count); 174 } 175 176 // jshorts, conjoint array, atomic on each jshort 177 static void arrayof_conjoint_jshorts(HeapWord* from, HeapWord* to, size_t count) { 178 assert_params_ok(from, to, LogBytesPerShort); 179 pd_arrayof_conjoint_jshorts(from, to, count); 180 } 181 182 // jints, conjoint array, atomic on each jint 183 static void arrayof_conjoint_jints(HeapWord* from, HeapWord* to, size_t count) { 184 assert_params_ok(from, to, LogBytesPerInt); 185 pd_arrayof_conjoint_jints(from, to, count); 186 } 187 188 // jlongs, conjoint array, atomic on each jlong 189 static void arrayof_conjoint_jlongs(HeapWord* from, HeapWord* to, size_t count) { 190 assert_params_ok(from, to, LogBytesPerLong); 191 pd_arrayof_conjoint_jlongs(from, to, count); 192 } 193 194 // oops, conjoint array, atomic on each oop 195 static void arrayof_conjoint_oops(HeapWord* from, HeapWord* to, size_t count) { 196 assert_params_ok(from, to, LogBytesPerHeapOop); 197 pd_arrayof_conjoint_oops(from, to, count); 198 } 199 200 // Known overlap methods 201 202 // Copy word-aligned words from higher to lower addresses, not atomic on each word 203 inline static void conjoint_words_to_lower(HeapWord* from, HeapWord* to, size_t byte_count) { 204 // byte_count is in bytes to check its alignment 205 assert_params_ok(from, to, LogHeapWordSize); 206 assert_byte_count_ok(byte_count, HeapWordSize); 207 208 size_t count = align_up(byte_count, HeapWordSize) >> LogHeapWordSize; 209 assert(to <= from || from + count <= to, "do not overwrite source data"); 210 211 while (count-- > 0) { 212 *to++ = *from++; 213 } 214 } 215 216 // Copy word-aligned words from lower to higher addresses, not atomic on each word 217 inline static void conjoint_words_to_higher(HeapWord* from, HeapWord* to, size_t byte_count) { 218 // byte_count is in bytes to check its alignment 219 assert_params_ok(from, to, LogHeapWordSize); 220 assert_byte_count_ok(byte_count, HeapWordSize); 221 222 size_t count = align_up(byte_count, HeapWordSize) >> LogHeapWordSize; 223 assert(from <= to || to + count <= from, "do not overwrite source data"); 224 225 from += count - 1; 226 to += count - 1; 227 while (count-- > 0) { 228 *to-- = *from--; 229 } 230 } 231 232 /** 233 * Copy elements 234 * 235 * @param src address of source 236 * @param dst address of destination 237 * @param byte_count number of bytes to copy 238 * @param elem_size size of the elements to copy-swap 239 */ 240 static void conjoint_copy(const void* src, void* dst, size_t byte_count, size_t elem_size); 241 242 /** 243 * Copy and *unconditionally* byte swap elements 244 * 245 * @param src address of source 246 * @param dst address of destination 247 * @param byte_count number of bytes to copy 248 * @param elem_size size of the elements to copy-swap 249 */ 250 static void conjoint_swap(const void* src, void* dst, size_t byte_count, size_t elem_size); 251 252 /** 253 * Copy and byte swap elements from the specified endian to the native (cpu) endian if needed (if they differ) 254 * 255 * @param src address of source 256 * @param dst address of destination 257 * @param byte_count number of bytes to copy 258 * @param elem_size size of the elements to copy-swap 259 */ 260 template <Endian::Order endian> 261 static void conjoint_swap_if_needed(const void* src, void* dst, size_t byte_count, size_t elem_size) { 262 if (Endian::NATIVE != endian) { 263 conjoint_swap(src, dst, byte_count, elem_size); 264 } else { 265 conjoint_copy(src, dst, byte_count, elem_size); 266 } 267 } 268 269 // Fill methods 270 271 // Fill word-aligned words, not atomic on each word 272 // set_words 273 static void fill_to_words(HeapWord* to, size_t count, juint value = 0) { 274 assert_params_ok(to, LogHeapWordSize); 275 pd_fill_to_words(to, count, value); 276 } 277 278 static void fill_to_aligned_words(HeapWord* to, size_t count, juint value = 0) { 279 assert_params_aligned(to); 280 pd_fill_to_aligned_words(to, count, value); 281 } 282 283 // Fill bytes 284 static void fill_to_bytes(void* to, size_t count, jubyte value = 0) { 285 pd_fill_to_bytes(to, count, value); 286 } 287 288 // Fill a span of memory. If the span is an integral number of aligned 289 // longs, words, or ints, store to those units atomically. 290 // The largest atomic transfer unit is 8 bytes, or the largest power 291 // of two which divides both to and size, whichever is smaller. 292 static void fill_to_memory_atomic(void* to, size_t size, jubyte value = 0); 293 294 // Zero-fill methods 295 296 // Zero word-aligned words, not atomic on each word 297 static void zero_to_words(HeapWord* to, size_t count) { 298 assert_params_ok(to, LogHeapWordSize); 299 pd_zero_to_words(to, count); 300 } 301 302 // Zero bytes 303 static void zero_to_bytes(void* to, size_t count) { 304 pd_zero_to_bytes(to, count); 305 } 306 307 private: 308 static bool params_disjoint(HeapWord* from, HeapWord* to, size_t count) { 309 if (from < to) { 310 return pointer_delta(to, from) >= count; 311 } 312 return pointer_delta(from, to) >= count; 313 } 314 315 // These methods raise a fatal if they detect a problem. 316 317 static void assert_disjoint(HeapWord* from, HeapWord* to, size_t count) { 318 #ifdef ASSERT 319 if (!params_disjoint(from, to, count)) 320 basic_fatal("source and dest overlap"); 321 #endif 322 } 323 324 static void assert_params_ok(void* from, void* to, intptr_t log_align) { 325 #ifdef ASSERT 326 if (mask_bits((uintptr_t)from, right_n_bits(log_align)) != 0) 327 basic_fatal("not aligned"); 328 if (mask_bits((uintptr_t)to, right_n_bits(log_align)) != 0) 329 basic_fatal("not aligned"); 330 #endif 331 } 332 333 static void assert_params_ok(HeapWord* to, intptr_t log_align) { 334 #ifdef ASSERT 335 if (mask_bits((uintptr_t)to, right_n_bits(log_align)) != 0) 336 basic_fatal("not word aligned"); 337 #endif 338 } 339 static void assert_params_aligned(HeapWord* from, HeapWord* to) { 340 #ifdef ASSERT 341 if (mask_bits((uintptr_t)from, BytesPerLong-1) != 0) 342 basic_fatal("not long aligned"); 343 if (mask_bits((uintptr_t)to, BytesPerLong-1) != 0) 344 basic_fatal("not long aligned"); 345 #endif 346 } 347 348 static void assert_params_aligned(HeapWord* to) { 349 #ifdef ASSERT 350 if (mask_bits((uintptr_t)to, BytesPerLong-1) != 0) 351 basic_fatal("not long aligned"); 352 #endif 353 } 354 355 static void assert_byte_count_ok(size_t byte_count, size_t unit_size) { 356 #ifdef ASSERT 357 if (!is_aligned(byte_count, unit_size)) { 358 basic_fatal("byte count must be aligned"); 359 } 360 #endif 361 } 362 363 // Platform dependent implementations of the above methods. 364 #include CPU_HEADER(copy) 365 366 }; 367 368 #endif // SHARE_VM_UTILITIES_COPY_HPP