< prev index next >

src/hotspot/cpu/x86/stubGenerator_x86_64.cpp

8198949_arraycopy

8  * This code is distributed in the hope that it will be useful, but WITHOUT                                                          
9  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or                                                             
10  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License                                                             
11  * version 2 for more details (a copy is included in the LICENSE file that                                                           
12  * accompanied this code).                                                                                                           
13  *                                                                                                                                   
14  * You should have received a copy of the GNU General Public License version                                                         
15  * 2 along with this work; if not, write to the Free Software Foundation,                                                            
16  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.                                                                     
17  *                                                                                                                                   
18  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA                                                           
19  * or visit www.oracle.com if you need additional information or have any                                                            
20  * questions.                                                                                                                        
21  *                                                                                                                                   
22  */                                                                                                                                  
23 
24 #include "precompiled.hpp"                                                                                                           
25 #include "asm/macroAssembler.hpp"                                                                                                    
26 #include "asm/macroAssembler.inline.hpp"                                                                                             
27 #include "ci/ciUtilities.hpp"                                                                                                        
28 #include "gc/shared/cardTable.hpp"                                                                                                   
29 #include "gc/shared/cardTableModRefBS.hpp"                                                                                           
30 #include "interpreter/interpreter.hpp"                                                                                               
31 #include "nativeInst_x86.hpp"                                                                                                        
32 #include "oops/instanceOop.hpp"                                                                                                      
33 #include "oops/method.hpp"                                                                                                           
34 #include "oops/objArrayKlass.hpp"                                                                                                    
35 #include "oops/oop.inline.hpp"                                                                                                       
36 #include "prims/methodHandles.hpp"                                                                                                   
37 #include "runtime/frame.inline.hpp"                                                                                                  
38 #include "runtime/handles.inline.hpp"                                                                                                
39 #include "runtime/sharedRuntime.hpp"                                                                                                 
40 #include "runtime/stubCodeGenerator.hpp"                                                                                             
41 #include "runtime/stubRoutines.hpp"                                                                                                  
42 #include "runtime/thread.inline.hpp"                                                                                                 
43 #ifdef COMPILER2                                                                                                                     
44 #include "opto/runtime.hpp"                                                                                                          
45 #endif                                                                                                                               
46 
47 // Declaration and definition of StubGenerator (no .hpp file).                                                                       
48 // For a more detailed description of the stub routine structure                                                                     

8  * This code is distributed in the hope that it will be useful, but WITHOUT
9  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
10  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
11  * version 2 for more details (a copy is included in the LICENSE file that
12  * accompanied this code).
13  *
14  * You should have received a copy of the GNU General Public License version
15  * 2 along with this work; if not, write to the Free Software Foundation,
16  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
17  *
18  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
19  * or visit www.oracle.com if you need additional information or have any
20  * questions.
21  *
22  */
23 
24 #include "precompiled.hpp"
25 #include "asm/macroAssembler.hpp"
26 #include "asm/macroAssembler.inline.hpp"
27 #include "ci/ciUtilities.hpp"
28 #include "gc/shared/barrierSet.hpp"
29 #include "gc/shared/barrierSetCodeGen.hpp"
30 #include "interpreter/interpreter.hpp"
31 #include "nativeInst_x86.hpp"
32 #include "oops/instanceOop.hpp"
33 #include "oops/method.hpp"
34 #include "oops/objArrayKlass.hpp"
35 #include "oops/oop.inline.hpp"
36 #include "prims/methodHandles.hpp"
37 #include "runtime/frame.inline.hpp"
38 #include "runtime/handles.inline.hpp"
39 #include "runtime/sharedRuntime.hpp"
40 #include "runtime/stubCodeGenerator.hpp"
41 #include "runtime/stubRoutines.hpp"
42 #include "runtime/thread.inline.hpp"
43 #ifdef COMPILER2
44 #include "opto/runtime.hpp"
45 #endif
46 
47 // Declaration and definition of StubGenerator (no .hpp file).
48 // For a more detailed description of the stub routine structure

1172     __ mov(rdi, rcx); // c_rarg0                                                                                                     
1173     __ mov(rsi, rdx); // c_rarg1                                                                                                     
1174     __ mov(rdx, r8);  // c_rarg2                                                                                                     
1175     if (nargs >= 4)                                                                                                                  
1176       __ mov(rcx, rax); // c_rarg3 (via rax)                                                                                         
1177 #else                                                                                                                                
1178     assert(c_rarg0 == rdi && c_rarg1 == rsi && c_rarg2 == rdx && c_rarg3 == rcx,                                                     
1179            "unexpected argument registers");                                                                                         
1180 #endif                                                                                                                               
1181   }                                                                                                                                  
1182 
1183   void restore_arg_regs() {                                                                                                          
1184     const Register saved_rdi = r9;                                                                                                   
1185     const Register saved_rsi = r10;                                                                                                  
1186 #ifdef _WIN64                                                                                                                        
1187     __ movptr(rdi, saved_rdi);                                                                                                       
1188     __ movptr(rsi, saved_rsi);                                                                                                       
1189 #endif                                                                                                                               
1190   }                                                                                                                                  
1191 
1192   // Generate code for an array write pre barrier                                                                                    
1193   //                                                                                                                                 
1194   //     addr    -  starting address                                                                                                 
1195   //     count   -  element count                                                                                                    
1196   //     tmp     - scratch register                                                                                                  
1197   //                                                                                                                                 
1198   //     Destroy no registers!                                                                                                       
1199   //                                                                                                                                 
1200   void  gen_write_ref_array_pre_barrier(Register addr, Register count, bool dest_uninitialized) {                                    
1201     BarrierSet* bs = Universe::heap()->barrier_set();                                                                                
1202     switch (bs->kind()) {                                                                                                            
1203       case BarrierSet::G1BarrierSet:                                                                                                 
1204         // With G1, don't generate the call if we statically know that the target in uninitialized                                   
1205         if (!dest_uninitialized) {                                                                                                   
1206           Label filtered;                                                                                                            
1207           Address in_progress(r15_thread, in_bytes(JavaThread::satb_mark_queue_offset() +                                            
1208                                                    SATBMarkQueue::byte_offset_of_active()));                                         
1209           // Is marking active?                                                                                                      
1210           if (in_bytes(SATBMarkQueue::byte_width_of_active()) == 4) {                                                                
1211             __ cmpl(in_progress, 0);                                                                                                 
1212           } else {                                                                                                                   
1213             assert(in_bytes(SATBMarkQueue::byte_width_of_active()) == 1, "Assumption");                                              
1214             __ cmpb(in_progress, 0);                                                                                                 
1215           }                                                                                                                          
1216           __ jcc(Assembler::equal, filtered);                                                                                        
1217                                                                                                                                      
1218            __ pusha();                      // push registers                                                                        
1219            if (count == c_rarg0) {                                                                                                   
1220              if (addr == c_rarg1) {                                                                                                  
1221                // exactly backwards!!                                                                                                
1222                __ xchgptr(c_rarg1, c_rarg0);                                                                                         
1223              } else {                                                                                                                
1224                __ movptr(c_rarg1, count);                                                                                            
1225                __ movptr(c_rarg0, addr);                                                                                             
1226              }                                                                                                                       
1227            } else {                                                                                                                  
1228              __ movptr(c_rarg0, addr);                                                                                               
1229              __ movptr(c_rarg1, count);                                                                                              
1230            }                                                                                                                         
1231            __ call_VM_leaf(CAST_FROM_FN_PTR(address, BarrierSet::static_write_ref_array_pre), 2);                                    
1232            __ popa();                                                                                                                
1233                                                                                                                                      
1234            __ bind(filtered);                                                                                                        
1235         }                                                                                                                            
1236          break;                                                                                                                      
1237       case BarrierSet::CardTableModRef:                                                                                              
1238         break;                                                                                                                       
1239       default:                                                                                                                       
1240         ShouldNotReachHere();                                                                                                        
1241                                                                                                                                      
1242     }                                                                                                                                
1243   }                                                                                                                                  
1244                                                                                                                                      
1245   //                                                                                                                                 
1246   // Generate code for an array write post barrier                                                                                   
1247   //                                                                                                                                 
1248   //  Input:                                                                                                                         
1249   //     start    - register containing starting address of destination array                                                        
1250   //     count    - elements count                                                                                                   
1251   //     scratch  - scratch register                                                                                                 
1252   //                                                                                                                                 
1253   //  The input registers are overwritten.                                                                                           
1254   //                                                                                                                                 
1255   void  gen_write_ref_array_post_barrier(Register start, Register count, Register scratch) {                                         
1256     assert_different_registers(start, count, scratch);                                                                               
1257     BarrierSet* bs = Universe::heap()->barrier_set();                                                                                
1258     switch (bs->kind()) {                                                                                                            
1259       case BarrierSet::G1BarrierSet:                                                                                                 
1260         {                                                                                                                            
1261           __ pusha();             // push registers (overkill)                                                                       
1262           if (c_rarg0 == count) { // On win64 c_rarg0 == rcx                                                                         
1263             assert_different_registers(c_rarg1, start);                                                                              
1264             __ mov(c_rarg1, count);                                                                                                  
1265             __ mov(c_rarg0, start);                                                                                                  
1266           } else {                                                                                                                   
1267             assert_different_registers(c_rarg0, count);                                                                              
1268             __ mov(c_rarg0, start);                                                                                                  
1269             __ mov(c_rarg1, count);                                                                                                  
1270           }                                                                                                                          
1271           __ call_VM_leaf(CAST_FROM_FN_PTR(address, BarrierSet::static_write_ref_array_post), 2);                                    
1272           __ popa();                                                                                                                 
1273         }                                                                                                                            
1274         break;                                                                                                                       
1275       case BarrierSet::CardTableModRef:                                                                                              
1276         {                                                                                                                            
1277           Label L_loop, L_done;                                                                                                      
1278           const Register end = count;                                                                                                
1279                                                                                                                                      
1280           __ testl(count, count);                                                                                                    
1281           __ jcc(Assembler::zero, L_done); // zero count - nothing to do                                                             
1282                                                                                                                                      
1283           __ leaq(end, Address(start, count, TIMES_OOP, 0));  // end == start+count*oop_size                                         
1284           __ subptr(end, BytesPerHeapOop); // end - 1 to make inclusive                                                              
1285           __ shrptr(start, CardTable::card_shift);                                                                                   
1286           __ shrptr(end,   CardTable::card_shift);                                                                                   
1287           __ subptr(end, start); // end --> cards count                                                                              
1288                                                                                                                                      
1289           int64_t disp = ci_card_table_address_as<int64_t>();                                                                        
1290           __ mov64(scratch, disp);                                                                                                   
1291           __ addptr(start, scratch);                                                                                                 
1292         __ BIND(L_loop);                                                                                                             
1293           __ movb(Address(start, count, Address::times_1), 0);                                                                       
1294           __ decrement(count);                                                                                                       
1295           __ jcc(Assembler::greaterEqual, L_loop);                                                                                   
1296         __ BIND(L_done);                                                                                                             
1297         }                                                                                                                            
1298         break;                                                                                                                       
1299       default:                                                                                                                       
1300         ShouldNotReachHere();                                                                                                        
1301                                                                                                                                      
1302     }                                                                                                                                
1303   }                                                                                                                                  
1304                                                                                                                                      
1305 
1306   // Copy big chunks forward                                                                                                         
1307   //                                                                                                                                 
1308   // Inputs:                                                                                                                         
1309   //   end_from     - source arrays end address                                                                                      
1310   //   end_to       - destination array end address                                                                                  
1311   //   qword_count  - 64-bits element count, negative                                                                                
1312   //   to           - scratch                                                                                                        
1313   //   L_copy_bytes - entry label                                                                                                    
1314   //   L_copy_8_bytes  - exit  label                                                                                                 
1315   //                                                                                                                                 
1316   void copy_bytes_forward(Register end_from, Register end_to,                                                                        
1317                              Register qword_count, Register to,                                                                      
1318                              Label& L_copy_bytes, Label& L_copy_8_bytes) {                                                           
1319     DEBUG_ONLY(__ stop("enter at entry label, not here"));                                                                           
1320     Label L_loop;                                                                                                                    
1321     __ align(OptoLoopAlignment);                                                                                                     
1322     if (UseUnalignedLoadStores) {                                                                                                    
1323       Label L_end;                                                                                                                   

1172     __ mov(rdi, rcx); // c_rarg0
1173     __ mov(rsi, rdx); // c_rarg1
1174     __ mov(rdx, r8);  // c_rarg2
1175     if (nargs >= 4)
1176       __ mov(rcx, rax); // c_rarg3 (via rax)
1177 #else
1178     assert(c_rarg0 == rdi && c_rarg1 == rsi && c_rarg2 == rdx && c_rarg3 == rcx,
1179            "unexpected argument registers");
1180 #endif
1181   }
1182 
1183   void restore_arg_regs() {
1184     const Register saved_rdi = r9;
1185     const Register saved_rsi = r10;
1186 #ifdef _WIN64
1187     __ movptr(rdi, saved_rdi);
1188     __ movptr(rsi, saved_rsi);
1189 #endif
1190   }
1191 

















































































































1192 
1193   // Copy big chunks forward
1194   //
1195   // Inputs:
1196   //   end_from     - source arrays end address
1197   //   end_to       - destination array end address
1198   //   qword_count  - 64-bits element count, negative
1199   //   to           - scratch
1200   //   L_copy_bytes - entry label
1201   //   L_copy_8_bytes  - exit  label
1202   //
1203   void copy_bytes_forward(Register end_from, Register end_to,
1204                              Register qword_count, Register to,
1205                              Label& L_copy_bytes, Label& L_copy_8_bytes) {
1206     DEBUG_ONLY(__ stop("enter at entry label, not here"));
1207     Label L_loop;
1208     __ align(OptoLoopAlignment);
1209     if (UseUnalignedLoadStores) {
1210       Label L_end;

1900   // cache line boundaries will still be loaded and stored atomicly.                                                                 
1901   //                                                                                                                                 
1902   // Side Effects:                                                                                                                   
1903   //   disjoint_int_copy_entry is set to the no-overlap entry point                                                                  
1904   //   used by generate_conjoint_int_oop_copy().                                                                                     
1905   //                                                                                                                                 
1906   address generate_disjoint_int_oop_copy(bool aligned, bool is_oop, address* entry,                                                  
1907                                          const char *name, bool dest_uninitialized = false) {                                        
1908     __ align(CodeEntryAlignment);                                                                                                    
1909     StubCodeMark mark(this, "StubRoutines", name);                                                                                   
1910     address start = __ pc();                                                                                                         
1911 
1912     Label L_copy_bytes, L_copy_8_bytes, L_copy_4_bytes, L_exit;                                                                      
1913     const Register from        = rdi;  // source array address                                                                       
1914     const Register to          = rsi;  // destination array address                                                                  
1915     const Register count       = rdx;  // elements count                                                                             
1916     const Register dword_count = rcx;                                                                                                
1917     const Register qword_count = count;                                                                                              
1918     const Register end_from    = from; // source array end address                                                                   
1919     const Register end_to      = to;   // destination array end address                                                              
1920     const Register saved_to    = r11;  // saved destination array address                                                            
1921     // End pointers are inclusive, and if count is not zero they point                                                               
1922     // to the last unit copied:  end_to[0] := end_from[0]                                                                            
1923 
1924     __ enter(); // required for proper stackwalking of RuntimeStub frame                                                             
1925     assert_clean_int(c_rarg2, rax);    // Make sure 'count' is clean int.                                                            
1926 
1927     if (entry != NULL) {                                                                                                             
1928       *entry = __ pc();                                                                                                              
1929       // caller can pass a 64-bit byte count here (from Unsafe.copyMemory)                                                           
1930       BLOCK_COMMENT("Entry:");                                                                                                       
1931     }                                                                                                                                
1932 
1933     setup_arg_regs(); // from => rdi, to => rsi, count => rdx                                                                        
1934                       // r9 and r10 may be used to save non-volatile registers                                                       
1935     if (is_oop) {                                                                                                                    
1936       __ movq(saved_to, to);                                                                                                         
1937       gen_write_ref_array_pre_barrier(to, count, dest_uninitialized);                                                                
                                                                                                                                     
                                                                                                                                     
                                                                                                                                     
                                                                                                                                     
                                                                                                                                     
                                                                                                                                     
1938     }                                                                                                                                
1939 
                                                                                                                                     
                                                                                                                                     
1940     // 'from', 'to' and 'count' are now valid                                                                                        
1941     __ movptr(dword_count, count);                                                                                                   
1942     __ shrptr(count, 1); // count => qword_count                                                                                     
1943 
1944     // Copy from low to high addresses.  Use 'to' as scratch.                                                                        
1945     __ lea(end_from, Address(from, qword_count, Address::times_8, -8));                                                              
1946     __ lea(end_to,   Address(to,   qword_count, Address::times_8, -8));                                                              
1947     __ negptr(qword_count);                                                                                                          
1948     __ jmp(L_copy_bytes);                                                                                                            
1949 
1950     // Copy trailing qwords                                                                                                          
1951   __ BIND(L_copy_8_bytes);                                                                                                           
1952     __ movq(rax, Address(end_from, qword_count, Address::times_8, 8));                                                               
1953     __ movq(Address(end_to, qword_count, Address::times_8, 8), rax);                                                                 
1954     __ increment(qword_count);                                                                                                       
1955     __ jcc(Assembler::notZero, L_copy_8_bytes);                                                                                      
1956 
1957     // Check for and copy trailing dword                                                                                             
1958   __ BIND(L_copy_4_bytes);                                                                                                           
1959     __ testl(dword_count, 1); // Only byte test since the value is 0 or 1                                                            
1960     __ jccb(Assembler::zero, L_exit);                                                                                                
1961     __ movl(rax, Address(end_from, 8));                                                                                              
1962     __ movl(Address(end_to, 8), rax);                                                                                                
1963 
1964   __ BIND(L_exit);                                                                                                                   
1965     if (is_oop) {                                                                                                                    
1966       gen_write_ref_array_post_barrier(saved_to, dword_count, rax);                                                                  
1967     }                                                                                                                                
1968     restore_arg_regs();                                                                                                              
1969     inc_counter_np(SharedRuntime::_jint_array_copy_ctr); // Update counter after rscratch1 is free                                   
1970     __ vzeroupper();                                                                                                                 
1971     __ xorptr(rax, rax); // return 0                                                                                                 
1972     __ leave(); // required for proper stackwalking of RuntimeStub frame                                                             
1973     __ ret(0);                                                                                                                       
1974 
1975     // Copy in multi-bytes chunks                                                                                                    
1976     copy_bytes_forward(end_from, end_to, qword_count, rax, L_copy_bytes, L_copy_8_bytes);                                            
1977     __ jmp(L_copy_4_bytes);                                                                                                          
1978 
1979     return start;                                                                                                                    
1980   }                                                                                                                                  
1981 
1982   // Arguments:                                                                                                                      
1983   //   aligned - true => Input and output aligned on a HeapWord == 8-byte boundary                                                   
1984   //             ignored                                                                                                             
1985   //   is_oop  - true => oop array, so generate store check code                                                                     
1986   //   name    - stub name string                                                                                                    

1787   // cache line boundaries will still be loaded and stored atomicly.
1788   //
1789   // Side Effects:
1790   //   disjoint_int_copy_entry is set to the no-overlap entry point
1791   //   used by generate_conjoint_int_oop_copy().
1792   //
1793   address generate_disjoint_int_oop_copy(bool aligned, bool is_oop, address* entry,
1794                                          const char *name, bool dest_uninitialized = false) {
1795     __ align(CodeEntryAlignment);
1796     StubCodeMark mark(this, "StubRoutines", name);
1797     address start = __ pc();
1798 
1799     Label L_copy_bytes, L_copy_8_bytes, L_copy_4_bytes, L_exit;
1800     const Register from        = rdi;  // source array address
1801     const Register to          = rsi;  // destination array address
1802     const Register count       = rdx;  // elements count
1803     const Register dword_count = rcx;
1804     const Register qword_count = count;
1805     const Register end_from    = from; // source array end address
1806     const Register end_to      = to;   // destination array end address

1807     // End pointers are inclusive, and if count is not zero they point
1808     // to the last unit copied:  end_to[0] := end_from[0]
1809 
1810     __ enter(); // required for proper stackwalking of RuntimeStub frame
1811     assert_clean_int(c_rarg2, rax);    // Make sure 'count' is clean int.
1812 
1813     if (entry != NULL) {
1814       *entry = __ pc();
1815       // caller can pass a 64-bit byte count here (from Unsafe.copyMemory)
1816       BLOCK_COMMENT("Entry:");
1817     }
1818 
1819     setup_arg_regs(); // from => rdi, to => rsi, count => rdx
1820                       // r9 and r10 may be used to save non-volatile registers
1821 
1822     BarrierSetCodeGen *bs = Universe::heap()->barrier_set()->code_gen();
1823     DecoratorSet decorators = ARRAYCOPY_DISJOINT;
1824     BasicType type = is_oop ? T_OBJECT : T_INT;
1825     if (dest_uninitialized) {
1826       decorators |= AS_DEST_NOT_INITIALIZED;
1827     }
1828     if (aligned) {
1829       decorators |= ARRAYCOPY_ALIGNED;
1830     }
1831 
1832     bs->arraycopy_prologue(_masm, decorators, type, from, to, count);
1833 
1834     // 'from', 'to' and 'count' are now valid
1835     __ movptr(dword_count, count);
1836     __ shrptr(count, 1); // count => qword_count
1837 
1838     // Copy from low to high addresses.  Use 'to' as scratch.
1839     __ lea(end_from, Address(from, qword_count, Address::times_8, -8));
1840     __ lea(end_to,   Address(to,   qword_count, Address::times_8, -8));
1841     __ negptr(qword_count);
1842     __ jmp(L_copy_bytes);
1843 
1844     // Copy trailing qwords
1845   __ BIND(L_copy_8_bytes);
1846     __ movq(rax, Address(end_from, qword_count, Address::times_8, 8));
1847     __ movq(Address(end_to, qword_count, Address::times_8, 8), rax);
1848     __ increment(qword_count);
1849     __ jcc(Assembler::notZero, L_copy_8_bytes);
1850 
1851     // Check for and copy trailing dword
1852   __ BIND(L_copy_4_bytes);
1853     __ testl(dword_count, 1); // Only byte test since the value is 0 or 1
1854     __ jccb(Assembler::zero, L_exit);
1855     __ movl(rax, Address(end_from, 8));
1856     __ movl(Address(end_to, 8), rax);
1857 
1858   __ BIND(L_exit);
1859     bs->arraycopy_epilogue(_masm, decorators, type, from, to, dword_count);


1860     restore_arg_regs();
1861     inc_counter_np(SharedRuntime::_jint_array_copy_ctr); // Update counter after rscratch1 is free
1862     __ vzeroupper();
1863     __ xorptr(rax, rax); // return 0
1864     __ leave(); // required for proper stackwalking of RuntimeStub frame
1865     __ ret(0);
1866 
1867     // Copy in multi-bytes chunks
1868     copy_bytes_forward(end_from, end_to, qword_count, rax, L_copy_bytes, L_copy_8_bytes);
1869     __ jmp(L_copy_4_bytes);
1870 
1871     return start;
1872   }
1873 
1874   // Arguments:
1875   //   aligned - true => Input and output aligned on a HeapWord == 8-byte boundary
1876   //             ignored
1877   //   is_oop  - true => oop array, so generate store check code
1878   //   name    - stub name string

2004     Label L_copy_bytes, L_copy_8_bytes, L_copy_2_bytes, L_exit;                                                                      
2005     const Register from        = rdi;  // source array address                                                                       
2006     const Register to          = rsi;  // destination array address                                                                  
2007     const Register count       = rdx;  // elements count                                                                             
2008     const Register dword_count = rcx;                                                                                                
2009     const Register qword_count = count;                                                                                              
2010 
2011     __ enter(); // required for proper stackwalking of RuntimeStub frame                                                             
2012     assert_clean_int(c_rarg2, rax);    // Make sure 'count' is clean int.                                                            
2013 
2014     if (entry != NULL) {                                                                                                             
2015       *entry = __ pc();                                                                                                              
2016        // caller can pass a 64-bit byte count here (from Unsafe.copyMemory)                                                          
2017       BLOCK_COMMENT("Entry:");                                                                                                       
2018     }                                                                                                                                
2019 
2020     array_overlap_test(nooverlap_target, Address::times_4);                                                                          
2021     setup_arg_regs(); // from => rdi, to => rsi, count => rdx                                                                        
2022                       // r9 and r10 may be used to save non-volatile registers                                                       
2023 
2024     if (is_oop) {                                                                                                                    
2025       // no registers are destroyed by this call                                                                                     
2026       gen_write_ref_array_pre_barrier(to, count, dest_uninitialized);                                                                
                                                                                                                                     
                                                                                                                                     
2027     }                                                                                                                                
                                                                                                                                     
                                                                                                                                     
                                                                                                                                     
                                                                                                                                     
                                                                                                                                     
                                                                                                                                     
2028 
2029     assert_clean_int(count, rax); // Make sure 'count' is clean int.                                                                 
2030     // 'from', 'to' and 'count' are now valid                                                                                        
2031     __ movptr(dword_count, count);                                                                                                   
2032     __ shrptr(count, 1); // count => qword_count                                                                                     
2033 
2034     // Copy from high to low addresses.  Use 'to' as scratch.                                                                        
2035 
2036     // Check for and copy trailing dword                                                                                             
2037     __ testl(dword_count, 1);                                                                                                        
2038     __ jcc(Assembler::zero, L_copy_bytes);                                                                                           
2039     __ movl(rax, Address(from, dword_count, Address::times_4, -4));                                                                  
2040     __ movl(Address(to, dword_count, Address::times_4, -4), rax);                                                                    
2041     __ jmp(L_copy_bytes);                                                                                                            
2042 
2043     // Copy trailing qwords                                                                                                          
2044   __ BIND(L_copy_8_bytes);                                                                                                           
2045     __ movq(rax, Address(from, qword_count, Address::times_8, -8));                                                                  
2046     __ movq(Address(to, qword_count, Address::times_8, -8), rax);                                                                    
2047     __ decrement(qword_count);                                                                                                       
2048     __ jcc(Assembler::notZero, L_copy_8_bytes);                                                                                      
2049 
2050     if (is_oop) {                                                                                                                    
2051       __ jmp(L_exit);                                                                                                                
2052     }                                                                                                                                
2053     restore_arg_regs();                                                                                                              
2054     inc_counter_np(SharedRuntime::_jint_array_copy_ctr); // Update counter after rscratch1 is free                                   
2055     __ xorptr(rax, rax); // return 0                                                                                                 
2056     __ vzeroupper();                                                                                                                 
2057     __ leave(); // required for proper stackwalking of RuntimeStub frame                                                             
2058     __ ret(0);                                                                                                                       
2059 
2060     // Copy in multi-bytes chunks                                                                                                    
2061     copy_bytes_backward(from, to, qword_count, rax, L_copy_bytes, L_copy_8_bytes);                                                   
2062 
2063   __ BIND(L_exit);                                                                                                                   
2064     if (is_oop) {                                                                                                                    
2065       gen_write_ref_array_post_barrier(to, dword_count, rax);                                                                        
2066     }                                                                                                                                
2067     restore_arg_regs();                                                                                                              
2068     inc_counter_np(SharedRuntime::_jint_array_copy_ctr); // Update counter after rscratch1 is free                                   
2069     __ xorptr(rax, rax); // return 0                                                                                                 
2070     __ vzeroupper();                                                                                                                 
2071     __ leave(); // required for proper stackwalking of RuntimeStub frame                                                             
2072     __ ret(0);                                                                                                                       
2073 
2074     return start;                                                                                                                    
2075   }                                                                                                                                  
2076 
2077   // Arguments:                                                                                                                      
2078   //   aligned - true => Input and output aligned on a HeapWord boundary == 8 bytes                                                  
2079   //             ignored                                                                                                             
2080   //   is_oop  - true => oop array, so generate store check code                                                                     
2081   //   name    - stub name string                                                                                                    
2082   //                                                                                                                                 
2083   // Inputs:                                                                                                                         
2084   //   c_rarg0   - source array address                                                                                              
2085   //   c_rarg1   - destination array address                                                                                         
2086   //   c_rarg2   - element count, treated as ssize_t, can be zero                                                                    
2087   //                                                                                                                                 
2088  // Side Effects:                                                                                                                    
2089   //   disjoint_oop_copy_entry or disjoint_long_copy_entry is set to the                                                             
2090   //   no-overlap entry point used by generate_conjoint_long_oop_copy().                                                             
2091   //                                                                                                                                 
2092   address generate_disjoint_long_oop_copy(bool aligned, bool is_oop, address *entry,                                                 
2093                                           const char *name, bool dest_uninitialized = false) {                                       
2094     __ align(CodeEntryAlignment);                                                                                                    
2095     StubCodeMark mark(this, "StubRoutines", name);                                                                                   
2096     address start = __ pc();                                                                                                         
2097 
2098     Label L_copy_bytes, L_copy_8_bytes, L_exit;                                                                                      
2099     const Register from        = rdi;  // source array address                                                                       
2100     const Register to          = rsi;  // destination array address                                                                  
2101     const Register qword_count = rdx;  // elements count                                                                             
2102     const Register end_from    = from; // source array end address                                                                   
2103     const Register end_to      = rcx;  // destination array end address                                                              
2104     const Register saved_to    = to;                                                                                                 
2105     const Register saved_count = r11;                                                                                                
2106     // End pointers are inclusive, and if count is not zero they point                                                               
2107     // to the last unit copied:  end_to[0] := end_from[0]                                                                            
2108 
2109     __ enter(); // required for proper stackwalking of RuntimeStub frame                                                             
2110     // Save no-overlap entry point for generate_conjoint_long_oop_copy()                                                             
2111     assert_clean_int(c_rarg2, rax);    // Make sure 'count' is clean int.                                                            
2112 
2113     if (entry != NULL) {                                                                                                             
2114       *entry = __ pc();                                                                                                              
2115       // caller can pass a 64-bit byte count here (from Unsafe.copyMemory)                                                           
2116       BLOCK_COMMENT("Entry:");                                                                                                       
2117     }                                                                                                                                
2118 
2119     setup_arg_regs(); // from => rdi, to => rsi, count => rdx                                                                        
2120                       // r9 and r10 may be used to save non-volatile registers                                                       
2121     // 'from', 'to' and 'qword_count' are now valid                                                                                  
2122     if (is_oop) {                                                                                                                    
2123       // Save to and count for store barrier                                                                                         
2124       __ movptr(saved_count, qword_count);                                                                                           
2125       // no registers are destroyed by this call                                                                                     
2126       gen_write_ref_array_pre_barrier(to, qword_count, dest_uninitialized);                                                          
                                                                                                                                     
                                                                                                                                     
                                                                                                                                     
                                                                                                                                     
2127     }                                                                                                                                
2128 
                                                                                                                                     
                                                                                                                                     
2129     // Copy from low to high addresses.  Use 'to' as scratch.                                                                        
2130     __ lea(end_from, Address(from, qword_count, Address::times_8, -8));                                                              
2131     __ lea(end_to,   Address(to,   qword_count, Address::times_8, -8));                                                              
2132     __ negptr(qword_count);                                                                                                          
2133     __ jmp(L_copy_bytes);                                                                                                            
2134 
2135     // Copy trailing qwords                                                                                                          
2136   __ BIND(L_copy_8_bytes);                                                                                                           
2137     __ movq(rax, Address(end_from, qword_count, Address::times_8, 8));                                                               
2138     __ movq(Address(end_to, qword_count, Address::times_8, 8), rax);                                                                 
2139     __ increment(qword_count);                                                                                                       
2140     __ jcc(Assembler::notZero, L_copy_8_bytes);                                                                                      
2141 
2142     if (is_oop) {                                                                                                                    
2143       __ jmp(L_exit);                                                                                                                
2144     } else {                                                                                                                         
2145       restore_arg_regs();                                                                                                            
2146       inc_counter_np(SharedRuntime::_jlong_array_copy_ctr); // Update counter after rscratch1 is free                                
2147       __ xorptr(rax, rax); // return 0                                                                                               
2148       __ vzeroupper();                                                                                                               
2149       __ leave(); // required for proper stackwalking of RuntimeStub frame                                                           
2150       __ ret(0);                                                                                                                     
2151     }                                                                                                                                
2152 
2153     // Copy in multi-bytes chunks                                                                                                    
2154     copy_bytes_forward(end_from, end_to, qword_count, rax, L_copy_bytes, L_copy_8_bytes);                                            
2155 
2156     if (is_oop) {                                                                                                                    
2157     __ BIND(L_exit);                                                                                                                 
2158       gen_write_ref_array_post_barrier(saved_to, saved_count, rax);                                                                  
2159     }                                                                                                                                
2160     restore_arg_regs();                                                                                                              
2161     if (is_oop) {                                                                                                                    
2162       inc_counter_np(SharedRuntime::_oop_array_copy_ctr); // Update counter after rscratch1 is free                                  
2163     } else {                                                                                                                         
2164       inc_counter_np(SharedRuntime::_jlong_array_copy_ctr); // Update counter after rscratch1 is free                                
2165     }                                                                                                                                
2166     __ vzeroupper();                                                                                                                 
2167     __ xorptr(rax, rax); // return 0                                                                                                 
2168     __ leave(); // required for proper stackwalking of RuntimeStub frame                                                             
2169     __ ret(0);                                                                                                                       
2170 
2171     return start;                                                                                                                    
2172   }                                                                                                                                  
2173 
2174   // Arguments:                                                                                                                      
2175   //   aligned - true => Input and output aligned on a HeapWord boundary == 8 bytes                                                  
2176   //             ignored                                                                                                             
2177   //   is_oop  - true => oop array, so generate store check code                                                                     
2178   //   name    - stub name string                                                                                                    

1896     Label L_copy_bytes, L_copy_8_bytes, L_copy_2_bytes, L_exit;
1897     const Register from        = rdi;  // source array address
1898     const Register to          = rsi;  // destination array address
1899     const Register count       = rdx;  // elements count
1900     const Register dword_count = rcx;
1901     const Register qword_count = count;
1902 
1903     __ enter(); // required for proper stackwalking of RuntimeStub frame
1904     assert_clean_int(c_rarg2, rax);    // Make sure 'count' is clean int.
1905 
1906     if (entry != NULL) {
1907       *entry = __ pc();
1908        // caller can pass a 64-bit byte count here (from Unsafe.copyMemory)
1909       BLOCK_COMMENT("Entry:");
1910     }
1911 
1912     array_overlap_test(nooverlap_target, Address::times_4);
1913     setup_arg_regs(); // from => rdi, to => rsi, count => rdx
1914                       // r9 and r10 may be used to save non-volatile registers
1915 
1916     BarrierSetCodeGen *bs = Universe::heap()->barrier_set()->code_gen();
1917     DecoratorSet decorators = 0;
1918     BasicType type = is_oop ? T_OBJECT : T_INT;
1919     if (dest_uninitialized) {
1920       decorators |= AS_DEST_NOT_INITIALIZED;
1921     }
1922     if (aligned) {
1923       decorators |= ARRAYCOPY_ALIGNED;
1924     }
1925 
1926     // no registers are destroyed by this call
1927     bs->arraycopy_prologue(_masm, decorators, type, from, to, count);
1928 
1929     assert_clean_int(count, rax); // Make sure 'count' is clean int.
1930     // 'from', 'to' and 'count' are now valid
1931     __ movptr(dword_count, count);
1932     __ shrptr(count, 1); // count => qword_count
1933 
1934     // Copy from high to low addresses.  Use 'to' as scratch.
1935 
1936     // Check for and copy trailing dword
1937     __ testl(dword_count, 1);
1938     __ jcc(Assembler::zero, L_copy_bytes);
1939     __ movl(rax, Address(from, dword_count, Address::times_4, -4));
1940     __ movl(Address(to, dword_count, Address::times_4, -4), rax);
1941     __ jmp(L_copy_bytes);
1942 
1943     // Copy trailing qwords
1944   __ BIND(L_copy_8_bytes);
1945     __ movq(rax, Address(from, qword_count, Address::times_8, -8));
1946     __ movq(Address(to, qword_count, Address::times_8, -8), rax);
1947     __ decrement(qword_count);
1948     __ jcc(Assembler::notZero, L_copy_8_bytes);
1949 
1950     if (is_oop) {
1951       __ jmp(L_exit);
1952     }
1953     restore_arg_regs();
1954     inc_counter_np(SharedRuntime::_jint_array_copy_ctr); // Update counter after rscratch1 is free
1955     __ xorptr(rax, rax); // return 0
1956     __ vzeroupper();
1957     __ leave(); // required for proper stackwalking of RuntimeStub frame
1958     __ ret(0);
1959 
1960     // Copy in multi-bytes chunks
1961     copy_bytes_backward(from, to, qword_count, rax, L_copy_bytes, L_copy_8_bytes);
1962 
1963   __ BIND(L_exit);
1964     bs->arraycopy_epilogue(_masm, decorators, type, from, to, dword_count);


1965     restore_arg_regs();
1966     inc_counter_np(SharedRuntime::_jint_array_copy_ctr); // Update counter after rscratch1 is free
1967     __ xorptr(rax, rax); // return 0
1968     __ vzeroupper();
1969     __ leave(); // required for proper stackwalking of RuntimeStub frame
1970     __ ret(0);
1971 
1972     return start;
1973   }
1974 
1975   // Arguments:
1976   //   aligned - true => Input and output aligned on a HeapWord boundary == 8 bytes
1977   //             ignored
1978   //   is_oop  - true => oop array, so generate store check code
1979   //   name    - stub name string
1980   //
1981   // Inputs:
1982   //   c_rarg0   - source array address
1983   //   c_rarg1   - destination array address
1984   //   c_rarg2   - element count, treated as ssize_t, can be zero
1985   //
1986  // Side Effects:
1987   //   disjoint_oop_copy_entry or disjoint_long_copy_entry is set to the
1988   //   no-overlap entry point used by generate_conjoint_long_oop_copy().
1989   //
1990   address generate_disjoint_long_oop_copy(bool aligned, bool is_oop, address *entry,
1991                                           const char *name, bool dest_uninitialized = false) {
1992     __ align(CodeEntryAlignment);
1993     StubCodeMark mark(this, "StubRoutines", name);
1994     address start = __ pc();
1995 
1996     Label L_copy_bytes, L_copy_8_bytes, L_exit;
1997     const Register from        = rdi;  // source array address
1998     const Register to          = rsi;  // destination array address
1999     const Register qword_count = rdx;  // elements count
2000     const Register end_from    = from; // source array end address
2001     const Register end_to      = rcx;  // destination array end address

2002     const Register saved_count = r11;
2003     // End pointers are inclusive, and if count is not zero they point
2004     // to the last unit copied:  end_to[0] := end_from[0]
2005 
2006     __ enter(); // required for proper stackwalking of RuntimeStub frame
2007     // Save no-overlap entry point for generate_conjoint_long_oop_copy()
2008     assert_clean_int(c_rarg2, rax);    // Make sure 'count' is clean int.
2009 
2010     if (entry != NULL) {
2011       *entry = __ pc();
2012       // caller can pass a 64-bit byte count here (from Unsafe.copyMemory)
2013       BLOCK_COMMENT("Entry:");
2014     }
2015 
2016     setup_arg_regs(); // from => rdi, to => rsi, count => rdx
2017                       // r9 and r10 may be used to save non-volatile registers
2018     // 'from', 'to' and 'qword_count' are now valid
2019 
2020     BarrierSetCodeGen *bs = Universe::heap()->barrier_set()->code_gen();
2021     DecoratorSet decorators = ARRAYCOPY_DISJOINT;
2022     BasicType type = is_oop ? T_OBJECT : T_LONG;
2023     if (dest_uninitialized) {
2024       decorators |= AS_DEST_NOT_INITIALIZED;
2025     }
2026     if (aligned) {
2027       decorators |= ARRAYCOPY_ALIGNED;
2028     }
2029 
2030     bs->arraycopy_prologue(_masm, decorators, type, from, to, qword_count);
2031 
2032     // Copy from low to high addresses.  Use 'to' as scratch.
2033     __ lea(end_from, Address(from, qword_count, Address::times_8, -8));
2034     __ lea(end_to,   Address(to,   qword_count, Address::times_8, -8));
2035     __ negptr(qword_count);
2036     __ jmp(L_copy_bytes);
2037 
2038     // Copy trailing qwords
2039   __ BIND(L_copy_8_bytes);
2040     __ movq(rax, Address(end_from, qword_count, Address::times_8, 8));
2041     __ movq(Address(end_to, qword_count, Address::times_8, 8), rax);
2042     __ increment(qword_count);
2043     __ jcc(Assembler::notZero, L_copy_8_bytes);
2044 
2045     if (is_oop) {
2046       __ jmp(L_exit);
2047     } else {
2048       restore_arg_regs();
2049       inc_counter_np(SharedRuntime::_jlong_array_copy_ctr); // Update counter after rscratch1 is free
2050       __ xorptr(rax, rax); // return 0
2051       __ vzeroupper();
2052       __ leave(); // required for proper stackwalking of RuntimeStub frame
2053       __ ret(0);
2054     }
2055 
2056     // Copy in multi-bytes chunks
2057     copy_bytes_forward(end_from, end_to, qword_count, rax, L_copy_bytes, L_copy_8_bytes);
2058 

2059     __ BIND(L_exit);
2060     bs->arraycopy_epilogue(_masm, decorators, type, from, to, qword_count);

2061     restore_arg_regs();
2062     if (is_oop) {
2063       inc_counter_np(SharedRuntime::_oop_array_copy_ctr); // Update counter after rscratch1 is free
2064     } else {
2065       inc_counter_np(SharedRuntime::_jlong_array_copy_ctr); // Update counter after rscratch1 is free
2066     }
2067     __ vzeroupper();
2068     __ xorptr(rax, rax); // return 0
2069     __ leave(); // required for proper stackwalking of RuntimeStub frame
2070     __ ret(0);
2071 
2072     return start;
2073   }
2074 
2075   // Arguments:
2076   //   aligned - true => Input and output aligned on a HeapWord boundary == 8 bytes
2077   //             ignored
2078   //   is_oop  - true => oop array, so generate store check code
2079   //   name    - stub name string

2191 
2192     Label L_copy_bytes, L_copy_8_bytes, L_exit;                                                                                      
2193     const Register from        = rdi;  // source array address                                                                       
2194     const Register to          = rsi;  // destination array address                                                                  
2195     const Register qword_count = rdx;  // elements count                                                                             
2196     const Register saved_count = rcx;                                                                                                
2197 
2198     __ enter(); // required for proper stackwalking of RuntimeStub frame                                                             
2199     assert_clean_int(c_rarg2, rax);    // Make sure 'count' is clean int.                                                            
2200 
2201     if (entry != NULL) {                                                                                                             
2202       *entry = __ pc();                                                                                                              
2203       // caller can pass a 64-bit byte count here (from Unsafe.copyMemory)                                                           
2204       BLOCK_COMMENT("Entry:");                                                                                                       
2205     }                                                                                                                                
2206 
2207     array_overlap_test(nooverlap_target, Address::times_8);                                                                          
2208     setup_arg_regs(); // from => rdi, to => rsi, count => rdx                                                                        
2209                       // r9 and r10 may be used to save non-volatile registers                                                       
2210     // 'from', 'to' and 'qword_count' are now valid                                                                                  
2211     if (is_oop) {                                                                                                                    
2212       // Save to and count for store barrier                                                                                         
2213       __ movptr(saved_count, qword_count);                                                                                           
2214       // No registers are destroyed by this call                                                                                     
2215       gen_write_ref_array_pre_barrier(to, saved_count, dest_uninitialized);                                                          
                                                                                                                                     
2216     }                                                                                                                                
                                                                                                                                     
                                                                                                                                     
                                                                                                                                     
                                                                                                                                     
                                                                                                                                     
2217 
2218     __ jmp(L_copy_bytes);                                                                                                            
2219 
2220     // Copy trailing qwords                                                                                                          
2221   __ BIND(L_copy_8_bytes);                                                                                                           
2222     __ movq(rax, Address(from, qword_count, Address::times_8, -8));                                                                  
2223     __ movq(Address(to, qword_count, Address::times_8, -8), rax);                                                                    
2224     __ decrement(qword_count);                                                                                                       
2225     __ jcc(Assembler::notZero, L_copy_8_bytes);                                                                                      
2226 
2227     if (is_oop) {                                                                                                                    
2228       __ jmp(L_exit);                                                                                                                
2229     } else {                                                                                                                         
2230       restore_arg_regs();                                                                                                            
2231       inc_counter_np(SharedRuntime::_jlong_array_copy_ctr); // Update counter after rscratch1 is free                                
2232       __ xorptr(rax, rax); // return 0                                                                                               
2233       __ vzeroupper();                                                                                                               
2234       __ leave(); // required for proper stackwalking of RuntimeStub frame                                                           
2235       __ ret(0);                                                                                                                     
2236     }                                                                                                                                
2237 
2238     // Copy in multi-bytes chunks                                                                                                    
2239     copy_bytes_backward(from, to, qword_count, rax, L_copy_bytes, L_copy_8_bytes);                                                   
2240 
2241     if (is_oop) {                                                                                                                    
2242     __ BIND(L_exit);                                                                                                                 
2243       gen_write_ref_array_post_barrier(to, saved_count, rax);                                                                        
2244     }                                                                                                                                
2245     restore_arg_regs();                                                                                                              
2246     if (is_oop) {                                                                                                                    
2247       inc_counter_np(SharedRuntime::_oop_array_copy_ctr); // Update counter after rscratch1 is free                                  
2248     } else {                                                                                                                         
2249       inc_counter_np(SharedRuntime::_jlong_array_copy_ctr); // Update counter after rscratch1 is free                                
2250     }                                                                                                                                
2251     __ vzeroupper();                                                                                                                 
2252     __ xorptr(rax, rax); // return 0                                                                                                 
2253     __ leave(); // required for proper stackwalking of RuntimeStub frame                                                             
2254     __ ret(0);                                                                                                                       
2255 
2256     return start;                                                                                                                    
2257   }                                                                                                                                  
2258 
2259 
2260   // Helper for generating a dynamic type check.                                                                                     
2261   // Smashes no registers.                                                                                                           
2262   void generate_type_check(Register sub_klass,                                                                                       
2263                            Register super_check_offset,                                                                              

2092 
2093     Label L_copy_bytes, L_copy_8_bytes, L_exit;
2094     const Register from        = rdi;  // source array address
2095     const Register to          = rsi;  // destination array address
2096     const Register qword_count = rdx;  // elements count
2097     const Register saved_count = rcx;
2098 
2099     __ enter(); // required for proper stackwalking of RuntimeStub frame
2100     assert_clean_int(c_rarg2, rax);    // Make sure 'count' is clean int.
2101 
2102     if (entry != NULL) {
2103       *entry = __ pc();
2104       // caller can pass a 64-bit byte count here (from Unsafe.copyMemory)
2105       BLOCK_COMMENT("Entry:");
2106     }
2107 
2108     array_overlap_test(nooverlap_target, Address::times_8);
2109     setup_arg_regs(); // from => rdi, to => rsi, count => rdx
2110                       // r9 and r10 may be used to save non-volatile registers
2111     // 'from', 'to' and 'qword_count' are now valid
2112 
2113     BarrierSetCodeGen *bs = Universe::heap()->barrier_set()->code_gen();
2114     DecoratorSet decorators = ARRAYCOPY_DISJOINT;
2115     BasicType type = is_oop ? T_OBJECT : T_LONG;
2116     if (dest_uninitialized) {
2117       decorators |= AS_DEST_NOT_INITIALIZED;
2118     }
2119     if (aligned) {
2120       decorators |= ARRAYCOPY_ALIGNED;
2121     }
2122 
2123     bs->arraycopy_prologue(_masm, decorators, type, from, to, qword_count);
2124 
2125     __ jmp(L_copy_bytes);
2126 
2127     // Copy trailing qwords
2128   __ BIND(L_copy_8_bytes);
2129     __ movq(rax, Address(from, qword_count, Address::times_8, -8));
2130     __ movq(Address(to, qword_count, Address::times_8, -8), rax);
2131     __ decrement(qword_count);
2132     __ jcc(Assembler::notZero, L_copy_8_bytes);
2133 
2134     if (is_oop) {
2135       __ jmp(L_exit);
2136     } else {
2137       restore_arg_regs();
2138       inc_counter_np(SharedRuntime::_jlong_array_copy_ctr); // Update counter after rscratch1 is free
2139       __ xorptr(rax, rax); // return 0
2140       __ vzeroupper();
2141       __ leave(); // required for proper stackwalking of RuntimeStub frame
2142       __ ret(0);
2143     }
2144 
2145     // Copy in multi-bytes chunks
2146     copy_bytes_backward(from, to, qword_count, rax, L_copy_bytes, L_copy_8_bytes);
2147 

2148     __ BIND(L_exit);
2149     bs->arraycopy_epilogue(_masm, decorators, type, from, to, qword_count);

2150     restore_arg_regs();
2151     if (is_oop) {
2152       inc_counter_np(SharedRuntime::_oop_array_copy_ctr); // Update counter after rscratch1 is free
2153     } else {
2154       inc_counter_np(SharedRuntime::_jlong_array_copy_ctr); // Update counter after rscratch1 is free
2155     }
2156     __ vzeroupper();
2157     __ xorptr(rax, rax); // return 0
2158     __ leave(); // required for proper stackwalking of RuntimeStub frame
2159     __ ret(0);
2160 
2161     return start;
2162   }
2163 
2164 
2165   // Helper for generating a dynamic type check.
2166   // Smashes no registers.
2167   void generate_type_check(Register sub_klass,
2168                            Register super_check_offset,

2371 #ifdef ASSERT                                                                                                                        
2372     BLOCK_COMMENT("assert consistent ckoff/ckval");                                                                                  
2373     // The ckoff and ckval must be mutually consistent,                                                                              
2374     // even though caller generates both.                                                                                            
2375     { Label L;                                                                                                                       
2376       int sco_offset = in_bytes(Klass::super_check_offset_offset());                                                                 
2377       __ cmpl(ckoff, Address(ckval, sco_offset));                                                                                    
2378       __ jcc(Assembler::equal, L);                                                                                                   
2379       __ stop("super_check_offset inconsistent");                                                                                    
2380       __ bind(L);                                                                                                                    
2381     }                                                                                                                                
2382 #endif //ASSERT                                                                                                                      
2383 
2384     // Loop-invariant addresses.  They are exclusive end pointers.                                                                   
2385     Address end_from_addr(from, length, TIMES_OOP, 0);                                                                               
2386     Address   end_to_addr(to,   length, TIMES_OOP, 0);                                                                               
2387     // Loop-variant addresses.  They assume post-incremented count < 0.                                                              
2388     Address from_element_addr(end_from, count, TIMES_OOP, 0);                                                                        
2389     Address   to_element_addr(end_to,   count, TIMES_OOP, 0);                                                                        
2390 
2391     gen_write_ref_array_pre_barrier(to, count, dest_uninitialized);                                                                  
                                                                                                                                     
                                                                                                                                     
                                                                                                                                     
                                                                                                                                     
                                                                                                                                     
                                                                                                                                     
                                                                                                                                     
2392 
2393     // Copy from low to high addresses, indexed from the end of each array.                                                          
2394     __ lea(end_from, end_from_addr);                                                                                                 
2395     __ lea(end_to,   end_to_addr);                                                                                                   
2396     __ movptr(r14_length, length);        // save a copy of the length                                                               
2397     assert(length == count, "");          // else fix next line:                                                                     
2398     __ negptr(count);                     // negate and test the length                                                              
2399     __ jcc(Assembler::notZero, L_load_element);                                                                                      
2400 
2401     // Empty array:  Nothing to do.                                                                                                  
2402     __ xorptr(rax, rax);                  // return 0 on (trivial) success                                                           
2403     __ jmp(L_done);                                                                                                                  
2404 
2405     // ======== begin loop ========                                                                                                  
2406     // (Loop is rotated; its entry is L_load_element.)                                                                               
2407     // Loop control:                                                                                                                 
2408     //   for (count = -count; count != 0; count++)                                                                                   
2409     // Base pointers src, dst are biased by 8*(count-1),to last element.                                                             
2410     __ align(OptoLoopAlignment);                                                                                                     

2276 #ifdef ASSERT
2277     BLOCK_COMMENT("assert consistent ckoff/ckval");
2278     // The ckoff and ckval must be mutually consistent,
2279     // even though caller generates both.
2280     { Label L;
2281       int sco_offset = in_bytes(Klass::super_check_offset_offset());
2282       __ cmpl(ckoff, Address(ckval, sco_offset));
2283       __ jcc(Assembler::equal, L);
2284       __ stop("super_check_offset inconsistent");
2285       __ bind(L);
2286     }
2287 #endif //ASSERT
2288 
2289     // Loop-invariant addresses.  They are exclusive end pointers.
2290     Address end_from_addr(from, length, TIMES_OOP, 0);
2291     Address   end_to_addr(to,   length, TIMES_OOP, 0);
2292     // Loop-variant addresses.  They assume post-incremented count < 0.
2293     Address from_element_addr(end_from, count, TIMES_OOP, 0);
2294     Address   to_element_addr(end_to,   count, TIMES_OOP, 0);
2295 
2296     BarrierSetCodeGen *bs = Universe::heap()->barrier_set()->code_gen();
2297     DecoratorSet decorators = ARRAYCOPY_CHECKCAST;
2298     BasicType type = T_OBJECT;
2299     if (dest_uninitialized) {
2300       decorators |= AS_DEST_NOT_INITIALIZED;
2301     }
2302 
2303     bs->arraycopy_prologue(_masm, decorators, type, from, to, count);
2304 
2305     // Copy from low to high addresses, indexed from the end of each array.
2306     __ lea(end_from, end_from_addr);
2307     __ lea(end_to,   end_to_addr);
2308     __ movptr(r14_length, length);        // save a copy of the length
2309     assert(length == count, "");          // else fix next line:
2310     __ negptr(count);                     // negate and test the length
2311     __ jcc(Assembler::notZero, L_load_element);
2312 
2313     // Empty array:  Nothing to do.
2314     __ xorptr(rax, rax);                  // return 0 on (trivial) success
2315     __ jmp(L_done);
2316 
2317     // ======== begin loop ========
2318     // (Loop is rotated; its entry is L_load_element.)
2319     // Loop control:
2320     //   for (count = -count; count != 0; count++)
2321     // Base pointers src, dst are biased by 8*(count-1),to last element.
2322     __ align(OptoLoopAlignment);

2424     generate_type_check(r11_klass, ckoff, ckval, L_store_element);                                                                   
2425     // ======== end loop ========                                                                                                    
2426 
2427     // It was a real error; we must depend on the caller to finish the job.                                                          
2428     // Register rdx = -1 * number of *remaining* oops, r14 = *total* oops.                                                           
2429     // Emit GC store barriers for the oops we have copied (r14 + rdx),                                                               
2430     // and report their number to the caller.                                                                                        
2431     assert_different_registers(rax, r14_length, count, to, end_to, rcx, rscratch1);                                                  
2432     Label L_post_barrier;                                                                                                            
2433     __ addptr(r14_length, count);     // K = (original - remaining) oops                                                             
2434     __ movptr(rax, r14_length);       // save the value                                                                              
2435     __ notptr(rax);                   // report (-1^K) to caller (does not affect flags)                                             
2436     __ jccb(Assembler::notZero, L_post_barrier);                                                                                     
2437     __ jmp(L_done); // K == 0, nothing was copied, skip post barrier                                                                 
2438 
2439     // Come here on success only.                                                                                                    
2440     __ BIND(L_do_card_marks);                                                                                                        
2441     __ xorptr(rax, rax);              // return 0 on success                                                                         
2442 
2443     __ BIND(L_post_barrier);                                                                                                         
2444     gen_write_ref_array_post_barrier(to, r14_length, rscratch1);                                                                     
2445 
2446     // Common exit point (success or failure).                                                                                       
2447     __ BIND(L_done);                                                                                                                 
2448     __ movptr(r13, Address(rsp, saved_r13_offset * wordSize));                                                                       
2449     __ movptr(r14, Address(rsp, saved_r14_offset * wordSize));                                                                       
2450     restore_arg_regs();                                                                                                              
2451     inc_counter_np(SharedRuntime::_checkcast_array_copy_ctr); // Update counter after rscratch1 is free                              
2452     __ leave(); // required for proper stackwalking of RuntimeStub frame                                                             
2453     __ ret(0);                                                                                                                       
2454 
2455     return start;                                                                                                                    
2456   }                                                                                                                                  
2457 
2458   //                                                                                                                                 
2459   //  Generate 'unsafe' array copy stub                                                                                              
2460   //  Though just as safe as the other stubs, it takes an unscaled                                                                   
2461   //  size_t argument instead of an element count.                                                                                   
2462   //                                                                                                                                 
2463   //  Input:                                                                                                                         

2336     generate_type_check(r11_klass, ckoff, ckval, L_store_element);
2337     // ======== end loop ========
2338 
2339     // It was a real error; we must depend on the caller to finish the job.
2340     // Register rdx = -1 * number of *remaining* oops, r14 = *total* oops.
2341     // Emit GC store barriers for the oops we have copied (r14 + rdx),
2342     // and report their number to the caller.
2343     assert_different_registers(rax, r14_length, count, to, end_to, rcx, rscratch1);
2344     Label L_post_barrier;
2345     __ addptr(r14_length, count);     // K = (original - remaining) oops
2346     __ movptr(rax, r14_length);       // save the value
2347     __ notptr(rax);                   // report (-1^K) to caller (does not affect flags)
2348     __ jccb(Assembler::notZero, L_post_barrier);
2349     __ jmp(L_done); // K == 0, nothing was copied, skip post barrier
2350 
2351     // Come here on success only.
2352     __ BIND(L_do_card_marks);
2353     __ xorptr(rax, rax);              // return 0 on success
2354 
2355     __ BIND(L_post_barrier);
2356     bs->arraycopy_epilogue(_masm, decorators, type, from, to, r14_length);
2357 
2358     // Common exit point (success or failure).
2359     __ BIND(L_done);
2360     __ movptr(r13, Address(rsp, saved_r13_offset * wordSize));
2361     __ movptr(r14, Address(rsp, saved_r14_offset * wordSize));
2362     restore_arg_regs();
2363     inc_counter_np(SharedRuntime::_checkcast_array_copy_ctr); // Update counter after rscratch1 is free
2364     __ leave(); // required for proper stackwalking of RuntimeStub frame
2365     __ ret(0);
2366 
2367     return start;
2368   }
2369 
2370   //
2371   //  Generate 'unsafe' array copy stub
2372   //  Though just as safe as the other stubs, it takes an unscaled
2373   //  size_t argument instead of an element count.
2374   //
2375   //  Input:
< prev index next >