1 /*
2 * Copyright (c) 2016, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 */
23
24 #include "precompiled.hpp"
25 #include "gc/g1/heapRegion.hpp"
26 #include "gc/shared/gcLocker.hpp"
27 #include "jvmci/compilerRuntime.hpp"
28 #include "jvmci/jvmciRuntime.hpp"
29 #include "oops/method.hpp"
30 #include "aot/aotLoader.hpp"
31 #include "aot/aotCodeHeap.hpp"
32
33 #include "runtime/os.hpp"
34 #include "runtime/sharedRuntime.hpp"
35 #include "runtime/vm_operations.hpp"
36
37 bool AOTCodeHeap::_narrow_oop_shift_initialized = false;
38 int AOTCodeHeap::_narrow_oop_shift = 0;
39 int AOTCodeHeap::_narrow_klass_shift = 0;
40
41 address AOTCodeHeap::load_symbol(const char *name) {
42 address symbol = (address) dlsym(_dl_handle, name);
43 if (symbol == NULL) {
44 tty->print_cr("Shared file %s error: missing %s", _name, name);
45 vm_exit(1);
46 }
47 return symbol;
48 }
49
50 Klass* AOTCodeHeap::get_klass_from_got(const char* klass_name, int klass_len, const Method* method) {
51 AOTKlassData* klass_data = (AOTKlassData*)load_symbol(klass_name);
52 Klass* k = (Klass*)_metaspace_got[klass_data->_got_index];
53 if (k == NULL) {
54 Thread* thread = Thread::current();
55 k = lookup_klass(klass_name, klass_len, method, thread);
56 // Note, exceptions are cleared.
57 if (k == NULL) {
58 fatal("Shared file %s error: klass %s should be resolved already", _name, klass_name);
59 vm_exit(1);
60 }
61 _metaspace_got[klass_data->_got_index] = k;
62 }
63 return k;
64 }
65
66 Klass* AOTCodeHeap::lookup_klass(const char* name, int len, const Method* method, Thread* thread) {
67 ResourceMark rm(thread);
68 assert(method != NULL, "incorrect call parameter");
69 methodHandle caller(thread, (Method*)method);
70
71 // Use class loader of aot method.
72 Handle loader(thread, caller->method_holder()->class_loader());
73 Handle protection_domain(thread, caller->method_holder()->protection_domain());
74
75 // Ignore wrapping L and ;
76 if (name[0] == 'L') {
77 assert(len > 2, "small name %s", name);
78 name++;
79 len -= 2;
80 }
81 TempNewSymbol sym = SymbolTable::probe(name, len);
82 if (sym == NULL) {
83 log_debug(aotclassresolve)("Probe failed for AOT class %s", name);
84 return NULL;
85 }
86 Klass* k = SystemDictionary::find_instance_or_array_klass(sym, loader, protection_domain, thread);
87 assert(!thread->has_pending_exception(), "should not throw");
88
89 if (k != NULL) {
90 log_info(aotclassresolve)("%s %s (lookup)", caller->method_holder()->external_name(), k->external_name());
91 }
92 return k;
93 }
94
95 void AOTCodeHeap::handle_config_error(const char* format, ...) {
96 if (PrintAOT) {
97 va_list ap;
98 va_start(ap, format);
99 tty->vprint_cr(format, ap);
100 va_end(ap);
101 }
102 if (UseAOTStrictLoading) {
103 vm_exit(1);
104 }
105 _valid = false;
106 }
107
108 void AOTCodeHeap::verify_flag(bool aot_flag, bool flag, const char* name) {
109 if (_valid && aot_flag != flag) {
110 handle_config_error("Shared file %s error: %s has different value '%s' from current '%s'", _name, name , (aot_flag ? "true" : "false"), (flag ? "true" : "false"));
111 }
112 }
113
114 void AOTCodeHeap::verify_flag(int aot_flag, int flag, const char* name) {
115 if (_valid && aot_flag != flag) {
116 handle_config_error("Shared file %s error: %s has different value '%d' from current '%d'", _name, name , aot_flag, flag);
117 }
118 }
119
120 void AOTCodeHeap::verify_config(const AOTHeader* header) {
121
122 GrowableArray<AOTCodeHeap*>* heaps = AOTLoader::heaps();
123 for (GrowableArrayIterator<AOTCodeHeap*> heap = heaps->begin(); heap != heaps->end(); ++heap) {
124 if ((*heap)->_config == _config) {
125 handle_config_error("AOT library %s already loaded.", (*heap)->_name);
126 return;
127 }
128 }
129
130 if (header->_version != AOTHeader::AOT_SHARED_VERSION) {
131 handle_config_error("Invalid version of the shared file %s. Expected %d but was %d", _name, header->_version, AOTHeader::AOT_SHARED_VERSION);
132 return;
133 }
134 // Debug VM has different layout of runtime and metadata structures
135 #ifdef ASSERT
136 verify_flag(_config->_debug_VM, true, "Debug VM version");
137 #else
138 verify_flag(!(_config->_debug_VM), true, "Product VM version");
139 #endif
140 // Check configuration size
141 verify_flag(_config->_config_size, AOTConfiguration::CONFIG_SIZE, "AOT configuration size");
142
143 // Check flags
144 verify_flag(_config->_useCompressedOops, UseCompressedOops, "UseCompressedOops");
145 verify_flag(_config->_useCompressedClassPointers, UseCompressedClassPointers, "UseCompressedClassPointers");
146 verify_flag(_config->_useG1GC, UseG1GC, "UseG1GC");
147 verify_flag(_config->_useCMSGC, UseConcMarkSweepGC, "UseConcMarkSweepGC");
148 verify_flag(_config->_useTLAB, UseTLAB, "UseTLAB");
149 verify_flag(_config->_useBiasedLocking, UseBiasedLocking, "UseBiasedLocking");
150 verify_flag(_config->_objectAlignment, ObjectAlignmentInBytes, "ObjectAlignmentInBytes");
151 verify_flag(_config->_contendedPaddingWidth, ContendedPaddingWidth, "ContendedPaddingWidth");
152 verify_flag(_config->_fieldsAllocationStyle, FieldsAllocationStyle, "FieldsAllocationStyle");
153 verify_flag(_config->_compactFields, CompactFields, "CompactFields");
154 verify_flag(_config->_enableContended, EnableContended, "EnableContended");
155 verify_flag(_config->_restrictContended, RestrictContended, "RestrictContended");
156
157 if (!TieredCompilation && _config->_tieredAOT) {
158 handle_config_error("Shared file %s error: Expected to run with tiered compilation on", _name);
159 }
160
161 // Shifts are static values which initialized by 0 until java heap initialization.
162 // AOT libs are loaded before heap initialized so shift values are not set.
163 // It is okay since ObjectAlignmentInBytes flag which defines shifts value is set before AOT libs are loaded.
164 // Set shifts value based on first AOT library config.
165 if (UseCompressedOops && _valid) {
166 if (!_narrow_oop_shift_initialized) {
167 _narrow_oop_shift = _config->_narrowOopShift;
168 if (UseCompressedClassPointers) { // It is set only if UseCompressedOops is set
169 _narrow_klass_shift = _config->_narrowKlassShift;
170 }
171 _narrow_oop_shift_initialized = true;
172 } else {
173 verify_flag(_config->_narrowOopShift, _narrow_oop_shift, "aot_config->_narrowOopShift");
174 if (UseCompressedClassPointers) { // It is set only if UseCompressedOops is set
175 verify_flag(_config->_narrowKlassShift, _narrow_klass_shift, "aot_config->_narrowKlassShift");
176 }
177 }
178 }
179 }
180
181 AOTCodeHeap::~AOTCodeHeap() {
182 free((void*) _name);
183 if (_classes != NULL) {
184 FREE_C_HEAP_ARRAY(AOTClass, _classes);
185 }
186 if (_code_to_aot != NULL) {
187 FREE_C_HEAP_ARRAY(CodeToAMethod, _code_to_aot);
188 }
189 }
190
191 AOTCodeHeap::AOTCodeHeap(void* handle, const char* name, int id) :
192 CodeHeap("CodeHeap 'AOT'", CodeBlobType::AOT), _valid(true), _dl_handle(handle), _dso_id(id), _classes(NULL), _code_to_aot(NULL) {
193 _name = (const char*) strdup(name);
194 _lib_symbols_initialized = false;
195 _aot_id = 0;
196
197 // Verify that VM runs with the same parameters as AOT tool.
198 _config = (AOTConfiguration*) load_symbol("JVM.config");
199 const AOTHeader* header = (AOTHeader*) load_symbol("JVM.header");
200
201 verify_config(header);
202
203 if (!_valid) {
204 if (PrintAOT) {
205 tty->print("%7d ", (int) tty->time_stamp().milliseconds());
206 tty->print_cr("%4d skipped %s aot library", _dso_id, _name);
207 }
208 return;
209 }
210
211 _class_count = header->_class_count;
212 _method_count = header->_method_count;
213
214 // Collect metaspace info: names -> address in .got section
215 _metaspace_names = (const char*) load_symbol("JVM.metaspace.names");
216 _method_metadata = (address) load_symbol("JVM.method.metadata");
217 _methods_offsets = (address) load_symbol("JVM.methods.offsets");
218 _klasses_offsets = (address) load_symbol("JVM.klasses.offsets");
219 _dependencies = (address) load_symbol("JVM.klasses.dependencies");
220 _code_space = (address) load_symbol("JVM.text");
221
222 // First cell is number of elements.
223 jlong* got_sect;
224 _metaspace_got = (Metadata**) load_symbol("JVM.metaspace.got");
225 _metaspace_got_size = header->_metaspace_got_size;
226
227 _metadata_got = (Metadata**) load_symbol("JVM.metadata.got");
228 _metadata_got_size = header->_metadata_got_size;
229
230 _oop_got = (oop*) load_symbol("JVM.oop.got");
231 _oop_got_size = header->_oop_got_size;
232
233 // Collect stubs info
234 _stubs_offsets = (int*) load_symbol("JVM.stubs.offsets");
235
236 // code segments table
237 _code_segments = (address) load_symbol("JVM.code.segments");
238
239 // method state
240 _method_state = (jlong*) load_symbol("JVM.method.state");
241
242 // Create a table for mapping classes
243 _classes = NEW_C_HEAP_ARRAY(AOTClass, _class_count, mtCode);
244 memset(_classes, 0, _class_count * sizeof(AOTClass));
245
246 // Create table for searching AOTCompiledMethod based on pc.
247 _code_to_aot = NEW_C_HEAP_ARRAY(CodeToAMethod, _method_count, mtCode);
248 memset(_code_to_aot, 0, _method_count * sizeof(CodeToAMethod));
249
250 _low_boundary = _code_space;
251 _memory.set_low_boundary((char *)_code_space);
252 _memory.set_high_boundary((char *)_code_space);
253 _memory.set_low((char *)_code_space);
254 _memory.set_high((char *)_code_space);
255
256 _segmap.set_low_boundary((char *)_code_segments);
257 _segmap.set_low((char *)_code_segments);
258
259 _log2_segment_size = exact_log2(_config->_codeSegmentSize);
260
261 // Register aot stubs
262 register_stubs();
263
264 if (PrintAOT || (PrintCompilation && PrintAOT)) {
265 tty->print("%7d ", (int) tty->time_stamp().milliseconds());
266 tty->print_cr("%4d loaded %s aot library", _dso_id, _name);
267 }
268 }
269
270 void AOTCodeHeap::publish_aot(instanceKlassHandle kh, methodHandle mh, AOTMethodData* method_data, int code_id) {
271 // The method may be explicitly excluded by the user.
272 if (CompilerOracle::should_exclude(mh)) {
273 return;
274 }
275
276 address code = method_data->_code;
277 const char* name = method_data->_name;
278 aot_metadata* meta = method_data->_meta;
279
280 if (meta->scopes_pcs_begin() == meta->scopes_pcs_end()) {
281 // When the AOT compiler compiles something big we fail to generate metadata
282 // in CodeInstaller::gather_metadata. In that case the scopes_pcs_begin == scopes_pcs_end.
283 // In all successful cases we always have 2 entries of scope pcs.
284 return;
285 }
286
287 jlong* state_adr = &_method_state[code_id];
288 address metadata_table = method_data->_metadata_table;
289 int metadata_size = method_data->_metadata_size;
290 assert(code_id < _method_count, "sanity");
291 _aot_id++;
292
293 #ifdef ASSERT
294 if (_aot_id > CIStop || _aot_id < CIStart) {
295 // Skip compilation
296 return;
297 }
298 #endif
299 // Check one more time.
300 if (_code_to_aot[code_id]._state == invalid) {
301 return;
302 }
303 AOTCompiledMethod *aot = new AOTCompiledMethod(code, mh(), meta, metadata_table, metadata_size, state_adr, this, name, code_id, _aot_id);
304 assert(_code_to_aot[code_id]._aot == NULL, "should be not initialized");
305 _code_to_aot[code_id]._aot = aot; // Should set this first
306 if (Atomic::cmpxchg(in_use, (jint*)&_code_to_aot[code_id]._state, not_set) != not_set) {
307 _code_to_aot[code_id]._aot = NULL; // Clean
308 } else { // success
309 // Publish method
310 #ifdef TIERED
311 mh->set_aot_code(aot);
312 #endif
313 Method::set_code(mh, aot);
314 if (PrintAOT || (PrintCompilation && PrintAOT)) {
315 aot->print_on(tty, NULL);
316 }
317 // Publish oop only after we are visible to CompiledMethodIterator
318 aot->set_oop(mh()->method_holder()->klass_holder());
319 }
320 }
321
322 void AOTCodeHeap::link_primitive_array_klasses() {
323 ResourceMark rm;
324 for (int i = T_BOOLEAN; i <= T_CONFLICT; i++) {
325 BasicType t = (BasicType)i;
326 if (is_java_primitive(t)) {
327 const Klass* arr_klass = Universe::typeArrayKlassObj(t);
328 AOTKlassData* klass_data = (AOTKlassData*) dlsym(_dl_handle, arr_klass->signature_name());
329 if (klass_data != NULL) {
330 // Set both GOT cells, resolved and initialized klass pointers.
331 // _got_index points to second cell - resolved klass pointer.
332 _metaspace_got[klass_data->_got_index-1] = (Metadata*)arr_klass; // Initialized
333 _metaspace_got[klass_data->_got_index ] = (Metadata*)arr_klass; // Resolved
334 if (PrintAOT) {
335 tty->print_cr("[Found %s in %s]", arr_klass->internal_name(), _name);
336 }
337 }
338 }
339 }
340 }
341
342 void AOTCodeHeap::register_stubs() {
343 int stubs_count = _stubs_offsets[0]; // contains number
344 _stubs_offsets++;
345 AOTMethodOffsets* stub_offsets = (AOTMethodOffsets*)_stubs_offsets;
346 for (int i = 0; i < stubs_count; ++i) {
347 const char* stub_name = _metaspace_names + stub_offsets[i]._name_offset;
348 address entry = _code_space + stub_offsets[i]._code_offset;
349 aot_metadata* meta = (aot_metadata *) (_method_metadata + stub_offsets[i]._meta_offset);
350 address metadata_table = (address)_metadata_got + stub_offsets[i]._metadata_got_offset;
351 int metadata_size = stub_offsets[i]._metadata_got_size;
352 int code_id = stub_offsets[i]._code_id;
353 assert(code_id < _method_count, "sanity");
354 jlong* state_adr = &_method_state[code_id];
355 int len = build_u2_from((address)stub_name);
356 stub_name += 2;
357 char* full_name = NEW_C_HEAP_ARRAY(char, len+5, mtCode);
358 memcpy(full_name, "AOT ", 4);
359 memcpy(full_name+4, stub_name, len);
360 full_name[len+4] = 0;
361 guarantee(_code_to_aot[code_id]._state != invalid, "stub %s can't be invalidated", full_name);
362 AOTCompiledMethod* aot = new AOTCompiledMethod(entry, NULL, meta, metadata_table, metadata_size, state_adr, this, full_name, code_id, i);
363 assert(_code_to_aot[code_id]._aot == NULL, "should be not initialized");
364 _code_to_aot[code_id]._aot = aot;
365 if (Atomic::cmpxchg(in_use, (jint*)&_code_to_aot[code_id]._state, not_set) != not_set) {
366 fatal("stab '%s' code state is %d", full_name, _code_to_aot[code_id]._state);
367 }
368 // Adjust code buffer boundaries only for stubs because they are last in the buffer.
369 adjust_boundaries(aot);
370 if (PrintAOT && Verbose) {
371 aot->print_on(tty, NULL);
372 }
373 }
374 }
375
376 #define SET_AOT_GLOBAL_SYMBOL_VALUE(AOTSYMNAME, AOTSYMTYPE, VMSYMVAL) \
377 { \
378 char* error; \
379 /* Clear any existing error */ \
380 dlerror(); \
381 AOTSYMTYPE * adr = (AOTSYMTYPE *) dlsym(_dl_handle, AOTSYMNAME); \
382 /* Check for any dlsym lookup error */ \
383 error = dlerror(); \
384 guarantee(error == NULL, "%s", error); \
385 *adr = (AOTSYMTYPE) VMSYMVAL; \
386 }
387
388 void AOTCodeHeap::link_graal_runtime_symbols() {
389 SET_AOT_GLOBAL_SYMBOL_VALUE("_aot_jvmci_runtime_monitorenter", address, JVMCIRuntime::monitorenter);
390 SET_AOT_GLOBAL_SYMBOL_VALUE("_aot_jvmci_runtime_monitorexit", address, JVMCIRuntime::monitorexit);
391 SET_AOT_GLOBAL_SYMBOL_VALUE("_aot_jvmci_runtime_log_object", address, JVMCIRuntime::log_object);
392 SET_AOT_GLOBAL_SYMBOL_VALUE("_aot_jvmci_runtime_log_printf", address, JVMCIRuntime::log_printf);
393 SET_AOT_GLOBAL_SYMBOL_VALUE("_aot_jvmci_runtime_log_primitive", address, JVMCIRuntime::log_primitive);
394 SET_AOT_GLOBAL_SYMBOL_VALUE("_aot_jvmci_runtime_new_instance", address, JVMCIRuntime::new_instance);
395 SET_AOT_GLOBAL_SYMBOL_VALUE("_aot_jvmci_runtime_new_array", address, JVMCIRuntime::new_array);
396 SET_AOT_GLOBAL_SYMBOL_VALUE("_aot_jvmci_runtime_new_multi_array", address, JVMCIRuntime::new_multi_array);
397 SET_AOT_GLOBAL_SYMBOL_VALUE("_aot_jvmci_runtime_dynamic_new_array", address, JVMCIRuntime::dynamic_new_array);
398 SET_AOT_GLOBAL_SYMBOL_VALUE("_aot_jvmci_runtime_validate_object", address, JVMCIRuntime::validate_object);
399 SET_AOT_GLOBAL_SYMBOL_VALUE("_aot_jvmci_runtime_write_barrier_pre", address, JVMCIRuntime::write_barrier_pre);
400 SET_AOT_GLOBAL_SYMBOL_VALUE("_aot_jvmci_runtime_identity_hash_code", address, JVMCIRuntime::identity_hash_code);
401 SET_AOT_GLOBAL_SYMBOL_VALUE("_aot_jvmci_runtime_write_barrier_post", address, JVMCIRuntime::write_barrier_post);
402 SET_AOT_GLOBAL_SYMBOL_VALUE("_aot_jvmci_runtime_dynamic_new_instance", address, JVMCIRuntime::dynamic_new_instance);
403 SET_AOT_GLOBAL_SYMBOL_VALUE("_aot_jvmci_runtime_thread_is_interrupted", address, JVMCIRuntime::thread_is_interrupted);
404 SET_AOT_GLOBAL_SYMBOL_VALUE("_aot_jvmci_runtime_exception_handler_for_pc", address, JVMCIRuntime::exception_handler_for_pc);
405 SET_AOT_GLOBAL_SYMBOL_VALUE("_aot_jvmci_runtime_test_deoptimize_call_int", address, JVMCIRuntime::test_deoptimize_call_int);
406 SET_AOT_GLOBAL_SYMBOL_VALUE("_aot_jvmci_runtime_throw_and_post_jvmti_exception", address, JVMCIRuntime::throw_and_post_jvmti_exception);
407 SET_AOT_GLOBAL_SYMBOL_VALUE("_aot_jvmci_runtime_throw_klass_external_name_exception", address, JVMCIRuntime::throw_klass_external_name_exception);
408 SET_AOT_GLOBAL_SYMBOL_VALUE("_aot_jvmci_runtime_throw_class_cast_exception", address, JVMCIRuntime::throw_class_cast_exception);
409 SET_AOT_GLOBAL_SYMBOL_VALUE("_aot_jvmci_runtime_vm_message", address, JVMCIRuntime::vm_message);
410 SET_AOT_GLOBAL_SYMBOL_VALUE("_aot_jvmci_runtime_vm_error", address, JVMCIRuntime::vm_error);
411 }
412
413 void AOTCodeHeap::link_shared_runtime_symbols() {
414 SET_AOT_GLOBAL_SYMBOL_VALUE("_resolve_static_entry", address, SharedRuntime::get_resolve_static_call_stub());
415 SET_AOT_GLOBAL_SYMBOL_VALUE("_resolve_virtual_entry", address, SharedRuntime::get_resolve_virtual_call_stub());
416 SET_AOT_GLOBAL_SYMBOL_VALUE("_resolve_opt_virtual_entry", address, SharedRuntime::get_resolve_opt_virtual_call_stub());
417 SET_AOT_GLOBAL_SYMBOL_VALUE("_aot_deopt_blob_unpack", address, SharedRuntime::deopt_blob()->unpack());
418 SET_AOT_GLOBAL_SYMBOL_VALUE("_aot_deopt_blob_uncommon_trap", address, SharedRuntime::deopt_blob()->uncommon_trap());
419 SET_AOT_GLOBAL_SYMBOL_VALUE("_aot_ic_miss_stub", address, SharedRuntime::get_ic_miss_stub());
420 SET_AOT_GLOBAL_SYMBOL_VALUE("_aot_handle_wrong_method_stub", address, SharedRuntime::get_handle_wrong_method_stub());
421 SET_AOT_GLOBAL_SYMBOL_VALUE("_aot_exception_handler_for_return_address", address, SharedRuntime::exception_handler_for_return_address);
422 SET_AOT_GLOBAL_SYMBOL_VALUE("_aot_register_finalizer", address, SharedRuntime::register_finalizer);
423 SET_AOT_GLOBAL_SYMBOL_VALUE("_aot_OSR_migration_end", address, SharedRuntime::OSR_migration_end);
424 SET_AOT_GLOBAL_SYMBOL_VALUE("_aot_resolve_string_by_symbol", address, CompilerRuntime::resolve_string_by_symbol);
425 SET_AOT_GLOBAL_SYMBOL_VALUE("_aot_resolve_klass_by_symbol", address, CompilerRuntime::resolve_klass_by_symbol);
426 SET_AOT_GLOBAL_SYMBOL_VALUE("_aot_resolve_method_by_symbol_and_load_counters", address, CompilerRuntime::resolve_method_by_symbol_and_load_counters);
427 SET_AOT_GLOBAL_SYMBOL_VALUE("_aot_initialize_klass_by_symbol", address, CompilerRuntime::initialize_klass_by_symbol);
428 SET_AOT_GLOBAL_SYMBOL_VALUE("_aot_invocation_event", address, CompilerRuntime::invocation_event);
429 SET_AOT_GLOBAL_SYMBOL_VALUE("_aot_backedge_event", address, CompilerRuntime::backedge_event);
430
431 SET_AOT_GLOBAL_SYMBOL_VALUE("_aot_shared_runtime_dpow", address, SharedRuntime::dpow);
432 SET_AOT_GLOBAL_SYMBOL_VALUE("_aot_shared_runtime_dexp", address, SharedRuntime::dexp);
433 SET_AOT_GLOBAL_SYMBOL_VALUE("_aot_shared_runtime_dcos", address, SharedRuntime::dcos);
434 SET_AOT_GLOBAL_SYMBOL_VALUE("_aot_shared_runtime_dsin", address, SharedRuntime::dsin);
435 SET_AOT_GLOBAL_SYMBOL_VALUE("_aot_shared_runtime_dtan", address, SharedRuntime::dtan);
436 SET_AOT_GLOBAL_SYMBOL_VALUE("_aot_shared_runtime_dlog", address, SharedRuntime::dlog);
437 SET_AOT_GLOBAL_SYMBOL_VALUE("_aot_shared_runtime_dlog10", address, SharedRuntime::dlog10);
438 }
439
440 void AOTCodeHeap::link_stub_routines_symbols() {
441 SET_AOT_GLOBAL_SYMBOL_VALUE("_aot_stub_routines_jbyte_arraycopy", address, StubRoutines::_jbyte_arraycopy);
442 SET_AOT_GLOBAL_SYMBOL_VALUE("_aot_stub_routines_jshort_arraycopy", address, StubRoutines::_jshort_arraycopy);
443 SET_AOT_GLOBAL_SYMBOL_VALUE("_aot_stub_routines_jint_arraycopy", address, StubRoutines::_jint_arraycopy);
444 SET_AOT_GLOBAL_SYMBOL_VALUE("_aot_stub_routines_jlong_arraycopy", address, StubRoutines::_jlong_arraycopy);
445 SET_AOT_GLOBAL_SYMBOL_VALUE("_aot_stub_routines_oop_arraycopy", address, StubRoutines::_oop_arraycopy);
446 SET_AOT_GLOBAL_SYMBOL_VALUE("_aot_stub_routines_oop_arraycopy_uninit", address, StubRoutines::_oop_arraycopy_uninit);
447
448 SET_AOT_GLOBAL_SYMBOL_VALUE("_aot_stub_routines_jbyte_disjoint_arraycopy", address, StubRoutines::_jbyte_disjoint_arraycopy);
449 SET_AOT_GLOBAL_SYMBOL_VALUE("_aot_stub_routines_jshort_disjoint_arraycopy", address, StubRoutines::_jshort_disjoint_arraycopy);
450 SET_AOT_GLOBAL_SYMBOL_VALUE("_aot_stub_routines_jint_disjoint_arraycopy", address, StubRoutines::_jint_disjoint_arraycopy);
451 SET_AOT_GLOBAL_SYMBOL_VALUE("_aot_stub_routines_jlong_disjoint_arraycopy", address, StubRoutines::_jlong_disjoint_arraycopy);
452 SET_AOT_GLOBAL_SYMBOL_VALUE("_aot_stub_routines_oop_disjoint_arraycopy", address, StubRoutines::_oop_disjoint_arraycopy);
453 SET_AOT_GLOBAL_SYMBOL_VALUE("_aot_stub_routines_oop_disjoint_arraycopy_uninit", address, StubRoutines::_oop_disjoint_arraycopy_uninit);
454
455 SET_AOT_GLOBAL_SYMBOL_VALUE("_aot_stub_routines_arrayof_jbyte_arraycopy", address, StubRoutines::_arrayof_jbyte_arraycopy);
456 SET_AOT_GLOBAL_SYMBOL_VALUE("_aot_stub_routines_arrayof_jshort_arraycopy", address, StubRoutines::_arrayof_jshort_arraycopy);
457 SET_AOT_GLOBAL_SYMBOL_VALUE("_aot_stub_routines_arrayof_jint_arraycopy", address, StubRoutines::_arrayof_jint_arraycopy);
458 SET_AOT_GLOBAL_SYMBOL_VALUE("_aot_stub_routines_arrayof_jlong_arraycopy", address, StubRoutines::_arrayof_jlong_arraycopy);
459 SET_AOT_GLOBAL_SYMBOL_VALUE("_aot_stub_routines_arrayof_oop_arraycopy", address, StubRoutines::_arrayof_oop_arraycopy);
460 SET_AOT_GLOBAL_SYMBOL_VALUE("_aot_stub_routines_arrayof_oop_arraycopy_uninit", address, StubRoutines::_arrayof_oop_arraycopy_uninit);
461
462 SET_AOT_GLOBAL_SYMBOL_VALUE("_aot_stub_routines_arrayof_jbyte_disjoint_arraycopy", address, StubRoutines::_arrayof_jbyte_disjoint_arraycopy);
463 SET_AOT_GLOBAL_SYMBOL_VALUE("_aot_stub_routines_arrayof_jshort_disjoint_arraycopy", address, StubRoutines::_arrayof_jshort_disjoint_arraycopy);
464 SET_AOT_GLOBAL_SYMBOL_VALUE("_aot_stub_routines_arrayof_jint_disjoint_arraycopy", address, StubRoutines::_arrayof_jint_disjoint_arraycopy);
465 SET_AOT_GLOBAL_SYMBOL_VALUE("_aot_stub_routines_arrayof_jlong_disjoint_arraycopy", address, StubRoutines::_arrayof_jlong_disjoint_arraycopy);
466 SET_AOT_GLOBAL_SYMBOL_VALUE("_aot_stub_routines_arrayof_oop_disjoint_arraycopy", address, StubRoutines::_arrayof_oop_disjoint_arraycopy);
467 SET_AOT_GLOBAL_SYMBOL_VALUE("_aot_stub_routines_arrayof_oop_disjoint_arraycopy_uninit", address, StubRoutines::_arrayof_oop_disjoint_arraycopy_uninit);
468
469 SET_AOT_GLOBAL_SYMBOL_VALUE("_aot_stub_routines_checkcast_arraycopy", address, StubRoutines::_checkcast_arraycopy);
470
471 SET_AOT_GLOBAL_SYMBOL_VALUE("_aot_stub_routines_aescrypt_encryptBlock", address, StubRoutines::_aescrypt_encryptBlock);
472 SET_AOT_GLOBAL_SYMBOL_VALUE("_aot_stub_routines_aescrypt_decryptBlock", address, StubRoutines::_aescrypt_decryptBlock);
473 SET_AOT_GLOBAL_SYMBOL_VALUE("_aot_stub_routines_cipherBlockChaining_encryptAESCrypt", address, StubRoutines::_cipherBlockChaining_encryptAESCrypt);
474 SET_AOT_GLOBAL_SYMBOL_VALUE("_aot_stub_routines_cipherBlockChaining_decryptAESCrypt", address, StubRoutines::_cipherBlockChaining_decryptAESCrypt);
475 SET_AOT_GLOBAL_SYMBOL_VALUE("_aot_stub_routines_update_bytes_crc32", address, StubRoutines::_updateBytesCRC32);
476 SET_AOT_GLOBAL_SYMBOL_VALUE("_aot_stub_routines_crc_table_adr", address, StubRoutines::_crc_table_adr);
477 }
478
479 void AOTCodeHeap::link_os_symbols() {
480 SET_AOT_GLOBAL_SYMBOL_VALUE("_aot_os_javaTimeMillis", address, os::javaTimeMillis);
481 SET_AOT_GLOBAL_SYMBOL_VALUE("_aot_os_javaTimeNanos", address, os::javaTimeNanos);
482 }
483
484 /*
485 * Link any global symbols in precompiled DSO with dlopen() _dl_handle
486 * dso_handle.
487 */
488
489 void AOTCodeHeap::link_global_lib_symbols() {
490 if (!_lib_symbols_initialized) {
491 _lib_symbols_initialized = true;
492
493 CollectedHeap* heap = Universe::heap();
494 CardTableModRefBS* ct = (CardTableModRefBS*)(heap->barrier_set());
495 SET_AOT_GLOBAL_SYMBOL_VALUE("_aot_card_table_address", address, ct->byte_map_base);
496 SET_AOT_GLOBAL_SYMBOL_VALUE("_aot_heap_top_address", address, (heap->supports_inline_contig_alloc() ? heap->top_addr() : NULL));
497 SET_AOT_GLOBAL_SYMBOL_VALUE("_aot_heap_end_address", address, (heap->supports_inline_contig_alloc() ? heap->end_addr() : NULL));
498 SET_AOT_GLOBAL_SYMBOL_VALUE("_aot_polling_page", address, os::get_polling_page());
499 SET_AOT_GLOBAL_SYMBOL_VALUE("_aot_narrow_klass_base_address", address, Universe::narrow_klass_base());
500 SET_AOT_GLOBAL_SYMBOL_VALUE("_aot_log_of_heap_region_grain_bytes", int, HeapRegion::LogOfHRGrainBytes);
501
502 link_shared_runtime_symbols();
503 link_stub_routines_symbols();
504 link_os_symbols();
505 link_graal_runtime_symbols();
506
507 // Link primitive array klasses.
508 link_primitive_array_klasses();
509 }
510 }
511
512 #ifndef PRODUCT
513 int AOTCodeHeap::klasses_seen = 0;
514 int AOTCodeHeap::aot_klasses_found = 0;
515 int AOTCodeHeap::aot_klasses_fp_miss = 0;
516 int AOTCodeHeap::aot_klasses_cl_miss = 0;
517 int AOTCodeHeap::aot_methods_found = 0;
518
519 void AOTCodeHeap::print_statistics() {
520 tty->print_cr("Classes seen: %d AOT classes found: %d AOT methods found: %d", klasses_seen, aot_klasses_found, aot_methods_found);
521 tty->print_cr("AOT fingerprint mismatches: %d AOT class loader mismatches: %d", aot_klasses_fp_miss, aot_klasses_cl_miss);
522 }
523 #endif
524
525 Method* AOTCodeHeap::find_method(KlassHandle klass, Thread* thread, const char* method_name) {
526 int method_name_len = build_u2_from((address)method_name);
527 method_name += 2;
528 const char* signature_name = method_name + method_name_len;
529 int signature_name_len = build_u2_from((address)signature_name);
530 signature_name += 2;
531 // The class should have been loaded so the method and signature should already be
532 // in the symbol table. If they're not there, the method doesn't exist.
533 TempNewSymbol name = SymbolTable::probe(method_name, method_name_len);
534 TempNewSymbol signature = SymbolTable::probe(signature_name, signature_name_len);
535
536 Method* m;
537 if (name == NULL || signature == NULL) {
538 m = NULL;
539 } else if (name == vmSymbols::object_initializer_name() ||
540 name == vmSymbols::class_initializer_name()) {
541 // Never search superclasses for constructors
542 if (klass->is_instance_klass()) {
543 m = InstanceKlass::cast(klass())->find_method(name, signature);
544 } else {
545 m = NULL;
546 }
547 } else {
548 m = klass->lookup_method(name, signature);
549 if (m == NULL && klass->is_instance_klass()) {
550 m = InstanceKlass::cast(klass())->lookup_method_in_ordered_interfaces(name, signature);
551 }
552 }
553 if (m == NULL) {
554 // Fatal error because we assume classes and methods should not be changed since aot compilation.
555 const char* klass_name = klass->external_name();
556 int klass_len = (int)strlen(klass_name);
557 char* meta_name = NEW_RESOURCE_ARRAY(char, klass_len + 1 + method_name_len + signature_name_len + 1);
558 memcpy(meta_name, klass_name, klass_len);
559 meta_name[klass_len] = '.';
560 memcpy(&meta_name[klass_len + 1], method_name, method_name_len);
561 memcpy(&meta_name[klass_len + 1 + method_name_len], signature_name, signature_name_len);
562 meta_name[klass_len + 1 + method_name_len + signature_name_len] = '\0';
563 Handle exception = Exceptions::new_exception(thread, vmSymbols::java_lang_NoSuchMethodError(), meta_name);
564 java_lang_Throwable::print(exception, tty);
565 tty->cr();
566 java_lang_Throwable::print_stack_trace(exception(), tty);
567 tty->cr();
568 fatal("Failed to find method '%s'", meta_name);
569 }
570 NOT_PRODUCT( aot_methods_found++; )
571 return m;
572 }
573
574 AOTKlassData* AOTCodeHeap::find_klass(InstanceKlass* ik) {
575 ResourceMark rm;
576 AOTKlassData* klass_data = (AOTKlassData*) dlsym(_dl_handle, ik->signature_name());
577 return klass_data;
578 }
579
580 bool AOTCodeHeap::is_dependent_method(Klass* dependee, AOTCompiledMethod* aot) {
581 InstanceKlass *dependee_ik = InstanceKlass::cast(dependee);
582 AOTKlassData* klass_data = find_klass(dependee_ik);
583 if (klass_data == NULL) {
584 return false; // no AOT records for this class - no dependencies
585 }
586 if (!dependee_ik->has_passed_fingerprint_check()) {
587 return false; // different class
588 }
589
590 int methods_offset = klass_data->_dependent_methods_offset;
591 if (methods_offset >= 0) {
592 address methods_cnt_adr = _dependencies + methods_offset;
593 int methods_cnt = *(int*)methods_cnt_adr;
594 int* indexes = (int*)(methods_cnt_adr + 4);
595 for (int i = 0; i < methods_cnt; ++i) {
596 int code_id = indexes[i];
597 if (_code_to_aot[code_id]._aot == aot) {
598 return true; // found dependent method
599 }
600 }
601 }
602 return false;
603 }
604
605 void AOTCodeHeap::sweep_dependent_methods(AOTKlassData* klass_data) {
606 // Make dependent methods non_entrant forever.
607 int methods_offset = klass_data->_dependent_methods_offset;
608 if (methods_offset >= 0) {
609 int marked = 0;
610 address methods_cnt_adr = _dependencies + methods_offset;
611 int methods_cnt = *(int*)methods_cnt_adr;
612 int* indexes = (int*)(methods_cnt_adr + 4);
613 for (int i = 0; i < methods_cnt; ++i) {
614 int code_id = indexes[i];
615 // Invalidate aot code.
616 if (Atomic::cmpxchg(invalid, (jint*)&_code_to_aot[code_id]._state, not_set) != not_set) {
617 if (_code_to_aot[code_id]._state == in_use) {
618 AOTCompiledMethod* aot = _code_to_aot[code_id]._aot;
619 assert(aot != NULL, "aot should be set");
620 if (!aot->is_runtime_stub()) { // Something is wrong - should not invalidate stubs.
621 aot->mark_for_deoptimization(false);
622 marked++;
623 }
624 }
625 }
626 }
627 if (marked > 0) {
628 VM_Deoptimize op;
629 VMThread::execute(&op);
630 }
631 }
632 }
633
634 bool AOTCodeHeap::load_klass_data(instanceKlassHandle kh, Thread* thread) {
635 ResourceMark rm;
636
637 NOT_PRODUCT( klasses_seen++; )
638
639 AOTKlassData* klass_data = find_klass(kh());
640 if (klass_data == NULL) {
641 return false;
642 }
643
644 if (!kh->has_passed_fingerprint_check()) {
645 NOT_PRODUCT( aot_klasses_fp_miss++; )
646 log_trace(aotclassfingerprint)("class %s%s has bad fingerprint in %s tid=" INTPTR_FORMAT,
647 kh->internal_name(), kh->is_shared() ? " (shared)" : "",
648 _name, p2i(thread));
649 sweep_dependent_methods(klass_data);
650 return false;
651 }
652
653 assert(klass_data->_class_id < _class_count, "invalid class id");
654 AOTClass* aot_class = &_classes[klass_data->_class_id];
655 if (aot_class->_classloader != NULL && aot_class->_classloader != kh->class_loader_data()) {
656 log_trace(aotclassload)("class %s in %s already loaded for classloader %p vs %p tid=" INTPTR_FORMAT,
657 kh->internal_name(), _name, aot_class->_classloader, kh->class_loader_data(), p2i(thread));
658 NOT_PRODUCT( aot_klasses_cl_miss++; )
659 return false;
660 }
661
662 NOT_PRODUCT( aot_klasses_found++; )
663
664 log_trace(aotclassload)("found %s in %s for classloader %p tid=" INTPTR_FORMAT, kh->internal_name(), _name, kh->class_loader_data(), p2i(thread));
665
666 aot_class->_classloader = kh->class_loader_data();
667 // Set klass's Resolve (second) got cell.
668 _metaspace_got[klass_data->_got_index] = kh();
669
670 // Initialize global symbols of the DSO to the correspondingVM symbol values.
671 link_global_lib_symbols();
672
673 int methods_offset = klass_data->_compiled_methods_offset;
674 if (methods_offset >= 0) {
675 address methods_cnt_adr = _methods_offsets + methods_offset;
676 int methods_cnt = *(int*)methods_cnt_adr;
677 // Collect data about compiled methods
678 AOTMethodData* methods_data = NEW_RESOURCE_ARRAY(AOTMethodData, methods_cnt);
679 AOTMethodOffsets* methods_offsets = (AOTMethodOffsets*)(methods_cnt_adr + 4);
680 for (int i = 0; i < methods_cnt; ++i) {
681 AOTMethodOffsets* method_offsets = &methods_offsets[i];
682 int code_id = method_offsets->_code_id;
683 if (_code_to_aot[code_id]._state == invalid) {
684 continue; // skip AOT methods slots which have been invalidated
685 }
686 AOTMethodData* method_data = &methods_data[i];
687 const char* aot_name = _metaspace_names + method_offsets->_name_offset;
688 method_data->_name = aot_name;
689 method_data->_code = _code_space + method_offsets->_code_offset;
690 method_data->_meta = (aot_metadata*)(_method_metadata + method_offsets->_meta_offset);
691 method_data->_metadata_table = (address)_metadata_got + method_offsets->_metadata_got_offset;
692 method_data->_metadata_size = method_offsets->_metadata_got_size;
693 // aot_name format: "<u2_size>Ljava/lang/ThreadGroup;<u2_size>addUnstarted<u2_size>()V"
694 int klass_len = build_u2_from((address)aot_name);
695 const char* method_name = aot_name + 2 + klass_len;
696 Method* m = AOTCodeHeap::find_method(kh, thread, method_name);
697 methodHandle mh(thread, m);
698 if (mh->code() != NULL) { // Does it have already compiled code?
699 continue; // Don't overwrite
700 }
701 publish_aot(kh, mh, method_data, code_id);
702 }
703 }
704 return true;
705 }
706
707 AOTCompiledMethod* AOTCodeHeap::next_in_use_at(int start) const {
708 for (int index = start; index < _method_count; index++) {
709 if (_code_to_aot[index]._state != in_use) {
710 continue; // Skip uninitialized entries.
711 }
712 AOTCompiledMethod* aot = _code_to_aot[index]._aot;
713 return aot;
714 }
715 return NULL;
716 }
717
718 void* AOTCodeHeap::first() const {
719 return next_in_use_at(0);
720 }
721
722 void* AOTCodeHeap::next(void* p) const {
723 AOTCompiledMethod *aot = (AOTCompiledMethod *)p;
724 int next_index = aot->method_index() + 1;
725 assert(next_index <= _method_count, "");
726 if (next_index == _method_count) {
727 return NULL;
728 }
729 return next_in_use_at(next_index);
730 }
731
732 void* AOTCodeHeap::find_start(void* p) const {
733 if (!contains(p)) {
734 return NULL;
735 }
736 size_t offset = pointer_delta(p, low_boundary(), 1);
737 // Use segments table
738 size_t seg_idx = offset / _config->_codeSegmentSize;
739 if ((int)(_code_segments[seg_idx]) == 0xff) {
740 return NULL;
741 }
742 while (_code_segments[seg_idx] > 0) {
743 seg_idx -= (int)_code_segments[seg_idx];
744 }
745 int code_offset = (int)seg_idx * _config->_codeSegmentSize;
746 int aot_index = *(int*)(_code_space + code_offset);
747 AOTCompiledMethod* aot = _code_to_aot[aot_index]._aot;
748 assert(aot != NULL, "should find registered aot method");
749 return aot;
750 }
751
752 AOTCompiledMethod* AOTCodeHeap::find_aot(address p) const {
753 assert(contains(p), "should be here");
754 return (AOTCompiledMethod *)find_start(p);
755 }
756
757 CodeBlob* AOTCodeHeap::find_blob_unsafe(void* start) const {
758 return (CodeBlob*)AOTCodeHeap::find_start(start);
759 }
760
761 void AOTCodeHeap::oops_do(OopClosure* f) {
762 for (int i = 0; i < _oop_got_size; i++) {
763 oop* p = &_oop_got[i];
764 if (*p == NULL) continue; // skip non-oops
765 f->do_oop(p);
766 }
767 for (int index = 0; index < _method_count; index++) {
768 if (_code_to_aot[index]._state != in_use) {
769 continue; // Skip uninitialized entries.
770 }
771 AOTCompiledMethod* aot = _code_to_aot[index]._aot;
772 aot->do_oops(f);
773 }
774 }
775
776 // Yes, this is faster than going through the relocations,
777 // but there are two problems:
778 // 1) GOT slots are sometimes patched with non-Metadata values
779 // 2) We don't want to scan metadata for dead methods
780 // Unfortunately we don't know if the metadata belongs to
781 // live aot methods or not, so process them all. If this
782 // is for mark_on_stack, some old methods may stick around
783 // forever instead of getting cleaned up.
784 void AOTCodeHeap::got_metadata_do(void f(Metadata*)) {
785 for (int i = 1; i < _metaspace_got_size; i++) {
786 Metadata** p = &_metaspace_got[i];
787 Metadata* md = *p;
788 if (md == NULL) continue; // skip non-oops
789 intptr_t meta = (intptr_t)md;
790 if (meta == -1) continue; // skip non-oops
791 if (Metaspace::contains(md)) {
792 f(md);
793 }
794 }
795 for (int i = 1; i < _metadata_got_size; i++) {
796 Metadata** p = &_metadata_got[i];
797 Metadata* md = *p;
798 intptr_t meta = (intptr_t)md;
799 if ((meta & 1) == 1) {
800 // already resolved
801 md = (Metadata*)(meta & ~1);
802 } else {
803 continue;
804 }
805 if (md == NULL) continue; // skip non-oops
806 if (Metaspace::contains(md)) {
807 f(md);
808 }
809 }
810 }
811
812 void AOTCodeHeap::cleanup_inline_caches() {
813 for (int index = 0; index < _method_count; index++) {
814 if (_code_to_aot[index]._state != in_use) {
815 continue; // Skip uninitialized entries.
816 }
817 AOTCompiledMethod* aot = _code_to_aot[index]._aot;
818 aot->cleanup_inline_caches();
819 }
820 }
821
822 #ifdef ASSERT
823 int AOTCodeHeap::verify_icholder_relocations() {
824 int count = 0;
825 for (int index = 0; index < _method_count; index++) {
826 if (_code_to_aot[index]._state != in_use) {
827 continue; // Skip uninitialized entries.
828 }
829 AOTCompiledMethod* aot = _code_to_aot[index]._aot;
830 count += aot->verify_icholder_relocations();
831 }
832 return count;
833 }
834 #endif
835
836 void AOTCodeHeap::flush_evol_dependents_on(instanceKlassHandle dependee) {
837 for (int index = 0; index < _method_count; index++) {
838 if (_code_to_aot[index]._state != in_use) {
839 continue; // Skip uninitialized entries.
840 }
841 AOTCompiledMethod* aot = _code_to_aot[index]._aot;
842 aot->flush_evol_dependents_on(dependee);
843 }
844 }
845
846 void AOTCodeHeap::metadata_do(void f(Metadata*)) {
847 for (int index = 0; index < _method_count; index++) {
848 if (_code_to_aot[index]._state != in_use) {
849 continue; // Skip uninitialized entries.
850 }
851 AOTCompiledMethod* aot = _code_to_aot[index]._aot;
852 if (aot->_is_alive()) {
853 aot->metadata_do(f);
854 }
855 }
856 #if 0
857 // With the marking above, this call doesn't seem to be needed
858 got_metadata_do(f);
859 #endif
860 }