src/hotspot/share/oops/method.cpp

@@ -495,10 +495,56 @@
   assert(k != NULL && !thread->has_pending_exception(), "can't resolve klass");
   return ValueKlass::cast(k);
 }
 #endif
 
+void Method::check_returning_vt(TRAPS) {
+  ResourceMark rm(THREAD);
+  SignatureStream ss(signature());
+  while (!ss.at_return_type()) {
+    ss.next();
+  }
+  assert(ss.is_object(), "should be called only if the return type is known to be an object");
+
+  Handle class_loader(THREAD, method_holder()->class_loader());
+  Handle protection_domain(THREAD, method_holder()->protection_domain());
+  Symbol* return_class_name = ss.as_symbol(CHECK);
+  Klass* k = SystemDictionary::find(return_class_name, class_loader, protection_domain, CHECK);
+  assert(!THREAD->has_pending_exception(), "can't resolve klass");
+
+  if (k != NULL) {
+    if (k->is_value()) {
+      set_known_returning_vt();
+    } else {
+      set_known_not_returning_vt();
+    }
+  } else {
+    //   set_known_not_returning_vt(); // <<- enable this and RarelyUsedValueUser.doit() will crash.
+    // We are still not sure if the return type is a VT. This could happen
+    // when we call a method whose return type has not yet been resolved:
+    //
+    //   public /*non-VT-aware*/ class Foo {
+    //      SomeType doit() { return null; }
+    //      public static void main(String args[]) {doit(); doit(); doit(); }
+    //   }
+    //
+    // Normally, if SomeType is a VT, it will be included in Foo's "ValueTypes"
+    // attribute, which means SomeType is eagerly resolved when Foo
+    // is loaded.
+    //
+    // However, we might have a Foo class that's not VT-aware, so it doesn't have
+    // a "ValueTypes" attribute, which means SomeType may not be loaded
+    // before Foo.doit() is called.
+    //
+    // Also, we cannot simply assume that SomeType is not a VT, because Foo may
+    // be a hand-written class that elides the "ValueTypes" attribute on purpose.
+    //
+    // This means that in some rare cases we might call check_returning_vt()
+    // repeatedly (3 times in the examples above).
+  }
+}
+
 bool Method::is_empty_method() const {
   return  code_size() == 1
       && *code_base() == Bytecodes::_return;
 }