--- old/make/test/JtregNativeHotspot.gmk 2019-06-28 14:09:28.163040063 -0700 +++ new/make/test/JtregNativeHotspot.gmk 2019-06-28 14:09:27.799035055 -0700 @@ -862,12 +862,13 @@ BUILD_HOTSPOT_JTREG_LIBRARIES_LIBS_libredefineClasses := -lpthread BUILD_HOTSPOT_JTREG_EXECUTABLES_LIBS_exeinvoke := -ljvm -lpthread BUILD_HOTSPOT_JTREG_EXECUTABLES_LIBS_exestack-gap := -ljvm -lpthread + BUILD_HOTSPOT_JTREG_EXECUTABLES_LIBS_exestack-tls := -ljvm BUILD_TEST_exeinvoke_exeinvoke.c_OPTIMIZATION := NONE BUILD_HOTSPOT_JTREG_EXECUTABLES_LIBS_exeFPRegs := -ldl BUILD_HOTSPOT_JTREG_LIBRARIES_LIBS_libAsyncGetCallTraceTest := -ldl else BUILD_HOTSPOT_JTREG_EXCLUDE += libtest-rw.c libtest-rwx.c libTestJNI.c \ - exeinvoke.c exestack-gap.c libAsyncGetCallTraceTest.cpp + exeinvoke.c exestack-gap.c exestack-tls.c libAsyncGetCallTraceTest.cpp endif BUILD_HOTSPOT_JTREG_EXECUTABLES_LIBS_exesigtest := -ljvm --- old/src/hotspot/os/linux/globals_linux.hpp 2019-06-28 14:09:29.147053600 -0700 +++ new/src/hotspot/os/linux/globals_linux.hpp 2019-06-28 14:09:28.787048648 -0700 @@ -64,9 +64,13 @@ \ product(bool, PreferContainerQuotaForCPUCount, true, \ "Calculate the container CPU availability based on the value" \ - " of quotas (if set), when true. Otherwise, use the CPU" \ + " of quotas (if set), when true. Otherwise, use the CPU" \ " shares value, provided it is less than quota.") \ \ + product(bool, AdjustStackSizeForTLS, false, \ + "Increase the thread stack size to include space for glibc " \ + "static thread-local storage (TLS) if true") \ + \ diagnostic(bool, DumpPrivateMappingsInCore, true, \ "If true, sets bit 2 of /proc/PID/coredump_filter, thus " \ "resulting in file-backed private mappings of the process to "\ --- old/src/hotspot/os/linux/os_linux.cpp 2019-06-28 14:09:30.151067412 -0700 +++ new/src/hotspot/os/linux/os_linux.cpp 2019-06-28 14:09:29.779062295 -0700 @@ -800,6 +800,71 @@ return 0; } +// On Linux, glibc places static TLS blocks (for __thread variables) on +// the thread stack. This decreases the stack size actually available +// to threads. +// +// For large static TLS sizes, this may cause threads to malfunction due +// to insufficient stack space. This is a well-known issue in glibc: +// http://sourceware.org/bugzilla/show_bug.cgi?id=11787. +// +// As a workaround, we call a private but assumed-stable glibc function, +// __pthread_get_minstack() to obtain the minstack size and derive the +// static TLS size from it. We then increase the user requested stack +// size by this TLS size. +// +// Due to compatibility concerns, this size adjustment is opt-in and +// controlled via AdjustStackSizeForTLS. +typedef size_t (*GetMinStack)(const pthread_attr_t *attr); + +GetMinStack _get_minstack_func = NULL; + +static void get_minstack_init() { + _get_minstack_func = + (GetMinStack)dlsym(RTLD_DEFAULT, "__pthread_get_minstack"); + log_info(os, thread)("Lookup of __pthread_get_minstack %s", + _get_minstack_func == NULL ? "failed" : "succeeded"); +} + +// Returns the size of the static TLS area glibc puts on thread stacks. +static size_t get_static_tls_area_size(const pthread_attr_t *attr) { + size_t tls_size = 0; + if (_get_minstack_func != NULL) { + // Obtain the pthread minstack size by calling __pthread_get_minstack. + size_t minstack_size = _get_minstack_func(attr); + + // Remove non-TLS area size included in minstack size returned + // by __pthread_get_minstack() to get the static TLS size. + // In glibc before 2.27, minstack size includes guard_size. + // In glibc 2.27 and later, guard_size is automatically added + // to the stack size by pthread_create and is no longer included + // in minstack size. In both cases, the guard_size is taken into + // account, so there is no need to adjust the result for that. + // + // Although __pthread_get_minstack() is a private glibc function, + // it is expected to have a stable behavior across future glibc + // versions while glibc still allocats the static TLS blocks off + // the stack. Following is glibc 2.28 __pthread_get_minstack(): + // + // size_t + // __pthread_get_minstack (const pthread_attr_t *attr) + // { + // return GLRO(dl_pagesize) + __static_tls_size + PTHREAD_STACK_MIN; + // } + // + // + // The following 'minstack_size > os::vm_page_size() + PTHREAD_STACK_MIN' + // if check is done for precaution. + if (minstack_size > (size_t)os::vm_page_size() + PTHREAD_STACK_MIN) { + tls_size = minstack_size - os::vm_page_size() - PTHREAD_STACK_MIN; + } + } + + log_info(os, thread)("Stack size adjustment for TLS is " SIZE_FORMAT, + tls_size); + return tls_size; +} + bool os::create_thread(Thread* thread, ThreadType thr_type, size_t req_stack_size) { assert(thread->osthread() == NULL, "caller responsible"); @@ -825,7 +890,7 @@ // Calculate stack size if it's not specified by caller. size_t stack_size = os::Posix::get_initial_stack_size(thr_type, req_stack_size); - // In the Linux NPTL pthread implementation the guard size mechanism + // In glibc versions prior to 2.7 the guard size mechanism // is not implemented properly. The posix standard requires adding // the size of the guard pages to the stack size, instead Linux // takes the space out of 'stacksize'. Thus we adapt the requested @@ -833,17 +898,27 @@ // behaviour. However, be careful not to end up with a size // of zero due to overflow. Don't add the guard page in that case. size_t guard_size = os::Linux::default_guard_size(thr_type); - if (stack_size <= SIZE_MAX - guard_size) { - stack_size += guard_size; + // Configure glibc guard page. Must happen before calling + // get_static_tls_area_size(), which uses the guard_size. + pthread_attr_setguardsize(&attr, guard_size); + + size_t stack_adjust_size = 0; + if (AdjustStackSizeForTLS) { + // Adjust the stack_size for on-stack TLS - see get_static_tls_area_size(). + stack_adjust_size += get_static_tls_area_size(&attr); + } else { + stack_adjust_size += guard_size; + } + + stack_adjust_size = align_up(stack_adjust_size, os::vm_page_size()); + if (stack_size <= SIZE_MAX - stack_adjust_size) { + stack_size += stack_adjust_size; } assert(is_aligned(stack_size, os::vm_page_size()), "stack_size not aligned"); int status = pthread_attr_setstacksize(&attr, stack_size); assert_status(status == 0, status, "pthread_attr_setstacksize"); - // Configure glibc guard page. - pthread_attr_setguardsize(&attr, os::Linux::default_guard_size(thr_type)); - ThreadState state; { @@ -5122,6 +5197,10 @@ jdk_misc_signal_init(); } + if (AdjustStackSizeForTLS) { + get_minstack_init(); + } + // Check and sets minimum stack sizes against command line options if (Posix::set_minimum_stack_sizes() == JNI_ERR) { return JNI_ERR; --- /dev/null 2019-05-22 09:10:35.508023926 -0700 +++ new/test/hotspot/jtreg/runtime/TLS/T.java 2019-06-28 14:09:30.863077207 -0700 @@ -0,0 +1,73 @@ +/* + * Copyright (c) 2019, Google Inc. All rights reserved. + * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +import java.lang.ProcessBuilder; +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; + +public class T { + public static boolean run() { + boolean res = false; + String echoInput = "foo"; + ProcessBuilder pb = new ProcessBuilder("echo", echoInput); + + try { + // Starting a ProcessBuilder causes the process reaper thread to be + // created. The process reaper thread has small stack size. In JDK + // 13, the REAPER_DEFAULT_STACKSIZE is 128K. With JDK 8, it is 32K. + // Using the process reaper thread can demonstrate the TLS problem. + // The reaper thread can fail with StackOverflow in one of the + // failure mode with certain TLS sizes. In another observed + // failure mode the VM fails to create a thread with error message + // 'Failed to start thread - pthread_create failed'. + System.out.println("Starting a new process ..."); + Process process = pb.start(); + process.waitFor(); + String echoOutput = output(process.getInputStream()); + System.out.println("Echo Output: " + echoOutput); + if (echoOutput.equals(echoInput)) { + res = true; + } else { + // 'res' is false, fail + System.out.println("Unexpected Echo output: " + echoOutput + + ", expects: " + echoInput); + } + } catch (Exception e) { + System.out.println(e.toString()); + e.printStackTrace(); + } + return res; + } + + private static String output(InputStream inputStream) throws IOException { + String s = ""; + try (BufferedReader br = + new BufferedReader(new InputStreamReader(inputStream))) { + s = br.readLine(); + } + return s; + } +} + --- /dev/null 2019-05-22 09:10:35.508023926 -0700 +++ new/test/hotspot/jtreg/runtime/TLS/exestack-tls.c 2019-06-28 14:09:31.771089699 -0700 @@ -0,0 +1,105 @@ +/* + * Copyright (c) 2019, Google Inc. All rights reserved. + * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +#include +#include +#include +#include + +// Declare the thread local variable(s) in the main executable. This can be +// used to demonstrate the issues associated with the on-stack static TLS blocks +// that may cause insufficient stack space. The dynamic TLS blocks for shared +// objects (such as a JNI library) loaded via dlopen are not allocated on stack. +__thread int tls[128 * 1024]; + +JNIEnv* create_vm(JavaVM **jvm, char* argTLS) { + JNIEnv* env; + JavaVMInitArgs args; + JavaVMOption options[3]; + args.version = JNI_VERSION_1_8; + args.nOptions = 3; + char classpath[4096]; + snprintf(classpath, sizeof classpath, + "-Djava.class.path=%s", getenv("CLASSPATH")); + options[0].optionString = classpath; + options[1].optionString = "-Xlog:os+thread=info"; + options[2].optionString = argTLS; + args.options = &options[0]; + args.ignoreUnrecognized = 0; + int rv; + rv = JNI_CreateJavaVM(jvm, (void**)&env, &args); + if (rv < 0) return NULL; + return env; +} + +int run(jboolean addTLS) { + JavaVM *jvm; + jclass testClass; + jmethodID runMethod; + char* argTLS; + int res = -1; + + if (addTLS) { + argTLS = "-XX:+AdjustStackSizeForTLS"; + } else { + argTLS = "-XX:-AdjustStackSizeForTLS"; // default + } + printf("Running test with %s ...\n", argTLS); + JNIEnv *env = create_vm(&jvm, argTLS); + + // Run T.run() and check result: + // - Expect T.run() to return 'true' when stack size is adjusted for TLS, + // return 0 if so + // - Expect T.run() to return 'false' if stack size is not adjusted for + // TLS, return 0 if so + // Return -1 (fail) for other cases + testClass = (*env)->FindClass(env, "T"); + runMethod = (*env)->GetStaticMethodID(env, testClass, "run", "()Z"); + if ((*env)->CallStaticBooleanMethod(env, testClass, runMethod, NULL)) { + if (addTLS) { + // expect T.run() to return 'true' + res = 0; + } + } else { + if (!addTLS) { + // expect T.run() to return 'false' + res = 0; + } + } + + if (res == 0) { + printf("Test passed with %s\n", argTLS); + } else { + printf("Test failed with %s\n", argTLS); + } + return res; +} + +int main(int argc, char **argv) { + if (argc == 2 && strcmp(argv[1], "-add_tls") == 0) { + return run(JNI_TRUE); + } else { + return run(JNI_FALSE); + } +} --- /dev/null 2019-05-22 09:10:35.508023926 -0700 +++ new/test/hotspot/jtreg/runtime/TLS/testtls.sh 2019-06-28 14:09:32.695102411 -0700 @@ -0,0 +1,47 @@ +# Copyright (c) 2019, Google Inc. All rights reserved. +# Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved. +# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. +# +# This code is free software; you can redistribute it and/or modify it +# under the terms of the GNU General Public License version 2 only, as +# published by the Free Software Foundation. +# +# This code is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License +# version 2 for more details (a copy is included in the LICENSE file that +# accompanied this code). +# +# You should have received a copy of the GNU General Public License version +# 2 along with this work; if not, write to the Free Software Foundation, +# Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. +# +# Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA +# or visit www.oracle.com if you need additional information or have any +# questions. +#!/bin/sh + +# +# @test testtls.sh +# @summary Test with extra TLS size. +# @requires os.family == "linux" +# @compile T.java +# @run shell testtls.sh +# + +if [ "${TESTSRC}" = "" ] +then + TESTSRC=${PWD} + echo "TESTSRC not set. Using "${TESTSRC}" as default" +fi +echo "TESTSRC=${TESTSRC}" +## Adding common setup Variables for running shell tests. +. ${TESTSRC}/../../test_env.sh + +LD_LIBRARY_PATH=.:${TESTJAVA}/lib/${VM_TYPE}:/usr/lib:$LD_LIBRARY_PATH +export LD_LIBRARY_PATH + +# Test 1) Run with stack size adjusted for TLS +${TESTNATIVEPATH}/stack-tls -add_tls || exit $? +# Test 2) Run with no stack size adjustment +${TESTNATIVEPATH}/stack-tls || exit $?