1 # 2 # Copyright (c) 2016, 2017, Oracle and/or its affiliates. All rights reserved. 3 # DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 4 # 5 # This code is free software; you can redistribute it and/or modify it 6 # under the terms of the GNU General Public License version 2 only, as 7 # published by the Free Software Foundation. Oracle designates this 8 # particular file as subject to the "Classpath" exception as provided 9 # by Oracle in the LICENSE file that accompanied this code. 10 # 11 # This code is distributed in the hope that it will be useful, but WITHOUT 12 # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 13 # FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 14 # version 2 for more details (a copy is included in the LICENSE file that 15 # accompanied this code). 16 # 17 # You should have received a copy of the GNU General Public License version 18 # 2 along with this work; if not, write to the Free Software Foundation, 19 # Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. 20 # 21 # Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 22 # or visit www.oracle.com if you need additional information or have any 23 # questions. 24 # 25 26 default: all 27 28 include $(SPEC) 29 include MakeBase.gmk 30 include FindTests.gmk 31 32 # We will always run multiple tests serially 33 .NOTPARALLEL: 34 35 ################################################################################ 36 # Parse global control variables 37 ################################################################################ 38 39 ifneq ($(TEST_VM_OPTS), ) 40 ifneq ($(TEST_OPTS), ) 41 TEST_OPTS := $(TEST_OPTS);VM_OPTIONS=$(TEST_VM_OPTS) 42 else 43 TEST_OPTS := VM_OPTIONS=$(TEST_VM_OPTS) 44 endif 45 endif 46 47 $(eval $(call ParseKeywordVariable, TEST_OPTS, \ 48 KEYWORDS := JOBS TIMEOUT, \ 49 STRING_KEYWORDS := VM_OPTIONS, \ 50 )) 51 52 # Helper function to propagate TEST_OPTS values. 53 # 54 # Note: No spaces are allowed around the arguments. 55 # Arg $1 The variable in TEST_OPTS to propagate 56 # Arg $2 The control variable to propagate it to 57 define SetTestOpt 58 ifneq ($$(TEST_OPTS_$1), ) 59 $2_$1 := $$(TEST_OPTS_$1) 60 endif 61 endef 62 63 # Setup _NT_SYMBOL_PATH on Windows 64 ifeq ($(OPENJDK_TARGET_OS), windows) 65 ifndef _NT_SYMBOL_PATH 66 # Can't use PathList here as it adds quotes around the value. 67 _NT_SYMBOL_PATH := \ 68 $(subst $(SPACE),;, $(foreach p, $(sort $(dir $(wildcard \ 69 $(addprefix $(SYMBOLS_IMAGE_DIR)/bin/, *.pdb */*.pdb)))), $(call FixPath, $p))) 70 export _NT_SYMBOL_PATH 71 $(info _NT_SYMBOL_PATH $(_NT_SYMBOL_PATH)) 72 endif 73 endif 74 75 ################################################################################ 76 # Hook to include the corresponding custom file, if present. 77 $(eval $(call IncludeCustomExtension, RunTests.gmk)) 78 ################################################################################ 79 80 TEST_RESULTS_DIR := $(OUTPUTDIR)/test-results 81 TEST_SUPPORT_DIR := $(OUTPUTDIR)/test-support 82 TEST_SUMMARY := $(TEST_RESULTS_DIR)/test-summary.txt 83 TEST_LAST_IDS := $(TEST_SUPPORT_DIR)/test-last-ids.txt 84 85 ifeq ($(CUSTOM_ROOT), ) 86 JTREG_TOPDIR := $(TOPDIR) 87 else 88 JTREG_TOPDIR := $(CUSTOM_ROOT) 89 endif 90 91 JTREG_FAILURE_HANDLER_DIR := $(TEST_IMAGE_DIR)/failure_handler 92 JTREG_FAILURE_HANDLER := $(JTREG_FAILURE_HANDLER_DIR)/jtregFailureHandler.jar 93 94 ifneq ($(wildcard $(JTREG_FAILURE_HANDLER)), ) 95 JTREG_FAILURE_HANDLER_OPTIONS := \ 96 -timeoutHandlerDir:$(JTREG_FAILURE_HANDLER) \ 97 -observerDir:$(JTREG_FAILURE_HANDLER) \ 98 -timeoutHandler:jdk.test.failurehandler.jtreg.GatherProcessInfoTimeoutHandler \ 99 -observer:jdk.test.failurehandler.jtreg.GatherDiagnosticInfoObserver \ 100 -timeoutHandlerTimeout:0 101 endif 102 103 GTEST_LAUNCHER_DIRS := $(patsubst %/gtestLauncher, %, $(wildcard $(TEST_IMAGE_DIR)/hotspot/gtest/*/gtestLauncher)) 104 GTEST_VARIANTS := $(strip $(patsubst $(TEST_IMAGE_DIR)/hotspot/gtest/%, %, $(GTEST_LAUNCHER_DIRS))) 105 106 ################################################################################ 107 # Parse control variables 108 ################################################################################ 109 110 ifneq ($(TEST_OPTS), ) 111 # Inform the user 112 $(info Running tests using TEST_OPTS control variable '$(TEST_OPTS)') 113 114 $(eval $(call SetTestOpt,VM_OPTIONS,JTREG)) 115 $(eval $(call SetTestOpt,VM_OPTIONS,GTEST)) 116 117 $(eval $(call SetTestOpt,JOBS,JTREG)) 118 $(eval $(call SetTestOpt,TIMEOUT,JTREG)) 119 endif 120 121 $(eval $(call ParseKeywordVariable, JTREG, \ 122 KEYWORDS := JOBS TIMEOUT TEST_MODE ASSERT VERBOSE RETAIN MAX_MEM, \ 123 STRING_KEYWORDS := OPTIONS JAVA_OPTIONS VM_OPTIONS, \ 124 )) 125 126 ifneq ($(JTREG), ) 127 # Inform the user 128 $(info Running tests using JTREG control variable '$(JTREG)') 129 endif 130 131 $(eval $(call ParseKeywordVariable, GTEST, \ 132 KEYWORDS := REPEAT, \ 133 STRING_KEYWORDS := OPTIONS VM_OPTIONS, \ 134 )) 135 136 ifneq ($(GTEST), ) 137 # Inform the user 138 $(info Running tests using GTEST control variable '$(GTEST)') 139 endif 140 141 142 ################################################################################ 143 # Component-specific Jtreg settings 144 ################################################################################ 145 146 ifeq ($(TEST_JOBS), 0) 147 # If TEST_JOBS is not specified, hotspot fallback default is 148 # min(num_cores / 2, 12). 149 hotspot_JTREG_JOBS := $(shell $(EXPR) $(NUM_CORES) / 2) 150 ifeq ($(hotspot_JTREG_JOBS), 0) 151 hotspot_JTREG_JOBS := 1 152 else ifeq ($(shell $(EXPR) $(hotspot_JTREG_JOBS) \> 12), 1) 153 hotspot_JTREG_JOBS := 12 154 endif 155 endif 156 157 hotspot_JTREG_MAX_MEM := 0 158 hotspot_JTREG_ASSERT := false 159 hotspot_JTREG_NATIVEPATH := $(TEST_IMAGE_DIR)/hotspot/jtreg/native 160 jdk_JTREG_NATIVEPATH := $(TEST_IMAGE_DIR)/jdk/jtreg/native 161 162 jdk_JTREG_PROBLEM_LIST += $(TOPDIR)/test/jdk/ProblemList.txt 163 jaxp_JTREG_PROBLEM_LIST += $(TOPDIR)/test/jaxp/ProblemList.txt 164 langtools_JTREG_PROBLEM_LIST += $(TOPDIR)/test/langtools/ProblemList.txt 165 nashorn_JTREG_PROBLEM_LIST += $(TOPDIR)/test/nashorn/ProblemList.txt 166 hotspot_JTREG_PROBLEM_LIST += $(TOPDIR)/test/hotspot/jtreg/ProblemList.txt 167 168 ################################################################################ 169 # Parse test selection 170 # 171 # The user has given a test selection in the TEST variable. We must parse it 172 # and determine what that means in terms of actual calls to the test framework. 173 # 174 # The parse functions take as argument a test specification as given by the 175 # user, and returns a fully qualified test descriptor if it was a match, or 176 # nothing if not. A single test specification can result in multiple test 177 # descriptors being returned. A valid test descriptor must always be accepted 178 # and returned identically. 179 ################################################################################ 180 181 # Helper function to determine if a test specification is a Gtest test 182 # 183 # It is a Gtest test if it is either "gtest", or "gtest:" followed by an optional 184 # test filter string, and an optional "/<variant>" to select a specific JVM 185 # variant. If no variant is specified, all found variants are tested. 186 define ParseGtestTestSelection 187 $(if $(filter gtest%, $1), \ 188 $(if $(filter gtest, $1), \ 189 $(addprefix gtest:all/, $(GTEST_VARIANTS)) \ 190 , \ 191 $(if $(strip $(or $(filter gtest/%, $1) $(filter gtest:/%, $1))), \ 192 $(patsubst gtest:/%, gtest:all/%, $(patsubst gtest/%, gtest:/%, $1)) \ 193 , \ 194 $(if $(filter gtest:%, $1), \ 195 $(if $(findstring /, $1), \ 196 $1 \ 197 , \ 198 $(addprefix $1/, $(GTEST_VARIANTS)) \ 199 ) \ 200 ) \ 201 ) \ 202 ) \ 203 ) 204 endef 205 206 # Helper function that removes the TOPDIR part 207 CleanupJtregPath = \ 208 $(strip $(patsubst %/, %, $(subst $(JTREG_TOPDIR)/,, $1))) 209 210 # Take a partial Jtreg root path and return a full, absolute path to that Jtreg 211 # root. Also support having "hotspot" as an alias for "hotspot/jtreg". 212 ExpandJtregRoot = \ 213 $(call CleanupJtregPath, $(wildcard \ 214 $(if $(filter /%, $1), \ 215 $(if $(wildcard $(strip $1)/TEST.ROOT), \ 216 $1 \ 217 ) \ 218 , \ 219 $(filter $(addprefix %, $1), $(JTREG_TESTROOTS) $(addsuffix /, $(JTREG_TESTROOTS))) \ 220 $(filter $(addprefix %, $(strip $1)/jtreg), $(JTREG_TESTROOTS) $(addsuffix /, $(JTREG_TESTROOTS))) \ 221 ) \ 222 )) 223 224 # Take a partial Jtreg test path and return a full, absolute path to that Jtreg 225 # test. Also support having "hotspot" as an alias for "hotspot/jtreg". 226 ExpandJtregPath = \ 227 $(if $(call ExpandJtregRoot, $1), \ 228 $(call ExpandJtregRoot, $1) \ 229 , \ 230 $(call CleanupJtregPath, $(wildcard \ 231 $(if $(filter /%, $1), \ 232 $1 \ 233 , \ 234 $(addsuffix /$(strip $1), $(JTREG_TESTROOTS) $(TEST_BASEDIRS)) \ 235 $(addsuffix $(strip $(patsubst hotspot/%, /hotspot/jtreg/%, $1)), $(JTREG_TESTROOTS) $(TEST_BASEDIRS)) \ 236 ) \ 237 )) \ 238 ) 239 240 # Helper function to determine if a test specification is a Jtreg test 241 # 242 # It is a Jtreg test if it optionally begins with jtreg:, and then is either 243 # an unspecified group name (possibly prefixed by :), or a group in a 244 # specified test root, or a path to a test or test directory, 245 # either absolute or relative to any of the TEST_BASEDIRS or test roots. 246 define ParseJtregTestSelection 247 $(eval TEST_NAME := $(strip $(patsubst jtreg:%, %, $1))) \ 248 $(if $(or $(findstring :, $(TEST_NAME)), $(findstring /, $(TEST_NAME))), , \ 249 $(eval TEST_NAME := :$(TEST_NAME)) \ 250 ) \ 251 $(if $(findstring :, $(TEST_NAME)), \ 252 $(if $(filter :%, $(TEST_NAME)), \ 253 $(eval TEST_GROUP := $(patsubst :%, %, $(TEST_NAME))) \ 254 $(eval TEST_ROOTS := $(foreach test_root, $(JTREG_TESTROOTS), \ 255 $(call CleanupJtregPath, $(test_root)))) \ 256 , \ 257 $(eval TEST_PATH := $(word 1, $(subst :, $(SPACE), $(TEST_NAME)))) \ 258 $(eval TEST_GROUP := $(word 2, $(subst :, $(SPACE), $(TEST_NAME)))) \ 259 $(eval TEST_ROOTS := $(call ExpandJtregRoot, $(TEST_PATH))) \ 260 ) \ 261 $(foreach test_root, $(TEST_ROOTS), \ 262 $(if $(filter /%, $(test_root)), \ 263 jtreg:$(test_root):$(TEST_GROUP) \ 264 , \ 265 $(if $(filter $(TEST_GROUP), $($(JTREG_TOPDIR)/$(test_root)_JTREG_TEST_GROUPS)), \ 266 jtreg:$(test_root):$(TEST_GROUP) \ 267 ) \ 268 ) \ 269 ) \ 270 , \ 271 $(eval TEST_PATHS := $(call ExpandJtregPath, $(TEST_NAME))) \ 272 $(foreach test_path, $(TEST_PATHS), \ 273 jtreg:$(test_path) \ 274 ) \ 275 ) 276 endef 277 278 # Helper function to determine if a test specification is a special test 279 # 280 # It is a special test if it is "special:" followed by a test name. 281 define ParseSpecialTestSelection 282 $(if $(filter special:%, $1), \ 283 $1 \ 284 ) 285 endef 286 287 ifeq ($(TEST), ) 288 $(info No test selection given in TEST!) 289 $(info Please use e.g. 'run-test TEST=tier1' or 'run-test-tier1') 290 $(info See doc/testing.[md|html] for help) 291 $(error Cannot continue) 292 endif 293 294 # Now intelligently convert the test selection given by the user in TEST 295 # into a list of fully qualified test descriptors of the tests to run. 296 TESTS_TO_RUN := 297 $(foreach test, $(TEST), \ 298 $(eval PARSED_TESTS := $(call ParseCustomTestSelection, $(test))) \ 299 $(if $(strip $(PARSED_TESTS)), , \ 300 $(eval PARSED_TESTS += $(call ParseGtestTestSelection, $(test))) \ 301 ) \ 302 $(if $(strip $(PARSED_TESTS)), , \ 303 $(eval PARSED_TESTS += $(call ParseJtregTestSelection, $(test))) \ 304 ) \ 305 $(if $(strip $(PARSED_TESTS)), , \ 306 $(eval PARSED_TESTS += $(call ParseSpecialTestSelection, $(test))) \ 307 ) \ 308 $(if $(strip $(PARSED_TESTS)), , \ 309 $(eval UNKNOWN_TEST := $(test)) \ 310 ) \ 311 $(eval TESTS_TO_RUN += $(PARSED_TESTS)) \ 312 ) 313 314 ifneq ($(UNKNOWN_TEST), ) 315 $(info Unknown test selection: '$(UNKNOWN_TEST)') 316 $(info See doc/testing.[md|html] for help) 317 $(error Cannot continue) 318 endif 319 320 TESTS_TO_RUN := $(strip $(TESTS_TO_RUN)) 321 322 323 # Present the result of our parsing to the user 324 $(info Test selection '$(TEST)', will run:) 325 $(foreach test, $(TESTS_TO_RUN), $(info * $(test))) 326 327 328 ################################################################################ 329 # Functions for setting up rules for running the selected tests 330 # 331 # The SetupRun*Test functions all have the same interface: 332 # 333 # Parameter 1 is the name of the rule. This is the test id, based on the test 334 # descriptor, and this is also used as variable prefix, and the targets 335 # generated are listed in a variable by that name. 336 # 337 # Remaining parameters are named arguments. Currently this is only: 338 # TEST -- The properly formatted fully qualified test descriptor 339 # 340 # After the rule named by the test id has been executed, the following 341 # variables will be available: 342 # testid_TOTAL - the total number of tests run 343 # testid_PASSED - the number of successful tests 344 # testid_FAILED - the number of failed tests 345 # testid_ERROR - the number of tests was neither successful or failed 346 # 347 ################################################################################ 348 349 ### Rules for Gtest 350 351 SetupRunGtestTest = $(NamedParamsMacroTemplate) 352 define SetupRunGtestTestBody 353 $1_TEST_RESULTS_DIR := $$(TEST_RESULTS_DIR)/$1 354 $1_TEST_SUPPORT_DIR := $$(TEST_SUPPORT_DIR)/$1 355 $1_EXITCODE := $$($1_TEST_RESULTS_DIR)/exitcode.txt 356 357 $1_VARIANT := $$(lastword $$(subst /, , $$($1_TEST))) 358 ifeq ($$(filter $$($1_VARIANT), $$(GTEST_VARIANTS)), ) 359 $$(error Invalid gtest variant '$$($1_VARIANT)'. Valid variants: $$(GTEST_VARIANTS)) 360 endif 361 $1_TEST_NAME := $$(strip $$(patsubst %/$$($1_VARIANT), %, \ 362 $$(patsubst gtest:%, %, $$($1_TEST)))) 363 ifneq ($$($1_TEST_NAME), all) 364 $1_GTEST_FILTER := --gtest_filter=$$($1_TEST_NAME)* 365 endif 366 367 ifneq ($$(GTEST_REPEAT), ) 368 $1_GTEST_REPEAT :=--gtest_repeat=$$(GTEST_REPEAT) 369 endif 370 371 run-test-$1: 372 $$(call LogWarn) 373 $$(call LogWarn, Running test '$$($1_TEST)') 374 $$(call MakeDir, $$($1_TEST_RESULTS_DIR) $$($1_TEST_SUPPORT_DIR)) 375 $$(call ExecuteWithLog, $$($1_TEST_SUPPORT_DIR)/gtest, \ 376 $$(FIXPATH) $$(TEST_IMAGE_DIR)/hotspot/gtest/$$($1_VARIANT)/gtestLauncher \ 377 -jdk $(JDK_IMAGE_DIR) $$($1_GTEST_FILTER) \ 378 --gtest_output=xml:$$($1_TEST_RESULTS_DIR)/gtest.xml \ 379 $$($1_GTEST_REPEAT) $$(GTEST_OPTIONS) $$(GTEST_VM_OPTIONS) \ 380 > >($(TEE) $$($1_TEST_RESULTS_DIR)/gtest.txt) \ 381 && $$(ECHO) $$$$? > $$($1_EXITCODE) \ 382 || $$(ECHO) $$$$? > $$($1_EXITCODE) \ 383 ) 384 385 $1_RESULT_FILE := $$($1_TEST_RESULTS_DIR)/gtest.txt 386 387 parse-test-$1: run-test-$1 388 $$(call LogWarn, Finished running test '$$($1_TEST)') 389 $$(call LogWarn, Test report is stored in $$(strip \ 390 $$(subst $$(TOPDIR)/, , $$($1_TEST_RESULTS_DIR)))) 391 $$(if $$(wildcard $$($1_RESULT_FILE)), \ 392 $$(eval $1_TOTAL := $$(shell $$(AWK) '/==========.* tests? from .* \ 393 test cases? ran/ { print $$$$2 }' $$($1_RESULT_FILE))) \ 394 $$(if $$($1_TOTAL), , $$(eval $1_TOTAL := 0)) \ 395 $$(eval $1_PASSED := $$(shell $$(AWK) '/\[ PASSED \] .* tests?./ \ 396 { print $$$$4 }' $$($1_RESULT_FILE))) \ 397 $$(if $$($1_PASSED), , $$(eval $1_PASSED := 0)) \ 398 $$(eval $1_FAILED := $$(shell $$(AWK) '/\[ FAILED \] .* tests?, \ 399 listed below/ { print $$$$4 }' $$($1_RESULT_FILE))) \ 400 $$(if $$($1_FAILED), , $$(eval $1_FAILED := 0)) \ 401 $$(eval $1_ERROR := $$(shell \ 402 $$(EXPR) $$($1_TOTAL) - $$($1_PASSED) - $$($1_FAILED))) \ 403 , \ 404 $$(eval $1_PASSED := 0) \ 405 $$(eval $1_FAILED := 0) \ 406 $$(eval $1_ERROR := 1) \ 407 $$(eval $1_TOTAL := 1) \ 408 ) 409 410 $1: run-test-$1 parse-test-$1 411 412 TARGETS += $1 413 endef 414 415 ################################################################################ 416 417 ### Rules for Jtreg 418 419 # Helper function for SetupRunJtregTest. Set a JTREG_* variable from, in order: 420 # 1) Specified by user on command line 421 # 2) Component-specific default 422 # 3) Generic default 423 # 424 # Note: No spaces are allowed around the arguments. 425 # Arg $1 The test ID (i.e. $1 in SetupRunJtregTest) 426 # Arg $2 Base variable, e.g. JTREG_JOBS 427 # Arg $3 The default value (optional) 428 define SetJtregValue 429 ifneq ($$($2), ) 430 $1_$2 := $$($2) 431 else 432 ifneq ($$($$($1_COMPONENT)_$2), ) 433 $1_$2 := $$($$($1_COMPONENT)_$2) 434 else 435 ifneq ($3, ) 436 $1_$2 := $3 437 endif 438 endif 439 endif 440 endef 441 442 SetupRunJtregTest = $(NamedParamsMacroTemplate) 443 define SetupRunJtregTestBody 444 $1_TEST_RESULTS_DIR := $$(TEST_RESULTS_DIR)/$1 445 $1_TEST_SUPPORT_DIR := $$(TEST_SUPPORT_DIR)/$1 446 $1_EXITCODE := $$($1_TEST_RESULTS_DIR)/exitcode.txt 447 448 $1_TEST_NAME := $$(strip $$(patsubst jtreg:%, %, $$($1_TEST))) 449 450 $1_COMPONENT := \ 451 $$(strip $$(foreach root, $$(JTREG_TESTROOTS), \ 452 $$(if $$(filter $$(root)%, $$(JTREG_TOPDIR)/$$($1_TEST_NAME)), \ 453 $$(lastword $$(subst /, $$(SPACE), $$(root))) \ 454 ) \ 455 )) 456 # This will work only as long as just hotspot has the additional "jtreg" directory 457 ifeq ($$($1_COMPONENT), jtreg) 458 $1_COMPONENT := hotspot 459 endif 460 461 ifeq ($$(JT_HOME), ) 462 $$(info Error: jtreg framework is not found.) 463 $$(info Please run configure using --with-jtreg.) 464 $$(error Cannot continue) 465 endif 466 467 # Unfortunately, we need different defaults for some JTREG values, 468 # depending on what component we're running. 469 470 # Convert JTREG_foo into $1_JTREG_foo with a suitable value. 471 $$(eval $$(call SetJtregValue,$1,JTREG_TEST_MODE,agentvm)) 472 $$(eval $$(call SetJtregValue,$1,JTREG_ASSERT,true)) 473 $$(eval $$(call SetJtregValue,$1,JTREG_MAX_MEM,512m)) 474 $$(eval $$(call SetJtregValue,$1,JTREG_NATIVEPATH)) 475 $$(eval $$(call SetJtregValue,$1,JTREG_BASIC_OPTIONS)) 476 $$(eval $$(call SetJtregValue,$1,JTREG_PROBLEM_LIST)) 477 478 ifneq ($(TEST_JOBS), 0) 479 # User has specified TEST_JOBS, use that as fallback default 480 $$(eval $$(call SetJtregValue,$1,JTREG_JOBS,$$(TEST_JOBS))) 481 else 482 # Use JOBS as default (except for hotspot) 483 $$(eval $$(call SetJtregValue,$1,JTREG_JOBS,$$(JOBS))) 484 endif 485 486 # Make sure MaxRAMPercentage is high enough to not cause OOM or swapping since 487 # we may end up with a lot of JVM's 488 $1_JTREG_MAX_RAM_PERCENTAGE := $$(shell $$(EXPR) 25 / $$($1_JTREG_JOBS)) 489 490 JTREG_TIMEOUT ?= 4 491 JTREG_VERBOSE ?= fail,error,summary 492 JTREG_RETAIN ?= fail,error 493 494 ifneq ($$($1_JTREG_MAX_MEM), 0) 495 $1_JTREG_BASIC_OPTIONS += -vmoption:-Xmx$$($1_JTREG_MAX_MEM) 496 $1_JTREG_LAUNCHER_OPTIONS += -Xmx$$($1_JTREG_MAX_MEM) 497 endif 498 499 $1_JTREG_BASIC_OPTIONS += -$$($1_JTREG_TEST_MODE) \ 500 -verbose:$$(JTREG_VERBOSE) -retain:$$(JTREG_RETAIN) \ 501 -concurrency:$$($1_JTREG_JOBS) -timeoutFactor:$$(JTREG_TIMEOUT) \ 502 -vmoption:-XX:MaxRAMPercentage=$$($1_JTREG_MAX_RAM_PERCENTAGE) 503 504 $1_JTREG_BASIC_OPTIONS += -automatic -keywords:\!ignore -ignore:quiet 505 506 # Make it possible to specify the JIB_DATA_DIR for tests using the 507 # JIB Artifact resolver 508 $1_JTREG_BASIC_OPTIONS += -e:JIB_DATA_DIR 509 # Some tests needs to find a boot JDK using the JDK8_HOME variable. 510 $1_JTREG_BASIC_OPTIONS += -e:JDK8_HOME=$$(BOOT_JDK) 511 # If running on Windows, propagate the _NT_SYMBOL_PATH to enable 512 # symbol lookup in hserr files 513 ifeq ($$(OPENJDK_TARGET_OS), windows) 514 $1_JTREG_BASIC_OPTIONS += -e:_NT_SYMBOL_PATH 515 endif 516 517 ifeq ($(ENABLE_LIBCLANG), yes) 518 $1_JTREG_BASIC_OPTIONS += \ 519 -Dclang.include.path=$(CLANG_INCLUDE_PATH) \ 520 -Dclang.lib.path=$(CLANG_LIB_PATH) 521 endif 522 523 $1_JTREG_BASIC_OPTIONS += \ 524 $$(addprefix -javaoption:, $$(JTREG_JAVA_OPTIONS)) \ 525 $$(addprefix -vmoption:, $$(JTREG_VM_OPTIONS)) \ 526 # 527 528 ifeq ($$($1_JTREG_ASSERT), true) 529 $1_JTREG_BASIC_OPTIONS += -ea -esa 530 endif 531 532 ifneq ($$($1_JTREG_NATIVEPATH), ) 533 $1_JTREG_BASIC_OPTIONS += -nativepath:$$($1_JTREG_NATIVEPATH) 534 endif 535 536 ifneq ($$($1_JTREG_PROBLEM_LIST), ) 537 $1_JTREG_BASIC_OPTIONS += $$(addprefix -exclude:, $$($1_JTREG_PROBLEM_LIST)) 538 endif 539 540 ifneq ($$(JIB_HOME), ) 541 $1_JTREG_BASIC_OPTIONS += -e:JIB_HOME=$$(JIB_HOME) 542 endif 543 544 $1_JTREG_BASIC_OPTIONS += -e:TEST_IMAGE_GRAAL_DIR=${TEST_IMAGE_DIR}/hotspot/jtreg/graal 545 546 ifneq ($$(JTREG_FAILURE_HANDLER_OPTIONS), ) 547 $1_JTREG_LAUNCHER_OPTIONS += -Djava.library.path="$(JTREG_FAILURE_HANDLER_DIR)" 548 endif 549 550 clean-workdir-$1: 551 $$(RM) -r $$($1_TEST_SUPPORT_DIR) 552 553 run-test-$1: clean-workdir-$1 554 $$(call LogWarn) 555 $$(call LogWarn, Running test '$$($1_TEST)') 556 $$(call MakeDir, $$($1_TEST_RESULTS_DIR) $$($1_TEST_SUPPORT_DIR)) 557 $$(call ExecuteWithLog, $$($1_TEST_SUPPORT_DIR)/jtreg, \ 558 $$(JAVA) $$($1_JTREG_LAUNCHER_OPTIONS) \ 559 -Dprogram=jtreg -jar $$(JT_HOME)/lib/jtreg.jar \ 560 $$($1_JTREG_BASIC_OPTIONS) \ 561 -testjdk:$$(JDK_IMAGE_DIR) \ 562 -dir:$$(JTREG_TOPDIR) \ 563 -reportDir:$$($1_TEST_RESULTS_DIR) \ 564 -workDir:$$($1_TEST_SUPPORT_DIR) \ 565 $$(JTREG_OPTIONS) \ 566 $$(JTREG_FAILURE_HANDLER_OPTIONS) \ 567 $$($1_TEST_NAME) \ 568 && $$(ECHO) $$$$? > $$($1_EXITCODE) \ 569 || $$(ECHO) $$$$? > $$($1_EXITCODE) \ 570 ) 571 572 $1_RESULT_FILE := $$($1_TEST_RESULTS_DIR)/text/stats.txt 573 574 parse-test-$1: run-test-$1 575 $$(call LogWarn, Finished running test '$$($1_TEST)') 576 $$(call LogWarn, Test report is stored in $$(strip \ 577 $$(subst $$(TOPDIR)/, , $$($1_TEST_RESULTS_DIR)))) 578 $$(if $$(wildcard $$($1_RESULT_FILE)), \ 579 $$(eval $1_PASSED := $$(shell $$(AWK) '{ gsub(/[,;]/, ""); \ 580 for (i=1; i<=NF; i++) { if ($$$$i == "passed:") \ 581 print $$$$(i+1) } }' $$($1_RESULT_FILE))) \ 582 $$(if $$($1_PASSED), , $$(eval $1_PASSED := 0)) \ 583 $$(eval $1_FAILED := $$(shell $$(AWK) '{gsub(/[,;]/, ""); \ 584 for (i=1; i<=NF; i++) { if ($$$$i == "failed:") \ 585 print $$$$(i+1) } }' $$($1_RESULT_FILE))) \ 586 $$(if $$($1_FAILED), , $$(eval $1_FAILED := 0)) \ 587 $$(eval $1_ERROR := $$(shell $$(AWK) '{gsub(/[,;]/, ""); \ 588 for (i=1; i<=NF; i++) { if ($$$$i == "error:") \ 589 print $$$$(i+1) } }' $$($1_RESULT_FILE))) \ 590 $$(if $$($1_ERROR), , $$(eval $1_ERROR := 0)) \ 591 $$(eval $1_TOTAL := $$(shell \ 592 $$(EXPR) $$($1_PASSED) + $$($1_FAILED) + $$($1_ERROR))) \ 593 , \ 594 $$(eval $1_PASSED := 0) \ 595 $$(eval $1_FAILED := 0) \ 596 $$(eval $1_ERROR := 1) \ 597 $$(eval $1_TOTAL := 1) \ 598 ) 599 600 $1: run-test-$1 parse-test-$1 601 602 TARGETS += $1 603 endef 604 605 ################################################################################ 606 607 ### Rules for special tests 608 609 SetupRunSpecialTest = $(NamedParamsMacroTemplate) 610 define SetupRunSpecialTestBody 611 $1_TEST_RESULTS_DIR := $$(TEST_RESULTS_DIR)/$1 612 $1_TEST_SUPPORT_DIR := $$(TEST_SUPPORT_DIR)/$1 613 $1_EXITCODE := $$($1_TEST_RESULTS_DIR)/exitcode.txt 614 615 $1_FULL_TEST_NAME := $$(strip $$(patsubst special:%, %, $$($1_TEST))) 616 ifneq ($$(findstring :, $$($1_FULL_TEST_NAME)), ) 617 $1_TEST_NAME := $$(firstword $$(subst :, ,$$($1_FULL_TEST_NAME))) 618 $1_TEST_ARGS := $$(strip $$(patsubst special:$$($1_TEST_NAME):%, %, $$($1_TEST))) 619 else 620 $1_TEST_NAME := $$($1_FULL_TEST_NAME) 621 $1_TEST_ARGS := 622 endif 623 624 ifeq ($$($1_TEST_NAME), hotspot-internal) 625 $1_TEST_COMMAND_LINE := \ 626 $$(JDK_IMAGE_DIR)/bin/java -XX:+ExecuteInternalVMTests \ 627 -XX:+ShowMessageBoxOnError -version 628 else ifeq ($$($1_TEST_NAME), failure-handler) 629 $1_TEST_COMMAND_LINE := \ 630 ($(CD) $(TOPDIR)/make/test && $(MAKE) $(MAKE_ARGS) -f \ 631 BuildFailureHandler.gmk test) 632 else ifeq ($$($1_TEST_NAME), make) 633 $1_TEST_COMMAND_LINE := \ 634 ($(CD) $(TOPDIR)/test/make && $(MAKE) $(MAKE_ARGS) -f \ 635 TestMake.gmk $$($1_TEST_ARGS)) 636 else 637 $$(error Invalid special test specification: $$($1_TEST_NAME)) 638 endif 639 640 run-test-$1: 641 $$(call LogWarn) 642 $$(call LogWarn, Running test '$$($1_TEST)') 643 $$(call MakeDir, $$($1_TEST_RESULTS_DIR) $$($1_TEST_SUPPORT_DIR)) 644 $$(call ExecuteWithLog, $$($1_TEST_SUPPORT_DIR)/test-execution, \ 645 $$($1_TEST_COMMAND_LINE) \ 646 > >($(TEE) $$($1_TEST_RESULTS_DIR)/test-output.txt) \ 647 && $$(ECHO) $$$$? > $$($1_EXITCODE) \ 648 || $$(ECHO) $$$$? > $$($1_EXITCODE) \ 649 ) 650 651 $1_RESULT_FILE := $$($1_TEST_RESULTS_DIR)/gtest.txt 652 653 # We can not parse the various "special" tests. 654 parse-test-$1: run-test-$1 655 $$(call LogWarn, Finished running test '$$($1_TEST)') 656 $$(call LogWarn, Test report is stored in $$(strip \ 657 $$(subst $$(TOPDIR)/, , $$($1_TEST_RESULTS_DIR)))) 658 $$(call LogWarn, Warning: Special test results are not properly parsed!) 659 $$(eval $1_PASSED := 0) 660 $$(eval $1_FAILED := 0) 661 $$(eval $1_ERROR := 0) 662 $$(eval $1_TOTAL := 0) 663 664 $1: run-test-$1 parse-test-$1 665 666 TARGETS += $1 667 endef 668 669 ################################################################################ 670 # Setup and execute make rules for all selected tests 671 ################################################################################ 672 673 # Helper function to determine which handler to use for the given test 674 UseGtestTestHandler = \ 675 $(if $(filter gtest:%, $1), true) 676 677 UseJtregTestHandler = \ 678 $(if $(filter jtreg:%, $1), true) 679 680 UseSpecialTestHandler = \ 681 $(if $(filter special:%, $1), true) 682 683 # Now process each test to run and setup a proper make rule 684 $(foreach test, $(TESTS_TO_RUN), \ 685 $(eval TEST_ID := $(shell $(ECHO) $(strip $(test)) | \ 686 $(TR) -cs '[a-z][A-Z][0-9]\n' '[_*1000]')) \ 687 $(eval ALL_TEST_IDS += $(TEST_ID)) \ 688 $(if $(call UseCustomTestHandler, $(test)), \ 689 $(eval $(call SetupRunCustomTest, $(TEST_ID), \ 690 TEST := $(test), \ 691 )) \ 692 ) \ 693 $(if $(call UseGtestTestHandler, $(test)), \ 694 $(eval $(call SetupRunGtestTest, $(TEST_ID), \ 695 TEST := $(test), \ 696 )) \ 697 ) \ 698 $(if $(call UseJtregTestHandler, $(test)), \ 699 $(eval $(call SetupRunJtregTest, $(TEST_ID), \ 700 TEST := $(test), \ 701 )) \ 702 ) \ 703 $(if $(call UseSpecialTestHandler, $(test)), \ 704 $(eval $(call SetupRunSpecialTest, $(TEST_ID), \ 705 TEST := $(test), \ 706 )) \ 707 ) \ 708 ) 709 710 # Sort also removes duplicates, so if there is any we'll get fewer words. 711 ifneq ($(words $(ALL_TEST_IDS)), $(words $(sort $(ALL_TEST_IDS)))) 712 $(error Duplicate test specification) 713 endif 714 715 716 ################################################################################ 717 # The main target for RunTests.gmk 718 ################################################################################ 719 720 # The SetupRun*Test functions have populated TARGETS. 721 722 TEST_FAILURE := false 723 724 run-test: $(TARGETS) 725 # Create and print a table of the result of all tests run 726 $(RM) $(TEST_SUMMARY).old 2> /dev/null 727 $(MV) $(TEST_SUMMARY) $(TEST_SUMMARY).old 2> /dev/null || true 728 $(RM) $(TEST_LAST_IDS).old 2> /dev/null 729 $(MV) $(TEST_LAST_IDS) $(TEST_LAST_IDS).old 2> /dev/null || true 730 $(ECHO) >> $(TEST_SUMMARY) ============================== 731 $(ECHO) >> $(TEST_SUMMARY) Test summary 732 $(ECHO) >> $(TEST_SUMMARY) ============================== 733 $(PRINTF) >> $(TEST_SUMMARY) "%2s %-49s %5s %5s %5s %5s %2s\n" " " \ 734 TEST TOTAL PASS FAIL ERROR " " 735 $(foreach test, $(TESTS_TO_RUN), \ 736 $(eval TEST_ID := $(shell $(ECHO) $(strip $(test)) | \ 737 $(TR) -cs '[a-z][A-Z][0-9]\n' '[_*1000]')) \ 738 $(ECHO) >> $(TEST_LAST_IDS) $(TEST_ID) $(NEWLINE) \ 739 $(eval NAME_PATTERN := $(shell $(ECHO) $(test) | $(TR) -c '\n' '[_*1000]')) \ 740 $(if $(filter __________________________________________________%, $(NAME_PATTERN)), \ 741 $(eval TEST_NAME := ) \ 742 $(PRINTF) >> $(TEST_SUMMARY) "%2s %-49s\n" " " "$(test)" $(NEWLINE) \ 743 , \ 744 $(eval TEST_NAME := $(test)) \ 745 ) \ 746 $(if $(filter $($(TEST_ID)_PASSED), $($(TEST_ID)_TOTAL)), \ 747 $(PRINTF) >> $(TEST_SUMMARY) "%2s %-49s %5d %5d %5d %5d %2s\n" \ 748 " " "$(TEST_NAME)" $($(TEST_ID)_TOTAL) $($(TEST_ID)_PASSED) \ 749 $($(TEST_ID)_FAILED) $($(TEST_ID)_ERROR) " " $(NEWLINE) \ 750 , \ 751 $(PRINTF) >> $(TEST_SUMMARY) "%2s %-49s %5d %5d %5d %5d %2s\n" \ 752 ">>" "$(TEST_NAME)" $($(TEST_ID)_TOTAL) $($(TEST_ID)_PASSED) \ 753 $($(TEST_ID)_FAILED) $($(TEST_ID)_ERROR) "<<" $(NEWLINE) \ 754 $(eval TEST_FAILURE := true) \ 755 ) \ 756 ) 757 $(ECHO) >> $(TEST_SUMMARY) ============================== 758 $(if $(filter true, $(TEST_FAILURE)), \ 759 $(ECHO) >> $(TEST_SUMMARY) TEST FAILURE $(NEWLINE) \ 760 $(MKDIR) -p $(MAKESUPPORT_OUTPUTDIR) $(NEWLINE) \ 761 $(TOUCH) $(MAKESUPPORT_OUTPUTDIR)/exit-with-error \ 762 , \ 763 $(ECHO) >> $(TEST_SUMMARY) TEST SUCCESS \ 764 ) 765 $(ECHO) 766 $(CAT) $(TEST_SUMMARY) 767 $(ECHO) 768 769 ################################################################################ 770 771 all: run-test 772 773 .PHONY: default all run-test $(TARGETS)