1 # 2 # Copyright (c) 2016, 2017, Oracle and/or its affiliates. All rights reserved. 3 # DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 4 # 5 # This code is free software; you can redistribute it and/or modify it 6 # under the terms of the GNU General Public License version 2 only, as 7 # published by the Free Software Foundation. Oracle designates this 8 # particular file as subject to the "Classpath" exception as provided 9 # by Oracle in the LICENSE file that accompanied this code. 10 # 11 # This code is distributed in the hope that it will be useful, but WITHOUT 12 # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 13 # FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 14 # version 2 for more details (a copy is included in the LICENSE file that 15 # accompanied this code). 16 # 17 # You should have received a copy of the GNU General Public License version 18 # 2 along with this work; if not, write to the Free Software Foundation, 19 # Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. 20 # 21 # Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 22 # or visit www.oracle.com if you need additional information or have any 23 # questions. 24 # 25 26 default: all 27 28 include $(SPEC) 29 include MakeBase.gmk 30 include FindTests.gmk 31 32 # We will always run multiple tests serially 33 .NOTPARALLEL: 34 35 # Directories to find jtreg tests relative to 36 JTREG_TEST_TOPDIRS := $(TOPDIR) $(JTREG_TESTROOTS) 37 38 # Hook to include the corresponding custom file, if present. 39 $(eval $(call IncludeCustomExtension, RunTests.gmk)) 40 41 TEST_RESULTS_DIR := $(OUTPUTDIR)/test-results 42 TEST_SUPPORT_DIR := $(OUTPUTDIR)/test-support 43 44 45 ################################################################################ 46 # Parse control variables 47 ################################################################################ 48 49 $(eval $(call ParseKeywordVariable, JTREG, \ 50 KEYWORDS := JOBS TIMEOUT TEST_MODE ASSERT VERBOSE RETAIN MAX_MEM, \ 51 STRING_KEYWORDS := OPTIONS JAVA_OPTIONS VM_OPTIONS, \ 52 )) 53 54 ifneq ($(JTREG), ) 55 # Inform the user 56 $(info Running tests using JTREG control variable '$(JTREG)') 57 endif 58 59 $(eval $(call ParseKeywordVariable, GTEST, \ 60 KEYWORDS := REPEAT, \ 61 STRING_KEYWORDS := OPTIONS, \ 62 )) 63 64 ifneq ($(GTEST), ) 65 # Inform the user 66 $(info Running tests using GTEST control variable '$(GTEST)') 67 endif 68 69 70 ################################################################################ 71 # Component-specific Jtreg settings 72 ################################################################################ 73 74 ifeq ($(TEST_JOBS), 0) 75 # If TEST_JOBS is not specified, hotspot fallback default is 76 # min(num_cores / 2, 12). 77 hotspot_JTREG_JOBS := $(shell $(EXPR) $(NUM_CORES) / 2) 78 ifeq ($(hotspot_JTREG_JOBS), 0) 79 hotspot_JTREG_JOBS := 1 80 else ifeq ($(shell $(EXPR) $(hotspot_JTREG_JOBS) \> 12), 1) 81 hotspot_JTREG_JOBS := 12 82 endif 83 endif 84 85 hotspot_JTREG_MAX_MEM := 0 86 hotspot_JTREG_ASSERT := false 87 hotspot_JTREG_NATIVEPATH := $(TEST_IMAGE_DIR)/hotspot/jtreg/native 88 jdk_JTREG_NATIVEPATH := $(TEST_IMAGE_DIR)/jdk/jtreg/native 89 90 91 ################################################################################ 92 # Parse test selection 93 # 94 # The user has given a test selection in the TEST variable. We must parse it 95 # and determine what that means in terms of actual calls to the test framework. 96 # 97 # The parse functions take as argument a test specification as given by the 98 # user, and returns a fully qualified test descriptor if it was a match, or 99 # nothing if not. A single test specification can result in multiple test 100 # descriptors being returned. A valid test descriptor must always be accepted 101 # and returned identically. 102 ################################################################################ 103 104 # Helper function to determine if a test specification is a Gtest test 105 # 106 # It is a Gtest test if it is either "gtest", or "gtest:" followed by an optional 107 # test filter string. 108 define ParseGtestTestSelection 109 $(if $(filter gtest%, $1), \ 110 $(if $(filter gtest, $1), \ 111 gtest:all \ 112 , \ 113 $(if $(filter gtest:, $1), \ 114 gtest:all \ 115 , \ 116 $1 \ 117 ) \ 118 ) \ 119 ) 120 endef 121 122 # Helper function to determine if a test specification is a Jtreg test 123 # 124 # It is a Jtreg test if it optionally begins with jtreg:, and then is either 125 # an unspecified group name (possibly prefixed by :), or a group in a 126 # specified test/<component> directory, or a path to a test or test directory, 127 # either absolute or relative to any of the JTREG_TEST_TOPDIRS. 128 define ParseJtregTestSelection 129 $(eval TEST_NAME := $(strip $(patsubst jtreg:%, %, $1))) \ 130 $(if $(or $(findstring :, $(TEST_NAME)), $(findstring /, $(TEST_NAME))), , \ 131 $(eval TEST_NAME := :$(TEST_NAME)) \ 132 ) \ 133 $(if $(findstring :, $(TEST_NAME)), \ 134 $(if $(filter :%, $(TEST_NAME)), \ 135 $(foreach root, $(JTREG_TESTROOTS), \ 136 $(if $(filter $(patsubst :%, %, $(TEST_NAME)), \ 137 $($(root)_JTREG_TEST_GROUPS)), \ 138 jtreg:$(root):$(patsubst :%,%,$(TEST_NAME)) \ 139 ) \ 140 ) \ 141 , \ 142 $(eval ROOT_PART := $(word 1, $(subst :, $(SPACE), $(TEST_NAME)))) \ 143 $(eval ROOT := $(filter $(addprefix %, $(ROOT_PART)), $(JTREG_TESTROOTS))) \ 144 $(eval GROUP := $(word 2, $(subst :, $(SPACE), $(TEST_NAME)))) \ 145 $(foreach root, $(ROOT), \ 146 $(if $(filter $(GROUP), $($(root)_JTREG_TEST_GROUPS)), \ 147 jtreg:$(root):$(GROUP) \ 148 ) \ 149 ) \ 150 ) \ 151 , \ 152 $(if $(filter /%, $(TEST_NAME)), \ 153 $(if $(wildcard $(TEST_NAME)), \ 154 jtreg:$(TEST_NAME) \ 155 ) \ 156 , \ 157 $(addprefix jtreg:, $(wildcard $(addsuffix /$(TEST_NAME), $(JTREG_TEST_TOPDIRS)))) \ 158 ) \ 159 ) 160 endef 161 162 ifeq ($(TEST), ) 163 $(info No test selection given in TEST!) 164 $(info Please use e.g. 'run-test TEST=tier1' or 'run-test-tier1') 165 $(info See common/doc/testing.[md|html] for help) 166 $(error Cannot continue) 167 endif 168 169 # Now intelligently convert the test selection given by the user in TEST 170 # into a list of fully qualified test descriptors of the tests to run. 171 TESTS_TO_RUN := 172 $(foreach test, $(TEST), \ 173 $(eval PARSED_TESTS := $(call ParseCustomTestSelection, $(test))) \ 174 $(if $(strip $(PARSED_TESTS)), , \ 175 $(eval PARSED_TESTS += $(call ParseGtestTestSelection, $(test))) \ 176 ) \ 177 $(if $(strip $(PARSED_TESTS)), , \ 178 $(eval PARSED_TESTS += $(call ParseJtregTestSelection, $(test))) \ 179 ) \ 180 $(if $(strip $(PARSED_TESTS)), , \ 181 $(eval UNKNOWN_TEST := $(test)) \ 182 ) \ 183 $(eval TESTS_TO_RUN += $(PARSED_TESTS)) \ 184 ) 185 186 ifneq ($(UNKNOWN_TEST), ) 187 $(info Unknown test selection: '$(UNKNOWN_TEST)') 188 $(info See common/doc/testing.[md|html] for help) 189 $(error Cannot continue) 190 endif 191 192 TESTS_TO_RUN := $(strip $(TESTS_TO_RUN)) 193 194 195 # Present the result of our parsing to the user 196 $(info Test selection '$(TEST)', will run:) 197 $(foreach test, $(TESTS_TO_RUN), $(info * $(test))) 198 199 200 ################################################################################ 201 # Functions for setting up rules for running the selected tests 202 # 203 # The SetupRun*Test functions all have the same interface: 204 # 205 # Parameter 1 is the name of the rule. This is the test id, based on the test 206 # descriptor, and this is also used as variable prefix, and the targets 207 # generated are listed in a variable by that name. 208 # 209 # Remaining parameters are named arguments. Currently this is only: 210 # TEST -- The properly formatted fully qualified test descriptor 211 # 212 # After the rule named by the test id has been executed, the following 213 # variables will be available: 214 # testid_TOTAL - the total number of tests run 215 # testid_PASSED - the number of successful tests 216 # testid_FAILED - the number of failed tests 217 # testid_ERROR - the number of tests was neither successful or failed 218 # 219 ################################################################################ 220 221 ### Rules for Gtest 222 223 SetupRunGtestTest = $(NamedParamsMacroTemplate) 224 define SetupRunGtestTestBody 225 $1_TEST_RESULTS_DIR := $$(TEST_RESULTS_DIR)/$1 226 $1_TEST_SUPPORT_DIR := $$(TEST_SUPPORT_DIR)/$1 227 228 $1_TEST_NAME := $$(strip $$(patsubst gtest:%, %, $$($1_TEST))) 229 ifneq ($$($1_TEST_NAME), all) 230 $1_GTEST_FILTER := --gtest_filter=$$($1_TEST_NAME)* 231 endif 232 233 ifneq ($$(GTEST_REPEAT), ) 234 $1_GTEST_REPEAT :=--gtest_repeat=$$(GTEST_REPEAT) 235 endif 236 237 run-test-$1: 238 $$(call LogWarn) 239 $$(call LogWarn, Running test '$$($1_TEST)') 240 $$(call MakeDir, $$($1_TEST_RESULTS_DIR) $$($1_TEST_SUPPORT_DIR)) 241 $$(call ExecuteWithLog, $$($1_TEST_SUPPORT_DIR)/gtest, \ 242 $$(FIXPATH) $$(TEST_IMAGE_DIR)/hotspot/gtest/server/gtestLauncher \ 243 -jdk $(JDK_IMAGE_DIR) $$($1_GTEST_FILTER) \ 244 --gtest_output=xml:$$($1_TEST_RESULTS_DIR)/gtest.xml \ 245 $$($1_GTEST_REPEAT) $$(GTEST_OPTIONS) \ 246 > >($(TEE) $$($1_TEST_RESULTS_DIR)/gtest.txt) || true ) 247 248 $1_RESULT_FILE := $$($1_TEST_RESULTS_DIR)/gtest.txt 249 250 parse-test-$1: run-test-$1 251 $$(call LogWarn, Finished running test '$$($1_TEST)') 252 $$(call LogWarn, Test report is stored in $$(strip \ 253 $$(subst $$(TOPDIR)/, , $$($1_TEST_RESULTS_DIR)))) 254 $$(eval $1_TOTAL := $$(shell $$(AWK) '/==========.* tests? from .* \ 255 test cases? ran/ { print $$$$2 }' $$($1_RESULT_FILE))) 256 $$(eval $1_PASSED := $$(shell $$(AWK) '/\[ PASSED \] .* tests?./ \ 257 { print $$$$4 }' $$($1_RESULT_FILE))) 258 $$(eval $1_FAILED := $$(shell $$(AWK) '/\[ FAILED \] .* tests?, \ 259 listed below/ { print $$$$4 }' $$($1_RESULT_FILE))) 260 $$(if $$($1_FAILED), , $$(eval $1_FAILED := 0)) 261 $$(eval $1_ERROR := $$(shell \ 262 $$(EXPR) $$($1_TOTAL) - $$($1_PASSED) - $$($1_FAILED))) 263 264 $1: run-test-$1 parse-test-$1 265 266 TARGETS += $1 267 endef 268 269 ################################################################################ 270 271 ### Rules for Jtreg 272 273 # Helper function for SetupRunJtregTest. Set a JTREG_* variable from, in order: 274 # 1) Specified by user on command line 275 # 2) Component-specific default 276 # 3) Generic default 277 # 278 # Note: No spaces are allowed around the arguments. 279 # Arg $1 The test ID (i.e. $1 in SetupRunJtregTest) 280 # Arg $2 Base variable, e.g. JTREG_JOBS 281 # Arg $3 The default value (optional) 282 define SetJtregValue 283 ifneq ($$($2), ) 284 $1_$2 := $$($2) 285 else 286 ifneq ($$($$($1_COMPONENT)_$2), ) 287 $1_$2 := $$($$($1_COMPONENT)_$2) 288 else 289 ifneq ($3, ) 290 $1_$2 := $3 291 endif 292 endif 293 endif 294 endef 295 296 SetupRunJtregTest = $(NamedParamsMacroTemplate) 297 define SetupRunJtregTestBody 298 $1_TEST_RESULTS_DIR := $$(TEST_RESULTS_DIR)/$1 299 $1_TEST_SUPPORT_DIR := $$(TEST_SUPPORT_DIR)/$1 300 301 $1_TEST_NAME := $$(strip $$(patsubst jtreg:%, %, $$($1_TEST))) 302 $1_COMPONENT := $$(firstword $$(subst /, $$(SPACE), \ 303 $$(patsubst test/%, %, $$($1_TEST_NAME)))) 304 305 ifeq ($$(JT_HOME), ) 306 $$(info Error: jtreg framework is not found.) 307 $$(info Please run configure using --with-jtreg.) 308 $$(error Cannot continue) 309 endif 310 311 # Unfortunately, we need different defaults for some JTREG values, 312 # depending on what component we're running. 313 314 # Convert JTREG_foo into $1_JTREG_foo with a suitable value. 315 $$(eval $$(call SetJtregValue,$1,JTREG_TEST_MODE,agentvm)) 316 $$(eval $$(call SetJtregValue,$1,JTREG_ASSERT,true)) 317 $$(eval $$(call SetJtregValue,$1,JTREG_MAX_MEM,512m)) 318 $$(eval $$(call SetJtregValue,$1,JTREG_NATIVEPATH)) 319 $$(eval $$(call SetJtregValue,$1,JTREG_BASIC_OPTIONS)) 320 321 ifneq ($(TEST_JOBS), 0) 322 # User has specified TEST_JOBS, use that as fallback default 323 $$(eval $$(call SetJtregValue,$1,JTREG_JOBS,$$(TEST_JOBS))) 324 else 325 # Use JOBS as default (except for hotspot) 326 $$(eval $$(call SetJtregValue,$1,JTREG_JOBS,$$(JOBS))) 327 endif 328 329 ifeq ($$(shell $$(EXPR) $$($1_JTREG_JOBS) \> 50), 1) 330 # Until CODETOOLS-7901892 is fixed, JTreg cannot handle more than 50 jobs 331 $1_JTREG_JOBS := 50 332 endif 333 334 # Make sure MaxRAMPercentage is high enough to not cause OOM or swapping since 335 # we may end up with a lot of JVM's 336 $1_JTREG_MAX_RAM_PERCENTAGE := $$(shell $$(EXPR) 25 / $$($1_JTREG_JOBS)) 337 338 JTREG_TIMEOUT ?= 4 339 JTREG_VERBOSE ?= fail,error,summary 340 JTREG_RETAIN ?= fail,error 341 342 ifneq ($$($1_JTREG_MAX_MEM), 0) 343 $1_JTREG_BASIC_OPTIONS += -vmoption:-Xmx$$($1_JTREG_MAX_MEM) 344 $1_JTREG_LAUNCHER_OPTIONS += -Xmx$$($1_JTREG_MAX_MEM) 345 endif 346 347 $1_JTREG_BASIC_OPTIONS += -$$($1_JTREG_TEST_MODE) \ 348 -verbose:$$(JTREG_VERBOSE) -retain:$$(JTREG_RETAIN) \ 349 -concurrency:$$($1_JTREG_JOBS) -timeoutFactor:$$(JTREG_TIMEOUT) \ 350 -vmoption:-XX:MaxRAMPercentage=$$($1_JTREG_MAX_RAM_PERCENTAGE) 351 352 $1_JTREG_BASIC_OPTIONS += -automatic -keywords:\!ignore -ignore:quiet 353 354 # Make it possible to specify the JIB_DATA_DIR for tests using the 355 # JIB Artifact resolver 356 $1_JTREG_BASIC_OPTIONS += -e:JIB_DATA_DIR 357 # Some tests needs to find a boot JDK using the JDK8_HOME variable. 358 $1_JTREG_BASIC_OPTIONS += -e:JDK8_HOME=$$(BOOT_JDK) 359 360 $1_JTREG_BASIC_OPTIONS += \ 361 $$(addprefix -javaoption:, $$(JTREG_JAVA_OPTIONS)) \ 362 $$(addprefix -vmoption:, $$(JTREG_VM_OPTIONS)) \ 363 # 364 365 ifeq ($$($1_JTREG_ASSERT), true) 366 $1_JTREG_BASIC_OPTIONS += -ea -esa 367 endif 368 369 ifneq ($$($1_JTREG_NATIVEPATH), ) 370 $1_JTREG_BASIC_OPTIONS += -nativepath:$$($1_JTREG_NATIVEPATH) 371 endif 372 373 ifneq ($$(JIB_JAR), ) 374 $1_JTREG_BASIC_OPTIONS += -cpa:$$(JIB_JAR) 375 endif 376 377 run-test-$1: 378 $$(call LogWarn) 379 $$(call LogWarn, Running test '$$($1_TEST)') 380 $$(call MakeDir, $$($1_TEST_RESULTS_DIR) $$($1_TEST_SUPPORT_DIR)) 381 $$(call ExecuteWithLog, $$($1_TEST_SUPPORT_DIR)/jtreg, \ 382 $$(JAVA) $$($1_JTREG_LAUNCHER_OPTIONS) \ 383 -Dprogram=jtreg -jar $$(JT_HOME)/lib/jtreg.jar \ 384 $$($1_JTREG_BASIC_OPTIONS) \ 385 -testjdk:$$(JDK_IMAGE_DIR) \ 386 -dir:$$(TOPDIR) \ 387 -reportDir:$$($1_TEST_RESULTS_DIR) \ 388 -workDir:$$($1_TEST_SUPPORT_DIR) \ 389 $$(JTREG_OPTIONS) \ 390 $$($1_TEST_NAME) || true ) 391 392 $1_RESULT_FILE := $$($1_TEST_RESULTS_DIR)/text/stats.txt 393 394 parse-test-$1: run-test-$1 395 $$(call LogWarn, Finished running test '$$($1_TEST)') 396 $$(call LogWarn, Test report is stored in $$(strip \ 397 $$(subst $$(TOPDIR)/, , $$($1_TEST_RESULTS_DIR)))) 398 $$(if $$(wildcard $$($1_RESULT_FILE)), \ 399 $$(eval $1_PASSED := $$(shell $$(AWK) '{ gsub(/[,;]/, ""); \ 400 for (i=1; i<=NF; i++) { if ($$$$i == "passed:") \ 401 print $$$$(i+1) } }' $$($1_RESULT_FILE))) \ 402 $$(if $$($1_PASSED), , $$(eval $1_PASSED := 0)) \ 403 $$(eval $1_FAILED := $$(shell $$(AWK) '{gsub(/[,;]/, ""); \ 404 for (i=1; i<=NF; i++) { if ($$$$i == "failed:") \ 405 print $$$$(i+1) } }' $$($1_RESULT_FILE))) \ 406 $$(if $$($1_FAILED), , $$(eval $1_FAILED := 0)) \ 407 $$(eval $1_ERROR := $$(shell $$(AWK) '{gsub(/[,;]/, ""); \ 408 for (i=1; i<=NF; i++) { if ($$$$i == "error:") \ 409 print $$$$(i+1) } }' $$($1_RESULT_FILE))) \ 410 $$(if $$($1_ERROR), , $$(eval $1_ERROR := 0)) \ 411 $$(eval $1_TOTAL := $$(shell \ 412 $$(EXPR) $$($1_PASSED) + $$($1_FAILED) + $$($1_ERROR))) \ 413 , \ 414 $$(eval $1_PASSED := 0) \ 415 $$(eval $1_FAILED := 0) \ 416 $$(eval $1_ERROR := 1) \ 417 $$(eval $1_TOTAL := 1) \ 418 ) 419 420 $1: run-test-$1 parse-test-$1 421 422 TARGETS += $1 423 endef 424 425 426 ################################################################################ 427 # Setup and execute make rules for all selected tests 428 ################################################################################ 429 430 # Helper function to determine which handler to use for the given test 431 UseGtestTestHandler = \ 432 $(if $(filter gtest:%, $1), true) 433 434 UseJtregTestHandler = \ 435 $(if $(filter jtreg:%, $1), true) 436 437 # Now process each test to run and setup a proper make rule 438 $(foreach test, $(TESTS_TO_RUN), \ 439 $(eval TEST_ID := $(shell $(ECHO) $(strip $(test)) | \ 440 $(TR) -cs '[a-z][A-Z][0-9]\n' '_')) \ 441 $(eval ALL_TEST_IDS += $(TEST_ID)) \ 442 $(if $(call UseCustomTestHandler, $(test)), \ 443 $(eval $(call SetupRunCustomTest, $(TEST_ID), \ 444 TEST := $(test), \ 445 )) \ 446 ) \ 447 $(if $(call UseGtestTestHandler, $(test)), \ 448 $(eval $(call SetupRunGtestTest, $(TEST_ID), \ 449 TEST := $(test), \ 450 )) \ 451 ) \ 452 $(if $(call UseJtregTestHandler, $(test)), \ 453 $(eval $(call SetupRunJtregTest, $(TEST_ID), \ 454 TEST := $(test), \ 455 )) \ 456 ) \ 457 ) 458 459 # Sort also removes duplicates, so if there is any we'll get fewer words. 460 ifneq ($(words $(ALL_TEST_IDS)), $(words $(sort $(ALL_TEST_IDS)))) 461 $(error Duplicate test specification) 462 endif 463 464 465 ################################################################################ 466 # The main target for RunTests.gmk 467 ################################################################################ 468 469 # The SetupRun*Test functions have populated TARGETS. 470 471 TEST_FAILURE := false 472 473 run-test: $(TARGETS) 474 # Print a table of the result of all tests run and their result 475 $(ECHO) 476 $(ECHO) ============================== 477 $(ECHO) Test summary 478 $(ECHO) ============================== 479 $(PRINTF) "%2s %-49s %5s %5s %5s %5s %2s\n" " " TEST \ 480 TOTAL PASS FAIL ERROR " " 481 $(foreach test, $(TESTS_TO_RUN), \ 482 $(eval TEST_ID := $(shell $(ECHO) $(strip $(test)) | \ 483 $(TR) -cs '[a-z][A-Z][0-9]\n' '_')) \ 484 $(if $(filter $($(TEST_ID)_PASSED), $($(TEST_ID)_TOTAL)), \ 485 $(PRINTF) "%2s %-49s %5d %5d %5d %5d %2s\n" " " "$(test)" \ 486 $($(TEST_ID)_TOTAL) $($(TEST_ID)_PASSED) $($(TEST_ID)_FAILED) \ 487 $($(TEST_ID)_ERROR) " " $(NEWLINE) \ 488 , \ 489 $(PRINTF) "%2s %-49s %5d %5d %5d %5d %2s\n" ">>" "$(test)" \ 490 $($(TEST_ID)_TOTAL) $($(TEST_ID)_PASSED) $($(TEST_ID)_FAILED) \ 491 $($(TEST_ID)_ERROR) "<<" $(NEWLINE) \ 492 $(eval TEST_FAILURE := true) \ 493 ) \ 494 ) 495 $(ECHO) ============================== 496 $(if $(filter true, $(TEST_FAILURE)), \ 497 $(ECHO) TEST FAILURE $(NEWLINE) \ 498 $(TOUCH) $(MAKESUPPORT_OUTPUTDIR)/exit-with-error \ 499 , \ 500 $(ECHO) TEST SUCCESS \ 501 ) 502 $(ECHO) 503 504 ################################################################################ 505 506 all: run-test 507 508 .PHONY: default all run-test $(TARGETS)