1 #
   2 # Copyright (c) 2016, 2020, Oracle and/or its affiliates. All rights reserved.
   3 # DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4 #
   5 # This code is free software; you can redistribute it and/or modify it
   6 # under the terms of the GNU General Public License version 2 only, as
   7 # published by the Free Software Foundation.  Oracle designates this
   8 # particular file as subject to the "Classpath" exception as provided
   9 # by Oracle in the LICENSE file that accompanied this code.
  10 #
  11 # This code is distributed in the hope that it will be useful, but WITHOUT
  12 # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  13 # FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  14 # version 2 for more details (a copy is included in the LICENSE file that
  15 # accompanied this code).
  16 #
  17 # You should have received a copy of the GNU General Public License version
  18 # 2 along with this work; if not, write to the Free Software Foundation,
  19 # Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  20 #
  21 # Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  22 # or visit www.oracle.com if you need additional information or have any
  23 # questions.
  24 #
  25 
  26 default: all
  27 
  28 include $(SPEC)
  29 include MakeBase.gmk
  30 include FindTests.gmk
  31 
  32 # We will always run multiple tests serially
  33 .NOTPARALLEL:
  34 
  35 ################################################################################
  36 # Parse global control variables
  37 ################################################################################
  38 
  39 ifneq ($(TEST_VM_OPTS), )
  40   ifneq ($(TEST_OPTS), )
  41     TEST_OPTS := $(TEST_OPTS);VM_OPTIONS=$(TEST_VM_OPTS)
  42   else
  43     TEST_OPTS := VM_OPTIONS=$(TEST_VM_OPTS)
  44   endif
  45 endif
  46 
  47 $(eval $(call ParseKeywordVariable, TEST_OPTS, \
  48     SINGLE_KEYWORDS := JOBS TIMEOUT_FACTOR JCOV JCOV_DIFF_CHANGESET, \
  49     STRING_KEYWORDS := VM_OPTIONS JAVA_OPTIONS AOT_MODULES, \
  50 ))
  51 
  52 # Helper function to propagate TEST_OPTS values.
  53 #
  54 # Note: No spaces are allowed around the arguments.
  55 # Arg $1 The variable in TEST_OPTS to propagate
  56 # Arg $2 The control variable to propagate it to
  57 define SetTestOpt
  58   ifneq ($$(TEST_OPTS_$1), )
  59     $2_$1 := $$(TEST_OPTS_$1)
  60   endif
  61 endef
  62 
  63 # Setup _NT_SYMBOL_PATH on Windows
  64 ifeq ($(call isTargetOs, windows), true)
  65   ifndef _NT_SYMBOL_PATH
  66     # Can't use PathList here as it adds quotes around the value.
  67     _NT_SYMBOL_PATH := \
  68         $(subst $(SPACE),;,$(strip \
  69             $(foreach p, $(sort $(dir $(wildcard \
  70                 $(addprefix $(SYMBOLS_IMAGE_DIR)/bin/, *.pdb */*.pdb)))), \
  71               $(call FixPath, $p) \
  72             ) \
  73         ))
  74     export _NT_SYMBOL_PATH
  75     $(call LogDebug, Rewriting _NT_SYMBOL_PATH to $(_NT_SYMBOL_PATH))
  76   endif
  77 endif
  78 
  79 ################################################################################
  80 # Hook to include the corresponding custom file, if present.
  81 $(eval $(call IncludeCustomExtension, RunTests.gmk))
  82 ################################################################################
  83 
  84 # This is the JDK that we will test
  85 JDK_UNDER_TEST := $(JDK_IMAGE_DIR)
  86 
  87 TEST_RESULTS_DIR := $(OUTPUTDIR)/test-results
  88 TEST_SUPPORT_DIR := $(OUTPUTDIR)/test-support
  89 TEST_SUMMARY := $(TEST_RESULTS_DIR)/test-summary.txt
  90 TEST_LAST_IDS := $(TEST_SUPPORT_DIR)/test-last-ids.txt
  91 
  92 ifeq ($(CUSTOM_ROOT), )
  93   JTREG_TOPDIR := $(TOPDIR)
  94 else
  95   JTREG_TOPDIR := $(CUSTOM_ROOT)
  96 endif
  97 
  98 JTREG_FAILURE_HANDLER_DIR := $(TEST_IMAGE_DIR)/failure_handler
  99 JTREG_FAILURE_HANDLER := $(JTREG_FAILURE_HANDLER_DIR)/jtregFailureHandler.jar
 100 
 101 JTREG_FAILURE_HANDLER_TIMEOUT ?= 0
 102 
 103 ifneq ($(wildcard $(JTREG_FAILURE_HANDLER)), )
 104   JTREG_FAILURE_HANDLER_OPTIONS := \
 105       -timeoutHandlerDir:$(JTREG_FAILURE_HANDLER) \
 106       -observerDir:$(JTREG_FAILURE_HANDLER) \
 107       -timeoutHandler:jdk.test.failurehandler.jtreg.GatherProcessInfoTimeoutHandler \
 108       -observer:jdk.test.failurehandler.jtreg.GatherDiagnosticInfoObserver \
 109       -timeoutHandlerTimeout:$(JTREG_FAILURE_HANDLER_TIMEOUT)
 110 endif
 111 
 112 GTEST_LAUNCHER_DIRS := $(patsubst %/gtestLauncher, %, \
 113     $(wildcard $(TEST_IMAGE_DIR)/hotspot/gtest/*/gtestLauncher))
 114 GTEST_VARIANTS := $(strip $(patsubst $(TEST_IMAGE_DIR)/hotspot/gtest/%, %, \
 115     $(GTEST_LAUNCHER_DIRS)))
 116 
 117 COV_ENVIRONMENT :=
 118 JTREG_COV_OPTIONS :=
 119 
 120 ifeq ($(TEST_OPTS_JCOV), true)
 121   JCOV_OUTPUT_DIR := $(TEST_RESULTS_DIR)/jcov-output
 122   JCOV_GRABBER_LOG := $(JCOV_OUTPUT_DIR)/grabber.log
 123   JCOV_RESULT_FILE := $(JCOV_OUTPUT_DIR)/result.xml
 124   JCOV_REPORT := $(JCOV_OUTPUT_DIR)/report
 125   JCOV_MEM_OPTIONS := -Xms64m -Xmx4g
 126 
 127   # Replace our normal test JDK with the JCov image.
 128   JDK_UNDER_TEST := $(JCOV_IMAGE_DIR)
 129 
 130   COV_ENVIRONMENT += JAVA_TOOL_OPTIONS="$(JCOV_MEM_OPTIONS)" \
 131       _JAVA_OPTIONS="$(JCOV_MEM_OPTIONS)"
 132   JTREG_COV_OPTIONS += -e:JAVA_TOOL_OPTIONS='$(JCOV_MEM_OPTIONS)' \
 133       -e:_JAVA_OPTIONS='$(JCOV_MEM_OPTIONS)'
 134 endif
 135 
 136 ifeq ($(GCOV_ENABLED), true)
 137   GCOV_OUTPUT_DIR := $(TEST_RESULTS_DIR)/gcov-output
 138   COV_ENVIRONMENT += GCOV_PREFIX="$(GCOV_OUTPUT_DIR)"
 139   JTREG_COV_OPTIONS += -e:GCOV_PREFIX="$(GCOV_OUTPUT_DIR)"
 140 endif
 141 
 142 ################################################################################
 143 # Optionally create AOT libraries for specified modules before running tests.
 144 # Note, this could not be done during JDK build time.
 145 ################################################################################
 146 
 147 # Note, this could not be done during JDK build time.
 148 
 149 # Parameter 1 is the name of the rule.
 150 #
 151 # Remaining parameters are named arguments.
 152 #   MODULE      The module to generate a library for
 153 #   BIN         Output directory in which to put the library
 154 #   VM_OPTIONS  List of JVM arguments to use when creating library
 155 #   OPTIONS_VAR Name of variable to put AOT java options in
 156 #   PREREQS_VAR Name of variable to put all AOT prerequisite rule targets in
 157 #               for test rules to depend on
 158 #
 159 SetupAotModule = $(NamedParamsMacroTemplate)
 160 define SetupAotModuleBody
 161   $1_AOT_LIB := $$($1_BIN)/$$(call SHARED_LIBRARY,$$($1_MODULE))
 162   $1_AOT_CCLIST := $$(wildcard $$(TOPDIR)/test/hotspot/jtreg/compiler/aot/scripts/$$($1_MODULE)-list.txt)
 163 
 164   # Create jaotc flags.
 165   # VM flags which don't affect AOT code generation are filtered out:
 166   # -Xcomp, -XX:+-TieredCompilation
 167   $1_JAOTC_OPTS := \
 168       -J-Xmx4g --info \
 169       $$(addprefix -J, $$(filter-out -Xcomp %TieredCompilation, $$($1_VM_OPTIONS))) \
 170       $$(addprefix --compile-commands$(SPACE), $$($1_AOT_CCLIST)) \
 171       --linker-path $$(LD_JAOTC) \
 172       #
 173 
 174   ifneq ($$(filter -ea, $$($1_VM_OPTIONS)), )
 175     $1_JAOTC_OPTS += --compile-with-assertions
 176   endif
 177 
 178   $$($1_AOT_LIB): $$(JDK_UNDER_TEST)/release \
 179       $$(call DependOnVariable, $1_JAOTC_OPTS) \
 180       $$(call DependOnVariable, JDK_UNDER_TEST)
 181         $$(call LogWarn, Generating $$(patsubst $$(OUTPUTDIR)/%, %, $$@))
 182         $$(call MakeTargetDir)
 183         $$(call ExecuteWithLog, $$@, \
 184             $((COV_ENVIRONMENT) \
 185             $$(FIXPATH) $$(JDK_UNDER_TEST)/bin/jaotc \
 186                 $$($1_JAOTC_OPTS) --output $$@ --module $$($1_MODULE) \
 187         )
 188         $$(call ExecuteWithLog, $$@.check, ( \
 189             $$(FIXPATH) $$(JDK_UNDER_TEST)/bin/java \
 190                 $$($1_VM_OPTIONS) -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions \
 191                 -XX:+PrintAOT -XX:+UseAOTStrictLoading \
 192                 -XX:AOTLibrary=$$@ -version \
 193                  > $$@.verify-aot \
 194         ))
 195 
 196   $1_AOT_OPTIONS += -XX:+UnlockExperimentalVMOptions
 197   $1_AOT_OPTIONS += -XX:AOTLibrary=$$($1_AOT_LIB)
 198   $1_AOT_TARGETS += $$($1_AOT_LIB)
 199 endef
 200 
 201 # Parameter 1 is the name of the rule.
 202 #
 203 # Remaining parameters are named arguments.
 204 #   MODULES     The modules to generate a library for
 205 #   VM_OPTIONS  List of JVM arguments to use when creating libraries
 206 #
 207 # After calling this, the following variables are defined
 208 #   $1_AOT_OPTIONS List of all java options needed to use the AOT libraries
 209 #   $1_AOT_TARGETS List of all targets that the test rule will need to depend on
 210 #
 211 SetupAot = $(NamedParamsMacroTemplate)
 212 define SetupAotBody
 213   $$(info Running with AOTd libraries for $$($1_MODULES))
 214   # Put aot libraries in a separate directory so they are not deleted between
 215   # test runs and may be reused between make invocations.
 216   $$(foreach m, $$($1_MODULES), \
 217     $$(eval $$(call SetupAotModule, $1_$$m, \
 218         MODULE := $$m, \
 219         BIN := $$(TEST_SUPPORT_DIR)/aot/$1, \
 220         VM_OPTIONS := $$($1_VM_OPTIONS), \
 221     )) \
 222     $$(eval $1_AOT_OPTIONS += $$($1_$$m_AOT_OPTIONS)) \
 223     $$(eval $1_AOT_TARGETS += $$($1_$$m_AOT_TARGETS)) \
 224   )
 225 endef
 226 
 227 ################################################################################
 228 # Setup global test running parameters
 229 ################################################################################
 230 
 231 # Each factor variable comes in 3 variants. The first one is reserved for users
 232 # to use on command line. The other two are for predifined configurations in JDL
 233 # and for machine specific configurations respectively.
 234 TEST_JOBS_FACTOR ?= 1
 235 TEST_JOBS_FACTOR_JDL ?= 1
 236 TEST_JOBS_FACTOR_MACHINE ?= 1
 237 
 238 ifeq ($(TEST_JOBS), 0)
 239   CORES_DIVIDER := 2
 240   ifeq ($(call isTargetCpuArch, sparc), true)
 241     # For smaller SPARC machines we see reasonable scaling of throughput up to
 242     # cpus/4 without affecting test reliability. On the bigger machines, cpus/4
 243     # causes intermittent timeouts.
 244     ifeq ($(shell $(EXPR) $(NUM_CORES) \> 16), 1)
 245       CORES_DIVIDER := 5
 246     else
 247       CORES_DIVIDER := 4
 248     endif
 249   endif
 250   # For some big multi-core machines with low ulimit -u setting we hit the max
 251   # threads/process limit. In such a setup the memory/cores-only-guided
 252   # TEST_JOBS config is insufficient. From experience a concurrency setting of
 253   # 14 works reasonably well for low ulimit values (<= 4096). Thus, use
 254   # divider 4096/14. For high ulimit -u values this shouldn't make a difference.
 255   ULIMIT_DIVIDER := (4096/14)
 256   PROC_ULIMIT := -1
 257   ifneq ($(OPENJDK_TARGET_OS), windows)
 258     PROC_ULIMIT := $(shell $(ULIMIT) -u)
 259     ifeq ($(PROC_ULIMIT), unlimited)
 260       PROC_ULIMIT := -1
 261     endif
 262   endif
 263   MEMORY_DIVIDER := 2048
 264   TEST_JOBS := $(shell $(AWK) \
 265     'BEGIN { \
 266       c = $(NUM_CORES) / $(CORES_DIVIDER); \
 267       m = $(MEMORY_SIZE) / $(MEMORY_DIVIDER); \
 268       u = $(PROC_ULIMIT); \
 269       if (u > -1) { \
 270         u = u / $(ULIMIT_DIVIDER); \
 271         if (u < c) c = u; \
 272       } \
 273       if (c > m) c = m; \
 274       c = c * $(TEST_JOBS_FACTOR); \
 275       c = c * $(TEST_JOBS_FACTOR_JDL); \
 276       c = c * $(TEST_JOBS_FACTOR_MACHINE); \
 277       if (c < 1) c = 1; \
 278       printf "%.0f", c; \
 279     }')
 280 endif
 281 
 282 ################################################################################
 283 # Parse control variables
 284 ################################################################################
 285 
 286 ifneq ($(TEST_OPTS), )
 287   # Inform the user
 288   $(info Running tests using TEST_OPTS control variable '$(TEST_OPTS)')
 289 endif
 290 
 291 ### Jtreg
 292 
 293 $(eval $(call SetTestOpt,VM_OPTIONS,JTREG))
 294 $(eval $(call SetTestOpt,JAVA_OPTIONS,JTREG))
 295 $(eval $(call SetTestOpt,AOT_MODULES,JTREG))
 296 
 297 $(eval $(call SetTestOpt,JOBS,JTREG))
 298 $(eval $(call SetTestOpt,TIMEOUT_FACTOR,JTREG))
 299 $(eval $(call SetTestOpt,FAILURE_HANDLER_TIMEOUT,JTREG))
 300 
 301 $(eval $(call ParseKeywordVariable, JTREG, \
 302     SINGLE_KEYWORDS := JOBS TIMEOUT_FACTOR FAILURE_HANDLER_TIMEOUT \
 303         TEST_MODE ASSERT VERBOSE RETAIN MAX_MEM RUN_PROBLEM_LISTS \
 304         RETRY_COUNT, \
 305     STRING_KEYWORDS := OPTIONS JAVA_OPTIONS VM_OPTIONS KEYWORDS \
 306         EXTRA_PROBLEM_LISTS AOT_MODULES, \
 307 ))
 308 
 309 ifneq ($(JTREG), )
 310   # Inform the user
 311   $(info Running tests using JTREG control variable '$(JTREG)')
 312 endif
 313 
 314 ### Gtest
 315 
 316 $(eval $(call SetTestOpt,VM_OPTIONS,GTEST))
 317 $(eval $(call SetTestOpt,JAVA_OPTIONS,GTEST))
 318 $(eval $(call SetTestOpt,AOT_MODULES,GTEST))
 319 
 320 $(eval $(call ParseKeywordVariable, GTEST, \
 321     SINGLE_KEYWORDS := REPEAT, \
 322     STRING_KEYWORDS := OPTIONS VM_OPTIONS JAVA_OPTIONS AOT_MODULES, \
 323 ))
 324 
 325 ifneq ($(GTEST), )
 326   # Inform the user
 327   $(info Running tests using GTEST control variable '$(GTEST)')
 328 endif
 329 
 330 ### Microbenchmarks
 331 
 332 $(eval $(call SetTestOpt,VM_OPTIONS,MICRO))
 333 $(eval $(call SetTestOpt,JAVA_OPTIONS,MICRO))
 334 
 335 $(eval $(call ParseKeywordVariable, MICRO, \
 336     SINGLE_KEYWORDS := ITER FORK TIME WARMUP_ITER WARMUP_TIME, \
 337     STRING_KEYWORDS := OPTIONS JAVA_OPTIONS VM_OPTIONS RESULTS_FORMAT TEST_JDK \
 338         BENCHMARKS_JAR, \
 339 ))
 340 
 341 ifneq ($(MICRO), )
 342   # Inform the user
 343   $(info Running tests using MICRO control variable '$(MICRO)')
 344 endif
 345 
 346 
 347 ################################################################################
 348 # Component-specific Jtreg settings
 349 ################################################################################
 350 
 351 hotspot_JTREG_MAX_MEM := 0
 352 hotspot_JTREG_ASSERT := false
 353 hotspot_JTREG_NATIVEPATH := $(TEST_IMAGE_DIR)/hotspot/jtreg/native
 354 jdk_JTREG_NATIVEPATH := $(TEST_IMAGE_DIR)/jdk/jtreg/native
 355 
 356 jdk_JTREG_PROBLEM_LIST += $(TOPDIR)/test/jdk/ProblemList.txt
 357 jaxp_JTREG_PROBLEM_LIST += $(TOPDIR)/test/jaxp/ProblemList.txt
 358 langtools_JTREG_PROBLEM_LIST += $(TOPDIR)/test/langtools/ProblemList.txt
 359 
 360 langtools_JTREG_MAX_MEM := 768m
 361 
 362 ################################################################################
 363 # Parse test selection
 364 #
 365 # The user has given a test selection in the TEST variable. We must parse it
 366 # and determine what that means in terms of actual calls to the test framework.
 367 #
 368 # The parse functions take as argument a test specification as given by the
 369 # user, and returns a fully qualified test descriptor if it was a match, or
 370 # nothing if not. A single test specification can result in multiple test
 371 # descriptors being returned. A valid test descriptor must always be accepted
 372 # and returned identically.
 373 ################################################################################
 374 
 375 # Helper function to determine if a test specification is a Gtest test
 376 #
 377 # It is a Gtest test if it is either "gtest", or "gtest:" followed by an optional
 378 # test filter string, and an optional "/<variant>" to select a specific JVM
 379 # variant. If no variant is specified, all found variants are tested.
 380 define ParseGtestTestSelection
 381   $(if $(filter gtest%, $1), \
 382     $(if $(filter gtest, $1), \
 383       $(addprefix gtest:all/, $(GTEST_VARIANTS)) \
 384     , \
 385       $(if $(strip $(or $(filter gtest/%, $1) $(filter gtest:/%, $1))), \
 386         $(patsubst gtest:/%, gtest:all/%, $(patsubst gtest/%, gtest:/%, $1)) \
 387       , \
 388         $(if $(filter gtest:%, $1), \
 389           $(if $(findstring /, $1), \
 390             $1 \
 391           , \
 392             $(addprefix $1/, $(GTEST_VARIANTS)) \
 393           ) \
 394         ) \
 395       ) \
 396     ) \
 397   )
 398 endef
 399 
 400 # Helper function to determine if a test specification is a microbenchmark test
 401 #
 402 # It is a microbenchmark test if it is either "micro", or "micro:" followed by
 403 # an optional test filter string.
 404 define ParseMicroTestSelection
 405   $(if $(filter micro%, $1), \
 406     $(if $(filter micro, $1), \
 407       micro:all \
 408     , \
 409       $(if $(filter micro:, $1), \
 410         micro:all \
 411       , \
 412         $1 \
 413       ) \
 414     ) \
 415   )
 416 endef
 417 
 418 # Helper function that removes the TOPDIR part
 419 CleanupJtregPath = \
 420   $(strip $(patsubst %/, %, $(subst $(JTREG_TOPDIR)/,, $1)))
 421 
 422 # Take a partial Jtreg root path and return a full, absolute path to that Jtreg
 423 # root. Also support having "hotspot" as an alias for "hotspot/jtreg".
 424 ExpandJtregRoot = \
 425   $(call CleanupJtregPath, $(wildcard \
 426     $(if $(filter /%, $1), \
 427       $(if $(wildcard $(strip $1)/TEST.ROOT), \
 428         $1 \
 429       ) \
 430     , \
 431       $(filter $(addprefix %, $1), $(JTREG_TESTROOTS) $(addsuffix /, $(JTREG_TESTROOTS))) \
 432       $(filter $(addprefix %, $(strip $1)/jtreg), $(JTREG_TESTROOTS) $(addsuffix /, $(JTREG_TESTROOTS))) \
 433     ) \
 434   ))
 435 
 436 # Take a partial Jtreg test path and return a full, absolute path to that Jtreg
 437 # test. Also support having "hotspot" as an alias for "hotspot/jtreg".
 438 ExpandJtregPath = \
 439   $(if $(call ExpandJtregRoot, $1), \
 440     $(call ExpandJtregRoot, $1) \
 441   , \
 442     $(call CleanupJtregPath, $(wildcard \
 443       $(if $(filter /%, $1), \
 444         $1 \
 445       , \
 446         $(addsuffix /$(strip $1), $(JTREG_TESTROOTS) $(TEST_BASEDIRS)) \
 447         $(addsuffix $(strip $(patsubst hotspot/%, /hotspot/jtreg/%, $1)), $(JTREG_TESTROOTS) $(TEST_BASEDIRS)) \
 448       ) \
 449     )) \
 450   )
 451 
 452 # Helper function to determine if a test specification is a Jtreg test
 453 #
 454 # It is a Jtreg test if it optionally begins with jtreg:, and then is either
 455 # an unspecified group name (possibly prefixed by :), or a group in a
 456 # specified test root, or a path to a test or test directory,
 457 # either absolute or relative to any of the TEST_BASEDIRS or test roots.
 458 define ParseJtregTestSelection
 459   $(eval TEST_NAME := $(strip $(patsubst jtreg:%, %, $1))) \
 460   $(if $(or $(findstring :, $(TEST_NAME)), $(findstring /, $(TEST_NAME))), , \
 461     $(eval TEST_NAME := :$(TEST_NAME)) \
 462   ) \
 463   $(if $(findstring :, $(TEST_NAME)), \
 464     $(if $(filter :%, $(TEST_NAME)), \
 465       $(eval TEST_GROUP := $(patsubst :%, %, $(TEST_NAME))) \
 466       $(eval TEST_ROOTS := $(foreach test_root, $(JTREG_TESTROOTS), \
 467           $(call CleanupJtregPath, $(test_root)))) \
 468     , \
 469       $(eval TEST_PATH := $(word 1, $(subst :, $(SPACE), $(TEST_NAME)))) \
 470       $(eval TEST_GROUP := $(word 2, $(subst :, $(SPACE), $(TEST_NAME)))) \
 471       $(eval TEST_ROOTS := $(call ExpandJtregRoot, $(TEST_PATH))) \
 472     ) \
 473     $(foreach test_root, $(TEST_ROOTS), \
 474       $(if $(filter /%, $(test_root)), \
 475         jtreg:$(test_root):$(TEST_GROUP) \
 476       , \
 477         $(if $(filter $(TEST_GROUP), $($(JTREG_TOPDIR)/$(test_root)_JTREG_TEST_GROUPS)), \
 478           jtreg:$(test_root):$(TEST_GROUP) \
 479         ) \
 480       ) \
 481     ) \
 482   , \
 483     $(eval TEST_PATHS := $(call ExpandJtregPath, $(TEST_NAME))) \
 484     $(foreach test_path, $(TEST_PATHS), \
 485       jtreg:$(test_path) \
 486     ) \
 487   )
 488 endef
 489 
 490 # Helper function to determine if a test specification is a special test
 491 #
 492 # It is a special test if it is "special:" followed by a test name,
 493 # if it is "make:" or "make-" followed by a make test, or any of the special
 494 # test names as a single word.
 495 define ParseSpecialTestSelection
 496   $(if $(filter special:%, $1), \
 497     $1 \
 498   ) \
 499   $(if $(filter make%, $1), \
 500     $(if $(filter make:%, $1), \
 501       special:$(strip $1) \
 502     ) \
 503     $(if $(filter make-%, $1), \
 504       special:$(patsubst make-%,make:%, $1) \
 505     ) \
 506     $(if $(filter make, $1), \
 507       special:make:all \
 508     )
 509   ) \
 510   $(if $(filter failure-handler, $1), \
 511     special:$(strip $1) \
 512   )
 513 endef
 514 
 515 ifeq ($(TEST), )
 516   $(info No test selection given in TEST!)
 517   $(info Please use e.g. 'make test TEST=tier1' or 'make test-tier1')
 518   $(info See doc/testing.[md|html] for help)
 519   $(error Cannot continue)
 520 endif
 521 
 522 # Now intelligently convert the test selection given by the user in TEST
 523 # into a list of fully qualified test descriptors of the tests to run.
 524 TESTS_TO_RUN :=
 525 $(foreach test, $(TEST), \
 526   $(eval PARSED_TESTS := $(call ParseCustomTestSelection, $(test))) \
 527   $(if $(strip $(PARSED_TESTS)), , \
 528     $(eval PARSED_TESTS += $(call ParseGtestTestSelection, $(test))) \
 529   ) \
 530   $(if $(strip $(PARSED_TESTS)), , \
 531     $(eval PARSED_TESTS += $(call ParseMicroTestSelection, $(test))) \
 532   ) \
 533   $(if $(strip $(PARSED_TESTS)), , \
 534     $(eval PARSED_TESTS += $(call ParseJtregTestSelection, $(test))) \
 535   ) \
 536   $(if $(strip $(PARSED_TESTS)), , \
 537     $(eval PARSED_TESTS += $(call ParseSpecialTestSelection, $(test))) \
 538   ) \
 539   $(if $(strip $(PARSED_TESTS)), , \
 540     $(eval UNKNOWN_TEST := $(test)) \
 541   ) \
 542   $(eval TESTS_TO_RUN += $(PARSED_TESTS)) \
 543 )
 544 
 545 ifneq ($(UNKNOWN_TEST), )
 546   $(info Unknown test selection: '$(UNKNOWN_TEST)')
 547   $(info See doc/testing.[md|html] for help)
 548   $(error Cannot continue)
 549 endif
 550 
 551 TESTS_TO_RUN := $(strip $(TESTS_TO_RUN))
 552 
 553 
 554 # Present the result of our parsing to the user
 555 $(info Test selection '$(TEST)', will run:)
 556 $(foreach test, $(TESTS_TO_RUN), $(info * $(test)))
 557 
 558 
 559 ################################################################################
 560 # Functions for setting up rules for running the selected tests
 561 #
 562 # The SetupRun*Test functions all have the same interface:
 563 #
 564 # Parameter 1 is the name of the rule. This is the test id, based on the test
 565 # descriptor, and this is also used as variable prefix, and the targets
 566 # generated are listed in a variable by that name.
 567 #
 568 # Remaining parameters are named arguments. Currently this is only:
 569 #   TEST -- The properly formatted fully qualified test descriptor
 570 #
 571 # After the rule named by the test id has been executed, the following
 572 # variables will be available:
 573 # testid_TOTAL - the total number of tests run
 574 # testid_PASSED - the number of successful tests
 575 # testid_FAILED - the number of failed tests
 576 # testid_ERROR - the number of tests was neither successful or failed
 577 #
 578 ################################################################################
 579 
 580 ### Rules for Gtest
 581 
 582 SetupRunGtestTest = $(NamedParamsMacroTemplate)
 583 define SetupRunGtestTestBody
 584   $1_TEST_RESULTS_DIR := $$(TEST_RESULTS_DIR)/$1
 585   $1_TEST_SUPPORT_DIR := $$(TEST_SUPPORT_DIR)/$1
 586   $1_EXITCODE := $$($1_TEST_RESULTS_DIR)/exitcode.txt
 587 
 588   $1_VARIANT :=  $$(lastword $$(subst /, , $$($1_TEST)))
 589   ifeq ($$(filter $$($1_VARIANT), $$(GTEST_VARIANTS)), )
 590     $$(error Invalid gtest variant '$$($1_VARIANT)'. Valid variants: $$(GTEST_VARIANTS))
 591   endif
 592   $1_TEST_NAME := $$(strip $$(patsubst %/$$($1_VARIANT), %, \
 593       $$(patsubst gtest:%, %, $$($1_TEST))))
 594   ifneq ($$($1_TEST_NAME), all)
 595     $1_GTEST_FILTER := --gtest_filter=$$($1_TEST_NAME)*
 596   endif
 597 
 598   ifneq ($$(GTEST_REPEAT), )
 599     $1_GTEST_REPEAT :=--gtest_repeat=$$(GTEST_REPEAT)
 600   endif
 601 
 602   ifneq ($$(GTEST_AOT_MODULES), )
 603     $$(eval $$(call SetupAot, $1, \
 604         MODULES := $$(GTEST_AOT_MODULES), \
 605         VM_OPTIONS := $$(GTEST_VM_OPTIONS) $$(GTEST_JAVA_OPTIONS), \
 606     ))
 607   endif
 608 
 609   run-test-$1: pre-run-test $$($1_AOT_TARGETS)
 610         $$(call LogWarn)
 611         $$(call LogWarn, Running test '$$($1_TEST)')
 612         $$(call MakeDir, $$($1_TEST_RESULTS_DIR) $$($1_TEST_SUPPORT_DIR))
 613         $$(call ExecuteWithLog, $$($1_TEST_SUPPORT_DIR)/gtest, ( \
 614             $$(FIXPATH) $$(TEST_IMAGE_DIR)/hotspot/gtest/$$($1_VARIANT)/gtestLauncher \
 615                 -jdk $(JDK_UNDER_TEST) $$($1_GTEST_FILTER) \
 616                 --gtest_output=xml:$$($1_TEST_RESULTS_DIR)/gtest.xml \
 617                 $$($1_GTEST_REPEAT) $$(GTEST_OPTIONS) $$(GTEST_VM_OPTIONS) \
 618                 $$(GTEST_JAVA_OPTIONS) $$($1_AOT_OPTIONS) \
 619                 > >($(TEE) $$($1_TEST_RESULTS_DIR)/gtest.txt) \
 620             && $$(ECHO) $$$$? > $$($1_EXITCODE) \
 621             || $$(ECHO) $$$$? > $$($1_EXITCODE) \
 622         ))
 623 
 624   $1_RESULT_FILE := $$($1_TEST_RESULTS_DIR)/gtest.txt
 625 
 626   parse-test-$1: run-test-$1
 627         $$(call LogWarn, Finished running test '$$($1_TEST)')
 628         $$(call LogWarn, Test report is stored in $$(strip \
 629             $$(subst $$(TOPDIR)/, , $$($1_TEST_RESULTS_DIR))))
 630         $$(if $$(wildcard $$($1_RESULT_FILE)), \
 631           $$(eval $1_TOTAL := $$(shell $$(AWK) '/==========.* tests? from .* \
 632               test cases? ran/ { print $$$$2 }' $$($1_RESULT_FILE))) \
 633           $$(if $$($1_TOTAL), , $$(eval $1_TOTAL := 0)) \
 634           $$(eval $1_PASSED := $$(shell $$(AWK) '/\[  PASSED  \] .* tests?./ \
 635               { print $$$$4 }' $$($1_RESULT_FILE))) \
 636           $$(if $$($1_PASSED), , $$(eval $1_PASSED := 0)) \
 637           $$(eval $1_FAILED := $$(shell $$(AWK) '/\[  FAILED  \] .* tests?, \
 638               listed below/ { print $$$$4 }' $$($1_RESULT_FILE))) \
 639           $$(if $$($1_FAILED), , $$(eval $1_FAILED := 0)) \
 640           $$(eval $1_ERROR := $$(shell \
 641               $$(EXPR) $$($1_TOTAL) - $$($1_PASSED) - $$($1_FAILED))) \
 642         , \
 643           $$(eval $1_PASSED := 0) \
 644           $$(eval $1_FAILED := 0) \
 645           $$(eval $1_ERROR := 1) \
 646           $$(eval $1_TOTAL := 1) \
 647         )
 648 
 649   $1: run-test-$1 parse-test-$1
 650 
 651   TARGETS += $1 run-test-$1 parse-test-$1
 652   TEST_TARGETS += parse-test-$1
 653 
 654 endef
 655 
 656 ################################################################################
 657 
 658 ### Rules for Microbenchmarks
 659 
 660 # Helper function for SetupRunMicroTest. Set a MICRO_* variable from, in order:
 661 # 1) Specified by user on command line
 662 # 2) Generic default
 663 #
 664 # Note: No spaces are allowed around the arguments.
 665 # Arg $1 The test ID (i.e. $1 in SetupRunMicroTest)
 666 # Arg $2 Base variable, e.g. MICRO_TEST_JDK
 667 # Arg $3 The default value (optional)
 668 define SetMicroValue
 669   ifneq ($$($2), )
 670     $1_$2 := $$($2)
 671   else
 672     ifneq ($3, )
 673       $1_$2 := $3
 674     endif
 675   endif
 676 endef
 677 
 678 SetupRunMicroTest = $(NamedParamsMacroTemplate)
 679 define SetupRunMicroTestBody
 680   $1_TEST_RESULTS_DIR := $$(TEST_RESULTS_DIR)/$1
 681   $1_TEST_SUPPORT_DIR := $$(TEST_SUPPORT_DIR)/$1
 682   $1_EXITCODE := $$($1_TEST_RESULTS_DIR)/exitcode.txt
 683 
 684   $1_TEST_NAME := $$(strip $$(patsubst micro:%, %, $$($1_TEST)))
 685 
 686   $$(eval $$(call SetMicroValue,$1,MICRO_BENCHMARKS_JAR,$$(TEST_IMAGE_DIR)/micro/benchmarks.jar))
 687   $$(eval $$(call SetMicroValue,$1,MICRO_TEST_JDK,$$(JDK_UNDER_TEST)))
 688   $$(eval $$(call SetMicroValue,$1,MICRO_JAVA_OPTIONS))
 689 
 690   # Current tests needs to open java.io
 691   $1_MICRO_JAVA_OPTIONS += --add-opens=java.base/java.io=ALL-UNNAMED
 692   # Set library path for native dependencies
 693   $1_MICRO_JAVA_OPTIONS += -Djava.library.path=$$(TEST_IMAGE_DIR)/micro/native
 694 
 695   # Save output as JSON or CSV file
 696   ifneq ($$(MICRO_RESULTS_FORMAT), )
 697     $1_MICRO_BASIC_OPTIONS += -rf $$(MICRO_RESULTS_FORMAT)
 698     $1_MICRO_BASIC_OPTIONS += -rff $$($1_TEST_RESULTS_DIR)/jmh-result.$(MICRO_RESULTS_FORMAT)
 699   endif
 700 
 701   ifneq ($$(MICRO_VM_OPTIONS)$$(MICRO_JAVA_OPTIONS), )
 702     JMH_JVM_ARGS := $$(MICRO_VM_OPTIONS) $$(MICRO_JAVA_OPTIONS)
 703     $1_MICRO_VM_OPTIONS := -jvmArgs $(call ShellQuote,$$(JMH_JVM_ARGS))
 704   endif
 705 
 706   ifneq ($$(MICRO_ITER), )
 707     $1_MICRO_ITER := -i $$(MICRO_ITER)
 708   endif
 709   ifneq ($$(MICRO_FORK), )
 710     $1_MICRO_FORK := -f $$(MICRO_FORK)
 711   endif
 712   ifneq ($$(MICRO_TIME), )
 713     $1_MICRO_TIME := -r $$(MICRO_TIME)
 714   endif
 715   ifneq ($$(MICRO_WARMUP_ITER), )
 716     $1_MICRO_WARMUP_ITER := -wi $$(MICRO_WARMUP_ITER)
 717   endif
 718   ifneq ($$(MICRO_WARMUP_TIME), )
 719     $1_MICRO_WARMUP_TIME := -w $$(MICRO_WARMUP_TIME)
 720   endif
 721 
 722   run-test-$1: pre-run-test
 723         $$(call LogWarn)
 724         $$(call LogWarn, Running test '$$($1_TEST)')
 725         $$(call MakeDir, $$($1_TEST_RESULTS_DIR) $$($1_TEST_SUPPORT_DIR))
 726         $$(call ExecuteWithLog, $$($1_TEST_SUPPORT_DIR)/micro, ( \
 727             $$(FIXPATH) $$($1_MICRO_TEST_JDK)/bin/java $$($1_MICRO_JAVA_OPTIONS) \
 728                 -jar $$($1_MICRO_BENCHMARKS_JAR) \
 729                 $$($1_MICRO_ITER) $$($1_MICRO_FORK) $$($1_MICRO_TIME) \
 730                 $$($1_MICRO_WARMUP_ITER) $$($1_MICRO_WARMUP_TIME) \
 731                 $$($1_MICRO_VM_OPTIONS) $$($1_MICRO_BASIC_OPTIONS) $$(MICRO_OPTIONS) \
 732                 $$($1_TEST_NAME) \
 733                 > >($(TEE) $$($1_TEST_RESULTS_DIR)/micro.txt) \
 734             && $$(ECHO) $$$$? > $$($1_EXITCODE) \
 735             || $$(ECHO) $$$$? > $$($1_EXITCODE) \
 736         ))
 737 
 738   $1_RESULT_FILE := $$($1_TEST_RESULTS_DIR)/micro.txt
 739 
 740   parse-test-$1: run-test-$1
 741         $$(call LogWarn, Finished running test '$$($1_TEST)')
 742         $$(call LogWarn, Test report is stored in $$(strip \
 743             $$(subst $$(TOPDIR)/, , $$($1_TEST_RESULTS_DIR))))
 744         $$(if $$(wildcard $$($1_EXITCODE)), \
 745           $$(eval $1_EXIT_CODE := $$(shell $$(CAT) $$($1_EXITCODE))) \
 746           $$(if $$(filter 0, $$($1_EXIT_CODE)), \
 747             $$(eval $1_PASSED := 1) \
 748             $$(eval $1_ERROR := 0) \
 749           , \
 750             $$(eval $1_PASSED := 0) \
 751             $$(eval $1_ERROR := 1) \
 752           ) \
 753           $$(eval $1_FAILED := 0) \
 754           $$(eval $1_TOTAL := $$(shell \
 755               $$(EXPR) $$($1_PASSED) + $$($1_ERROR))) \
 756         , \
 757           $$(eval $1_PASSED := 0) \
 758           $$(eval $1_FAILED := 0) \
 759           $$(eval $1_ERROR := 1) \
 760           $$(eval $1_TOTAL := 1) \
 761         )
 762 
 763   $1: run-test-$1 parse-test-$1
 764 
 765   TARGETS += $1 run-test-$1 parse-test-$1
 766   TEST_TARGETS += parse-test-$1
 767 
 768 endef
 769 
 770 ################################################################################
 771 
 772 ### Rules for Jtreg
 773 
 774 # Helper function for SetupRunJtregTest. Set a JTREG_* variable from, in order:
 775 # 1) Specified by user on command line
 776 # 2) Component-specific default
 777 # 3) Generic default
 778 #
 779 # Note: No spaces are allowed around the arguments.
 780 # Arg $1 The test ID (i.e. $1 in SetupRunJtregTest)
 781 # Arg $2 Base variable, e.g. JTREG_JOBS
 782 # Arg $3 The default value (optional)
 783 define SetJtregValue
 784   ifneq ($$($2), )
 785     $1_$2 := $$($2)
 786   else
 787     ifneq ($$($$($1_COMPONENT)_$2), )
 788       $1_$2 := $$($$($1_COMPONENT)_$2)
 789     else
 790       ifneq ($3, )
 791         $1_$2 := $3
 792       endif
 793     endif
 794   endif
 795 endef
 796 
 797 SetupRunJtregTest = $(NamedParamsMacroTemplate)
 798 define SetupRunJtregTestBody
 799   $1_TEST_RESULTS_DIR := $$(TEST_RESULTS_DIR)/$1
 800   $1_TEST_SUPPORT_DIR := $$(TEST_SUPPORT_DIR)/$1
 801   $1_EXITCODE := $$($1_TEST_RESULTS_DIR)/exitcode.txt
 802 
 803   $1_TEST_NAME := $$(strip $$(patsubst jtreg:%, %, $$($1_TEST)))
 804 
 805   $1_TEST_ROOT := \
 806       $$(strip $$(foreach root, $$(JTREG_TESTROOTS), \
 807         $$(if $$(filter $$(root)%, $$(JTREG_TOPDIR)/$$($1_TEST_NAME)), $$(root)) \
 808       ))
 809   $1_COMPONENT := $$(lastword $$(subst /, $$(SPACE), $$($1_TEST_ROOT)))
 810   # This will work only as long as just hotspot has the additional "jtreg" directory
 811   ifeq ($$($1_COMPONENT), jtreg)
 812     $1_COMPONENT := hotspot
 813   endif
 814 
 815   ifeq ($$(JT_HOME), )
 816     $$(info Error: jtreg framework is not found.)
 817     $$(info Please run configure using --with-jtreg.)
 818     $$(error Cannot continue)
 819   endif
 820 
 821   # Unfortunately, we need different defaults for some JTREG values,
 822   # depending on what component we're running.
 823 
 824   # Convert JTREG_foo into $1_JTREG_foo with a suitable value.
 825   $$(eval $$(call SetJtregValue,$1,JTREG_TEST_MODE,agentvm))
 826   $$(eval $$(call SetJtregValue,$1,JTREG_ASSERT,true))
 827   $$(eval $$(call SetJtregValue,$1,JTREG_MAX_MEM,512m))
 828   $$(eval $$(call SetJtregValue,$1,JTREG_NATIVEPATH))
 829   $$(eval $$(call SetJtregValue,$1,JTREG_BASIC_OPTIONS))
 830   $$(eval $$(call SetJtregValue,$1,JTREG_PROBLEM_LIST))
 831 
 832   # Only the problem list for the current test root should be used.
 833   $1_JTREG_PROBLEM_LIST := $$(filter $$($1_TEST_ROOT)%, $$($1_JTREG_PROBLEM_LIST))
 834 
 835   ifneq ($(TEST_JOBS), 0)
 836     $$(eval $$(call SetJtregValue,$1,JTREG_JOBS,$$(TEST_JOBS)))
 837   else
 838     $$(eval $$(call SetJtregValue,$1,JTREG_JOBS,$$(JOBS)))
 839   endif
 840 
 841   # Make sure MaxRAMPercentage is high enough to not cause OOM or swapping since
 842   # we may end up with a lot of JVM's
 843   $1_JTREG_MAX_RAM_PERCENTAGE := $$(shell $$(EXPR) 25 / $$($1_JTREG_JOBS))
 844 
 845   # SPARC is in general slower per core so need to scale up timeouts a bit.
 846   ifeq ($(call isTargetCpuArch, sparc), true)
 847     JTREG_TIMEOUT_FACTOR ?= 8
 848   else
 849     JTREG_TIMEOUT_FACTOR ?= 4
 850   endif
 851   JTREG_VERBOSE ?= fail,error,summary
 852   JTREG_RETAIN ?= fail,error
 853   JTREG_RUN_PROBLEM_LISTS ?= false
 854   JTREG_RETRY_COUNT ?= 0
 855 
 856   ifneq ($$($1_JTREG_MAX_MEM), 0)
 857     $1_JTREG_BASIC_OPTIONS += -vmoption:-Xmx$$($1_JTREG_MAX_MEM)
 858     $1_JTREG_LAUNCHER_OPTIONS += -Xmx$$($1_JTREG_MAX_MEM)
 859   endif
 860 
 861   $1_JTREG_BASIC_OPTIONS += -$$($1_JTREG_TEST_MODE) \
 862       -verbose:$$(JTREG_VERBOSE) -retain:$$(JTREG_RETAIN) \
 863       -concurrency:$$($1_JTREG_JOBS) -timeoutFactor:$$(JTREG_TIMEOUT_FACTOR) \
 864       -vmoption:-XX:MaxRAMPercentage=$$($1_JTREG_MAX_RAM_PERCENTAGE)
 865 
 866   $1_JTREG_BASIC_OPTIONS += -automatic -ignore:quiet
 867 
 868   # Make it possible to specify the JIB_DATA_DIR for tests using the
 869   # JIB Artifact resolver
 870   $1_JTREG_BASIC_OPTIONS += -e:JIB_DATA_DIR
 871   # Some tests needs to find a boot JDK using the JDK8_HOME variable.
 872   $1_JTREG_BASIC_OPTIONS += -e:JDK8_HOME=$$(BOOT_JDK)
 873   # If running on Windows, propagate the _NT_SYMBOL_PATH to enable
 874   # symbol lookup in hserr files
 875   ifeq ($$(call isTargetOs, windows), true)
 876     $1_JTREG_BASIC_OPTIONS += -e:_NT_SYMBOL_PATH
 877   endif
 878 
 879   $1_JTREG_BASIC_OPTIONS += \
 880       $$(addprefix -javaoption:, $$(JTREG_JAVA_OPTIONS)) \
 881       $$(addprefix -vmoption:, $$(JTREG_VM_OPTIONS)) \
 882       #
 883 
 884   ifeq ($$($1_JTREG_ASSERT), true)
 885     $1_JTREG_BASIC_OPTIONS += -ea -esa
 886   endif
 887 
 888   ifneq ($$($1_JTREG_NATIVEPATH), )
 889     $1_JTREG_BASIC_OPTIONS += -nativepath:$$($1_JTREG_NATIVEPATH)
 890   endif
 891 
 892   ifeq ($$(JTREG_RUN_PROBLEM_LISTS), true)
 893     JTREG_PROBLEM_LIST_PREFIX := -match:
 894   else
 895     JTREG_PROBLEM_LIST_PREFIX := -exclude:
 896   endif
 897 
 898   ifneq ($$($1_JTREG_PROBLEM_LIST), )
 899     $1_JTREG_BASIC_OPTIONS += $$(addprefix $$(JTREG_PROBLEM_LIST_PREFIX), $$($1_JTREG_PROBLEM_LIST))
 900   endif
 901 
 902   ifneq ($$(JTREG_EXTRA_PROBLEM_LISTS), )
 903     # Accept both absolute paths as well as relative to the current test root.
 904     $1_JTREG_BASIC_OPTIONS += $$(addprefix $$(JTREG_PROBLEM_LIST_PREFIX), $$(wildcard \
 905         $$(JTREG_EXTRA_PROBLEM_LISTS) \
 906         $$(addprefix $$($1_TEST_ROOT)/, $$(JTREG_EXTRA_PROBLEM_LISTS)) \
 907     ))
 908   endif
 909 
 910   ifneq ($$(JIB_HOME), )
 911     $1_JTREG_BASIC_OPTIONS += -e:JIB_HOME=$$(JIB_HOME)
 912   endif
 913 
 914   $1_JTREG_BASIC_OPTIONS += -e:TEST_IMAGE_DIR=$(TEST_IMAGE_DIR)
 915   $1_JTREG_BASIC_OPTIONS += -e:TEST_IMAGE_GRAAL_DIR=$(TEST_IMAGE_DIR)/hotspot/jtreg/graal
 916 
 917   ifneq ($$(JTREG_FAILURE_HANDLER_OPTIONS), )
 918     $1_JTREG_LAUNCHER_OPTIONS += -Djava.library.path="$(JTREG_FAILURE_HANDLER_DIR)"
 919   endif
 920 
 921   ifneq ($$(JTREG_KEYWORDS), )
 922     # The keywords string may contain problematic characters and may be quoted
 923     # already when it arrives here. Remove any existing quotes and replace them
 924     # with one set of single quotes.
 925     $1_JTREG_KEYWORDS := \
 926         $$(strip $$(subst $$(SQUOTE),,$$(subst $$(DQUOTE),,$$(JTREG_KEYWORDS))))
 927     ifneq ($$($1_JTREG_KEYWORDS), )
 928       $1_JTREG_BASIC_OPTIONS += -k:'$$($1_JTREG_KEYWORDS)'
 929     endif
 930   endif
 931 
 932   ifneq ($$(JTREG_AOT_MODULES), )
 933     $$(eval $$(call SetupAot, $1, \
 934         MODULES := $$(JTREG_AOT_MODULES), \
 935         VM_OPTIONS := $$(JTREG_VM_OPTIONS) $$(JTREG_JAVA_OPTIONS), \
 936     ))
 937   endif
 938 
 939   ifneq ($$($1_AOT_OPTIONS), )
 940     $1_JTREG_BASIC_OPTIONS += -vmoptions:"$$($1_AOT_OPTIONS)"
 941   endif
 942 
 943   clean-workdir-$1:
 944         $$(RM) -r $$($1_TEST_SUPPORT_DIR)
 945 
 946   $1_COMMAND_LINE := \
 947       $$(JAVA) $$($1_JTREG_LAUNCHER_OPTIONS) \
 948           -Dprogram=jtreg -jar $$(JT_HOME)/lib/jtreg.jar \
 949           $$($1_JTREG_BASIC_OPTIONS) \
 950           -testjdk:$$(JDK_UNDER_TEST) \
 951           -dir:$$(JTREG_TOPDIR) \
 952           -reportDir:$$($1_TEST_RESULTS_DIR) \
 953           -workDir:$$($1_TEST_SUPPORT_DIR) \
 954           -status:$$$${JTREG_STATUS} \
 955           $$(JTREG_OPTIONS) \
 956           $$(JTREG_FAILURE_HANDLER_OPTIONS) \
 957           $$(JTREG_COV_OPTIONS) \
 958           $$($1_TEST_NAME) \
 959       && $$(ECHO) $$$$? > $$($1_EXITCODE) \
 960       || $$(ECHO) $$$$? > $$($1_EXITCODE)
 961 
 962 
 963   ifneq ($$(JTREG_RETRY_COUNT), 0)
 964     $1_COMMAND_LINE := \
 965         for i in {0..$$(JTREG_RETRY_COUNT)}; do \
 966           if [ "$$$$i" != 0 ]; then \
 967             $$(PRINTF) "\nRetrying Jtreg run. Attempt: $$$$i\n"; \
 968           fi; \
 969           $$($1_COMMAND_LINE); \
 970           if [ "`$$(CAT) $$($1_EXITCODE)`" = "0" ]; then \
 971             break; \
 972           fi; \
 973           export JTREG_STATUS="-status:error,fail"; \
 974         done
 975   endif
 976 
 977   run-test-$1: pre-run-test clean-workdir-$1 $$($1_AOT_TARGETS)
 978         $$(call LogWarn)
 979         $$(call LogWarn, Running test '$$($1_TEST)')
 980         $$(call MakeDir, $$($1_TEST_RESULTS_DIR) $$($1_TEST_SUPPORT_DIR))
 981         $$(call ExecuteWithLog, $$($1_TEST_SUPPORT_DIR)/jtreg, ( \
 982             $$(COV_ENVIRONMENT) $$($1_COMMAND_LINE) \
 983         ))
 984 
 985   $1_RESULT_FILE := $$($1_TEST_RESULTS_DIR)/text/stats.txt
 986 
 987   parse-test-$1: run-test-$1
 988         $$(call LogWarn, Finished running test '$$($1_TEST)')
 989         $$(call LogWarn, Test report is stored in $$(strip \
 990             $$(subst $$(TOPDIR)/, , $$($1_TEST_RESULTS_DIR))))
 991         $$(if $$(wildcard $$($1_RESULT_FILE)), \
 992           $$(eval $1_PASSED := $$(shell $$(AWK) '{ gsub(/[,;]/, ""); \
 993               for (i=1; i<=NF; i++) { if ($$$$i == "passed:") \
 994               print $$$$(i+1) } }' $$($1_RESULT_FILE))) \
 995           $$(if $$($1_PASSED), , $$(eval $1_PASSED := 0)) \
 996           $$(eval $1_FAILED := $$(shell $$(AWK) '{gsub(/[,;]/, ""); \
 997               for (i=1; i<=NF; i++) { if ($$$$i == "failed:") \
 998               print $$$$(i+1) } }' $$($1_RESULT_FILE))) \
 999           $$(if $$($1_FAILED), , $$(eval $1_FAILED := 0)) \
1000           $$(eval $1_ERROR := $$(shell $$(AWK) '{gsub(/[,;]/, ""); \
1001               for (i=1; i<=NF; i++) { if ($$$$i == "error:") \
1002               print $$$$(i+1) } }' $$($1_RESULT_FILE))) \
1003           $$(if $$($1_ERROR), , $$(eval $1_ERROR := 0)) \
1004           $$(eval $1_TOTAL := $$(shell \
1005               $$(EXPR) $$($1_PASSED) + $$($1_FAILED) + $$($1_ERROR))) \
1006         , \
1007           $$(eval $1_PASSED := 0) \
1008           $$(eval $1_FAILED := 0) \
1009           $$(eval $1_ERROR := 1) \
1010           $$(eval $1_TOTAL := 1) \
1011         )
1012 
1013   $1: run-test-$1 parse-test-$1 clean-workdir-$1
1014 
1015   TARGETS += $1 run-test-$1 parse-test-$1 clean-workdir-$1
1016   TEST_TARGETS += parse-test-$1
1017 
1018 endef
1019 
1020 ################################################################################
1021 
1022 ### Rules for special tests
1023 
1024 SetupRunSpecialTest = $(NamedParamsMacroTemplate)
1025 define SetupRunSpecialTestBody
1026   $1_TEST_RESULTS_DIR := $$(TEST_RESULTS_DIR)/$1
1027   $1_TEST_SUPPORT_DIR := $$(TEST_SUPPORT_DIR)/$1
1028   $1_EXITCODE := $$($1_TEST_RESULTS_DIR)/exitcode.txt
1029 
1030   $1_FULL_TEST_NAME := $$(strip $$(patsubst special:%, %, $$($1_TEST)))
1031   ifneq ($$(findstring :, $$($1_FULL_TEST_NAME)), )
1032     $1_TEST_NAME := $$(firstword $$(subst :, ,$$($1_FULL_TEST_NAME)))
1033     $1_TEST_ARGS := $$(strip $$(patsubst special:$$($1_TEST_NAME):%, %, $$($1_TEST)))
1034   else
1035     $1_TEST_NAME := $$($1_FULL_TEST_NAME)
1036     $1_TEST_ARGS :=
1037   endif
1038 
1039   ifeq ($$($1_TEST_NAME), failure-handler)
1040     ifeq ($(BUILD_FAILURE_HANDLER), true)
1041       $1_TEST_COMMAND_LINE := \
1042           ($(CD) $(TOPDIR)/make/test && $(MAKE) $(MAKE_ARGS) -f \
1043           BuildFailureHandler.gmk test)
1044     else
1045       $$(error Cannot test failure handler if it is not built)
1046     endif
1047   else ifeq ($$($1_TEST_NAME), make)
1048     $1_TEST_COMMAND_LINE := \
1049         ($(CD) $(TOPDIR)/test/make && $(MAKE) $(MAKE_ARGS) -f \
1050         TestMake.gmk $$($1_TEST_ARGS) TEST_SUPPORT_DIR="$$($1_TEST_SUPPORT_DIR)")
1051   else
1052     $$(error Invalid special test specification: $$($1_TEST_NAME))
1053   endif
1054 
1055   run-test-$1: pre-run-test
1056         $$(call LogWarn)
1057         $$(call LogWarn, Running test '$$($1_TEST)')
1058         $$(call MakeDir, $$($1_TEST_RESULTS_DIR) $$($1_TEST_SUPPORT_DIR))
1059         $$(call ExecuteWithLog, $$($1_TEST_SUPPORT_DIR)/test-execution, ( \
1060             $$($1_TEST_COMMAND_LINE) \
1061                 > >($(TEE) $$($1_TEST_RESULTS_DIR)/test-output.txt) \
1062             && $$(ECHO) $$$$? > $$($1_EXITCODE) \
1063             || $$(ECHO) $$$$? > $$($1_EXITCODE) \
1064         ))
1065 
1066   $1_RESULT_FILE := $$($1_TEST_RESULTS_DIR)/gtest.txt
1067 
1068   # We can not parse the various "special" tests.
1069   parse-test-$1: run-test-$1
1070         $$(call LogWarn, Finished running test '$$($1_TEST)')
1071         $$(call LogWarn, Test report is stored in $$(strip \
1072             $$(subst $$(TOPDIR)/, , $$($1_TEST_RESULTS_DIR))))
1073         $$(call LogWarn, Warning: Special test results are not properly parsed!)
1074         $$(eval $1_PASSED := $$(shell \
1075           if [ `$(CAT) $$($1_EXITCODE)` = "0" ]; then $(ECHO) 1; else $(ECHO) 0; fi \
1076         ))
1077         $$(eval $1_FAILED := $$(shell \
1078           if [ `$(CAT) $$($1_EXITCODE)` = "0" ]; then $(ECHO) 0; else $(ECHO) 1; fi \
1079         ))
1080         $$(eval $1_ERROR := 0)
1081         $$(eval $1_TOTAL := 1)
1082 
1083   $1: run-test-$1 parse-test-$1
1084 
1085   TARGETS += $1 run-test-$1 parse-test-$1
1086   TEST_TARGETS += parse-test-$1
1087 
1088 endef
1089 
1090 ################################################################################
1091 # Setup and execute make rules for all selected tests
1092 ################################################################################
1093 
1094 # Helper function to determine which handler to use for the given test
1095 UseGtestTestHandler = \
1096   $(if $(filter gtest:%, $1), true)
1097 
1098 UseMicroTestHandler = \
1099   $(if $(filter micro:%, $1), true)
1100 
1101 UseJtregTestHandler = \
1102   $(if $(filter jtreg:%, $1), true)
1103 
1104 UseSpecialTestHandler = \
1105   $(if $(filter special:%, $1), true)
1106 
1107 # Now process each test to run and setup a proper make rule
1108 $(foreach test, $(TESTS_TO_RUN), \
1109   $(eval TEST_ID := $(shell $(ECHO) $(strip $(test)) | \
1110       $(TR) -cs '[a-z][A-Z][0-9]\n' '[_*1000]')) \
1111   $(eval ALL_TEST_IDS += $(TEST_ID)) \
1112   $(if $(call UseCustomTestHandler, $(test)), \
1113     $(eval $(call SetupRunCustomTest, $(TEST_ID), \
1114         TEST := $(test), \
1115     )) \
1116   ) \
1117   $(if $(call UseGtestTestHandler, $(test)), \
1118     $(eval $(call SetupRunGtestTest, $(TEST_ID), \
1119         TEST := $(test), \
1120     )) \
1121   ) \
1122   $(if $(call UseMicroTestHandler, $(test)), \
1123     $(eval $(call SetupRunMicroTest, $(TEST_ID), \
1124         TEST := $(test), \
1125     )) \
1126   ) \
1127   $(if $(call UseJtregTestHandler, $(test)), \
1128     $(eval $(call SetupRunJtregTest, $(TEST_ID), \
1129         TEST := $(test), \
1130     )) \
1131   ) \
1132   $(if $(call UseSpecialTestHandler, $(test)), \
1133     $(eval $(call SetupRunSpecialTest, $(TEST_ID), \
1134         TEST := $(test), \
1135     )) \
1136   ) \
1137 )
1138 
1139 # Sort also removes duplicates, so if there is any we'll get fewer words.
1140 ifneq ($(words $(ALL_TEST_IDS)), $(words $(sort $(ALL_TEST_IDS))))
1141   $(error Duplicate test specification)
1142 endif
1143 
1144 
1145 ################################################################################
1146 # The main target for RunTests.gmk
1147 ################################################################################
1148 
1149 #
1150 # Provide hooks for adding functionality before and after all tests are run.
1151 #
1152 
1153 $(call LogInfo, RunTest setup starting)
1154 
1155 # This target depends on all actual test having been run (TEST_TARGETS has beeen
1156 # populated by the SetupRun*Test functions). If you need to provide a teardown
1157 # hook, you must let it depend on this target.
1158 run-all-tests: $(TEST_TARGETS)
1159         $(call LogInfo, RunTest teardown starting)
1160 
1161 # This is an abstract target that will be run before any actual tests. Add your
1162 # target as a dependency to thisif you need "setup" type functionality executed
1163 # before all tests.
1164 pre-run-test:
1165         $(call LogInfo, RunTest setup done)
1166 
1167 # This is an abstract target that will be run after all actual tests, but before
1168 # the test summary. If you need "teardown" type functionality, add your target
1169 # as a dependency on this, and let the teardown target depend on run-all-tests.
1170 post-run-test: run-all-tests
1171         $(call LogInfo, RunTest teardown done)
1172 
1173 #
1174 # Create and print a table of the result of all tests run
1175 #
1176 TEST_FAILURE := false
1177 
1178 run-test-report: post-run-test
1179         $(RM) $(TEST_SUMMARY).old 2> /dev/null
1180         $(MV) $(TEST_SUMMARY) $(TEST_SUMMARY).old 2> /dev/null || true
1181         $(RM) $(TEST_LAST_IDS).old 2> /dev/null
1182         $(MV) $(TEST_LAST_IDS) $(TEST_LAST_IDS).old 2> /dev/null || true
1183         $(ECHO) >> $(TEST_SUMMARY) ==============================
1184         $(ECHO) >> $(TEST_SUMMARY) Test summary
1185         $(ECHO) >> $(TEST_SUMMARY) ==============================
1186         $(PRINTF) >> $(TEST_SUMMARY) "%2s %-49s %5s %5s %5s %5s %2s\n" "  " \
1187             TEST TOTAL PASS FAIL ERROR " "
1188         $(foreach test, $(TESTS_TO_RUN), \
1189           $(eval TEST_ID := $(shell $(ECHO) $(strip $(test)) | \
1190               $(TR) -cs '[a-z][A-Z][0-9]\n' '[_*1000]')) \
1191             $(ECHO) >> $(TEST_LAST_IDS) $(TEST_ID) $(NEWLINE) \
1192           $(eval NAME_PATTERN := $(shell $(ECHO) $(test) | $(TR) -c '\n' '[_*1000]')) \
1193           $(if $(filter __________________________________________________%, $(NAME_PATTERN)), \
1194             $(eval TEST_NAME := ) \
1195             $(PRINTF) >> $(TEST_SUMMARY) "%2s %-49s\n" "  " "$(test)"  $(NEWLINE) \
1196           , \
1197             $(eval TEST_NAME := $(test)) \
1198           ) \
1199           $(if $(filter $($(TEST_ID)_PASSED), $($(TEST_ID)_TOTAL)), \
1200             $(PRINTF) >> $(TEST_SUMMARY) "%2s %-49s %5d %5d %5d %5d %2s\n" \
1201                 "  " "$(TEST_NAME)" $($(TEST_ID)_TOTAL) $($(TEST_ID)_PASSED) \
1202                 $($(TEST_ID)_FAILED) $($(TEST_ID)_ERROR) "  " $(NEWLINE) \
1203           , \
1204             $(PRINTF) >> $(TEST_SUMMARY) "%2s %-49s %5d %5d %5d %5d %2s\n" \
1205                  ">>" "$(TEST_NAME)" $($(TEST_ID)_TOTAL) $($(TEST_ID)_PASSED) \
1206                 $($(TEST_ID)_FAILED) $($(TEST_ID)_ERROR) "<<" $(NEWLINE) \
1207             $(eval TEST_FAILURE := true) \
1208           ) \
1209         )
1210         $(ECHO) >> $(TEST_SUMMARY) ==============================
1211         $(if $(filter true, $(TEST_FAILURE)), \
1212           $(ECHO) >> $(TEST_SUMMARY) TEST FAILURE $(NEWLINE) \
1213           $(MKDIR) -p $(MAKESUPPORT_OUTPUTDIR) $(NEWLINE) \
1214           $(TOUCH) $(MAKESUPPORT_OUTPUTDIR)/exit-with-error \
1215         , \
1216           $(ECHO) >> $(TEST_SUMMARY) TEST SUCCESS \
1217         )
1218         $(ECHO)
1219         $(CAT) $(TEST_SUMMARY)
1220         $(ECHO)
1221 
1222 # The main run-test target
1223 run-test: run-test-report
1224 
1225 TARGETS += run-all-tests pre-run-test post-run-test run-test-report run-test
1226 
1227 ################################################################################
1228 # Setup JCov
1229 ################################################################################
1230 
1231 ifeq ($(TEST_OPTS_JCOV), true)
1232 
1233   jcov-do-start-grabber:
1234         $(call MakeDir, $(JCOV_OUTPUT_DIR))
1235         if $(JAVA) -jar $(JCOV_HOME)/lib/jcov.jar GrabberManager -status 1>/dev/null 2>&1 ; then \
1236           $(JAVA) -jar $(JCOV_HOME)/lib/jcov.jar GrabberManager -stop -stoptimeout 3600 ; \
1237         fi
1238         $(JAVA) -Xmx4g -jar $(JCOV_HOME)/lib/jcov.jar Grabber -v -t \
1239             $(JCOV_IMAGE_DIR)/template.xml -o $(JCOV_RESULT_FILE) \
1240             1>$(JCOV_GRABBER_LOG) 2>&1 &
1241 
1242   jcov-start-grabber: jcov-do-start-grabber
1243         $(call LogWarn, Starting JCov Grabber...)
1244         $(JAVA) -jar $(JCOV_HOME)/lib/jcov.jar GrabberManager -t 600 -wait
1245 
1246   jcov-stop-grabber:
1247         $(call LogWarn, Stopping JCov Grabber...)
1248         $(JAVA) -jar $(JCOV_HOME)/lib/jcov.jar GrabberManager -stop -stoptimeout 3600
1249 
1250   JCOV_REPORT_TITLE := JDK code coverage report<br/>
1251   ifneq ($(JCOV_FILTERS), )
1252     JCOV_REPORT_TITLE += Code filters: $(JCOV_FILTERS)<br>
1253   endif
1254   JCOV_REPORT_TITLE += Tests: $(TEST)
1255 
1256   jcov-gen-report: jcov-stop-grabber
1257         $(call LogWarn, Generating JCov report ...)
1258         $(JAVA) -Xmx4g -jar $(JCOV_HOME)/lib/jcov.jar RepGen -sourcepath \
1259             `$(ECHO) $(TOPDIR)/src/*/share/classes/ | $(TR) ' ' ':'` -fmt html \
1260             $(JCOV_FILTERS) \
1261             -mainReportTitle "$(JCOV_REPORT_TITLE)" \
1262             -o $(JCOV_REPORT) $(JCOV_RESULT_FILE)
1263 
1264   TARGETS += jcov-do-start-grabber jcov-start-grabber jcov-stop-grabber \
1265       jcov-gen-report
1266 
1267   ifneq ($(TEST_OPTS_JCOV_DIFF_CHANGESET), )
1268 
1269     JCOV_SOURCE_DIFF := $(JCOV_OUTPUT_DIR)/source_diff
1270     JCOV_DIFF_COVERAGE_REPORT := $(JCOV_OUTPUT_DIR)/diff_coverage_report
1271 
1272     ifneq ($(and $(HG), $(wildcard $(TOPDIR)/.hg)), )
1273       DIFF_COMMAND := $(HG) -R $(TOPDIR) diff -r $(TEST_OPTS_JCOV_DIFF_CHANGESET) > $(JCOV_SOURCE_DIFF)
1274     else ifneq ($(and $(GIT), $(wildcard $(TOPDIR)/.git)), )
1275       DIFF_COMMAND := $(GIT) -C $(TOPDIR) diff $(TEST_OPTS_JCOV_DIFF_CHANGESET) > $(JCOV_SOURCE_DIFF)
1276     else
1277       $(info Error: Must be either hg or git source tree for diff coverage.)
1278       $(error Neither hg nor git source tree.)
1279     endif
1280 
1281     jcov-gen-diffcoverage: jcov-stop-grabber
1282         $(call LogWarn, Generating diff coverage with changeset $(TEST_OPTS_JCOV_DIFF_CHANGESET) ... )
1283         $(DIFF_COMMAND)
1284         $(JAVA) -Xmx4g -jar $(JCOV_HOME)/lib/jcov.jar \
1285           DiffCoverage -replaceDiff "src/.*/classes/:" -all \
1286             $(JCOV_RESULT_FILE) $(JCOV_SOURCE_DIFF) > \
1287             $(JCOV_DIFF_COVERAGE_REPORT)
1288 
1289     TARGETS += jcov-gen-diffcoverage
1290 
1291   endif
1292 
1293   # Hook this into the framework at appropriate places
1294   pre-run-test: jcov-start-grabber
1295 
1296   post-run-test: jcov-gen-report
1297 
1298   ifneq ($(TEST_OPTS_JCOV_DIFF_CHANGESET), )
1299 
1300     post-run-test: jcov-gen-diffcoverage
1301 
1302   endif
1303 
1304   jcov-stop-grabber: run-all-tests
1305 
1306 endif
1307 
1308 ################################################################################
1309 
1310 all: run-test
1311 
1312 .PHONY: default all $(TARGETS)