1 #
   2 # Copyright (c) 2016, 2018, Oracle and/or its affiliates. All rights reserved.
   3 # DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4 #
   5 # This code is free software; you can redistribute it and/or modify it
   6 # under the terms of the GNU General Public License version 2 only, as
   7 # published by the Free Software Foundation.  Oracle designates this
   8 # particular file as subject to the "Classpath" exception as provided
   9 # by Oracle in the LICENSE file that accompanied this code.
  10 #
  11 # This code is distributed in the hope that it will be useful, but WITHOUT
  12 # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  13 # FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  14 # version 2 for more details (a copy is included in the LICENSE file that
  15 # accompanied this code).
  16 #
  17 # You should have received a copy of the GNU General Public License version
  18 # 2 along with this work; if not, write to the Free Software Foundation,
  19 # Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  20 #
  21 # Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  22 # or visit www.oracle.com if you need additional information or have any
  23 # questions.
  24 #
  25 
  26 default: all
  27 
  28 include $(SPEC)
  29 include MakeBase.gmk
  30 include FindTests.gmk
  31 
  32 # We will always run multiple tests serially
  33 .NOTPARALLEL:
  34 
  35 ################################################################################
  36 # Parse global control variables
  37 ################################################################################
  38 
  39 ifneq ($(TEST_VM_OPTS), )
  40   ifneq ($(TEST_OPTS), )
  41     TEST_OPTS := $(TEST_OPTS);VM_OPTIONS=$(TEST_VM_OPTS)
  42   else
  43     TEST_OPTS := VM_OPTIONS=$(TEST_VM_OPTS)
  44   endif
  45 endif
  46 
  47 $(eval $(call ParseKeywordVariable, TEST_OPTS, \
  48     SINGLE_KEYWORDS := JOBS TIMEOUT_FACTOR AOT_MODULES, \
  49     STRING_KEYWORDS := VM_OPTIONS JAVA_OPTIONS, \
  50 ))
  51 
  52 # Helper function to propagate TEST_OPTS values.
  53 #
  54 # Note: No spaces are allowed around the arguments.
  55 # Arg $1 The variable in TEST_OPTS to propagate
  56 # Arg $2 The control variable to propagate it to
  57 define SetTestOpt
  58   ifneq ($$(TEST_OPTS_$1), )
  59     $2_$1 := $$(TEST_OPTS_$1)
  60   endif
  61 endef
  62 
  63 # Setup _NT_SYMBOL_PATH on Windows
  64 ifeq ($(OPENJDK_TARGET_OS), windows)
  65   ifndef _NT_SYMBOL_PATH
  66     # Can't use PathList here as it adds quotes around the value.
  67     _NT_SYMBOL_PATH := \
  68         $(subst $(SPACE),;,$(strip \
  69             $(foreach p, $(sort $(dir $(wildcard \
  70                 $(addprefix $(SYMBOLS_IMAGE_DIR)/bin/, *.pdb */*.pdb)))), \
  71               $(call FixPath, $p) \
  72             ) \
  73         ))
  74     export _NT_SYMBOL_PATH
  75     $(info _NT_SYMBOL_PATH=$(_NT_SYMBOL_PATH))
  76   endif
  77 endif
  78 
  79 ################################################################################
  80 # Hook to include the corresponding custom file, if present.
  81 $(eval $(call IncludeCustomExtension, RunTests.gmk))
  82 ################################################################################
  83 
  84 TEST_RESULTS_DIR := $(OUTPUTDIR)/test-results
  85 TEST_SUPPORT_DIR := $(OUTPUTDIR)/test-support
  86 TEST_SUMMARY := $(TEST_RESULTS_DIR)/test-summary.txt
  87 TEST_LAST_IDS := $(TEST_SUPPORT_DIR)/test-last-ids.txt
  88 
  89 ifeq ($(CUSTOM_ROOT), )
  90   JTREG_TOPDIR := $(TOPDIR)
  91 else
  92   JTREG_TOPDIR := $(CUSTOM_ROOT)
  93 endif
  94 
  95 JTREG_FAILURE_HANDLER_DIR := $(TEST_IMAGE_DIR)/failure_handler
  96 JTREG_FAILURE_HANDLER := $(JTREG_FAILURE_HANDLER_DIR)/jtregFailureHandler.jar
  97 
  98 ifneq ($(wildcard $(JTREG_FAILURE_HANDLER)), )
  99   JTREG_FAILURE_HANDLER_OPTIONS := \
 100       -timeoutHandlerDir:$(JTREG_FAILURE_HANDLER) \
 101       -observerDir:$(JTREG_FAILURE_HANDLER) \
 102       -timeoutHandler:jdk.test.failurehandler.jtreg.GatherProcessInfoTimeoutHandler \
 103       -observer:jdk.test.failurehandler.jtreg.GatherDiagnosticInfoObserver \
 104       -timeoutHandlerTimeout:0
 105 endif
 106 
 107 GTEST_LAUNCHER_DIRS := $(patsubst %/gtestLauncher, %, $(wildcard $(TEST_IMAGE_DIR)/hotspot/gtest/*/gtestLauncher))
 108 GTEST_VARIANTS := $(strip $(patsubst $(TEST_IMAGE_DIR)/hotspot/gtest/%, %, $(GTEST_LAUNCHER_DIRS)))
 109 
 110 ################################################################################
 111 # Optionally create AOT libraries for specified modules before running tests.
 112 # Note, this could not be done during JDK build time.
 113 ################################################################################
 114 
 115 # Note, this could not be done during JDK build time.
 116 
 117 # Parameter 1 is the name of the rule.
 118 #
 119 # Remaining parameters are named arguments.
 120 #   MODULE      The module to generate a library for
 121 #   BIN         Output directory in which to put the library
 122 #   VM_OPTIONS  List of JVM arguments to use when creating library
 123 #   OPTIONS_VAR Name of variable to put AOT java options in
 124 #   PREREQS_VAR Name of variable to put all AOT prerequisite rule targets in
 125 #               for test rules to depend on
 126 #
 127 SetupAotModule = $(NamedParamsMacroTemplate)
 128 define SetupAotModuleBody
 129   $1_AOT_LIB := $$($1_BIN)/$$(call SHARED_LIBRARY,$$($1_MODULE))
 130   $1_AOT_CCLIST := $$(wildcard $$(TOPDIR)/test/hotspot/jtreg/compiler/aot/scripts/$$($1_MODULE)-list.txt)
 131 
 132   ifeq ($(OPENJDK_TARGET_OS), windows)
 133     $1_LD := $$(addsuffix $$(EXE_SUFFIX), $$(filter-out $$(FIXPATH), $$(LD)))
 134   else
 135     $1_LD := $$(LD)
 136   endif
 137 
 138   $1_JAOTC_OPTS := \
 139       -J-Xmx4g --info \
 140       $$(addprefix -J, $$($1_VM_OPTIONS)) \
 141       $$(addprefix --compile-commands$(SPACE), $$($1_AOT_CCLIST)) \
 142       --linker-path $$($1_LD) \
 143       #
 144 
 145   ifneq ($$(filter -ea, $$($1_VM_OPTIONS)), )
 146     $1_JAOTC_OPTS += --compile-with-assertions
 147   endif
 148 
 149   $$($1_AOT_LIB): $$(JDK_IMAGE_DIR)/release \
 150       $$(call DependOnVariable, $1_JAOTC_OPTS) \
 151       $$(call DependOnVariable, JDK_IMAGE_DIR)
 152         $$(call LogWarn, Generating $$(patsubst $$(OUTPUTDIR)/%, %, $$@))
 153         $$(call MakeTargetDir)
 154         $$(call ExecuteWithLog, $$@, \
 155             $$(FIXPATH) $$(JDK_IMAGE_DIR)/bin/jaotc \
 156             $$($1_JAOTC_OPTS) --output $$@ --module $$($1_MODULE) \
 157         )
 158         $$(call ExecuteWithLog, $$@.check, \
 159           $$(FIXPATH) $$(JDK_IMAGE_DIR)/bin/java \
 160           $$($1_VM_OPTIONS) -XX:+PrintAOT -XX:+UseAOTStrictLoading -XX:AOTLibrary=$$@ -version \
 161           > $$@.verify-aot \
 162         )
 163 
 164   $1_AOT_OPTIONS += -XX:AOTLibrary=$$($1_AOT_LIB)
 165   $1_AOT_TARGETS += $$($1_AOT_LIB)
 166 endef
 167 
 168 # Parameter 1 is the name of the rule.
 169 #
 170 # Remaining parameters are named arguments.
 171 #   MODULES     The modules to generate a library for
 172 #   VM_OPTIONS  List of JVM arguments to use when creating libraries
 173 #
 174 # After calling this, the following variables are defined
 175 #   $1_AOT_OPTIONS List of all java options needed to use the AOT libraries
 176 #   $1_AOT_TARGETS List of all targets that the test rule will need to depend on
 177 #
 178 SetupAot = $(NamedParamsMacroTemplate)
 179 define SetupAotBody
 180   $$(info Running with AOTd libraries for $$($1_MODULES))
 181   # Put aot libraries in a separate directory so they are not deleted between
 182   # test runs and may be reused between make invocations.
 183   $$(foreach m, $$($1_MODULES), \
 184     $$(eval $$(call SetupAotModule, $1_$$m, \
 185         MODULE := $$m, \
 186         BIN := $$(TEST_SUPPORT_DIR)/aot/$1, \
 187         VM_OPTIONS := $$($1_VM_OPTIONS), \
 188     )) \
 189     $$(eval $1_AOT_OPTIONS += $$($1_$$m_AOT_OPTIONS)) \
 190     $$(eval $1_AOT_TARGETS += $$($1_$$m_AOT_TARGETS)) \
 191   )
 192 endef
 193 
 194 ################################################################################
 195 # Setup global test running parameters
 196 ################################################################################
 197 
 198 # Each factor variable comes in 3 variants. The first one is reserved for users
 199 # to use on command line. The other two are for predifined configurations in JDL
 200 # and for machine specific configurations respectively.
 201 TEST_JOBS_FACTOR ?= 1
 202 TEST_JOBS_FACTOR_JDL ?= 1
 203 TEST_JOBS_FACTOR_MACHINE ?= 1
 204 
 205 ifeq ($(TEST_JOBS), 0)
 206   CORES_DIVIDER := 2
 207   ifeq ($(OPENJDK_TARGET_CPU_ARCH), sparc)
 208     # For smaller SPARC machines we see reasonable scaling of throughput up to
 209     # cpus/4 without affecting test reliability. On the bigger machines, cpus/4
 210     # causes intermittent timeouts.
 211     ifeq ($(shell $(EXPR) $(NUM_CORES) \> 16), 1)
 212       CORES_DIVIDER := 5
 213     else
 214       CORES_DIVIDER := 4
 215     endif
 216   endif
 217   MEMORY_DIVIDER := 2048
 218   TEST_JOBS := $(shell $(AWK) \
 219     'BEGIN { \
 220       c = $(NUM_CORES) / $(CORES_DIVIDER); \
 221       m = $(MEMORY_SIZE) / $(MEMORY_DIVIDER); \
 222       if (c > m) c = m; \
 223       c = c * $(TEST_JOBS_FACTOR); \
 224       c = c * $(TEST_JOBS_FACTOR_JDL); \
 225       c = c * $(TEST_JOBS_FACTOR_MACHINE); \
 226       if (c < 1) c = 1; \
 227       printf "%.0f", c; \
 228     }')
 229 endif
 230 
 231 ################################################################################
 232 # Parse control variables
 233 ################################################################################
 234 
 235 ifneq ($(TEST_OPTS), )
 236   # Inform the user
 237   $(info Running tests using TEST_OPTS control variable '$(TEST_OPTS)')
 238 endif
 239 
 240 $(eval $(call SetTestOpt,VM_OPTIONS,JTREG))
 241 $(eval $(call SetTestOpt,JAVA_OPTIONS,JTREG))
 242 $(eval $(call SetTestOpt,VM_OPTIONS,GTEST))
 243 $(eval $(call SetTestOpt,JAVA_OPTIONS,GTEST))
 244 
 245 $(eval $(call SetTestOpt,AOT_MODULES,JTREG))
 246 $(eval $(call SetTestOpt,AOT_MODULES,GTEST))
 247 
 248 $(eval $(call SetTestOpt,JOBS,JTREG))
 249 $(eval $(call SetTestOpt,TIMEOUT_FACTOR,JTREG))
 250 
 251 $(eval $(call ParseKeywordVariable, JTREG, \
 252     SINGLE_KEYWORDS := JOBS TIMEOUT_FACTOR TEST_MODE ASSERT VERBOSE RETAIN MAX_MEM \
 253         EXTRA_PROBLEM_LISTS KEYWORDS AOT_MODULES, \
 254     STRING_KEYWORDS := OPTIONS JAVA_OPTIONS VM_OPTIONS, \
 255 ))
 256 
 257 ifneq ($(JTREG), )
 258   # Inform the user
 259   $(info Running tests using JTREG control variable '$(JTREG)')
 260 endif
 261 
 262 $(eval $(call ParseKeywordVariable, GTEST, \
 263     SINGLE_KEYWORDS := REPEAT AOT_MODULES, \
 264     STRING_KEYWORDS := OPTIONS VM_OPTIONS JAVA_OPTIONS, \
 265 ))
 266 
 267 ifneq ($(GTEST), )
 268   # Inform the user
 269   $(info Running tests using GTEST control variable '$(GTEST)')
 270 endif
 271 
 272 
 273 ################################################################################
 274 # Component-specific Jtreg settings
 275 ################################################################################
 276 
 277 hotspot_JTREG_MAX_MEM := 0
 278 hotspot_JTREG_ASSERT := false
 279 hotspot_JTREG_NATIVEPATH := $(TEST_IMAGE_DIR)/hotspot/jtreg/native
 280 jdk_JTREG_NATIVEPATH := $(TEST_IMAGE_DIR)/jdk/jtreg/native
 281 
 282 jdk_JTREG_PROBLEM_LIST += $(TOPDIR)/test/jdk/ProblemList.txt
 283 jaxp_JTREG_PROBLEM_LIST += $(TOPDIR)/test/jaxp/ProblemList.txt
 284 langtools_JTREG_PROBLEM_LIST += $(TOPDIR)/test/langtools/ProblemList.txt
 285 nashorn_JTREG_PROBLEM_LIST += $(TOPDIR)/test/nashorn/ProblemList.txt
 286 hotspot_JTREG_PROBLEM_LIST += $(TOPDIR)/test/hotspot/jtreg/ProblemList.txt
 287 
 288 langtools_JTREG_MAX_MEM := 768m
 289 
 290 ################################################################################
 291 # Parse test selection
 292 #
 293 # The user has given a test selection in the TEST variable. We must parse it
 294 # and determine what that means in terms of actual calls to the test framework.
 295 #
 296 # The parse functions take as argument a test specification as given by the
 297 # user, and returns a fully qualified test descriptor if it was a match, or
 298 # nothing if not. A single test specification can result in multiple test
 299 # descriptors being returned. A valid test descriptor must always be accepted
 300 # and returned identically.
 301 ################################################################################
 302 
 303 # Helper function to determine if a test specification is a Gtest test
 304 #
 305 # It is a Gtest test if it is either "gtest", or "gtest:" followed by an optional
 306 # test filter string, and an optional "/<variant>" to select a specific JVM
 307 # variant. If no variant is specified, all found variants are tested.
 308 define ParseGtestTestSelection
 309   $(if $(filter gtest%, $1), \
 310     $(if $(filter gtest, $1), \
 311       $(addprefix gtest:all/, $(GTEST_VARIANTS)) \
 312     , \
 313       $(if $(strip $(or $(filter gtest/%, $1) $(filter gtest:/%, $1))), \
 314         $(patsubst gtest:/%, gtest:all/%, $(patsubst gtest/%, gtest:/%, $1)) \
 315       , \
 316         $(if $(filter gtest:%, $1), \
 317           $(if $(findstring /, $1), \
 318             $1 \
 319           , \
 320             $(addprefix $1/, $(GTEST_VARIANTS)) \
 321           ) \
 322         ) \
 323       ) \
 324     ) \
 325   )
 326 endef
 327 
 328 # Helper function that removes the TOPDIR part
 329 CleanupJtregPath = \
 330   $(strip $(patsubst %/, %, $(subst $(JTREG_TOPDIR)/,, $1)))
 331 
 332 # Take a partial Jtreg root path and return a full, absolute path to that Jtreg
 333 # root. Also support having "hotspot" as an alias for "hotspot/jtreg".
 334 ExpandJtregRoot = \
 335   $(call CleanupJtregPath, $(wildcard \
 336     $(if $(filter /%, $1), \
 337       $(if $(wildcard $(strip $1)/TEST.ROOT), \
 338         $1 \
 339       ) \
 340     , \
 341       $(filter $(addprefix %, $1), $(JTREG_TESTROOTS) $(addsuffix /, $(JTREG_TESTROOTS))) \
 342       $(filter $(addprefix %, $(strip $1)/jtreg), $(JTREG_TESTROOTS) $(addsuffix /, $(JTREG_TESTROOTS))) \
 343     ) \
 344   ))
 345 
 346 # Take a partial Jtreg test path and return a full, absolute path to that Jtreg
 347 # test. Also support having "hotspot" as an alias for "hotspot/jtreg".
 348 ExpandJtregPath = \
 349   $(if $(call ExpandJtregRoot, $1), \
 350     $(call ExpandJtregRoot, $1) \
 351   , \
 352     $(call CleanupJtregPath, $(wildcard \
 353       $(if $(filter /%, $1), \
 354         $1 \
 355       , \
 356         $(addsuffix /$(strip $1), $(JTREG_TESTROOTS) $(TEST_BASEDIRS)) \
 357         $(addsuffix $(strip $(patsubst hotspot/%, /hotspot/jtreg/%, $1)), $(JTREG_TESTROOTS) $(TEST_BASEDIRS)) \
 358       ) \
 359     )) \
 360   )
 361 
 362 # Helper function to determine if a test specification is a Jtreg test
 363 #
 364 # It is a Jtreg test if it optionally begins with jtreg:, and then is either
 365 # an unspecified group name (possibly prefixed by :), or a group in a
 366 # specified test root, or a path to a test or test directory,
 367 # either absolute or relative to any of the TEST_BASEDIRS or test roots.
 368 define ParseJtregTestSelection
 369   $(eval TEST_NAME := $(strip $(patsubst jtreg:%, %, $1))) \
 370   $(if $(or $(findstring :, $(TEST_NAME)), $(findstring /, $(TEST_NAME))), , \
 371     $(eval TEST_NAME := :$(TEST_NAME)) \
 372   ) \
 373   $(if $(findstring :, $(TEST_NAME)), \
 374     $(if $(filter :%, $(TEST_NAME)), \
 375       $(eval TEST_GROUP := $(patsubst :%, %, $(TEST_NAME))) \
 376       $(eval TEST_ROOTS := $(foreach test_root, $(JTREG_TESTROOTS), \
 377           $(call CleanupJtregPath, $(test_root)))) \
 378     , \
 379       $(eval TEST_PATH := $(word 1, $(subst :, $(SPACE), $(TEST_NAME)))) \
 380       $(eval TEST_GROUP := $(word 2, $(subst :, $(SPACE), $(TEST_NAME)))) \
 381       $(eval TEST_ROOTS := $(call ExpandJtregRoot, $(TEST_PATH))) \
 382     ) \
 383     $(foreach test_root, $(TEST_ROOTS), \
 384       $(if $(filter /%, $(test_root)), \
 385         jtreg:$(test_root):$(TEST_GROUP) \
 386       , \
 387         $(if $(filter $(TEST_GROUP), $($(JTREG_TOPDIR)/$(test_root)_JTREG_TEST_GROUPS)), \
 388           jtreg:$(test_root):$(TEST_GROUP) \
 389         ) \
 390       ) \
 391     ) \
 392   , \
 393     $(eval TEST_PATHS := $(call ExpandJtregPath, $(TEST_NAME))) \
 394     $(foreach test_path, $(TEST_PATHS), \
 395       jtreg:$(test_path) \
 396     ) \
 397   )
 398 endef
 399 
 400 # Helper function to determine if a test specification is a special test
 401 #
 402 # It is a special test if it is "special:" followed by a test name,
 403 # if it is "make:" or "make-" followed by a make test, or any of the special test names
 404 # as a single word.
 405 define ParseSpecialTestSelection
 406   $(if $(filter special:%, $1), \
 407     $1 \
 408   ) \
 409   $(if $(filter make%, $1), \
 410     $(if $(filter make:%, $1), \
 411       special:$(strip $1) \
 412     ) \
 413     $(if $(filter make-%, $1), \
 414       special:$(patsubst make-%,make:%, $1) \
 415     ) \
 416     $(if $(filter make, $1), \
 417       special:make:all \
 418     )
 419   ) \
 420   $(if $(filter failure-handler, $1), \
 421     special:$(strip $1) \
 422   )
 423 endef
 424 
 425 ifeq ($(TEST), )
 426   $(info No test selection given in TEST!)
 427   $(info Please use e.g. 'make test TEST=tier1' or 'make test-tier1')
 428   $(info See doc/testing.[md|html] for help)
 429   $(error Cannot continue)
 430 endif
 431 
 432 # Now intelligently convert the test selection given by the user in TEST
 433 # into a list of fully qualified test descriptors of the tests to run.
 434 TESTS_TO_RUN :=
 435 $(foreach test, $(TEST), \
 436   $(eval PARSED_TESTS := $(call ParseCustomTestSelection, $(test))) \
 437   $(if $(strip $(PARSED_TESTS)), , \
 438     $(eval PARSED_TESTS += $(call ParseGtestTestSelection, $(test))) \
 439   ) \
 440   $(if $(strip $(PARSED_TESTS)), , \
 441     $(eval PARSED_TESTS += $(call ParseJtregTestSelection, $(test))) \
 442   ) \
 443   $(if $(strip $(PARSED_TESTS)), , \
 444     $(eval PARSED_TESTS += $(call ParseSpecialTestSelection, $(test))) \
 445   ) \
 446   $(if $(strip $(PARSED_TESTS)), , \
 447     $(eval UNKNOWN_TEST := $(test)) \
 448   ) \
 449   $(eval TESTS_TO_RUN += $(PARSED_TESTS)) \
 450 )
 451 
 452 ifneq ($(UNKNOWN_TEST), )
 453   $(info Unknown test selection: '$(UNKNOWN_TEST)')
 454   $(info See doc/testing.[md|html] for help)
 455   $(error Cannot continue)
 456 endif
 457 
 458 TESTS_TO_RUN := $(strip $(TESTS_TO_RUN))
 459 
 460 
 461 # Present the result of our parsing to the user
 462 $(info Test selection '$(TEST)', will run:)
 463 $(foreach test, $(TESTS_TO_RUN), $(info * $(test)))
 464 
 465 
 466 ################################################################################
 467 # Functions for setting up rules for running the selected tests
 468 #
 469 # The SetupRun*Test functions all have the same interface:
 470 #
 471 # Parameter 1 is the name of the rule. This is the test id, based on the test
 472 # descriptor, and this is also used as variable prefix, and the targets
 473 # generated are listed in a variable by that name.
 474 #
 475 # Remaining parameters are named arguments. Currently this is only:
 476 #   TEST -- The properly formatted fully qualified test descriptor
 477 #
 478 # After the rule named by the test id has been executed, the following
 479 # variables will be available:
 480 # testid_TOTAL - the total number of tests run
 481 # testid_PASSED - the number of successful tests
 482 # testid_FAILED - the number of failed tests
 483 # testid_ERROR - the number of tests was neither successful or failed
 484 #
 485 ################################################################################
 486 
 487 ### Rules for Gtest
 488 
 489 SetupRunGtestTest = $(NamedParamsMacroTemplate)
 490 define SetupRunGtestTestBody
 491   $1_TEST_RESULTS_DIR := $$(TEST_RESULTS_DIR)/$1
 492   $1_TEST_SUPPORT_DIR := $$(TEST_SUPPORT_DIR)/$1
 493   $1_EXITCODE := $$($1_TEST_RESULTS_DIR)/exitcode.txt
 494 
 495   $1_VARIANT :=  $$(lastword $$(subst /, , $$($1_TEST)))
 496   ifeq ($$(filter $$($1_VARIANT), $$(GTEST_VARIANTS)), )
 497     $$(error Invalid gtest variant '$$($1_VARIANT)'. Valid variants: $$(GTEST_VARIANTS))
 498   endif
 499   $1_TEST_NAME := $$(strip $$(patsubst %/$$($1_VARIANT), %, \
 500       $$(patsubst gtest:%, %, $$($1_TEST))))
 501   ifneq ($$($1_TEST_NAME), all)
 502     $1_GTEST_FILTER := --gtest_filter=$$($1_TEST_NAME)*
 503   endif
 504 
 505   ifneq ($$(GTEST_REPEAT), )
 506     $1_GTEST_REPEAT :=--gtest_repeat=$$(GTEST_REPEAT)
 507   endif
 508 
 509   ifneq ($$(GTEST_AOT_MODULES), )
 510     $$(eval $$(call SetupAot, $1, \
 511         MODULES := $$(GTEST_AOT_MODULES), \
 512         VM_OPTIONS := $$(GTEST_VM_OPTIONS) $$(GTEST_JAVA_OPTIONS), \
 513     ))
 514   endif
 515 
 516   run-test-$1: $$($1_AOT_TARGETS)
 517         $$(call LogWarn)
 518         $$(call LogWarn, Running test '$$($1_TEST)')
 519         $$(call MakeDir, $$($1_TEST_RESULTS_DIR) $$($1_TEST_SUPPORT_DIR))
 520         $$(call ExecuteWithLog, $$($1_TEST_SUPPORT_DIR)/gtest, \
 521             $$(FIXPATH) $$(TEST_IMAGE_DIR)/hotspot/gtest/$$($1_VARIANT)/gtestLauncher \
 522                 -jdk $(JDK_IMAGE_DIR) $$($1_GTEST_FILTER) \
 523                 --gtest_output=xml:$$($1_TEST_RESULTS_DIR)/gtest.xml \
 524                 $$($1_GTEST_REPEAT) $$(GTEST_OPTIONS) $$(GTEST_VM_OPTIONS) \
 525                 $$(GTEST_JAVA_OPTIONS) $$($1_AOT_OPTIONS) \
 526                 > >($(TEE) $$($1_TEST_RESULTS_DIR)/gtest.txt) \
 527             && $$(ECHO) $$$$? > $$($1_EXITCODE) \
 528             || $$(ECHO) $$$$? > $$($1_EXITCODE) \
 529         )
 530 
 531   $1_RESULT_FILE := $$($1_TEST_RESULTS_DIR)/gtest.txt
 532 
 533   parse-test-$1: run-test-$1
 534         $$(call LogWarn, Finished running test '$$($1_TEST)')
 535         $$(call LogWarn, Test report is stored in $$(strip \
 536             $$(subst $$(TOPDIR)/, , $$($1_TEST_RESULTS_DIR))))
 537         $$(if $$(wildcard $$($1_RESULT_FILE)), \
 538           $$(eval $1_TOTAL := $$(shell $$(AWK) '/==========.* tests? from .* \
 539               test cases? ran/ { print $$$$2 }' $$($1_RESULT_FILE))) \
 540           $$(if $$($1_TOTAL), , $$(eval $1_TOTAL := 0)) \
 541           $$(eval $1_PASSED := $$(shell $$(AWK) '/\[  PASSED  \] .* tests?./ \
 542               { print $$$$4 }' $$($1_RESULT_FILE))) \
 543           $$(if $$($1_PASSED), , $$(eval $1_PASSED := 0)) \
 544           $$(eval $1_FAILED := $$(shell $$(AWK) '/\[  FAILED  \] .* tests?, \
 545               listed below/ { print $$$$4 }' $$($1_RESULT_FILE))) \
 546           $$(if $$($1_FAILED), , $$(eval $1_FAILED := 0)) \
 547           $$(eval $1_ERROR := $$(shell \
 548               $$(EXPR) $$($1_TOTAL) - $$($1_PASSED) - $$($1_FAILED))) \
 549         , \
 550           $$(eval $1_PASSED := 0) \
 551           $$(eval $1_FAILED := 0) \
 552           $$(eval $1_ERROR := 1) \
 553           $$(eval $1_TOTAL := 1) \
 554         )
 555 
 556   $1: run-test-$1 parse-test-$1
 557 
 558   TARGETS += $1
 559 endef
 560 
 561 ################################################################################
 562 
 563 ### Rules for Jtreg
 564 
 565 # Helper function for SetupRunJtregTest. Set a JTREG_* variable from, in order:
 566 # 1) Specified by user on command line
 567 # 2) Component-specific default
 568 # 3) Generic default
 569 #
 570 # Note: No spaces are allowed around the arguments.
 571 # Arg $1 The test ID (i.e. $1 in SetupRunJtregTest)
 572 # Arg $2 Base variable, e.g. JTREG_JOBS
 573 # Arg $3 The default value (optional)
 574 define SetJtregValue
 575   ifneq ($$($2), )
 576     $1_$2 := $$($2)
 577   else
 578     ifneq ($$($$($1_COMPONENT)_$2), )
 579       $1_$2 := $$($$($1_COMPONENT)_$2)
 580     else
 581       ifneq ($3, )
 582         $1_$2 := $3
 583       endif
 584     endif
 585   endif
 586 endef
 587 
 588 SetupRunJtregTest = $(NamedParamsMacroTemplate)
 589 define SetupRunJtregTestBody
 590   $1_TEST_RESULTS_DIR := $$(TEST_RESULTS_DIR)/$1
 591   $1_TEST_SUPPORT_DIR := $$(TEST_SUPPORT_DIR)/$1
 592   $1_EXITCODE := $$($1_TEST_RESULTS_DIR)/exitcode.txt
 593 
 594   $1_TEST_NAME := $$(strip $$(patsubst jtreg:%, %, $$($1_TEST)))
 595 
 596   $1_TEST_ROOT := \
 597       $$(strip $$(foreach root, $$(JTREG_TESTROOTS), \
 598         $$(if $$(filter $$(root)%, $$(JTREG_TOPDIR)/$$($1_TEST_NAME)), $$(root)) \
 599       ))
 600   $1_COMPONENT := $$(lastword $$(subst /, $$(SPACE), $$($1_TEST_ROOT)))
 601   # This will work only as long as just hotspot has the additional "jtreg" directory
 602   ifeq ($$($1_COMPONENT), jtreg)
 603     $1_COMPONENT := hotspot
 604   endif
 605 
 606   ifeq ($$(JT_HOME), )
 607     $$(info Error: jtreg framework is not found.)
 608     $$(info Please run configure using --with-jtreg.)
 609     $$(error Cannot continue)
 610   endif
 611 
 612   # Unfortunately, we need different defaults for some JTREG values,
 613   # depending on what component we're running.
 614 
 615   # Convert JTREG_foo into $1_JTREG_foo with a suitable value.
 616   $$(eval $$(call SetJtregValue,$1,JTREG_TEST_MODE,agentvm))
 617   $$(eval $$(call SetJtregValue,$1,JTREG_ASSERT,true))
 618   $$(eval $$(call SetJtregValue,$1,JTREG_MAX_MEM,512m))
 619   $$(eval $$(call SetJtregValue,$1,JTREG_NATIVEPATH))
 620   $$(eval $$(call SetJtregValue,$1,JTREG_BASIC_OPTIONS))
 621   $$(eval $$(call SetJtregValue,$1,JTREG_PROBLEM_LIST))
 622 
 623   # Only the problem list for the current test root should be used.
 624   $1_JTREG_PROBLEM_LIST := $$(filter $$($1_TEST_ROOT)%, $$($1_JTREG_PROBLEM_LIST))
 625 
 626   ifneq ($(TEST_JOBS), 0)
 627     $$(eval $$(call SetJtregValue,$1,JTREG_JOBS,$$(TEST_JOBS)))
 628   else
 629     $$(eval $$(call SetJtregValue,$1,JTREG_JOBS,$$(JOBS)))
 630   endif
 631 
 632   # Make sure MaxRAMPercentage is high enough to not cause OOM or swapping since
 633   # we may end up with a lot of JVM's
 634   $1_JTREG_MAX_RAM_PERCENTAGE := $$(shell $$(EXPR) 25 / $$($1_JTREG_JOBS))
 635 
 636   # SPARC is in general slower per core so need to scale up timeouts a bit.
 637   ifeq ($(OPENJDK_TARGET_CPU_ARCH), sparc)
 638     JTREG_TIMEOUT_FACTOR ?= 8
 639   else
 640     JTREG_TIMEOUT_FACTOR ?= 4
 641   endif
 642   JTREG_VERBOSE ?= fail,error,summary
 643   JTREG_RETAIN ?= fail,error
 644 
 645   ifneq ($$($1_JTREG_MAX_MEM), 0)
 646     $1_JTREG_BASIC_OPTIONS += -vmoption:-Xmx$$($1_JTREG_MAX_MEM)
 647     $1_JTREG_LAUNCHER_OPTIONS += -Xmx$$($1_JTREG_MAX_MEM)
 648   endif
 649 
 650   $1_JTREG_BASIC_OPTIONS += -$$($1_JTREG_TEST_MODE) \
 651       -verbose:$$(JTREG_VERBOSE) -retain:$$(JTREG_RETAIN) \
 652       -concurrency:$$($1_JTREG_JOBS) -timeoutFactor:$$(JTREG_TIMEOUT_FACTOR) \
 653       -vmoption:-XX:MaxRAMPercentage=$$($1_JTREG_MAX_RAM_PERCENTAGE)
 654 
 655   $1_JTREG_BASIC_OPTIONS += -automatic -ignore:quiet
 656 
 657   # Make it possible to specify the JIB_DATA_DIR for tests using the
 658   # JIB Artifact resolver
 659   $1_JTREG_BASIC_OPTIONS += -e:JIB_DATA_DIR
 660   # Some tests needs to find a boot JDK using the JDK8_HOME variable.
 661   $1_JTREG_BASIC_OPTIONS += -e:JDK8_HOME=$$(BOOT_JDK)
 662   # If running on Windows, propagate the _NT_SYMBOL_PATH to enable
 663   # symbol lookup in hserr files
 664   ifeq ($$(OPENJDK_TARGET_OS), windows)
 665     $1_JTREG_BASIC_OPTIONS += -e:_NT_SYMBOL_PATH
 666   endif
 667 
 668   $1_JTREG_BASIC_OPTIONS += \
 669       $$(addprefix -javaoption:, $$(JTREG_JAVA_OPTIONS)) \
 670       $$(addprefix -vmoption:, $$(JTREG_VM_OPTIONS)) \
 671       #
 672 
 673   ifeq ($$($1_JTREG_ASSERT), true)
 674     $1_JTREG_BASIC_OPTIONS += -ea -esa
 675   endif
 676 
 677   ifneq ($$($1_JTREG_NATIVEPATH), )
 678     $1_JTREG_BASIC_OPTIONS += -nativepath:$$($1_JTREG_NATIVEPATH)
 679   endif
 680 
 681   ifneq ($$($1_JTREG_PROBLEM_LIST), )
 682     $1_JTREG_BASIC_OPTIONS += $$(addprefix -exclude:, $$($1_JTREG_PROBLEM_LIST))
 683   endif
 684 
 685   ifneq ($$(JTREG_EXTRA_PROBLEM_LISTS), )
 686     # Accept both absolute paths as well as relative to the current test root.
 687     $1_JTREG_BASIC_OPTIONS += $$(addprefix -exclude:, $$(wildcard \
 688         $$(JTREG_EXTRA_PROBLEM_LISTS) \
 689         $$(addprefix $$($1_TEST_ROOT)/, $$(JTREG_EXTRA_PROBLEM_LISTS)) \
 690     ))
 691   endif
 692 
 693   ifneq ($$(JIB_HOME), )
 694     $1_JTREG_BASIC_OPTIONS += -e:JIB_HOME=$$(JIB_HOME)
 695   endif
 696 
 697   $1_JTREG_BASIC_OPTIONS += -e:TEST_IMAGE_GRAAL_DIR=${TEST_IMAGE_DIR}/hotspot/jtreg/graal
 698 
 699   ifneq ($$(JTREG_FAILURE_HANDLER_OPTIONS), )
 700     $1_JTREG_LAUNCHER_OPTIONS += -Djava.library.path="$(JTREG_FAILURE_HANDLER_DIR)"
 701   endif
 702 
 703   ifneq ($$(JTREG_KEYWORDS), )
 704     # The keywords string may contain problematic characters and may be quoted
 705     # already when it arrives here. Remove any existing quotes and replace them
 706     # with one set of single quotes.
 707     $1_JTREG_KEYWORDS := \
 708         $$(strip $$(subst $$(SQUOTE),,$$(subst $$(DQUOTE),,$$(JTREG_KEYWORDS))))
 709     ifneq ($$($1_JTREG_KEYWORDS), )
 710       $1_JTREG_BASIC_OPTIONS += -k:'$$($1_JTREG_KEYWORDS)'
 711     endif
 712   endif
 713 
 714   ifneq ($$(JTREG_AOT_MODULES), )
 715     $$(eval $$(call SetupAot, $1, \
 716         MODULES := $$(JTREG_AOT_MODULES), \
 717         VM_OPTIONS := $$(JTREG_VM_OPTIONS) $$(JTREG_JAVA_OPTIONS), \
 718     ))
 719   endif
 720 
 721   ifneq ($$($1_AOT_OPTIONS), )
 722     $1_JTREG_BASIC_OPTIONS += -vmoptions:"$$($1_AOT_OPTIONS)"
 723   endif
 724 
 725   clean-workdir-$1:
 726         $$(RM) -r $$($1_TEST_SUPPORT_DIR)
 727 
 728   run-test-$1: clean-workdir-$1 $$($1_AOT_TARGETS)
 729         $$(call LogWarn)
 730         $$(call LogWarn, Running test '$$($1_TEST)')
 731         $$(call MakeDir, $$($1_TEST_RESULTS_DIR) $$($1_TEST_SUPPORT_DIR))
 732         $$(call ExecuteWithLog, $$($1_TEST_SUPPORT_DIR)/jtreg, \
 733             $$(JAVA) $$($1_JTREG_LAUNCHER_OPTIONS) \
 734                 -Dprogram=jtreg -jar $$(JT_HOME)/lib/jtreg.jar \
 735                 $$($1_JTREG_BASIC_OPTIONS) \
 736                 -testjdk:$$(JDK_IMAGE_DIR) \
 737                 -dir:$$(JTREG_TOPDIR) \
 738                 -reportDir:$$($1_TEST_RESULTS_DIR) \
 739                 -workDir:$$($1_TEST_SUPPORT_DIR) \
 740                 $$(JTREG_OPTIONS) \
 741                 $$(JTREG_FAILURE_HANDLER_OPTIONS) \
 742                 $$($1_TEST_NAME) \
 743             && $$(ECHO) $$$$? > $$($1_EXITCODE) \
 744             || $$(ECHO) $$$$? > $$($1_EXITCODE) \
 745         )
 746 
 747   $1_RESULT_FILE := $$($1_TEST_RESULTS_DIR)/text/stats.txt
 748 
 749   parse-test-$1: run-test-$1
 750         $$(call LogWarn, Finished running test '$$($1_TEST)')
 751         $$(call LogWarn, Test report is stored in $$(strip \
 752             $$(subst $$(TOPDIR)/, , $$($1_TEST_RESULTS_DIR))))
 753         $$(if $$(wildcard $$($1_RESULT_FILE)), \
 754           $$(eval $1_PASSED := $$(shell $$(AWK) '{ gsub(/[,;]/, ""); \
 755               for (i=1; i<=NF; i++) { if ($$$$i == "passed:") \
 756               print $$$$(i+1) } }' $$($1_RESULT_FILE))) \
 757           $$(if $$($1_PASSED), , $$(eval $1_PASSED := 0)) \
 758           $$(eval $1_FAILED := $$(shell $$(AWK) '{gsub(/[,;]/, ""); \
 759               for (i=1; i<=NF; i++) { if ($$$$i == "failed:") \
 760               print $$$$(i+1) } }' $$($1_RESULT_FILE))) \
 761           $$(if $$($1_FAILED), , $$(eval $1_FAILED := 0)) \
 762           $$(eval $1_ERROR := $$(shell $$(AWK) '{gsub(/[,;]/, ""); \
 763               for (i=1; i<=NF; i++) { if ($$$$i == "error:") \
 764               print $$$$(i+1) } }' $$($1_RESULT_FILE))) \
 765           $$(if $$($1_ERROR), , $$(eval $1_ERROR := 0)) \
 766           $$(eval $1_TOTAL := $$(shell \
 767               $$(EXPR) $$($1_PASSED) + $$($1_FAILED) + $$($1_ERROR))) \
 768         , \
 769           $$(eval $1_PASSED := 0) \
 770           $$(eval $1_FAILED := 0) \
 771           $$(eval $1_ERROR := 1) \
 772           $$(eval $1_TOTAL := 1) \
 773         )
 774 
 775   $1: run-test-$1 parse-test-$1
 776 
 777   TARGETS += $1
 778 endef
 779 
 780 ################################################################################
 781 
 782 ### Rules for special tests
 783 
 784 SetupRunSpecialTest = $(NamedParamsMacroTemplate)
 785 define SetupRunSpecialTestBody
 786   $1_TEST_RESULTS_DIR := $$(TEST_RESULTS_DIR)/$1
 787   $1_TEST_SUPPORT_DIR := $$(TEST_SUPPORT_DIR)/$1
 788   $1_EXITCODE := $$($1_TEST_RESULTS_DIR)/exitcode.txt
 789 
 790   $1_FULL_TEST_NAME := $$(strip $$(patsubst special:%, %, $$($1_TEST)))
 791   ifneq ($$(findstring :, $$($1_FULL_TEST_NAME)), )
 792     $1_TEST_NAME := $$(firstword $$(subst :, ,$$($1_FULL_TEST_NAME)))
 793     $1_TEST_ARGS := $$(strip $$(patsubst special:$$($1_TEST_NAME):%, %, $$($1_TEST)))
 794   else
 795     $1_TEST_NAME := $$($1_FULL_TEST_NAME)
 796     $1_TEST_ARGS :=
 797   endif
 798 
 799   ifeq ($$($1_TEST_NAME), failure-handler)
 800     ifeq ($(BUILD_FAILURE_HANDLER), true)
 801       $1_TEST_COMMAND_LINE := \
 802           ($(CD) $(TOPDIR)/make/test && $(MAKE) $(MAKE_ARGS) -f \
 803           BuildFailureHandler.gmk test)
 804     else
 805       $$(error Cannot test failure handler if it is not built)
 806     endif
 807   else ifeq ($$($1_TEST_NAME), make)
 808     $1_TEST_COMMAND_LINE := \
 809         ($(CD) $(TOPDIR)/test/make && $(MAKE) $(MAKE_ARGS) -f \
 810         TestMake.gmk $$($1_TEST_ARGS))
 811   else
 812     $$(error Invalid special test specification: $$($1_TEST_NAME))
 813   endif
 814 
 815   run-test-$1: $(TEST_PREREQS)
 816         $$(call LogWarn)
 817         $$(call LogWarn, Running test '$$($1_TEST)')
 818         $$(call MakeDir, $$($1_TEST_RESULTS_DIR) $$($1_TEST_SUPPORT_DIR))
 819         $$(call ExecuteWithLog, $$($1_TEST_SUPPORT_DIR)/test-execution, \
 820             $$($1_TEST_COMMAND_LINE) \
 821                 > >($(TEE) $$($1_TEST_RESULTS_DIR)/test-output.txt) \
 822             && $$(ECHO) $$$$? > $$($1_EXITCODE) \
 823             || $$(ECHO) $$$$? > $$($1_EXITCODE) \
 824         )
 825 
 826   $1_RESULT_FILE := $$($1_TEST_RESULTS_DIR)/gtest.txt
 827 
 828   # We can not parse the various "special" tests.
 829   parse-test-$1: run-test-$1
 830         $$(call LogWarn, Finished running test '$$($1_TEST)')
 831         $$(call LogWarn, Test report is stored in $$(strip \
 832             $$(subst $$(TOPDIR)/, , $$($1_TEST_RESULTS_DIR))))
 833         $$(call LogWarn, Warning: Special test results are not properly parsed!)
 834         $$(eval $1_PASSED := 0)
 835         $$(eval $1_FAILED := 0)
 836         $$(eval $1_ERROR := 0)
 837         $$(eval $1_TOTAL := 0)
 838 
 839   $1: run-test-$1 parse-test-$1
 840 
 841   TARGETS += $1
 842 endef
 843 
 844 ################################################################################
 845 # Setup and execute make rules for all selected tests
 846 ################################################################################
 847 
 848 # Helper function to determine which handler to use for the given test
 849 UseGtestTestHandler = \
 850   $(if $(filter gtest:%, $1), true)
 851 
 852 UseJtregTestHandler = \
 853   $(if $(filter jtreg:%, $1), true)
 854 
 855 UseSpecialTestHandler = \
 856   $(if $(filter special:%, $1), true)
 857 
 858 # Now process each test to run and setup a proper make rule
 859 $(foreach test, $(TESTS_TO_RUN), \
 860   $(eval TEST_ID := $(shell $(ECHO) $(strip $(test)) | \
 861       $(TR) -cs '[a-z][A-Z][0-9]\n' '[_*1000]')) \
 862   $(eval ALL_TEST_IDS += $(TEST_ID)) \
 863   $(if $(call UseCustomTestHandler, $(test)), \
 864     $(eval $(call SetupRunCustomTest, $(TEST_ID), \
 865         TEST := $(test), \
 866     )) \
 867   ) \
 868   $(if $(call UseGtestTestHandler, $(test)), \
 869     $(eval $(call SetupRunGtestTest, $(TEST_ID), \
 870         TEST := $(test), \
 871     )) \
 872   ) \
 873   $(if $(call UseJtregTestHandler, $(test)), \
 874     $(eval $(call SetupRunJtregTest, $(TEST_ID), \
 875         TEST := $(test), \
 876     )) \
 877   ) \
 878   $(if $(call UseSpecialTestHandler, $(test)), \
 879     $(eval $(call SetupRunSpecialTest, $(TEST_ID), \
 880         TEST := $(test), \
 881     )) \
 882   ) \
 883 )
 884 
 885 # Sort also removes duplicates, so if there is any we'll get fewer words.
 886 ifneq ($(words $(ALL_TEST_IDS)), $(words $(sort $(ALL_TEST_IDS))))
 887   $(error Duplicate test specification)
 888 endif
 889 
 890 
 891 ################################################################################
 892 # The main target for RunTests.gmk
 893 ################################################################################
 894 
 895 # The SetupRun*Test functions have populated TARGETS.
 896 
 897 TEST_FAILURE := false
 898 
 899 run-test: $(TARGETS)
 900         # Create and print a table of the result of all tests run
 901         $(RM) $(TEST_SUMMARY).old 2> /dev/null
 902         $(MV) $(TEST_SUMMARY) $(TEST_SUMMARY).old 2> /dev/null || true
 903         $(RM) $(TEST_LAST_IDS).old 2> /dev/null
 904         $(MV) $(TEST_LAST_IDS) $(TEST_LAST_IDS).old 2> /dev/null || true
 905         $(ECHO) >> $(TEST_SUMMARY) ==============================
 906         $(ECHO) >> $(TEST_SUMMARY) Test summary
 907         $(ECHO) >> $(TEST_SUMMARY) ==============================
 908         $(PRINTF) >> $(TEST_SUMMARY) "%2s %-49s %5s %5s %5s %5s %2s\n" "  " \
 909             TEST TOTAL PASS FAIL ERROR " "
 910         $(foreach test, $(TESTS_TO_RUN), \
 911           $(eval TEST_ID := $(shell $(ECHO) $(strip $(test)) | \
 912               $(TR) -cs '[a-z][A-Z][0-9]\n' '[_*1000]')) \
 913             $(ECHO) >> $(TEST_LAST_IDS) $(TEST_ID) $(NEWLINE) \
 914           $(eval NAME_PATTERN := $(shell $(ECHO) $(test) | $(TR) -c '\n' '[_*1000]')) \
 915           $(if $(filter __________________________________________________%, $(NAME_PATTERN)), \
 916             $(eval TEST_NAME := ) \
 917             $(PRINTF) >> $(TEST_SUMMARY) "%2s %-49s\n" "  " "$(test)"  $(NEWLINE) \
 918           , \
 919             $(eval TEST_NAME := $(test)) \
 920           ) \
 921           $(if $(filter $($(TEST_ID)_PASSED), $($(TEST_ID)_TOTAL)), \
 922             $(PRINTF) >> $(TEST_SUMMARY) "%2s %-49s %5d %5d %5d %5d %2s\n" \
 923                 "  " "$(TEST_NAME)" $($(TEST_ID)_TOTAL) $($(TEST_ID)_PASSED) \
 924                 $($(TEST_ID)_FAILED) $($(TEST_ID)_ERROR) "  " $(NEWLINE) \
 925           , \
 926             $(PRINTF) >> $(TEST_SUMMARY) "%2s %-49s %5d %5d %5d %5d %2s\n" \
 927                  ">>" "$(TEST_NAME)" $($(TEST_ID)_TOTAL) $($(TEST_ID)_PASSED) \
 928                 $($(TEST_ID)_FAILED) $($(TEST_ID)_ERROR) "<<" $(NEWLINE) \
 929             $(eval TEST_FAILURE := true) \
 930           ) \
 931         )
 932         $(ECHO) >> $(TEST_SUMMARY) ==============================
 933         $(if $(filter true, $(TEST_FAILURE)), \
 934           $(ECHO) >> $(TEST_SUMMARY) TEST FAILURE $(NEWLINE) \
 935           $(MKDIR) -p $(MAKESUPPORT_OUTPUTDIR) $(NEWLINE) \
 936           $(TOUCH) $(MAKESUPPORT_OUTPUTDIR)/exit-with-error \
 937         , \
 938           $(ECHO) >> $(TEST_SUMMARY) TEST SUCCESS \
 939         )
 940         $(ECHO)
 941         $(CAT) $(TEST_SUMMARY)
 942         $(ECHO)
 943 
 944 ################################################################################
 945 
 946 all: run-test
 947 
 948 .PHONY: default all run-test $(TARGETS)