1 #
   2 # Copyright (c) 2016, 2017, Oracle and/or its affiliates. All rights reserved.
   3 # DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4 #
   5 # This code is free software; you can redistribute it and/or modify it
   6 # under the terms of the GNU General Public License version 2 only, as
   7 # published by the Free Software Foundation.  Oracle designates this
   8 # particular file as subject to the "Classpath" exception as provided
   9 # by Oracle in the LICENSE file that accompanied this code.
  10 #
  11 # This code is distributed in the hope that it will be useful, but WITHOUT
  12 # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  13 # FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  14 # version 2 for more details (a copy is included in the LICENSE file that
  15 # accompanied this code).
  16 #
  17 # You should have received a copy of the GNU General Public License version
  18 # 2 along with this work; if not, write to the Free Software Foundation,
  19 # Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  20 #
  21 # Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  22 # or visit www.oracle.com if you need additional information or have any
  23 # questions.
  24 #
  25 
  26 default: all
  27 
  28 include $(SPEC)
  29 include MakeBase.gmk
  30 include FindTests.gmk
  31 
  32 # We will always run multiple tests serially
  33 .NOTPARALLEL:
  34 
  35 # Directories to find jtreg tests relative to
  36 JTREG_TEST_TOPDIRS := $(TOPDIR) $(JTREG_TESTROOTS)
  37 
  38 # Hook to include the corresponding custom file, if present.
  39 $(eval $(call IncludeCustomExtension, RunTests.gmk))
  40 
  41 TEST_RESULTS_DIR := $(OUTPUTDIR)/test-results
  42 TEST_SUPPORT_DIR := $(OUTPUTDIR)/test-support
  43 
  44 
  45 ################################################################################
  46 # Parse control variables
  47 ################################################################################
  48 
  49 $(eval $(call ParseKeywordVariable, JTREG, \
  50     KEYWORDS := JOBS TIMEOUT TEST_MODE ASSERT VERBOSE RETAIN MAX_MEM, \
  51     STRING_KEYWORDS := OPTIONS JAVA_OPTIONS VM_OPTIONS, \
  52 ))
  53 
  54 ifneq ($(JTREG), )
  55   # Inform the user
  56   $(info Running tests using JTREG control variable '$(JTREG)')
  57 endif
  58 
  59 $(eval $(call ParseKeywordVariable, GTEST, \
  60     KEYWORDS := REPEAT, \
  61     STRING_KEYWORDS := OPTIONS, \
  62 ))
  63 
  64 ifneq ($(GTEST), )
  65   # Inform the user
  66   $(info Running tests using GTEST control variable '$(GTEST)')
  67 endif
  68 
  69 
  70 ################################################################################
  71 # Component-specific Jtreg settings
  72 ################################################################################
  73 
  74 ifeq ($(TEST_JOBS), 0)
  75   # If TEST_JOBS is not specified, hotspot fallback default is
  76   # min(num_cores / 2, 12).
  77   hotspot_JTREG_JOBS := $(shell $(EXPR) $(NUM_CORES) / 2)
  78   ifeq ($(hotspot_JTREG_JOBS), 0)
  79     hotspot_JTREG_JOBS := 1
  80   else ifeq ($(shell $(EXPR) $(hotspot_JTREG_JOBS) \> 12), 1)
  81     hotspot_JTREG_JOBS := 12
  82   endif
  83 endif
  84 
  85 hotspot_JTREG_MAX_MEM := 0
  86 hotspot_JTREG_ASSERT := false
  87 hotspot_JTREG_NATIVEPATH := $(TEST_IMAGE_DIR)/hotspot/jtreg/native
  88 jdk_JTREG_NATIVEPATH := $(TEST_IMAGE_DIR)/jdk/jtreg/native
  89 
  90 
  91 ################################################################################
  92 # Parse test selection
  93 #
  94 # The user has given a test selection in the TEST variable. We must parse it
  95 # and determine what that means in terms of actual calls to the test framework.
  96 #
  97 # The parse functions take as argument a test specification as given by the
  98 # user, and returns a fully qualified test descriptor if it was a match, or
  99 # nothing if not. A single test specification can result in multiple test
 100 # descriptors being returned. A valid test descriptor must always be accepted
 101 # and returned identically.
 102 ################################################################################
 103 
 104 # Helper function to determine if a test specification is a Gtest test
 105 #
 106 # It is a Gtest test if it is either "gtest", or "gtest:" followed by an optional
 107 # test filter string.
 108 define ParseGtestTestSelection
 109   $(if $(filter gtest%, $1), \
 110     $(if $(filter gtest, $1), \
 111       gtest:all \
 112     , \
 113       $(if $(filter gtest:, $1), \
 114         gtest:all \
 115       , \
 116         $1 \
 117       ) \
 118     ) \
 119   )
 120 endef
 121 
 122 # Helper function to determine if a test specification is a Jtreg test
 123 #
 124 # It is a Jtreg test if it optionally begins with jtreg:, and then is either
 125 # an unspecified group name (possibly prefixed by :), or a group in a
 126 # specified test/<component> directory, or a path to a test or test directory,
 127 # either absolute or relative to any of the JTREG_TEST_TOPDIRS.
 128 define ParseJtregTestSelection
 129   $(eval TEST_NAME := $(strip $(patsubst jtreg:%, %, $1))) \
 130   $(if $(or $(findstring :, $(TEST_NAME)), $(findstring /, $(TEST_NAME))), , \
 131     $(eval TEST_NAME := :$(TEST_NAME)) \
 132   ) \
 133   $(if $(findstring :, $(TEST_NAME)), \
 134     $(if $(filter :%, $(TEST_NAME)), \
 135       $(foreach root, $(JTREG_TESTROOTS), \
 136         $(if $(filter $(patsubst :%, %, $(TEST_NAME)), \
 137             $($(root)_JTREG_TEST_GROUPS)), \
 138           jtreg:$(root):$(patsubst :%,%,$(TEST_NAME)) \
 139         ) \
 140       ) \
 141     , \
 142       $(eval ROOT_PART := $(word 1, $(subst :, $(SPACE), $(TEST_NAME)))) \
 143       $(eval ROOT := $(filter $(addprefix %, $(ROOT_PART)), $(JTREG_TESTROOTS))) \
 144       $(eval GROUP := $(word 2, $(subst :, $(SPACE), $(TEST_NAME)))) \
 145       $(foreach root, $(ROOT), \
 146         $(if $(filter $(GROUP), $($(root)_JTREG_TEST_GROUPS)), \
 147           jtreg:$(root):$(GROUP) \
 148         ) \
 149       ) \
 150     ) \
 151   , \
 152     $(if $(filter /%, $(TEST_NAME)), \
 153       $(if $(wildcard $(TEST_NAME)), \
 154         jtreg:$(TEST_NAME) \
 155       ) \
 156     , \
 157       $(addprefix jtreg:, $(wildcard $(addsuffix /$(TEST_NAME), $(JTREG_TEST_TOPDIRS)))) \
 158     ) \
 159   )
 160 endef
 161 
 162 ifeq ($(TEST), )
 163   $(info No test selection given in TEST!)
 164   $(info Please use e.g. 'run-test TEST=tier1' or 'run-test-tier1')
 165   $(info See common/doc/testing.[md|html] for help)
 166   $(error Cannot continue)
 167 endif
 168 
 169 # Now intelligently convert the test selection given by the user in TEST
 170 # into a list of fully qualified test descriptors of the tests to run.
 171 TESTS_TO_RUN :=
 172 $(foreach test, $(TEST), \
 173   $(eval PARSED_TESTS := $(call ParseCustomTestSelection, $(test))) \
 174   $(if $(strip $(PARSED_TESTS)), , \
 175     $(eval PARSED_TESTS += $(call ParseGtestTestSelection, $(test))) \
 176   ) \
 177   $(if $(strip $(PARSED_TESTS)), , \
 178     $(eval PARSED_TESTS += $(call ParseJtregTestSelection, $(test))) \
 179   ) \
 180   $(if $(strip $(PARSED_TESTS)), , \
 181     $(eval UNKNOWN_TEST := $(test)) \
 182   ) \
 183   $(eval TESTS_TO_RUN += $(PARSED_TESTS)) \
 184 )
 185 
 186 ifneq ($(UNKNOWN_TEST), )
 187   $(info Unknown test selection: '$(UNKNOWN_TEST)')
 188   $(info See common/doc/testing.[md|html] for help)
 189   $(error Cannot continue)
 190 endif
 191 
 192 TESTS_TO_RUN := $(strip $(TESTS_TO_RUN))
 193 
 194 
 195 # Present the result of our parsing to the user
 196 $(info Test selection '$(TEST)', will run:)
 197 $(foreach test, $(TESTS_TO_RUN), $(info * $(test)))
 198 
 199 
 200 ################################################################################
 201 # Functions for setting up rules for running the selected tests
 202 #
 203 # The SetupRun*Test functions all have the same interface:
 204 #
 205 # Parameter 1 is the name of the rule. This is the test id, based on the test
 206 # descriptor, and this is also used as variable prefix, and the targets
 207 # generated are listed in a variable by that name.
 208 #
 209 # Remaining parameters are named arguments. Currently this is only:
 210 #   TEST -- The properly formatted fully qualified test descriptor
 211 #
 212 # After the rule named by the test id has been executed, the following
 213 # variables will be available:
 214 # testid_TOTAL - the total number of tests run
 215 # testid_PASSED - the number of successful tests
 216 # testid_FAILED - the number of failed tests
 217 # testid_ERROR - the number of tests was neither successful or failed
 218 #
 219 ################################################################################
 220 
 221 ### Rules for Gtest
 222 
 223 SetupRunGtestTest = $(NamedParamsMacroTemplate)
 224 define SetupRunGtestTestBody
 225   $1_TEST_RESULTS_DIR := $$(TEST_RESULTS_DIR)/$1
 226   $1_TEST_SUPPORT_DIR := $$(TEST_SUPPORT_DIR)/$1
 227 
 228   $1_TEST_NAME := $$(strip $$(patsubst gtest:%, %, $$($1_TEST)))
 229   ifneq ($$($1_TEST_NAME), all)
 230     $1_GTEST_FILTER := --gtest_filter=$$($1_TEST_NAME)*
 231   endif
 232 
 233   ifneq ($$(GTEST_REPEAT), )
 234     $1_GTEST_REPEAT :=--gtest_repeat=$$(GTEST_REPEAT)
 235   endif
 236 
 237   run-test-$1:
 238         $$(call LogWarn)
 239         $$(call LogWarn, Running test '$$($1_TEST)')
 240         $$(call MakeDir, $$($1_TEST_RESULTS_DIR) $$($1_TEST_SUPPORT_DIR))
 241         $$(call ExecuteWithLog, $$($1_TEST_SUPPORT_DIR)/gtest, \
 242             $$(FIXPATH) $$(TEST_IMAGE_DIR)/hotspot/gtest/server/gtestLauncher \
 243             -jdk $(JDK_IMAGE_DIR) $$($1_GTEST_FILTER) \
 244             --gtest_output=xml:$$($1_TEST_RESULTS_DIR)/gtest.xml \
 245             $$($1_GTEST_REPEAT) $$(GTEST_OPTIONS) \
 246             > >($(TEE) $$($1_TEST_RESULTS_DIR)/gtest.txt) || true )
 247 
 248   $1_RESULT_FILE := $$($1_TEST_RESULTS_DIR)/gtest.txt
 249 
 250   parse-test-$1: run-test-$1
 251         $$(call LogWarn, Finished running test '$$($1_TEST)')
 252         $$(call LogWarn, Test report is stored in $$(strip \
 253             $$(subst $$(TOPDIR)/, , $$($1_TEST_RESULTS_DIR))))
 254         $$(eval $1_TOTAL := $$(shell $$(AWK) '/==========.* tests? from .* \
 255             test cases? ran/ { print $$$$2 }' $$($1_RESULT_FILE)))
 256         $$(eval $1_PASSED := $$(shell $$(AWK) '/\[  PASSED  \] .* tests?./ \
 257             { print $$$$4 }' $$($1_RESULT_FILE)))
 258         $$(eval $1_FAILED := $$(shell $$(AWK) '/\[  FAILED  \] .* tests?, \
 259             listed below/ { print $$$$4 }' $$($1_RESULT_FILE)))
 260         $$(if $$($1_FAILED), , $$(eval $1_FAILED := 0))
 261         $$(eval $1_ERROR := $$(shell \
 262             $$(EXPR) $$($1_TOTAL) - $$($1_PASSED) - $$($1_FAILED)))
 263 
 264   $1: run-test-$1 parse-test-$1
 265 
 266   TARGETS += $1
 267 endef
 268 
 269 ################################################################################
 270 
 271 ### Rules for Jtreg
 272 
 273 # Helper function for SetupRunJtregTest. Set a JTREG_* variable from, in order:
 274 # 1) Specified by user on command line
 275 # 2) Component-specific default
 276 # 3) Generic default
 277 #
 278 # Note: No spaces are allowed around the arguments.
 279 # Arg $1 The test ID (i.e. $1 in SetupRunJtregTest)
 280 # Arg $2 Base variable, e.g. JTREG_JOBS
 281 # Arg $3 The default value (optional)
 282 define SetJtregValue
 283   ifneq ($$($2), )
 284     $1_$2 := $$($2)
 285   else
 286     ifneq ($$($$($1_COMPONENT)_$2), )
 287       $1_$2 := $$($$($1_COMPONENT)_$2)
 288     else
 289       ifneq ($3, )
 290         $1_$2 := $3
 291       endif
 292     endif
 293   endif
 294 endef
 295 
 296 SetupRunJtregTest = $(NamedParamsMacroTemplate)
 297 define SetupRunJtregTestBody
 298   $1_TEST_RESULTS_DIR := $$(TEST_RESULTS_DIR)/$1
 299   $1_TEST_SUPPORT_DIR := $$(TEST_SUPPORT_DIR)/$1
 300 
 301   $1_TEST_NAME := $$(strip $$(patsubst jtreg:%, %, $$($1_TEST)))
 302   $1_COMPONENT := $$(firstword $$(subst /, $$(SPACE), \
 303       $$(patsubst test/%, %, $$($1_TEST_NAME))))
 304 
 305   ifeq ($$(JT_HOME), )
 306     $$(info Error: jtreg framework is not found.)
 307     $$(info Please run configure using --with-jtreg.)
 308     $$(error Cannot continue)
 309   endif
 310 
 311   # Unfortunately, we need different defaults for some JTREG values,
 312   # depending on what component we're running.
 313 
 314   # Convert JTREG_foo into $1_JTREG_foo with a suitable value.
 315   $$(eval $$(call SetJtregValue,$1,JTREG_TEST_MODE,agentvm))
 316   $$(eval $$(call SetJtregValue,$1,JTREG_ASSERT,true))
 317   $$(eval $$(call SetJtregValue,$1,JTREG_MAX_MEM,512m))
 318   $$(eval $$(call SetJtregValue,$1,JTREG_NATIVEPATH))
 319   $$(eval $$(call SetJtregValue,$1,JTREG_BASIC_OPTIONS))
 320 
 321   ifneq ($(TEST_JOBS), 0)
 322     # User has specified TEST_JOBS, use that as fallback default
 323     $$(eval $$(call SetJtregValue,$1,JTREG_JOBS,$$(TEST_JOBS)))
 324   else
 325     # Use JOBS as default (except for hotspot)
 326     $$(eval $$(call SetJtregValue,$1,JTREG_JOBS,$$(JOBS)))
 327   endif
 328 
 329   ifeq ($$(shell $$(EXPR) $$($1_JTREG_JOBS) \> 50), 1)
 330     # Until CODETOOLS-7901892 is fixed, JTreg cannot handle more than 50 jobs
 331     $1_JTREG_JOBS := 50
 332   endif
 333 
 334   # Make sure MaxRAMPercentage is high enough to not cause OOM or swapping since
 335   # we may end up with a lot of JVM's
 336   $1_JTREG_MAX_RAM_PERCENTAGE := $$(shell $$(EXPR) 25 / $$($1_JTREG_JOBS))
 337 
 338   JTREG_TIMEOUT ?= 4
 339   JTREG_VERBOSE ?= fail,error,summary
 340   JTREG_RETAIN ?= fail,error
 341 
 342   ifneq ($$($1_JTREG_MAX_MEM), 0)
 343     $1_JTREG_BASIC_OPTIONS += -vmoption:-Xmx$$($1_JTREG_MAX_MEM)
 344     $1_JTREG_LAUNCHER_OPTIONS += -Xmx$$($1_JTREG_MAX_MEM)
 345   endif
 346 
 347   $1_JTREG_BASIC_OPTIONS += -$$($1_JTREG_TEST_MODE) \
 348       -verbose:$$(JTREG_VERBOSE) -retain:$$(JTREG_RETAIN) \
 349       -concurrency:$$($1_JTREG_JOBS) -timeoutFactor:$$(JTREG_TIMEOUT) \
 350       -vmoption:-XX:MaxRAMPercentage=$$($1_JTREG_MAX_RAM_PERCENTAGE)
 351 
 352   $1_JTREG_BASIC_OPTIONS += -automatic -keywords:\!ignore -ignore:quiet
 353 
 354   # Make it possible to specify the JIB_DATA_DIR for tests using the
 355   # JIB Artifact resolver
 356   $1_JTREG_BASIC_OPTIONS += -e:JIB_DATA_DIR
 357   # Some tests needs to find a boot JDK using the JDK8_HOME variable.
 358   $1_JTREG_BASIC_OPTIONS += -e:JDK8_HOME=$$(BOOT_JDK)
 359 
 360   $1_JTREG_BASIC_OPTIONS += \
 361       $$(addprefix -javaoption:, $$(JTREG_JAVA_OPTIONS)) \
 362       $$(addprefix -vmoption:, $$(JTREG_VM_OPTIONS)) \
 363       #
 364 
 365   ifeq ($$($1_JTREG_ASSERT), true)
 366     $1_JTREG_BASIC_OPTIONS += -ea -esa
 367   endif
 368 
 369   ifneq ($$($1_JTREG_NATIVEPATH), )
 370     $1_JTREG_BASIC_OPTIONS += -nativepath:$$($1_JTREG_NATIVEPATH)
 371   endif
 372 
 373   ifneq ($$(JIB_JAR), )
 374     $1_JTREG_BASIC_OPTIONS += -cpa:$$(JIB_JAR)
 375   endif
 376 
 377   run-test-$1:
 378         $$(call LogWarn)
 379         $$(call LogWarn, Running test '$$($1_TEST)')
 380         $$(call MakeDir, $$($1_TEST_RESULTS_DIR) $$($1_TEST_SUPPORT_DIR))
 381         $$(call ExecuteWithLog, $$($1_TEST_SUPPORT_DIR)/jtreg, \
 382             $$(JAVA) $$($1_JTREG_LAUNCHER_OPTIONS) \
 383                 -Dprogram=jtreg -jar $$(JT_HOME)/lib/jtreg.jar \
 384                 $$($1_JTREG_BASIC_OPTIONS) \
 385                 -testjdk:$$(JDK_IMAGE_DIR) \
 386                 -dir:$$(TOPDIR) \
 387                 -reportDir:$$($1_TEST_RESULTS_DIR) \
 388                 -workDir:$$($1_TEST_SUPPORT_DIR) \
 389                 $$(JTREG_OPTIONS) \
 390                 $$($1_TEST_NAME) || true )
 391 
 392   $1_RESULT_FILE := $$($1_TEST_RESULTS_DIR)/text/stats.txt
 393 
 394   parse-test-$1: run-test-$1
 395         $$(call LogWarn, Finished running test '$$($1_TEST)')
 396         $$(call LogWarn, Test report is stored in $$(strip \
 397             $$(subst $$(TOPDIR)/, , $$($1_TEST_RESULTS_DIR))))
 398         $$(if  $$(wildcard $$($1_RESULT_FILE)), \
 399           $$(eval $1_PASSED := $$(shell $$(AWK) '{ gsub(/[,;]/, ""); \
 400               for (i=1; i<=NF; i++) { if ($$$$i == "passed:") \
 401               print $$$$(i+1) } }' $$($1_RESULT_FILE))) \
 402           $$(if $$($1_PASSED), , $$(eval $1_PASSED := 0)) \
 403           $$(eval $1_FAILED := $$(shell $$(AWK) '{gsub(/[,;]/, ""); \
 404               for (i=1; i<=NF; i++) { if ($$$$i == "failed:") \
 405               print $$$$(i+1) } }' $$($1_RESULT_FILE))) \
 406           $$(if $$($1_FAILED), , $$(eval $1_FAILED := 0)) \
 407           $$(eval $1_ERROR := $$(shell $$(AWK) '{gsub(/[,;]/, ""); \
 408               for (i=1; i<=NF; i++) { if ($$$$i == "error:") \
 409               print $$$$(i+1) } }' $$($1_RESULT_FILE))) \
 410           $$(if $$($1_ERROR), , $$(eval $1_ERROR := 0)) \
 411           $$(eval $1_TOTAL := $$(shell \
 412               $$(EXPR) $$($1_PASSED) + $$($1_FAILED) + $$($1_ERROR))) \
 413         , \
 414           $$(eval $1_PASSED := 0) \
 415           $$(eval $1_FAILED := 0) \
 416           $$(eval $1_ERROR := 1) \
 417           $$(eval $1_TOTAL := 1) \
 418         )
 419 
 420   $1: run-test-$1 parse-test-$1
 421 
 422   TARGETS += $1
 423 endef
 424 
 425 
 426 ################################################################################
 427 # Setup and execute make rules for all selected tests
 428 ################################################################################
 429 
 430 # Helper function to determine which handler to use for the given test
 431 UseGtestTestHandler = \
 432   $(if $(filter gtest:%, $1), true)
 433 
 434 UseJtregTestHandler = \
 435   $(if $(filter jtreg:%, $1), true)
 436 
 437 # Now process each test to run and setup a proper make rule
 438 $(foreach test, $(TESTS_TO_RUN), \
 439   $(eval TEST_ID := $(shell $(ECHO) $(strip $(test)) | \
 440       $(TR) -cs '[a-z][A-Z][0-9]\n' '_')) \
 441   $(eval ALL_TEST_IDS += $(TEST_ID)) \
 442   $(if $(call UseCustomTestHandler, $(test)), \
 443     $(eval $(call SetupRunCustomTest, $(TEST_ID), \
 444         TEST := $(test), \
 445     )) \
 446   ) \
 447   $(if $(call UseGtestTestHandler, $(test)), \
 448     $(eval $(call SetupRunGtestTest, $(TEST_ID), \
 449         TEST := $(test), \
 450     )) \
 451   ) \
 452   $(if $(call UseJtregTestHandler, $(test)), \
 453     $(eval $(call SetupRunJtregTest, $(TEST_ID), \
 454         TEST := $(test), \
 455     )) \
 456   ) \
 457 )
 458 
 459 # Sort also removes duplicates, so if there is any we'll get fewer words.
 460 ifneq ($(words $(ALL_TEST_IDS)), $(words $(sort $(ALL_TEST_IDS))))
 461   $(error Duplicate test specification)
 462 endif
 463 
 464 
 465 ################################################################################
 466 # The main target for RunTests.gmk
 467 ################################################################################
 468 
 469 # The SetupRun*Test functions have populated TARGETS.
 470 
 471 TEST_FAILURE := false
 472 
 473 run-test: $(TARGETS)
 474         # Print a table of the result of all tests run and their result
 475         $(ECHO)
 476         $(ECHO) ==============================
 477         $(ECHO) Test summary
 478         $(ECHO) ==============================
 479         $(PRINTF) "%2s %-49s %5s %5s %5s %5s %2s\n" "  " TEST \
 480             TOTAL PASS FAIL ERROR " "
 481         $(foreach test, $(TESTS_TO_RUN), \
 482           $(eval TEST_ID := $(shell $(ECHO) $(strip $(test)) | \
 483               $(TR) -cs '[a-z][A-Z][0-9]\n' '_')) \
 484           $(if $(filter $($(TEST_ID)_PASSED), $($(TEST_ID)_TOTAL)), \
 485             $(PRINTF) "%2s %-49s %5d %5d %5d %5d %2s\n" "  " "$(test)" \
 486                 $($(TEST_ID)_TOTAL) $($(TEST_ID)_PASSED) $($(TEST_ID)_FAILED) \
 487                 $($(TEST_ID)_ERROR) "  " $(NEWLINE) \
 488           , \
 489             $(PRINTF) "%2s %-49s %5d %5d %5d %5d %2s\n" ">>" "$(test)" \
 490                 $($(TEST_ID)_TOTAL) $($(TEST_ID)_PASSED) $($(TEST_ID)_FAILED) \
 491                 $($(TEST_ID)_ERROR) "<<" $(NEWLINE) \
 492             $(eval TEST_FAILURE := true) \
 493           ) \
 494         )
 495         $(ECHO) ==============================
 496         $(if $(filter true, $(TEST_FAILURE)), \
 497           $(ECHO) TEST FAILURE $(NEWLINE) \
 498           $(TOUCH) $(MAKESUPPORT_OUTPUTDIR)/exit-with-error \
 499         , \
 500           $(ECHO) TEST SUCCESS \
 501         )
 502         $(ECHO)
 503 
 504 ################################################################################
 505 
 506 all: run-test
 507 
 508 .PHONY: default all run-test $(TARGETS)