28 include $(SPEC)
29 include MakeBase.gmk
30 include FindTests.gmk
31
32 # We will always run multiple tests serially
33 .NOTPARALLEL:
34
35 ################################################################################
36 # Parse global control variables
37 ################################################################################
38
39 ifneq ($(TEST_VM_OPTS), )
40 ifneq ($(TEST_OPTS), )
41 TEST_OPTS := $(TEST_OPTS);VM_OPTIONS=$(TEST_VM_OPTS)
42 else
43 TEST_OPTS := VM_OPTIONS=$(TEST_VM_OPTS)
44 endif
45 endif
46
47 $(eval $(call ParseKeywordVariable, TEST_OPTS, \
48 KEYWORDS := JOBS TIMEOUT, \
49 STRING_KEYWORDS := VM_OPTIONS, \
50 ))
51
52 # Helper function to propagate TEST_OPTS values.
53 #
54 # Note: No spaces are allowed around the arguments.
55 # Arg $1 The variable in TEST_OPTS to propagate
56 # Arg $2 The control variable to propagate it to
57 define SetTestOpt
58 ifneq ($$(TEST_OPTS_$1), )
59 $2_$1 := $$(TEST_OPTS_$1)
60 endif
61 endef
62
63 # Setup _NT_SYMBOL_PATH on Windows
64 ifeq ($(OPENJDK_TARGET_OS), windows)
65 ifndef _NT_SYMBOL_PATH
66 # Can't use PathList here as it adds quotes around the value.
67 _NT_SYMBOL_PATH := \
68 $(subst $(SPACE),;, $(foreach p, $(sort $(dir $(wildcard \
69 $(addprefix $(SYMBOLS_IMAGE_DIR)/bin/, *.pdb */*.pdb)))), $(call FixPath, $p)))
87 else
88 JTREG_TOPDIR := $(CUSTOM_ROOT)
89 endif
90
91 JTREG_FAILURE_HANDLER_DIR := $(TEST_IMAGE_DIR)/failure_handler
92 JTREG_FAILURE_HANDLER := $(JTREG_FAILURE_HANDLER_DIR)/jtregFailureHandler.jar
93
94 ifneq ($(wildcard $(JTREG_FAILURE_HANDLER)), )
95 JTREG_FAILURE_HANDLER_OPTIONS := \
96 -timeoutHandlerDir:$(JTREG_FAILURE_HANDLER) \
97 -observerDir:$(JTREG_FAILURE_HANDLER) \
98 -timeoutHandler:jdk.test.failurehandler.jtreg.GatherProcessInfoTimeoutHandler \
99 -observer:jdk.test.failurehandler.jtreg.GatherDiagnosticInfoObserver \
100 -timeoutHandlerTimeout:0
101 endif
102
103 GTEST_LAUNCHER_DIRS := $(patsubst %/gtestLauncher, %, $(wildcard $(TEST_IMAGE_DIR)/hotspot/gtest/*/gtestLauncher))
104 GTEST_VARIANTS := $(strip $(patsubst $(TEST_IMAGE_DIR)/hotspot/gtest/%, %, $(GTEST_LAUNCHER_DIRS)))
105
106 ################################################################################
107 # Parse control variables
108 ################################################################################
109
110 ifneq ($(TEST_OPTS), )
111 # Inform the user
112 $(info Running tests using TEST_OPTS control variable '$(TEST_OPTS)')
113
114 $(eval $(call SetTestOpt,VM_OPTIONS,JTREG))
115 $(eval $(call SetTestOpt,VM_OPTIONS,GTEST))
116
117 $(eval $(call SetTestOpt,JOBS,JTREG))
118 $(eval $(call SetTestOpt,TIMEOUT,JTREG))
119 endif
120
121 $(eval $(call ParseKeywordVariable, JTREG, \
122 KEYWORDS := JOBS TIMEOUT TEST_MODE ASSERT VERBOSE RETAIN MAX_MEM, \
123 STRING_KEYWORDS := OPTIONS JAVA_OPTIONS VM_OPTIONS, \
124 ))
125
126 ifneq ($(JTREG), )
127 # Inform the user
128 $(info Running tests using JTREG control variable '$(JTREG)')
129 endif
130
131 $(eval $(call ParseKeywordVariable, GTEST, \
132 KEYWORDS := REPEAT, \
133 STRING_KEYWORDS := OPTIONS VM_OPTIONS, \
134 ))
135
136 ifneq ($(GTEST), )
137 # Inform the user
138 $(info Running tests using GTEST control variable '$(GTEST)')
139 endif
140
141
142 ################################################################################
143 # Component-specific Jtreg settings
144 ################################################################################
145
146 ifeq ($(TEST_JOBS), 0)
147 # If TEST_JOBS is not specified, hotspot fallback default is
148 # min(num_cores / 2, 12).
149 hotspot_JTREG_JOBS := $(shell $(EXPR) $(NUM_CORES) / 2)
150 ifeq ($(hotspot_JTREG_JOBS), 0)
151 hotspot_JTREG_JOBS := 1
152 else ifeq ($(shell $(EXPR) $(hotspot_JTREG_JOBS) \> 12), 1)
153 hotspot_JTREG_JOBS := 12
154 endif
155 endif
156
157 hotspot_JTREG_MAX_MEM := 0
158 hotspot_JTREG_ASSERT := false
159 hotspot_JTREG_NATIVEPATH := $(TEST_IMAGE_DIR)/hotspot/jtreg/native
160 jdk_JTREG_NATIVEPATH := $(TEST_IMAGE_DIR)/jdk/jtreg/native
161
162 jdk_JTREG_PROBLEM_LIST += $(TOPDIR)/test/jdk/ProblemList.txt
163 jaxp_JTREG_PROBLEM_LIST += $(TOPDIR)/test/jaxp/ProblemList.txt
164 langtools_JTREG_PROBLEM_LIST += $(TOPDIR)/test/langtools/ProblemList.txt
165 nashorn_JTREG_PROBLEM_LIST += $(TOPDIR)/test/nashorn/ProblemList.txt
166 hotspot_JTREG_PROBLEM_LIST += $(TOPDIR)/test/hotspot/jtreg/ProblemList.txt
167
168 ################################################################################
169 # Parse test selection
170 #
171 # The user has given a test selection in the TEST variable. We must parse it
172 # and determine what that means in terms of actual calls to the test framework.
173 #
174 # The parse functions take as argument a test specification as given by the
175 # user, and returns a fully qualified test descriptor if it was a match, or
176 # nothing if not. A single test specification can result in multiple test
177 # descriptors being returned. A valid test descriptor must always be accepted
178 # and returned identically.
179 ################################################################################
180
181 # Helper function to determine if a test specification is a Gtest test
182 #
183 # It is a Gtest test if it is either "gtest", or "gtest:" followed by an optional
184 # test filter string, and an optional "/<variant>" to select a specific JVM
185 # variant. If no variant is specified, all found variants are tested.
186 define ParseGtestTestSelection
187 $(if $(filter gtest%, $1), \
351 SetupRunGtestTest = $(NamedParamsMacroTemplate)
352 define SetupRunGtestTestBody
353 $1_TEST_RESULTS_DIR := $$(TEST_RESULTS_DIR)/$1
354 $1_TEST_SUPPORT_DIR := $$(TEST_SUPPORT_DIR)/$1
355 $1_EXITCODE := $$($1_TEST_RESULTS_DIR)/exitcode.txt
356
357 $1_VARIANT := $$(lastword $$(subst /, , $$($1_TEST)))
358 ifeq ($$(filter $$($1_VARIANT), $$(GTEST_VARIANTS)), )
359 $$(error Invalid gtest variant '$$($1_VARIANT)'. Valid variants: $$(GTEST_VARIANTS))
360 endif
361 $1_TEST_NAME := $$(strip $$(patsubst %/$$($1_VARIANT), %, \
362 $$(patsubst gtest:%, %, $$($1_TEST))))
363 ifneq ($$($1_TEST_NAME), all)
364 $1_GTEST_FILTER := --gtest_filter=$$($1_TEST_NAME)*
365 endif
366
367 ifneq ($$(GTEST_REPEAT), )
368 $1_GTEST_REPEAT :=--gtest_repeat=$$(GTEST_REPEAT)
369 endif
370
371 run-test-$1:
372 $$(call LogWarn)
373 $$(call LogWarn, Running test '$$($1_TEST)')
374 $$(call MakeDir, $$($1_TEST_RESULTS_DIR) $$($1_TEST_SUPPORT_DIR))
375 $$(call ExecuteWithLog, $$($1_TEST_SUPPORT_DIR)/gtest, \
376 $$(FIXPATH) $$(TEST_IMAGE_DIR)/hotspot/gtest/$$($1_VARIANT)/gtestLauncher \
377 -jdk $(JDK_IMAGE_DIR) $$($1_GTEST_FILTER) \
378 --gtest_output=xml:$$($1_TEST_RESULTS_DIR)/gtest.xml \
379 $$($1_GTEST_REPEAT) $$(GTEST_OPTIONS) $$(GTEST_VM_OPTIONS) \
380 > >($(TEE) $$($1_TEST_RESULTS_DIR)/gtest.txt) \
381 && $$(ECHO) $$$$? > $$($1_EXITCODE) \
382 || $$(ECHO) $$$$? > $$($1_EXITCODE) \
383 )
384
385 $1_RESULT_FILE := $$($1_TEST_RESULTS_DIR)/gtest.txt
386
387 parse-test-$1: run-test-$1
388 $$(call LogWarn, Finished running test '$$($1_TEST)')
389 $$(call LogWarn, Test report is stored in $$(strip \
390 $$(subst $$(TOPDIR)/, , $$($1_TEST_RESULTS_DIR))))
391 $$(if $$(wildcard $$($1_RESULT_FILE)), \
392 $$(eval $1_TOTAL := $$(shell $$(AWK) '/==========.* tests? from .* \
393 test cases? ran/ { print $$$$2 }' $$($1_RESULT_FILE))) \
394 $$(if $$($1_TOTAL), , $$(eval $1_TOTAL := 0)) \
395 $$(eval $1_PASSED := $$(shell $$(AWK) '/\[ PASSED \] .* tests?./ \
396 { print $$$$4 }' $$($1_RESULT_FILE))) \
397 $$(if $$($1_PASSED), , $$(eval $1_PASSED := 0)) \
398 $$(eval $1_FAILED := $$(shell $$(AWK) '/\[ FAILED \] .* tests?, \
399 listed below/ { print $$$$4 }' $$($1_RESULT_FILE))) \
430 $1_$2 := $$($2)
431 else
432 ifneq ($$($$($1_COMPONENT)_$2), )
433 $1_$2 := $$($$($1_COMPONENT)_$2)
434 else
435 ifneq ($3, )
436 $1_$2 := $3
437 endif
438 endif
439 endif
440 endef
441
442 SetupRunJtregTest = $(NamedParamsMacroTemplate)
443 define SetupRunJtregTestBody
444 $1_TEST_RESULTS_DIR := $$(TEST_RESULTS_DIR)/$1
445 $1_TEST_SUPPORT_DIR := $$(TEST_SUPPORT_DIR)/$1
446 $1_EXITCODE := $$($1_TEST_RESULTS_DIR)/exitcode.txt
447
448 $1_TEST_NAME := $$(strip $$(patsubst jtreg:%, %, $$($1_TEST)))
449
450 $1_COMPONENT := \
451 $$(strip $$(foreach root, $$(JTREG_TESTROOTS), \
452 $$(if $$(filter $$(root)%, $$(JTREG_TOPDIR)/$$($1_TEST_NAME)), \
453 $$(lastword $$(subst /, $$(SPACE), $$(root))) \
454 ) \
455 ))
456 # This will work only as long as just hotspot has the additional "jtreg" directory
457 ifeq ($$($1_COMPONENT), jtreg)
458 $1_COMPONENT := hotspot
459 endif
460
461 ifeq ($$(JT_HOME), )
462 $$(info Error: jtreg framework is not found.)
463 $$(info Please run configure using --with-jtreg.)
464 $$(error Cannot continue)
465 endif
466
467 # Unfortunately, we need different defaults for some JTREG values,
468 # depending on what component we're running.
469
470 # Convert JTREG_foo into $1_JTREG_foo with a suitable value.
471 $$(eval $$(call SetJtregValue,$1,JTREG_TEST_MODE,agentvm))
472 $$(eval $$(call SetJtregValue,$1,JTREG_ASSERT,true))
473 $$(eval $$(call SetJtregValue,$1,JTREG_MAX_MEM,512m))
474 $$(eval $$(call SetJtregValue,$1,JTREG_NATIVEPATH))
475 $$(eval $$(call SetJtregValue,$1,JTREG_BASIC_OPTIONS))
476 $$(eval $$(call SetJtregValue,$1,JTREG_PROBLEM_LIST))
477
478 ifneq ($(TEST_JOBS), 0)
479 # User has specified TEST_JOBS, use that as fallback default
480 $$(eval $$(call SetJtregValue,$1,JTREG_JOBS,$$(TEST_JOBS)))
481 else
482 # Use JOBS as default (except for hotspot)
483 $$(eval $$(call SetJtregValue,$1,JTREG_JOBS,$$(JOBS)))
484 endif
485
486 # Make sure MaxRAMPercentage is high enough to not cause OOM or swapping since
487 # we may end up with a lot of JVM's
488 $1_JTREG_MAX_RAM_PERCENTAGE := $$(shell $$(EXPR) 25 / $$($1_JTREG_JOBS))
489
490 JTREG_TIMEOUT ?= 4
491 JTREG_VERBOSE ?= fail,error,summary
492 JTREG_RETAIN ?= fail,error
493
494 ifneq ($$($1_JTREG_MAX_MEM), 0)
495 $1_JTREG_BASIC_OPTIONS += -vmoption:-Xmx$$($1_JTREG_MAX_MEM)
496 $1_JTREG_LAUNCHER_OPTIONS += -Xmx$$($1_JTREG_MAX_MEM)
497 endif
498
499 $1_JTREG_BASIC_OPTIONS += -$$($1_JTREG_TEST_MODE) \
500 -verbose:$$(JTREG_VERBOSE) -retain:$$(JTREG_RETAIN) \
501 -concurrency:$$($1_JTREG_JOBS) -timeoutFactor:$$(JTREG_TIMEOUT) \
502 -vmoption:-XX:MaxRAMPercentage=$$($1_JTREG_MAX_RAM_PERCENTAGE)
503
504 $1_JTREG_BASIC_OPTIONS += -automatic -keywords:\!ignore -ignore:quiet
505
506 # Make it possible to specify the JIB_DATA_DIR for tests using the
507 # JIB Artifact resolver
508 $1_JTREG_BASIC_OPTIONS += -e:JIB_DATA_DIR
509 # Some tests needs to find a boot JDK using the JDK8_HOME variable.
510 $1_JTREG_BASIC_OPTIONS += -e:JDK8_HOME=$$(BOOT_JDK)
511 # If running on Windows, propagate the _NT_SYMBOL_PATH to enable
512 # symbol lookup in hserr files
513 ifeq ($$(OPENJDK_TARGET_OS), windows)
514 $1_JTREG_BASIC_OPTIONS += -e:_NT_SYMBOL_PATH
515 endif
516
517 $1_JTREG_BASIC_OPTIONS += \
518 $$(addprefix -javaoption:, $$(JTREG_JAVA_OPTIONS)) \
519 $$(addprefix -vmoption:, $$(JTREG_VM_OPTIONS)) \
520 #
521
522 ifeq ($$($1_JTREG_ASSERT), true)
523 $1_JTREG_BASIC_OPTIONS += -ea -esa
524 endif
525
526 ifneq ($$($1_JTREG_NATIVEPATH), )
527 $1_JTREG_BASIC_OPTIONS += -nativepath:$$($1_JTREG_NATIVEPATH)
528 endif
529
530 ifneq ($$($1_JTREG_PROBLEM_LIST), )
531 $1_JTREG_BASIC_OPTIONS += $$(addprefix -exclude:, $$($1_JTREG_PROBLEM_LIST))
532 endif
533
534 ifneq ($$(JIB_HOME), )
535 $1_JTREG_BASIC_OPTIONS += -e:JIB_HOME=$$(JIB_HOME)
536 endif
537
538 $1_JTREG_BASIC_OPTIONS += -e:TEST_IMAGE_GRAAL_DIR=${TEST_IMAGE_DIR}/hotspot/jtreg/graal
539
540 ifneq ($$(JTREG_FAILURE_HANDLER_OPTIONS), )
541 $1_JTREG_LAUNCHER_OPTIONS += -Djava.library.path="$(JTREG_FAILURE_HANDLER_DIR)"
542 endif
543
544 clean-workdir-$1:
545 $$(RM) -r $$($1_TEST_SUPPORT_DIR)
546
547 run-test-$1: clean-workdir-$1
548 $$(call LogWarn)
549 $$(call LogWarn, Running test '$$($1_TEST)')
550 $$(call MakeDir, $$($1_TEST_RESULTS_DIR) $$($1_TEST_SUPPORT_DIR))
551 $$(call ExecuteWithLog, $$($1_TEST_SUPPORT_DIR)/jtreg, \
552 $$(JAVA) $$($1_JTREG_LAUNCHER_OPTIONS) \
553 -Dprogram=jtreg -jar $$(JT_HOME)/lib/jtreg.jar \
554 $$($1_JTREG_BASIC_OPTIONS) \
555 -testjdk:$$(JDK_IMAGE_DIR) \
556 -dir:$$(JTREG_TOPDIR) \
557 -reportDir:$$($1_TEST_RESULTS_DIR) \
558 -workDir:$$($1_TEST_SUPPORT_DIR) \
559 $$(JTREG_OPTIONS) \
560 $$(JTREG_FAILURE_HANDLER_OPTIONS) \
561 $$($1_TEST_NAME) \
562 && $$(ECHO) $$$$? > $$($1_EXITCODE) \
563 || $$(ECHO) $$$$? > $$($1_EXITCODE) \
564 )
565
566 $1_RESULT_FILE := $$($1_TEST_RESULTS_DIR)/text/stats.txt
567
614 $1_TEST_NAME := $$($1_FULL_TEST_NAME)
615 $1_TEST_ARGS :=
616 endif
617
618 ifeq ($$($1_TEST_NAME), hotspot-internal)
619 $1_TEST_COMMAND_LINE := \
620 $$(JDK_IMAGE_DIR)/bin/java -XX:+ExecuteInternalVMTests \
621 -XX:+ShowMessageBoxOnError -version
622 else ifeq ($$($1_TEST_NAME), failure-handler)
623 $1_TEST_COMMAND_LINE := \
624 ($(CD) $(TOPDIR)/make/test && $(MAKE) $(MAKE_ARGS) -f \
625 BuildFailureHandler.gmk test)
626 else ifeq ($$($1_TEST_NAME), make)
627 $1_TEST_COMMAND_LINE := \
628 ($(CD) $(TOPDIR)/test/make && $(MAKE) $(MAKE_ARGS) -f \
629 TestMake.gmk $$($1_TEST_ARGS))
630 else
631 $$(error Invalid special test specification: $$($1_TEST_NAME))
632 endif
633
634 run-test-$1:
635 $$(call LogWarn)
636 $$(call LogWarn, Running test '$$($1_TEST)')
637 $$(call MakeDir, $$($1_TEST_RESULTS_DIR) $$($1_TEST_SUPPORT_DIR))
638 $$(call ExecuteWithLog, $$($1_TEST_SUPPORT_DIR)/test-execution, \
639 $$($1_TEST_COMMAND_LINE) \
640 > >($(TEE) $$($1_TEST_RESULTS_DIR)/test-output.txt) \
641 && $$(ECHO) $$$$? > $$($1_EXITCODE) \
642 || $$(ECHO) $$$$? > $$($1_EXITCODE) \
643 )
644
645 $1_RESULT_FILE := $$($1_TEST_RESULTS_DIR)/gtest.txt
646
647 # We can not parse the various "special" tests.
648 parse-test-$1: run-test-$1
649 $$(call LogWarn, Finished running test '$$($1_TEST)')
650 $$(call LogWarn, Test report is stored in $$(strip \
651 $$(subst $$(TOPDIR)/, , $$($1_TEST_RESULTS_DIR))))
652 $$(call LogWarn, Warning: Special test results are not properly parsed!)
653 $$(eval $1_PASSED := 0)
654 $$(eval $1_FAILED := 0)
|
28 include $(SPEC)
29 include MakeBase.gmk
30 include FindTests.gmk
31
32 # We will always run multiple tests serially
33 .NOTPARALLEL:
34
35 ################################################################################
36 # Parse global control variables
37 ################################################################################
38
39 ifneq ($(TEST_VM_OPTS), )
40 ifneq ($(TEST_OPTS), )
41 TEST_OPTS := $(TEST_OPTS);VM_OPTIONS=$(TEST_VM_OPTS)
42 else
43 TEST_OPTS := VM_OPTIONS=$(TEST_VM_OPTS)
44 endif
45 endif
46
47 $(eval $(call ParseKeywordVariable, TEST_OPTS, \
48 SINGLE_KEYWORDS := JOBS TIMEOUT_FACTOR, \
49 STRING_KEYWORDS := VM_OPTIONS JAVA_OPTIONS, \
50 ))
51
52 # Helper function to propagate TEST_OPTS values.
53 #
54 # Note: No spaces are allowed around the arguments.
55 # Arg $1 The variable in TEST_OPTS to propagate
56 # Arg $2 The control variable to propagate it to
57 define SetTestOpt
58 ifneq ($$(TEST_OPTS_$1), )
59 $2_$1 := $$(TEST_OPTS_$1)
60 endif
61 endef
62
63 # Setup _NT_SYMBOL_PATH on Windows
64 ifeq ($(OPENJDK_TARGET_OS), windows)
65 ifndef _NT_SYMBOL_PATH
66 # Can't use PathList here as it adds quotes around the value.
67 _NT_SYMBOL_PATH := \
68 $(subst $(SPACE),;, $(foreach p, $(sort $(dir $(wildcard \
69 $(addprefix $(SYMBOLS_IMAGE_DIR)/bin/, *.pdb */*.pdb)))), $(call FixPath, $p)))
87 else
88 JTREG_TOPDIR := $(CUSTOM_ROOT)
89 endif
90
91 JTREG_FAILURE_HANDLER_DIR := $(TEST_IMAGE_DIR)/failure_handler
92 JTREG_FAILURE_HANDLER := $(JTREG_FAILURE_HANDLER_DIR)/jtregFailureHandler.jar
93
94 ifneq ($(wildcard $(JTREG_FAILURE_HANDLER)), )
95 JTREG_FAILURE_HANDLER_OPTIONS := \
96 -timeoutHandlerDir:$(JTREG_FAILURE_HANDLER) \
97 -observerDir:$(JTREG_FAILURE_HANDLER) \
98 -timeoutHandler:jdk.test.failurehandler.jtreg.GatherProcessInfoTimeoutHandler \
99 -observer:jdk.test.failurehandler.jtreg.GatherDiagnosticInfoObserver \
100 -timeoutHandlerTimeout:0
101 endif
102
103 GTEST_LAUNCHER_DIRS := $(patsubst %/gtestLauncher, %, $(wildcard $(TEST_IMAGE_DIR)/hotspot/gtest/*/gtestLauncher))
104 GTEST_VARIANTS := $(strip $(patsubst $(TEST_IMAGE_DIR)/hotspot/gtest/%, %, $(GTEST_LAUNCHER_DIRS)))
105
106 ################################################################################
107 # Setup global test running parameters
108 ################################################################################
109
110 # Each factor variable comes in 3 variants. The first one is reserved for users
111 # to use on command line. The other two are for predifined configurations in JDL
112 # and for machine specific configurations respectively.
113 TEST_JOBS_FACTOR ?= 1
114 TEST_JOBS_FACTOR_JDL ?= 1
115 TEST_JOBS_FACTOR_MACHINE ?= 1
116
117 ifeq ($(TEST_JOBS), 0)
118 # Concurrency based on min(cores / 2, 12) * TEST_JOBS_FACTOR
119 TEST_JOBS := $(shell $(AWK) \
120 'BEGIN { \
121 c = $(NUM_CORES) / 2; \
122 if (c > 12) c = 12; \
123 c = c * $(TEST_JOBS_FACTOR); \
124 c = c * $(TEST_JOBS_FACTOR_JDL); \
125 c = c * $(TEST_JOBS_FACTOR_MACHINE); \
126 if (c < 1) c = 1; \
127 printf "%.0f", c; \
128 }')
129 endif
130
131 ################################################################################
132 # Parse control variables
133 ################################################################################
134
135 ifneq ($(TEST_OPTS), )
136 # Inform the user
137 $(info Running tests using TEST_OPTS control variable '$(TEST_OPTS)')
138 endif
139
140 $(eval $(call SetTestOpt,VM_OPTIONS,JTREG))
141 $(eval $(call SetTestOpt,JAVA_OPTIONS,JTREG))
142 $(eval $(call SetTestOpt,VM_OPTIONS,GTEST))
143 $(eval $(call SetTestOpt,JAVA_OPTIONS,GTEST))
144
145 $(eval $(call SetTestOpt,JOBS,JTREG))
146 $(eval $(call SetTestOpt,TIMEOUT_FACTOR,JTREG))
147
148 $(eval $(call ParseKeywordVariable, JTREG, \
149 SINGLE_KEYWORDS := JOBS TIMEOUT_FACTOR TEST_MODE ASSERT VERBOSE RETAIN MAX_MEM \
150 EXTRA_PROBLEM_LISTS KEYWORDS, \
151 STRING_KEYWORDS := OPTIONS JAVA_OPTIONS VM_OPTIONS, \
152 ))
153
154 ifneq ($(JTREG), )
155 # Inform the user
156 $(info Running tests using JTREG control variable '$(JTREG)')
157 endif
158
159 $(eval $(call ParseKeywordVariable, GTEST, \
160 SINGLE_KEYWORDS := REPEAT, \
161 STRING_KEYWORDS := OPTIONS VM_OPTIONS JAVA_OPTIONS, \
162 ))
163
164 ifneq ($(GTEST), )
165 # Inform the user
166 $(info Running tests using GTEST control variable '$(GTEST)')
167 endif
168
169
170 ################################################################################
171 # Component-specific Jtreg settings
172 ################################################################################
173
174 hotspot_JTREG_MAX_MEM := 0
175 hotspot_JTREG_ASSERT := false
176 hotspot_JTREG_NATIVEPATH := $(TEST_IMAGE_DIR)/hotspot/jtreg/native
177 jdk_JTREG_NATIVEPATH := $(TEST_IMAGE_DIR)/jdk/jtreg/native
178
179 jdk_JTREG_PROBLEM_LIST += $(TOPDIR)/test/jdk/ProblemList.txt
180 jaxp_JTREG_PROBLEM_LIST += $(TOPDIR)/test/jaxp/ProblemList.txt
181 langtools_JTREG_PROBLEM_LIST += $(TOPDIR)/test/langtools/ProblemList.txt
182 nashorn_JTREG_PROBLEM_LIST += $(TOPDIR)/test/nashorn/ProblemList.txt
183 hotspot_JTREG_PROBLEM_LIST += $(TOPDIR)/test/hotspot/jtreg/ProblemList.txt
184
185 langtools_JTREG_MAX_MEM := 768m
186
187 ################################################################################
188 # Parse test selection
189 #
190 # The user has given a test selection in the TEST variable. We must parse it
191 # and determine what that means in terms of actual calls to the test framework.
192 #
193 # The parse functions take as argument a test specification as given by the
194 # user, and returns a fully qualified test descriptor if it was a match, or
195 # nothing if not. A single test specification can result in multiple test
196 # descriptors being returned. A valid test descriptor must always be accepted
197 # and returned identically.
198 ################################################################################
199
200 # Helper function to determine if a test specification is a Gtest test
201 #
202 # It is a Gtest test if it is either "gtest", or "gtest:" followed by an optional
203 # test filter string, and an optional "/<variant>" to select a specific JVM
204 # variant. If no variant is specified, all found variants are tested.
205 define ParseGtestTestSelection
206 $(if $(filter gtest%, $1), \
370 SetupRunGtestTest = $(NamedParamsMacroTemplate)
371 define SetupRunGtestTestBody
372 $1_TEST_RESULTS_DIR := $$(TEST_RESULTS_DIR)/$1
373 $1_TEST_SUPPORT_DIR := $$(TEST_SUPPORT_DIR)/$1
374 $1_EXITCODE := $$($1_TEST_RESULTS_DIR)/exitcode.txt
375
376 $1_VARIANT := $$(lastword $$(subst /, , $$($1_TEST)))
377 ifeq ($$(filter $$($1_VARIANT), $$(GTEST_VARIANTS)), )
378 $$(error Invalid gtest variant '$$($1_VARIANT)'. Valid variants: $$(GTEST_VARIANTS))
379 endif
380 $1_TEST_NAME := $$(strip $$(patsubst %/$$($1_VARIANT), %, \
381 $$(patsubst gtest:%, %, $$($1_TEST))))
382 ifneq ($$($1_TEST_NAME), all)
383 $1_GTEST_FILTER := --gtest_filter=$$($1_TEST_NAME)*
384 endif
385
386 ifneq ($$(GTEST_REPEAT), )
387 $1_GTEST_REPEAT :=--gtest_repeat=$$(GTEST_REPEAT)
388 endif
389
390 run-test-$1: $(TEST_PREREQS)
391 $$(call LogWarn)
392 $$(call LogWarn, Running test '$$($1_TEST)')
393 $$(call MakeDir, $$($1_TEST_RESULTS_DIR) $$($1_TEST_SUPPORT_DIR))
394 $$(call ExecuteWithLog, $$($1_TEST_SUPPORT_DIR)/gtest, \
395 $$(FIXPATH) $$(TEST_IMAGE_DIR)/hotspot/gtest/$$($1_VARIANT)/gtestLauncher \
396 -jdk $(JDK_IMAGE_DIR) $$($1_GTEST_FILTER) \
397 --gtest_output=xml:$$($1_TEST_RESULTS_DIR)/gtest.xml \
398 $$($1_GTEST_REPEAT) $$(GTEST_OPTIONS) $$(GTEST_VM_OPTIONS) \
399 $$($1_GTEST_JAVA_OPTIONS) \
400 > >($(TEE) $$($1_TEST_RESULTS_DIR)/gtest.txt) \
401 && $$(ECHO) $$$$? > $$($1_EXITCODE) \
402 || $$(ECHO) $$$$? > $$($1_EXITCODE) \
403 )
404
405 $1_RESULT_FILE := $$($1_TEST_RESULTS_DIR)/gtest.txt
406
407 parse-test-$1: run-test-$1
408 $$(call LogWarn, Finished running test '$$($1_TEST)')
409 $$(call LogWarn, Test report is stored in $$(strip \
410 $$(subst $$(TOPDIR)/, , $$($1_TEST_RESULTS_DIR))))
411 $$(if $$(wildcard $$($1_RESULT_FILE)), \
412 $$(eval $1_TOTAL := $$(shell $$(AWK) '/==========.* tests? from .* \
413 test cases? ran/ { print $$$$2 }' $$($1_RESULT_FILE))) \
414 $$(if $$($1_TOTAL), , $$(eval $1_TOTAL := 0)) \
415 $$(eval $1_PASSED := $$(shell $$(AWK) '/\[ PASSED \] .* tests?./ \
416 { print $$$$4 }' $$($1_RESULT_FILE))) \
417 $$(if $$($1_PASSED), , $$(eval $1_PASSED := 0)) \
418 $$(eval $1_FAILED := $$(shell $$(AWK) '/\[ FAILED \] .* tests?, \
419 listed below/ { print $$$$4 }' $$($1_RESULT_FILE))) \
450 $1_$2 := $$($2)
451 else
452 ifneq ($$($$($1_COMPONENT)_$2), )
453 $1_$2 := $$($$($1_COMPONENT)_$2)
454 else
455 ifneq ($3, )
456 $1_$2 := $3
457 endif
458 endif
459 endif
460 endef
461
462 SetupRunJtregTest = $(NamedParamsMacroTemplate)
463 define SetupRunJtregTestBody
464 $1_TEST_RESULTS_DIR := $$(TEST_RESULTS_DIR)/$1
465 $1_TEST_SUPPORT_DIR := $$(TEST_SUPPORT_DIR)/$1
466 $1_EXITCODE := $$($1_TEST_RESULTS_DIR)/exitcode.txt
467
468 $1_TEST_NAME := $$(strip $$(patsubst jtreg:%, %, $$($1_TEST)))
469
470 $1_TEST_ROOT := \
471 $$(strip $$(foreach root, $$(JTREG_TESTROOTS), \
472 $$(if $$(filter $$(root)%, $$(JTREG_TOPDIR)/$$($1_TEST_NAME)), $$(root)) \
473 ))
474 $1_COMPONENT := $$(lastword $$(subst /, $$(SPACE), $$($1_TEST_ROOT)))
475 # This will work only as long as just hotspot has the additional "jtreg" directory
476 ifeq ($$($1_COMPONENT), jtreg)
477 $1_COMPONENT := hotspot
478 endif
479
480 ifeq ($$(JT_HOME), )
481 $$(info Error: jtreg framework is not found.)
482 $$(info Please run configure using --with-jtreg.)
483 $$(error Cannot continue)
484 endif
485
486 # Unfortunately, we need different defaults for some JTREG values,
487 # depending on what component we're running.
488
489 # Convert JTREG_foo into $1_JTREG_foo with a suitable value.
490 $$(eval $$(call SetJtregValue,$1,JTREG_TEST_MODE,agentvm))
491 $$(eval $$(call SetJtregValue,$1,JTREG_ASSERT,true))
492 $$(eval $$(call SetJtregValue,$1,JTREG_MAX_MEM,512m))
493 $$(eval $$(call SetJtregValue,$1,JTREG_NATIVEPATH))
494 $$(eval $$(call SetJtregValue,$1,JTREG_BASIC_OPTIONS))
495 $$(eval $$(call SetJtregValue,$1,JTREG_PROBLEM_LIST))
496
497 # Only the problem list for the current test root should be used.
498 $1_JTREG_PROBLEM_LIST := $$(filter $$($1_TEST_ROOT)%, $$($1_JTREG_PROBLEM_LIST))
499
500 ifneq ($(TEST_JOBS), 0)
501 $$(eval $$(call SetJtregValue,$1,JTREG_JOBS,$$(TEST_JOBS)))
502 else
503 $$(eval $$(call SetJtregValue,$1,JTREG_JOBS,$$(JOBS)))
504 endif
505
506 # Make sure MaxRAMPercentage is high enough to not cause OOM or swapping since
507 # we may end up with a lot of JVM's
508 $1_JTREG_MAX_RAM_PERCENTAGE := $$(shell $$(EXPR) 25 / $$($1_JTREG_JOBS))
509
510 JTREG_TIMEOUT_FACTOR ?= 4
511 JTREG_VERBOSE ?= fail,error,summary
512 JTREG_RETAIN ?= fail,error
513
514 ifneq ($$($1_JTREG_MAX_MEM), 0)
515 $1_JTREG_BASIC_OPTIONS += -vmoption:-Xmx$$($1_JTREG_MAX_MEM)
516 $1_JTREG_LAUNCHER_OPTIONS += -Xmx$$($1_JTREG_MAX_MEM)
517 endif
518
519 $1_JTREG_BASIC_OPTIONS += -$$($1_JTREG_TEST_MODE) \
520 -verbose:$$(JTREG_VERBOSE) -retain:$$(JTREG_RETAIN) \
521 -concurrency:$$($1_JTREG_JOBS) -timeoutFactor:$$(JTREG_TIMEOUT_FACTOR) \
522 -vmoption:-XX:MaxRAMPercentage=$$($1_JTREG_MAX_RAM_PERCENTAGE)
523
524 $1_JTREG_BASIC_OPTIONS += -automatic -ignore:quiet
525
526 # Make it possible to specify the JIB_DATA_DIR for tests using the
527 # JIB Artifact resolver
528 $1_JTREG_BASIC_OPTIONS += -e:JIB_DATA_DIR
529 # Some tests needs to find a boot JDK using the JDK8_HOME variable.
530 $1_JTREG_BASIC_OPTIONS += -e:JDK8_HOME=$$(BOOT_JDK)
531 # If running on Windows, propagate the _NT_SYMBOL_PATH to enable
532 # symbol lookup in hserr files
533 ifeq ($$(OPENJDK_TARGET_OS), windows)
534 $1_JTREG_BASIC_OPTIONS += -e:_NT_SYMBOL_PATH
535 endif
536
537 $1_JTREG_BASIC_OPTIONS += \
538 $$(addprefix -javaoption:, $$(JTREG_JAVA_OPTIONS)) \
539 $$(addprefix -vmoption:, $$(JTREG_VM_OPTIONS)) \
540 #
541
542 ifeq ($$($1_JTREG_ASSERT), true)
543 $1_JTREG_BASIC_OPTIONS += -ea -esa
544 endif
545
546 ifneq ($$($1_JTREG_NATIVEPATH), )
547 $1_JTREG_BASIC_OPTIONS += -nativepath:$$($1_JTREG_NATIVEPATH)
548 endif
549
550 ifneq ($$($1_JTREG_PROBLEM_LIST), )
551 $1_JTREG_BASIC_OPTIONS += $$(addprefix -exclude:, $$($1_JTREG_PROBLEM_LIST))
552 endif
553
554 ifneq ($$(JTREG_EXTRA_PROBLEM_LISTS), )
555 # Accept both absolute paths as well as relative to the current test root.
556 $1_JTREG_BASIC_OPTIONS += $$(addprefix -exclude:, $$(wildcard \
557 $$(JTREG_EXTRA_PROBLEM_LISTS) \
558 $$(addprefix $$($1_TEST_ROOT)/, $$(JTREG_EXTRA_PROBLEM_LISTS)) \
559 ))
560 endif
561
562 ifneq ($$(JIB_HOME), )
563 $1_JTREG_BASIC_OPTIONS += -e:JIB_HOME=$$(JIB_HOME)
564 endif
565
566 $1_JTREG_BASIC_OPTIONS += -e:TEST_IMAGE_GRAAL_DIR=${TEST_IMAGE_DIR}/hotspot/jtreg/graal
567
568 ifneq ($$(JTREG_FAILURE_HANDLER_OPTIONS), )
569 $1_JTREG_LAUNCHER_OPTIONS += -Djava.library.path="$(JTREG_FAILURE_HANDLER_DIR)"
570 endif
571
572 ifneq ($$(JTREG_KEYWORDS), )
573 # The keywords string may contain problematic characters and may be quoted
574 # already when it arrives here. Remove any existing quotes and replace them
575 # with one set of single quotes.
576 $1_JTREG_KEYWORDS := \
577 $$(strip $$(subst $$(SQUOTE),,$$(subst $$(DQUOTE),,$$(JTREG_KEYWORDS))))
578 ifneq ($$($1_JTREG_KEYWORDS), )
579 $1_JTREG_BASIC_OPTIONS += -k:'$$($1_JTREG_KEYWORDS)'
580 endif
581 endif
582
583 clean-workdir-$1:
584 $$(RM) -r $$($1_TEST_SUPPORT_DIR)
585
586 run-test-$1: clean-workdir-$1 $(TEST_PREREQS)
587 $$(call LogWarn)
588 $$(call LogWarn, Running test '$$($1_TEST)')
589 $$(call MakeDir, $$($1_TEST_RESULTS_DIR) $$($1_TEST_SUPPORT_DIR))
590 $$(call ExecuteWithLog, $$($1_TEST_SUPPORT_DIR)/jtreg, \
591 $$(JAVA) $$($1_JTREG_LAUNCHER_OPTIONS) \
592 -Dprogram=jtreg -jar $$(JT_HOME)/lib/jtreg.jar \
593 $$($1_JTREG_BASIC_OPTIONS) \
594 -testjdk:$$(JDK_IMAGE_DIR) \
595 -dir:$$(JTREG_TOPDIR) \
596 -reportDir:$$($1_TEST_RESULTS_DIR) \
597 -workDir:$$($1_TEST_SUPPORT_DIR) \
598 $$(JTREG_OPTIONS) \
599 $$(JTREG_FAILURE_HANDLER_OPTIONS) \
600 $$($1_TEST_NAME) \
601 && $$(ECHO) $$$$? > $$($1_EXITCODE) \
602 || $$(ECHO) $$$$? > $$($1_EXITCODE) \
603 )
604
605 $1_RESULT_FILE := $$($1_TEST_RESULTS_DIR)/text/stats.txt
606
653 $1_TEST_NAME := $$($1_FULL_TEST_NAME)
654 $1_TEST_ARGS :=
655 endif
656
657 ifeq ($$($1_TEST_NAME), hotspot-internal)
658 $1_TEST_COMMAND_LINE := \
659 $$(JDK_IMAGE_DIR)/bin/java -XX:+ExecuteInternalVMTests \
660 -XX:+ShowMessageBoxOnError -version
661 else ifeq ($$($1_TEST_NAME), failure-handler)
662 $1_TEST_COMMAND_LINE := \
663 ($(CD) $(TOPDIR)/make/test && $(MAKE) $(MAKE_ARGS) -f \
664 BuildFailureHandler.gmk test)
665 else ifeq ($$($1_TEST_NAME), make)
666 $1_TEST_COMMAND_LINE := \
667 ($(CD) $(TOPDIR)/test/make && $(MAKE) $(MAKE_ARGS) -f \
668 TestMake.gmk $$($1_TEST_ARGS))
669 else
670 $$(error Invalid special test specification: $$($1_TEST_NAME))
671 endif
672
673 run-test-$1: $(TEST_PREREQS)
674 $$(call LogWarn)
675 $$(call LogWarn, Running test '$$($1_TEST)')
676 $$(call MakeDir, $$($1_TEST_RESULTS_DIR) $$($1_TEST_SUPPORT_DIR))
677 $$(call ExecuteWithLog, $$($1_TEST_SUPPORT_DIR)/test-execution, \
678 $$($1_TEST_COMMAND_LINE) \
679 > >($(TEE) $$($1_TEST_RESULTS_DIR)/test-output.txt) \
680 && $$(ECHO) $$$$? > $$($1_EXITCODE) \
681 || $$(ECHO) $$$$? > $$($1_EXITCODE) \
682 )
683
684 $1_RESULT_FILE := $$($1_TEST_RESULTS_DIR)/gtest.txt
685
686 # We can not parse the various "special" tests.
687 parse-test-$1: run-test-$1
688 $$(call LogWarn, Finished running test '$$($1_TEST)')
689 $$(call LogWarn, Test report is stored in $$(strip \
690 $$(subst $$(TOPDIR)/, , $$($1_TEST_RESULTS_DIR))))
691 $$(call LogWarn, Warning: Special test results are not properly parsed!)
692 $$(eval $1_PASSED := 0)
693 $$(eval $1_FAILED := 0)
|