1 #
2 # Makefile to run jtreg and other tests
3 #
4
5 # Product builds and langtools builds
6 #
7 # A full product build (or "control" build) creates a complete JDK image.
8 # To test a product build, set TESTJAVA to the path for the image.
9 #
10 # A langtools build just builds the langtools components of a JDK.
11 # To test a langtools build, set TESTJAVA to the path for a recent JDK
12 # build, and set TESTBOOTCLASSPATH to the compiled langtools classes --
13 # for example build/classes or dist/lib/classes.jar.
14
15 # JPRT
16 # JPRT may invoke this Makefile directly, as part of a langtools build,
17 # or indirectly, via FOREST/test/Makefile, as part of a control build.
18
19 # Get OS/ARCH specifics
20 OSNAME = $(shell uname -s)
21 ifeq ($(OSNAME), SunOS)
22 SLASH_JAVA = /java
23 PLATFORM = solaris
24 ARCH = $(shell uname -p)
25 ifeq ($(ARCH), i386)
26 ARCH=i586
27 endif
28 endif
29 ifeq ($(OSNAME), Linux)
30 SLASH_JAVA = /java
31 PLATFORM = linux
32 ARCH = $(shell uname -m)
33 ifeq ($(ARCH), i386)
34 ARCH=i586
35 endif
36 endif
37 ifeq ($(OSNAME), Darwin)
38 PLATFORM = bsd
39 ARCH = $(shell uname -m)
40 ifeq ($(ARCH), i386)
41 ARCH=i586
42 endif
43 endif
44 ifeq ($(OSNAME), Windows_NT)
45 # MKS
46 PLATFORM=windows
47 endif
48 ifeq ($(PLATFORM),)
49 PLATFORM = windows
50 CYGPATH = | cygpath -m -s -f -
51 endif
52
53 ifeq ($(PLATFORM), windows)
54 SLASH_JAVA = J:
55 ifeq ($(word 1, $(PROCESSOR_IDENTIFIER)),ia64)
56 ARCH=ia64
57 else
58 ifeq ($(word 1, $(PROCESSOR_IDENTIFIER)),AMD64)
59 ARCH=x64
60 else
61 ifeq ($(word 1, $(PROCESSOR_IDENTIFIER)),EM64T)
62 ARCH=x64
63 else
64 ARCH=i586
65 endif
66 endif
67 endif
68 EXE_SUFFIX=.exe
69 endif
70
71 # Root of this test area (important to use full paths in some places)
72 TEST_ROOT := $(shell pwd $(CYGPATH) )
73
74 # Default bundle of all test results (passed or not) (JPRT only)
75 ifdef JPRT_JOB_ID
76 JPRT_CLEAN = clean
77 JPRT_ARCHIVE_BUNDLE = $(TEST_ROOT)/JPRT_ARCHIVE_BUNDLE.zip
78 endif
79
80 ifeq ($(PLATFORM), windows)
81 SLASH_JAVA = J:
82 else
83 SLASH_JAVA = /java
84 endif
85
86 # Default JTREG to run
87 ifndef JTREG_HOME
88 ifdef JPRT_JTREG_HOME
89 JTREG_HOME = $(JPRT_JTREG_HOME)
90 else ifdef JT_HOME
91 JTREG_HOME = $(JT_HOME)
92 else
93 JTREG_HOME = $(SLASH_JAVA)/re/jtreg/4.2/promoted/latest/
94 endif
95 endif
96 JTREG = $(JTREG_HOME)/bin/jtreg
97 JTDIFF = $(JTREG_HOME)/bin/jtdiff
98
99 # Problematic tests to be excluded
100 PROBLEM_LISTS=ProblemList.txt
101
102 # Create exclude list for this platform and arch
103 ifdef NO_EXCLUDES
104 JTREG_EXCLUSIONS =
105 else
106 JTREG_EXCLUSIONS = $(PROBLEM_LISTS:%=-exclude:%)
107 endif
108
109 # Default JCK to run
110 ifndef JCK_HOME
111 ifdef JPRT_JCK_HOME
112 JCK_HOME = $(JPRT_JCK_HOME)
113 else
114 JCK_HOME = $(SLASH_JAVA)/re/jck/8/promoted/latest/binaries
115 endif
116 endif
117
118 # Default JDK for JTREG and JCK
119 #
120 # JT_JAVA is the version of java used to run jtreg/JCK.
121 #
122 ifndef JT_JAVA
123 ifdef JPRT_JAVA_HOME
124 JT_JAVA = $(JPRT_JAVA_HOME)
125 else
126 JT_JAVA = $(SLASH_JAVA)/re/jdk/1.9.0/archive/fcs/binaries/$(PLATFORM)-$(ARCH)
127 endif
128 endif
129
130 # Default JDK to test
131 ifdef JPRT_IMPORT_PRODUCT_HOME
132 TESTJAVA = $(JPRT_IMPORT_PRODUCT_HOME)
133 else
134 TESTJAVA = $(SLASH_JAVA)/re/jdk/1.9.0/promoted/latest/binaries/$(PLATFORM)-$(ARCH)
135 endif
136
137 # PRODUCT_HOME is a JPRT variable pointing to a directory containing the output from
138 # make/Makefile
139 # For langtools, this is a directory containing build and dist
140 # For a control build, this is build/$(PRODUCT)-$(ARCH)/XYZ-image
141 # (i.e, j2sdk-image or jdk-module-image)
142 ifdef PRODUCT_HOME
143 ifeq ($(shell [ -r $(PRODUCT_HOME)/dist/lib/classes.jar ]; echo $$?),0)
144 TESTBOOTCLASSPATH = $(PRODUCT_HOME)/dist/lib/classes.jar
145 endif
146 ifeq ($(shell [ -r $(PRODUCT_HOME)/bin/javac$(EXE_SUFFIX) ]; echo $$?),0)
147 TESTJAVA = $(PRODUCT_HOME)
148 endif
149 endif
150
151 ifdef TESTBOOTCLASSPATH
152 JTREG_OPTIONS += -Xbootclasspath/p:$(TESTBOOTCLASSPATH)
153 ### In the following, -refvmoptions is an undocumented option
154 ### The following does not work JCK 7 b30 2/6/2010. Awaiting b31.
155 JCK_OPTIONS += \
156 -vmoptions:-Xbootclasspath/p:$(TESTBOOTCLASSPATH) \
157 -refvmoptions:-Xbootclasspath/p:$(TESTBOOTCLASSPATH)
158 endif
159
160 # Set the max memory for jtreg target test JVMs
161 JTREG_TESTVM_MEMORY_OPTION = -vmoption:-Xmx768m
162 JTREG_OPTIONS += $(JTREG_TESTVM_MEMORY_OPTION)
163
164 # Retain all files for failing tests
165 JTREG_OPTIONS += -retain:fail,error
166
167 ifdef EXTRA_JTREG_OPTIONS
168 JTREG_OPTIONS += $(EXTRA_JTREG_OPTIONS)
169 endif
170
171 # Concurrency is the number of tests that can execute at once.
172 # On an otherwise empty machine, suggest setting to (#cpus + 2)
173 # If unset, the default is (#cpus)
174 ### RFE: determine and use #cpus
175 ifdef CONCURRENCY
176 JTREG_OPTIONS += -agentvm -concurrency:$(CONCURRENCY)
177 else
178 JTREG_OPTIONS += -agentvm
179 endif
180
181 ifdef JCK_CONCURRENCY
182 JCK_OPTIONS += -concurrency:$(JCK_CONCURRENCY)
183 endif
184
185 # JCK is executed using "Multi-JVM Group Mode", which is a hybrid
186 # of otherVM and sameVM modes. New JVMs are created and reused for
187 # a number of tests, then eventually discarded and a new one started.
188 # This amortizes the JVM startup time. The "group size" defines
189 # how many tests are run in a JVM before it is replaced.
190 # If unset, the default is 100.
191 JCK_GROUP_SIZE = 1000
192 ifdef JCK_GROUP_SIZE
193 JCK_COMPILER_OPTIONS += \
194 -jtoptions:-Ejck.env.compiler.testCompile.groupMode.groupSize=$(JCK_GROUP_SIZE) \
195 -jtoptions:-Ejck.env.compiler.compRefExecute.groupMode.groupSize=$(JCK_GROUP_SIZE)
196 ### The following is not supported. Awaiting RFE 6924287
197 ### 6924287: Jck4Jdk: Allow to configure test group size for group mode via simple command line option
198 ### JCK_RUNTIME_OPTIONS += \
199 ### -jtoptions:-Ejck.env.runtime.testCompile.groupMode.groupSize=$(JCK_GROUP_SIZE)
200 endif
201
202 # Timeouts -- by default, increase test timeouts when running on JPRT
203 ifdef JPRT_JOB_ID
204 ifndef JTREG_TIMEOUT_FACTOR
205 JTREG_TIMEOUT_FACTOR = 3
206 endif
207 endif
208 ifdef JTREG_TIMEOUT_FACTOR
209 JTREG_OPTIONS += -timeoutFactor:$(JTREG_TIMEOUT_FACTOR)
210 endif
211
212 ifdef JCK_TIMEOUT_FACTOR
213 JCK_OPTIONS += -timeout:$(JCK_TIMEOUT_FACTOR)
214 endif
215
216 # Default verbosity setting for jtreg
217 JTREG_VERBOSE ?= fail,error,time
218
219 # Default verbosity setting for jck
220 JCK_VERBOSE ?= non-pass
221
222 # Assertions: some tests show failures when assertions are enabled.
223 # Since javac is typically loaded via the bootclassloader (either via TESTJAVA
224 # or TESTBOOTCLASSPATH), you may need -esa to enable assertions in javac.
225 JTREG_OPTIONS += $(ASSERTION_OPTIONS)
226 JCK_OPTIONS += $(ASSERTION_OPTIONS:%=-vmoptions:%)
227
228 # Include shared options
229 JCK_COMPILER_OPTIONS += $(JCK_OPTIONS)
230 JCK_RUNTIME_OPTIONS += $(JCK_OPTIONS)
231
232 # Exit codes:
233 # jtreg, jck: 0: OK, 1: tests failed, 2: tests error; 3+: SERIOUS
234 FATAL_JTREG_EXIT = 3
235 FATAL_JCK_EXIT = 3
236 # jtdiff: 0: OK, 1: differences found; 2+: SERIOUS
237 FATAL_JTDIFF_EXIT = 2
238 #
239 # Exit -- used for final "normal" exit from "make". Redefine to "true" to avoid
240 # having make exit with non-zero return code.
241 EXIT = exit
242 # Function to exit shell if exit code of preceding command is greater than or equal
243 # to a given level. Redefine function or preceding FATAL_*_EXIT codes as needed.
244 EXIT_IF_FATAL = status=$$?; if [ $$status -ge $(1) ]; then exit $$status ; fi
245
246 # Root of all test results
247 TEST_OUTPUT_DIR ?= $(TEST_ROOT)/../build/$(PLATFORM)-$(ARCH)/test/langtools
248 ABS_TEST_OUTPUT_DIR := \
249 $(shell mkdir -p $(TEST_OUTPUT_DIR); \
250 cd $(TEST_OUTPUT_DIR); \
251 pwd $(CYGPATH))
252 # Subdirectories for different test runs
253 JTREG_OUTPUT_DIR = $(ABS_TEST_OUTPUT_DIR)/jtreg
254 JCK_COMPILER_OUTPUT_DIR = $(ABS_TEST_OUTPUT_DIR)/jck-compiler
255 JCK_RUNTIME_OUTPUT_DIR = $(ABS_TEST_OUTPUT_DIR)/jck-runtime-Xcompile
256
257 # Is the test JVM 32-bit?
258 DATA_MODEL := \
259 $(shell $(TESTJAVA)/bin/java -XshowSettings:properties -version 2>&1 | \
260 grep 'sun\.arch\.data\.model' | \
261 awk '{print $$3}')
262 ifeq ($(DATA_MODEL), 32)
263 # Set the GC options for test vms having a smaller address space
264 JTREG_GC_OPTION = -vmoption:-XX:+UseSerialGC
265 JTREG_OPTIONS += $(JTREG_GC_OPTION)
266 endif
267
268 # Default make rule -- warning, may take a while
269 all: $(JPRT_CLEAN) jtreg-tests jck-compiler-tests jck-runtime-tests $(JPRT_ARCHIVE_BUNDLE) all-summary
270 @echo "Testing completed successfully"
271
272 jtreg apt javac javadoc javah javap jdeps: $(JPRT_CLEAN) jtreg-tests $(JPRT_ARCHIVE_BUNDLE) jtreg-summary
273 @echo "Testing completed successfully"
274
275 jck-compiler: $(JPRT_CLEAN) jck-compiler-tests $(JPRT_ARCHIVE_BUNDLE) jck-compiler-summary
276 @echo "Testing completed successfully"
277
278 jck-runtime: $(JPRT_CLEAN) jck-runtime-tests $(JPRT_ARCHIVE_BUNDLE) jck-runtime-summary
279 @echo "Testing completed successfully"
280
281 # a way to select tests from outside
282 # works for targets 'jtreg', 'jck-compiler', and 'jck-runtime'
283 ifdef TEST_SELECTION
284 JTREG_TESTDIRS = $(TEST_SELECTION)
285 JCK_COMPILER_TESTDIRS = $(TEST_SELECTION)
286 JCK_RUNTIME_TESTDIRS = $(TEST_SELECTION)
287 endif
288
289 # for use with JPRT -testrule
290 all: JTREG_TESTDIRS = .
291 jtreg: JTREG_TESTDIRS ?= .
292 apt: JTREG_TESTDIRS = tools/apt
293 javac: JTREG_TESTDIRS = tools/javac
294 javadoc: JTREG_TESTDIRS = tools/javadoc com/sun/javadoc
295 javah: JTREG_TESTDIRS = tools/javah
296 javap: JTREG_TESTDIRS = tools/javap
297 jdeps: JTREG_TESTDIRS = tools/jdeps
298
299
300 # Run jtreg tests
301 #
302 # JTREG_HOME
303 # Installed location of jtreg
304 # JT_JAVA
305 # Version of java used to run jtreg. Should normally be the same as TESTJAVA
306 # TESTJAVA
307 # Version of java to be tested.
308 # JTREG_VERBOSE
309 # Verbosity setting for jtreg
310 # JTREG_OPTIONS
311 # Additional options for jtreg
312 # JTREG_TESTDIRS
313 # Directories of tests to be run
314 # JTREG_OUTPUT_DIR
315 # Where to write the results
316 # JTREG_REFERENCE
317 # (Optional) reference results (e.g. work, report or summary.txt)
318 #
319 jtreg_tests: jtreg-tests
320 jtreg-tests: check-jtreg FRC
321 @rm -f -r $(JTREG_OUTPUT_DIR)/JTwork $(JTREG_OUTPUT_DIR)/JTreport \
322 $(JTREG_OUTPUT_DIR)/diff.html $(JTREG_OUTPUT_DIR)/status.txt
323 @mkdir -p $(JTREG_OUTPUT_DIR)
324 ( JT_JAVA=$(JT_JAVA) $(JTREG) \
325 -a -ignore:quiet $(if $(JTREG_VERBOSE),-v:$(JTREG_VERBOSE)) \
326 -r:$(JTREG_OUTPUT_DIR)/JTreport \
327 -w:$(JTREG_OUTPUT_DIR)/JTwork \
328 -jdk:$(TESTJAVA) \
329 $(JAVA_ARGS:%=-vmoption:%) \
330 $(JTREG_EXCLUSIONS) \
331 $(JTREG_OPTIONS) \
332 $(JTREG_TESTDIRS) \
333 || ( $(call EXIT_IF_FATAL,$(FATAL_JTREG_EXIT)) ; \
334 echo $$status > $(JTREG_OUTPUT_DIR)/status.txt \
335 ) \
336 ) 2>&1 | tee $(JTREG_OUTPUT_DIR)/output.txt
337 ifdef JTREG_REFERENCE
338 JT_JAVA=$(JT_JAVA) $(JTDIFF) -o $(JTREG_OUTPUT_DIR)/diff.html \
339 $(JTREG_REFERENCE) $(JTREG_OUTPUT_DIR)/JTreport \
340 || ( $(call EXIT_IF_FATAL,$(FATAL_JTDIFF_EXIT)) )
341 endif
342
343 jtreg-summary: FRC
344 @if [ -r $(JTREG_OUTPUT_DIR)/status.txt ]; then \
345 echo ; echo "Summary of jtreg test failures" ; \
346 cat $(JTREG_OUTPUT_DIR)/JTreport/text/summary.txt | \
347 grep -v 'Not run' | grep -v 'Passed' ; \
348 echo ; \
349 $(EXIT) `cat $(JTREG_OUTPUT_DIR)/status.txt` ; \
350 fi
351
352 # Check to make sure these directories exist
353 check-jtreg: $(PRODUCT_HOME) $(JTREG)
354
355
356 # Run JCK-compiler tests
357 #
358 # JCK_HOME
359 # Installed location of JCK: should include JCK-compiler, and JCK-extras
360 # Default is JCK 8.
361 # JT_JAVA
362 # Version of java used to run JCK. Should normally be the same as TESTJAVA
363 # Default is JDK 7
364 # TESTJAVA
365 # Version of java to be tested.
366 # JCK_VERBOSE
367 # Verbosity setting for jtjck
368 # JCK_COMPILER_OPTIONS
369 # Additional options for JCK-compiler
370 # JCK_COMPILER_TESTDIRS
371 # Directories of tests to be run
372 # JCK_COMPILER_OUTPUT_DIR
373 # Where to write the results
374 # JCK_COMPILER_REFERENCE
375 # (Optional) reference results (e.g. work, report or summary.txt)
376 #
377 jck-compiler-tests: check-jck FRC
378 @rm -f -r $(JCK_COMPILER_OUTPUT_DIR)/work $(JCK_COMPILER_OUTPUT_DIR)/report \
379 $(JCK_COMPILER_OUTPUT_DIR)/diff.html $(JCK_COMPILER_OUTPUT_DIR)/status.txt
380 @mkdir -p $(JCK_COMPILER_OUTPUT_DIR)
381 $(JT_JAVA)/bin/java -Xmx1024m \
382 -jar $(JCK_HOME)/JCK-compiler-9/lib/jtjck.jar \
383 $(if $(JCK_VERBOSE),$(if $(filter $(JCK_VERBOSE),summary),-v,-v:$(JCK_VERBOSE))) \
384 -r:$(JCK_COMPILER_OUTPUT_DIR)/report \
385 -w:$(JCK_COMPILER_OUTPUT_DIR)/work \
386 -jdk:$(TESTJAVA) \
387 $(JCK_COMPILER_OPTIONS) \
388 $(JCK_COMPILER_TESTDIRS) \
389 || ( $(call EXIT_IF_FATAL,$(FATAL_JCK_EXIT)) ; \
390 echo $$status > $(JCK_COMPILER_OUTPUT_DIR)/status.txt \
391 )
392 ifdef JCK_COMPILER_REFERENCE
393 JT_JAVA=$(JT_JAVA) $(JTDIFF) -o $(JCK_COMPILER_OUTPUT_DIR)/diff.html \
394 $(JCK_COMPILER_REFERENCE) $(JCK_COMPILER_OUTPUT_DIR)/report \
395 || ( $(call EXIT_IF_FATAL,$(FATAL_JTDIFF_EXIT)) )
396 endif
397
398 jck-compiler-summary: FRC
399 @if [ -r $(JCK_COMPILER_OUTPUT_DIR)/status.txt ]; then \
400 echo ; echo "Summary of JCK-compiler test failures" ; \
401 cat $(JCK_COMPILER_OUTPUT_DIR)/report/text/summary.txt | \
402 grep -v 'Not run' | grep -v 'Passed' ; \
403 echo ; \
404 $(EXIT) `cat $(JCK_COMPILER_OUTPUT_DIR)/status.txt` ; \
405 fi
406
407 # Run JCK-runtime tests in -Xcompile mode
408 # This is a special mode to test javac by compiling the tests in the JCK-runtime test suite
409 # Normal JCK-runtime invocation belongs in the jdk/ repository.
410 #
411 # JCK_HOME
412 # Installed location of JCK: should include JCK-compiler, JCK-runtime and JCK-extras
413 # JT_JAVA
414 # Version of java used to run JCK. Should normally be the same as TESTJAVA
415 # TESTJAVA
416 # Version of java to be tested.
417 # JCK_VERBOSE
418 # Verbosity setting for jtjck
419 # JCK_RUNTIME_OPTIONS
420 # Additional options for JCK-runtime
421 # JCK_RUNTIME_TESTDIRS
422 # Directories of tests to be run
423 # JCK_RUNTIME_OUTPUT_DIR
424 # Where to write the results
425 # JCK_RUNTIME_REFERENCE
426 # (Optional) reference results (e.g. work, report or summary.txt)
427 #
428 jck-runtime-tests: check-jck FRC
429 @rm -f -r $(JCK_RUNTIME_OUTPUT_DIR)/work $(JCK_RUNTIME_OUTPUT_DIR)/report \
430 $(JCK_RUNTIME_OUTPUT_DIR)/diff.html $(JCK_RUNTIME_OUTPUT_DIR)/status.txt
431 @mkdir -p $(JCK_RUNTIME_OUTPUT_DIR)
432 $(JT_JAVA)/bin/java -Xmx1024m \
433 -jar $(JCK_HOME)/JCK-runtime-9/lib/jtjck.jar \
434 $(if $(JCK_VERBOSE),$(if $(filter $(JCK_VERBOSE),summary),-v,-v:$(JCK_VERBOSE))) \
435 -r:$(JCK_RUNTIME_OUTPUT_DIR)/report \
436 -w:$(JCK_RUNTIME_OUTPUT_DIR)/work \
437 -jdk:$(TESTJAVA) \
438 -Xcompile \
439 $(JCK_RUNTIME_OPTIONS) \
440 $(JCK_RUNTIME_TESTDIRS) \
441 || ( $(call EXIT_IF_FATAL,$(FATAL_JCK_EXIT)) ; \
442 echo $$status > $(JCK_RUNTIME_OUTPUT_DIR)/status.txt \
443 )
444 ifdef JCK_RUNTIME_REFERENCE
445 JT_JAVA=$(JT_JAVA) $(JTDIFF) -o $(JCK_RUNTIME_OUTPUT_DIR)/diff.html \
446 $(JCK_RUNTIME_REFERENCE) $(JCK_RUNTIME_OUTPUT_DIR)/report \
447 || ( $(call EXIT_IF_FATAL,$(FATAL_JTDIFF_EXIT)) )
448 endif
449
450 jck-runtime-summary: FRC
451 @if [ -r $(JCK_RUNTIME_OUTPUT_DIR)/status.txt ]; then \
452 echo ; echo "Summary of JCK-runtime test failures" ; \
453 cat $(JCK_RUNTIME_OUTPUT_DIR)/report/text/summary.txt | \
454 grep -v 'Not run' | grep -v 'Passed' ; \
455 echo ; \
456 $(EXIT) `cat $(JCK_RUNTIME_OUTPUT_DIR)/status.txt` ; \
457 fi
458
459 # Check to make sure these directories exist
460 check-jck:
461 @if [ ! -d '$(JCK_HOME)' ]; then \
462 echo "JCK_HOME $(JCK_HOME) missing" ; \
463 $(EXIT) 1 ; \
464 fi
465 @if [ ! -d '$(PRODUCT_HOME)' ]; then \
466 echo "PRODUCT_HOME $(PRODUCT_HOME) missing" ; \
467 $(EXIT) 1 ; \
468 fi
469
470 all-summary: FRC
471 @if [ -n "`find $(TEST_OUTPUT_DIR) -name status.txt`" ]; then
472 echo ; echo "Summary of test failures" ; \
473 cat `find $(TEST_OUTPUT_DIR) -name summary.txt` | \
474 grep -v 'Not run' | grep -v 'Passed' ; \
475 echo ; \
476 $(EXIT) 1
477 fi
478
479 # Bundle up the results
480 $(JPRT_ARCHIVE_BUNDLE): FRC
481 @rm -f $@
482 @mkdir -p $(@D)
483 ( cd $(TEST_OUTPUT_DIR) && zip -q -r $@ . )
484
485 # Cleanup
486 clean:
487 rm -f $(JPRT_ARCHIVE_BUNDLE)
488
489 # Used to force a target rules to run
490 FRC:
491
492 # Phony targets (e.g. these are not filenames)
493 .PHONY: all clean \
494 jtreg javac javadoc javah javap jdeps jtreg-tests jtreg-summary check-jtreg \
495 jck-compiler jck-compiler-tests jck-compiler-summary \
496 jck-runtime jck-runtime-tests jck-runtime-summary check-jck
497
498 # No use of suffix rules
499 .SUFFIXES:
500