1 # 2 # Makefile to run jtreg and other tests 3 # 4 5 # Product builds and langtools builds 6 # 7 # A full product build (or "control" build) creates a complete JDK image. 8 # To test a product build, set TESTJAVA to the path for the image. 9 # 10 # A langtools build just builds the langtools components of a JDK. 11 # To test a langtools build, set TESTJAVA to the path for a recent JDK 12 # build, and set TESTBOOTCLASSPATH to the compiled langtools classes -- 13 # for example build/classes or dist/lib/classes.jar. 14 15 # JPRT 16 # JPRT may invoke this Makefile directly, as part of a langtools build, 17 # or indirectly, via FOREST/test/Makefile, as part of a control build. 18 19 # Get OS/ARCH specifics 20 OSNAME = $(shell uname -s) 21 ifeq ($(OSNAME), SunOS) 22 SLASH_JAVA = /java 23 PLATFORM = solaris 24 ARCH = $(shell uname -p) 25 ifeq ($(ARCH), i386) 26 ARCH=i586 27 endif 28 endif 29 ifeq ($(OSNAME), Linux) 30 SLASH_JAVA = /java 31 PLATFORM = linux 32 ARCH = $(shell uname -m) 33 ifeq ($(ARCH), i386) 34 ARCH=i586 35 endif 36 endif 37 ifeq ($(OSNAME), Darwin) 38 PLATFORM = bsd 39 ARCH = $(shell uname -m) 40 ifeq ($(ARCH), i386) 41 ARCH=i586 42 endif 43 endif 44 ifeq ($(OSNAME), Windows_NT) 45 # MKS 46 PLATFORM=windows 47 endif 48 ifeq ($(PLATFORM),) 49 PLATFORM = windows 50 CYGPATH = | cygpath -m -s -f - 51 endif 52 53 ifeq ($(PLATFORM), windows) 54 SLASH_JAVA = J: 55 ifeq ($(word 1, $(PROCESSOR_IDENTIFIER)),ia64) 56 ARCH=ia64 57 else 58 ifeq ($(word 1, $(PROCESSOR_IDENTIFIER)),AMD64) 59 ARCH=x64 60 else 61 ifeq ($(word 1, $(PROCESSOR_IDENTIFIER)),EM64T) 62 ARCH=x64 63 else 64 ARCH=i586 65 endif 66 endif 67 endif 68 EXE_SUFFIX=.exe 69 endif 70 71 # Root of this test area (important to use full paths in some places) 72 TEST_ROOT := $(shell pwd $(CYGPATH) ) 73 74 # Default bundle of all test results (passed or not) (JPRT only) 75 ifdef JPRT_JOB_ID 76 JPRT_CLEAN = clean 77 JPRT_ARCHIVE_BUNDLE = $(TEST_ROOT)/JPRT_ARCHIVE_BUNDLE.zip 78 endif 79 80 ifeq ($(PLATFORM), windows) 81 SLASH_JAVA = J: 82 else 83 SLASH_JAVA = /java 84 endif 85 86 # Default JTREG to run 87 ifndef JTREG_HOME 88 ifdef JPRT_JTREG_HOME 89 JTREG_HOME = $(JPRT_JTREG_HOME) 90 else ifdef JT_HOME 91 JTREG_HOME = $(JT_HOME) 92 else 93 JTREG_HOME = $(SLASH_JAVA)/re/jtreg/4.2/promoted/latest/ 94 endif 95 endif 96 JTREG = $(JTREG_HOME)/bin/jtreg 97 JTDIFF = $(JTREG_HOME)/bin/jtdiff 98 99 # Problematic tests to be excluded 100 PROBLEM_LISTS=ProblemList.txt 101 102 # Create exclude list for this platform and arch 103 ifdef NO_EXCLUDES 104 JTREG_EXCLUSIONS = 105 else 106 JTREG_EXCLUSIONS = $(PROBLEM_LISTS:%=-exclude:%) 107 endif 108 109 # Default JCK to run 110 ifndef JCK_HOME 111 ifdef JPRT_JCK_HOME 112 JCK_HOME = $(JPRT_JCK_HOME) 113 else 114 JCK_HOME = $(SLASH_JAVA)/re/jck/8/promoted/latest/binaries 115 endif 116 endif 117 118 # Default JDK for JTREG and JCK 119 # 120 # JT_JAVA is the version of java used to run jtreg/JCK. 121 # 122 ifndef JT_JAVA 123 ifdef JPRT_JAVA_HOME 124 JT_JAVA = $(JPRT_JAVA_HOME) 125 else 126 JT_JAVA = $(SLASH_JAVA)/re/jdk/1.9.0/archive/fcs/binaries/$(PLATFORM)-$(ARCH) 127 endif 128 endif 129 130 # Default JDK to test 131 ifdef JPRT_IMPORT_PRODUCT_HOME 132 TESTJAVA = $(JPRT_IMPORT_PRODUCT_HOME) 133 else 134 TESTJAVA = $(SLASH_JAVA)/re/jdk/1.9.0/promoted/latest/binaries/$(PLATFORM)-$(ARCH) 135 endif 136 137 # PRODUCT_HOME is a JPRT variable pointing to a directory containing the output from 138 # make/Makefile 139 # For langtools, this is a directory containing build and dist 140 # For a control build, this is build/$(PRODUCT)-$(ARCH)/XYZ-image 141 # (i.e, j2sdk-image or jdk-module-image) 142 ifdef PRODUCT_HOME 143 ifeq ($(shell [ -r $(PRODUCT_HOME)/dist/lib/classes.jar ]; echo $$?),0) 144 TESTBOOTCLASSPATH = $(PRODUCT_HOME)/dist/lib/classes.jar 145 endif 146 ifeq ($(shell [ -r $(PRODUCT_HOME)/bin/javac$(EXE_SUFFIX) ]; echo $$?),0) 147 TESTJAVA = $(PRODUCT_HOME) 148 endif 149 endif 150 151 ifdef TESTBOOTCLASSPATH 152 JTREG_OPTIONS += -Xbootclasspath/p:$(TESTBOOTCLASSPATH) 153 ### In the following, -refvmoptions is an undocumented option 154 ### The following does not work JCK 7 b30 2/6/2010. Awaiting b31. 155 JCK_OPTIONS += \ 156 -vmoptions:-Xbootclasspath/p:$(TESTBOOTCLASSPATH) \ 157 -refvmoptions:-Xbootclasspath/p:$(TESTBOOTCLASSPATH) 158 endif 159 160 ifeq ($(ARCH_DATA_MODEL),32) 161 # Set the GC options for test vms 162 JTREG_GC_OPTION = -vmoption:-XX:+UseSerialGC 163 JTREG_OPTIONS += $(JTREG_GC_OPTION) 164 endif 165 # Set the max memory for jtreg target test JVMs 166 JTREG_TESTVM_MEMORY_OPTION = -vmoption:-Xmx768m 167 JTREG_OPTIONS += $(JTREG_TESTVM_MEMORY_OPTION) 168 169 ifdef EXTRA_JTREG_OPTIONS 170 JTREG_OPTIONS += $(EXTRA_JTREG_OPTIONS) 171 endif 172 173 # Concurrency is the number of tests that can execute at once. 174 # On an otherwise empty machine, suggest setting to (#cpus + 2) 175 # If unset, the default is (#cpus) 176 ### RFE: determine and use #cpus 177 ifdef CONCURRENCY 178 JTREG_OPTIONS += -agentvm -concurrency:$(CONCURRENCY) 179 else 180 JTREG_OPTIONS += -agentvm 181 endif 182 183 ifdef JCK_CONCURRENCY 184 JCK_OPTIONS += -concurrency:$(JCK_CONCURRENCY) 185 endif 186 187 # JCK is executed using "Multi-JVM Group Mode", which is a hybrid 188 # of otherVM and sameVM modes. New JVMs are created and reused for 189 # a number of tests, then eventually discarded and a new one started. 190 # This amortizes the JVM startup time. The "group size" defines 191 # how many tests are run in a JVM before it is replaced. 192 # If unset, the default is 100. 193 JCK_GROUP_SIZE = 1000 194 ifdef JCK_GROUP_SIZE 195 JCK_COMPILER_OPTIONS += \ 196 -jtoptions:-Ejck.env.compiler.testCompile.groupMode.groupSize=$(JCK_GROUP_SIZE) \ 197 -jtoptions:-Ejck.env.compiler.compRefExecute.groupMode.groupSize=$(JCK_GROUP_SIZE) 198 ### The following is not supported. Awaiting RFE 6924287 199 ### 6924287: Jck4Jdk: Allow to configure test group size for group mode via simple command line option 200 ### JCK_RUNTIME_OPTIONS += \ 201 ### -jtoptions:-Ejck.env.runtime.testCompile.groupMode.groupSize=$(JCK_GROUP_SIZE) 202 endif 203 204 # Timeouts -- by default, increase test timeouts when running on JPRT 205 ifdef JPRT_JOB_ID 206 ifndef JTREG_TIMEOUT_FACTOR 207 JTREG_TIMEOUT_FACTOR = 3 208 endif 209 endif 210 ifdef JTREG_TIMEOUT_FACTOR 211 JTREG_OPTIONS += -timeoutFactor:$(JTREG_TIMEOUT_FACTOR) 212 endif 213 214 ifdef JCK_TIMEOUT_FACTOR 215 JCK_OPTIONS += -timeout:$(JCK_TIMEOUT_FACTOR) 216 endif 217 218 # Default verbosity setting for jtreg 219 JTREG_VERBOSE ?= fail,error,nopass 220 221 # Default verbosity setting for jck 222 JCK_VERBOSE ?= non-pass 223 224 # Assertions: some tests show failures when assertions are enabled. 225 # Since javac is typically loaded via the bootclassloader (either via TESTJAVA 226 # or TESTBOOTCLASSPATH), you may need -esa to enable assertions in javac. 227 JTREG_OPTIONS += $(ASSERTION_OPTIONS) 228 JCK_OPTIONS += $(ASSERTION_OPTIONS:%=-vmoptions:%) 229 230 # Include shared options 231 JCK_COMPILER_OPTIONS += $(JCK_OPTIONS) 232 JCK_RUNTIME_OPTIONS += $(JCK_OPTIONS) 233 234 # Exit codes: 235 # jtreg, jck: 0: OK, 1: tests failed, 2: tests error; 3+: SERIOUS 236 FATAL_JTREG_EXIT = 3 237 FATAL_JCK_EXIT = 3 238 # jtdiff: 0: OK, 1: differences found; 2+: SERIOUS 239 FATAL_JTDIFF_EXIT = 2 240 # 241 # Exit -- used for final "normal" exit from "make". Redefine to "true" to avoid 242 # having make exit with non-zero return code. 243 EXIT = exit 244 # Function to exit shell if exit code of preceding command is greater than or equal 245 # to a given level. Redefine function or preceding FATAL_*_EXIT codes as needed. 246 EXIT_IF_FATAL = status=$$?; if [ $$status -ge $(1) ]; then exit $$status ; fi 247 248 # Root of all test results 249 TEST_OUTPUT_DIR = $(TEST_ROOT)/../build/$(PLATFORM)-$(ARCH)/test/langtools 250 ABS_TEST_OUTPUT_DIR := \ 251 $(shell mkdir -p $(TEST_OUTPUT_DIR); \ 252 cd $(TEST_OUTPUT_DIR); \ 253 pwd $(CYGPATH)) 254 # Subdirectories for different test runs 255 JTREG_OUTPUT_DIR = $(ABS_TEST_OUTPUT_DIR)/jtreg 256 JCK_COMPILER_OUTPUT_DIR = $(ABS_TEST_OUTPUT_DIR)/jck-compiler 257 JCK_RUNTIME_OUTPUT_DIR = $(ABS_TEST_OUTPUT_DIR)/jck-runtime-Xcompile 258 259 # Default make rule -- warning, may take a while 260 all: $(JPRT_CLEAN) jtreg-tests jck-compiler-tests jck-runtime-tests $(JPRT_ARCHIVE_BUNDLE) all-summary 261 @echo "Testing completed successfully" 262 263 jtreg apt javac javadoc javah javap jdeps: $(JPRT_CLEAN) jtreg-tests $(JPRT_ARCHIVE_BUNDLE) jtreg-summary 264 @echo "Testing completed successfully" 265 266 jck-compiler: $(JPRT_CLEAN) jck-compiler-tests $(JPRT_ARCHIVE_BUNDLE) jck-compiler-summary 267 @echo "Testing completed successfully" 268 269 jck-runtime: $(JPRT_CLEAN) jck-runtime-tests $(JPRT_ARCHIVE_BUNDLE) jck-runtime-summary 270 @echo "Testing completed successfully" 271 272 # a way to select tests from outside 273 # works for targets 'jtreg', 'jck-compiler', and 'jck-runtime' 274 ifdef TEST_SELECTION 275 JTREG_TESTDIRS = $(TEST_SELECTION) 276 JCK_COMPILER_TESTDIRS = $(TEST_SELECTION) 277 JCK_RUNTIME_TESTDIRS = $(TEST_SELECTION) 278 endif 279 280 # for use with JPRT -testrule 281 all: JTREG_TESTDIRS = . 282 jtreg: JTREG_TESTDIRS ?= . 283 apt: JTREG_TESTDIRS = tools/apt 284 javac: JTREG_TESTDIRS = tools/javac 285 javadoc: JTREG_TESTDIRS = tools/javadoc com/sun/javadoc 286 javah: JTREG_TESTDIRS = tools/javah 287 javap: JTREG_TESTDIRS = tools/javap 288 jdeps: JTREG_TESTDIRS = tools/jdeps 289 290 291 # Run jtreg tests 292 # 293 # JTREG_HOME 294 # Installed location of jtreg 295 # JT_JAVA 296 # Version of java used to run jtreg. Should normally be the same as TESTJAVA 297 # TESTJAVA 298 # Version of java to be tested. 299 # JTREG_VERBOSE 300 # Verbosity setting for jtreg 301 # JTREG_OPTIONS 302 # Additional options for jtreg 303 # JTREG_TESTDIRS 304 # Directories of tests to be run 305 # JTREG_OUTPUT_DIR 306 # Where to write the results 307 # JTREG_REFERENCE 308 # (Optional) reference results (e.g. work, report or summary.txt) 309 # 310 jtreg_tests: jtreg-tests 311 jtreg-tests: check-jtreg FRC 312 @rm -f -r $(JTREG_OUTPUT_DIR)/JTwork $(JTREG_OUTPUT_DIR)/JTreport \ 313 $(JTREG_OUTPUT_DIR)/diff.html $(JTREG_OUTPUT_DIR)/status.txt 314 @mkdir -p $(JTREG_OUTPUT_DIR) 315 JT_JAVA=$(JT_JAVA) $(JTREG) \ 316 -a -ignore:quiet $(if $(JTREG_VERBOSE),-v:$(JTREG_VERBOSE)) \ 317 -r:$(JTREG_OUTPUT_DIR)/JTreport \ 318 -w:$(JTREG_OUTPUT_DIR)/JTwork \ 319 -jdk:$(TESTJAVA) \ 320 $(JAVA_ARGS:%=-vmoption:%) \ 321 $(JTREG_EXCLUSIONS) \ 322 $(JTREG_OPTIONS) \ 323 $(JTREG_TESTDIRS) \ 324 || ( $(call EXIT_IF_FATAL,$(FATAL_JTREG_EXIT)) ; \ 325 echo $$status > $(JTREG_OUTPUT_DIR)/status.txt \ 326 ) 327 ifdef JTREG_REFERENCE 328 JT_JAVA=$(JT_JAVA) $(JTDIFF) -o $(JTREG_OUTPUT_DIR)/diff.html \ 329 $(JTREG_REFERENCE) $(JTREG_OUTPUT_DIR)/JTreport \ 330 || ( $(call EXIT_IF_FATAL,$(FATAL_JTDIFF_EXIT)) ) 331 endif 332 333 jtreg-summary: FRC 334 @if [ -r $(JTREG_OUTPUT_DIR)/status.txt ]; then \ 335 echo ; echo "Summary of jtreg test failures" ; \ 336 cat $(JTREG_OUTPUT_DIR)/JTreport/text/summary.txt | \ 337 grep -v 'Not run' | grep -v 'Passed' ; \ 338 echo ; \ 339 $(EXIT) `cat $(JTREG_OUTPUT_DIR)/status.txt` ; \ 340 fi 341 342 # Check to make sure these directories exist 343 check-jtreg: $(PRODUCT_HOME) $(JTREG) 344 345 346 # Run JCK-compiler tests 347 # 348 # JCK_HOME 349 # Installed location of JCK: should include JCK-compiler, and JCK-extras 350 # Default is JCK 8. 351 # JT_JAVA 352 # Version of java used to run JCK. Should normally be the same as TESTJAVA 353 # Default is JDK 7 354 # TESTJAVA 355 # Version of java to be tested. 356 # JCK_VERBOSE 357 # Verbosity setting for jtjck 358 # JCK_COMPILER_OPTIONS 359 # Additional options for JCK-compiler 360 # JCK_COMPILER_TESTDIRS 361 # Directories of tests to be run 362 # JCK_COMPILER_OUTPUT_DIR 363 # Where to write the results 364 # JCK_COMPILER_REFERENCE 365 # (Optional) reference results (e.g. work, report or summary.txt) 366 # 367 jck-compiler-tests: check-jck FRC 368 @rm -f -r $(JCK_COMPILER_OUTPUT_DIR)/work $(JCK_COMPILER_OUTPUT_DIR)/report \ 369 $(JCK_COMPILER_OUTPUT_DIR)/diff.html $(JCK_COMPILER_OUTPUT_DIR)/status.txt 370 @mkdir -p $(JCK_COMPILER_OUTPUT_DIR) 371 $(JT_JAVA)/bin/java -Xmx512m \ 372 -jar $(JCK_HOME)/JCK-compiler-9/lib/jtjck.jar \ 373 $(if $(JCK_VERBOSE),$(if $(filter $(JCK_VERBOSE),summary),-v,-v:$(JCK_VERBOSE))) \ 374 -r:$(JCK_COMPILER_OUTPUT_DIR)/report \ 375 -w:$(JCK_COMPILER_OUTPUT_DIR)/work \ 376 -jdk:$(TESTJAVA) \ 377 $(JCK_COMPILER_OPTIONS) \ 378 $(JCK_COMPILER_TESTDIRS) \ 379 || ( $(call EXIT_IF_FATAL,$(FATAL_JCK_EXIT)) ; \ 380 echo $$status > $(JCK_COMPILER_OUTPUT_DIR)/status.txt \ 381 ) 382 ifdef JCK_COMPILER_REFERENCE 383 JT_JAVA=$(JT_JAVA) $(JTDIFF) -o $(JCK_COMPILER_OUTPUT_DIR)/diff.html \ 384 $(JCK_COMPILER_REFERENCE) $(JCK_COMPILER_OUTPUT_DIR)/report \ 385 || ( $(call EXIT_IF_FATAL,$(FATAL_JTDIFF_EXIT)) ) 386 endif 387 388 jck-compiler-summary: FRC 389 @if [ -r $(JCK_COMPILER_OUTPUT_DIR)/status.txt ]; then \ 390 echo ; echo "Summary of JCK-compiler test failures" ; \ 391 cat $(JCK_COMPILER_OUTPUT_DIR)/report/text/summary.txt | \ 392 grep -v 'Not run' | grep -v 'Passed' ; \ 393 echo ; \ 394 $(EXIT) `cat $(JCK_COMPILER_OUTPUT_DIR)/status.txt` ; \ 395 fi 396 397 # Run JCK-runtime tests in -Xcompile mode 398 # This is a special mode to test javac by compiling the tests in the JCK-runtime test suite 399 # Normal JCK-runtime invocation belongs in the jdk/ repository. 400 # 401 # JCK_HOME 402 # Installed location of JCK: should include JCK-compiler, JCK-runtime and JCK-extras 403 # JT_JAVA 404 # Version of java used to run JCK. Should normally be the same as TESTJAVA 405 # TESTJAVA 406 # Version of java to be tested. 407 # JCK_VERBOSE 408 # Verbosity setting for jtjck 409 # JCK_RUNTIME_OPTIONS 410 # Additional options for JCK-runtime 411 # JCK_RUNTIME_TESTDIRS 412 # Directories of tests to be run 413 # JCK_RUNTIME_OUTPUT_DIR 414 # Where to write the results 415 # JCK_RUNTIME_REFERENCE 416 # (Optional) reference results (e.g. work, report or summary.txt) 417 # 418 jck-runtime-tests: check-jck FRC 419 @rm -f -r $(JCK_RUNTIME_OUTPUT_DIR)/work $(JCK_RUNTIME_OUTPUT_DIR)/report \ 420 $(JCK_RUNTIME_OUTPUT_DIR)/diff.html $(JCK_RUNTIME_OUTPUT_DIR)/status.txt 421 @mkdir -p $(JCK_RUNTIME_OUTPUT_DIR) 422 $(JT_JAVA)/bin/java -Xmx512m \ 423 -jar $(JCK_HOME)/JCK-runtime-9/lib/jtjck.jar \ 424 $(if $(JCK_VERBOSE),$(if $(filter $(JCK_VERBOSE),summary),-v,-v:$(JCK_VERBOSE))) \ 425 -r:$(JCK_RUNTIME_OUTPUT_DIR)/report \ 426 -w:$(JCK_RUNTIME_OUTPUT_DIR)/work \ 427 -jdk:$(TESTJAVA) \ 428 -Xcompile \ 429 $(JCK_RUNTIME_OPTIONS) \ 430 $(JCK_RUNTIME_TESTDIRS) \ 431 || ( $(call EXIT_IF_FATAL,$(FATAL_JCK_EXIT)) ; \ 432 echo $$status > $(JCK_RUNTIME_OUTPUT_DIR)/status.txt \ 433 ) 434 ifdef JCK_RUNTIME_REFERENCE 435 JT_JAVA=$(JT_JAVA) $(JTDIFF) -o $(JCK_RUNTIME_OUTPUT_DIR)/diff.html \ 436 $(JCK_RUNTIME_REFERENCE) $(JCK_RUNTIME_OUTPUT_DIR)/report \ 437 || ( $(call EXIT_IF_FATAL,$(FATAL_JTDIFF_EXIT)) ) 438 endif 439 440 jck-runtime-summary: FRC 441 @if [ -r $(JCK_RUNTIME_OUTPUT_DIR)/status.txt ]; then \ 442 echo ; echo "Summary of JCK-runtime test failures" ; \ 443 cat $(JCK_RUNTIME_OUTPUT_DIR)/report/text/summary.txt | \ 444 grep -v 'Not run' | grep -v 'Passed' ; \ 445 echo ; \ 446 $(EXIT) `cat $(JCK_RUNTIME_OUTPUT_DIR)/status.txt` ; \ 447 fi 448 449 # Check to make sure these directories exist 450 check-jck: 451 @if [ ! -d '$(JCK_HOME)' ]; then \ 452 echo "JCK_HOME $(JCK_HOME) missing" ; \ 453 $(EXIT) 1 ; \ 454 fi 455 @if [ ! -d '$(PRODUCT_HOME)' ]; then \ 456 echo "PRODUCT_HOME $(PRODUCT_HOME) missing" ; \ 457 $(EXIT) 1 ; \ 458 fi 459 460 all-summary: FRC 461 @if [ -n "`find $(TEST_OUTPUT_DIR) -name status.txt`" ]; then 462 echo ; echo "Summary of test failures" ; \ 463 cat `find $(TEST_OUTPUT_DIR) -name summary.txt` | \ 464 grep -v 'Not run' | grep -v 'Passed' ; \ 465 echo ; \ 466 $(EXIT) 1 467 fi 468 469 # Bundle up the results 470 $(JPRT_ARCHIVE_BUNDLE): FRC 471 @rm -f $@ 472 @mkdir -p $(@D) 473 ( cd $(TEST_OUTPUT_DIR) && zip -q -r $@ . ) 474 475 # Cleanup 476 clean: 477 rm -f $(JPRT_ARCHIVE_BUNDLE) 478 479 # Used to force a target rules to run 480 FRC: 481 482 # Phony targets (e.g. these are not filenames) 483 .PHONY: all clean \ 484 jtreg javac javadoc javah javap jdeps jtreg-tests jtreg-summary check-jtreg \ 485 jck-compiler jck-compiler-tests jck-compiler-summary \ 486 jck-runtime jck-runtime-tests jck-runtime-summary check-jck 487 488 # No use of suffix rules 489 .SUFFIXES: 490