1 # 2 # Makefile to run jtreg and other tests 3 # 4 5 # Product builds and langtools builds 6 # 7 # A full product build (or "control" build) creates a complete JDK image. 8 # To test a product build, set TESTJAVA to the path for the image. 9 # 10 # A langtools build just builds the langtools components of a JDK. 11 # To test a langtools build, set TESTJAVA to the path for a recent JDK 12 # build, and set TESTBOOTCLASSPATH to the compiled langtools classes -- 13 # for example build/classes or dist/lib/classes.jar. 14 15 # JPRT 16 # JPRT may invoke this Makefile directly, as part of a langtools build, 17 # or indirectly, via FOREST/test/Makefile, as part of a control build. 18 19 # Get OS/ARCH specifics 20 OSNAME = $(shell uname -s) 21 ifeq ($(OSNAME), SunOS) 22 SLASH_JAVA = /java 23 PLATFORM = solaris 24 ARCH = $(shell uname -p) 25 ifeq ($(ARCH), i386) 26 ARCH=i586 27 endif 28 endif 29 ifeq ($(OSNAME), Linux) 30 SLASH_JAVA = /java 31 PLATFORM = linux 32 ARCH = $(shell uname -m) 33 ifeq ($(ARCH), i386) 34 ARCH=i586 35 endif 36 endif 37 ifeq ($(OSNAME), Darwin) 38 PLATFORM = bsd 39 ARCH = $(shell uname -m) 40 ifeq ($(ARCH), i386) 41 ARCH=i586 42 endif 43 endif 44 ifeq ($(OSNAME), Windows_NT) 45 # MKS 46 PLATFORM=windows 47 endif 48 ifeq ($(PLATFORM),) 49 PLATFORM = windows 50 CYGPATH = | cygpath -m -s -f - 51 endif 52 53 ifeq ($(PLATFORM), windows) 54 SLASH_JAVA = J: 55 ifeq ($(word 1, $(PROCESSOR_IDENTIFIER)),ia64) 56 ARCH=ia64 57 else 58 ifeq ($(word 1, $(PROCESSOR_IDENTIFIER)),AMD64) 59 ARCH=x64 60 else 61 ifeq ($(word 1, $(PROCESSOR_IDENTIFIER)),EM64T) 62 ARCH=x64 63 else 64 ARCH=i586 65 endif 66 endif 67 endif 68 EXE_SUFFIX=.exe 69 endif 70 71 # Root of this test area (important to use full paths in some places) 72 TEST_ROOT := $(shell pwd $(CYGPATH) ) 73 74 # Default bundle of all test results (passed or not) (JPRT only) 75 ifdef JPRT_JOB_ID 76 JPRT_CLEAN = clean 77 JPRT_ARCHIVE_BUNDLE = $(TEST_ROOT)/JPRT_ARCHIVE_BUNDLE.zip 78 endif 79 80 ifeq ($(PLATFORM), windows) 81 SLASH_JAVA = J: 82 else 83 SLASH_JAVA = /java 84 endif 85 86 # Default JTREG to run 87 ifdef JPRT_JTREG_HOME 88 JTREG_HOME = $(JPRT_JTREG_HOME) 89 else ifdef JT_HOME 90 JTREG_HOME = $(JT_HOME) 91 else 92 JTREG_HOME = $(SLASH_JAVA)/re/jtreg/4.1-jigsaw/nightly/binaries/jtreg/ 93 endif 94 JTREG = $(JTREG_HOME)/bin/jtreg 95 JTDIFF = $(JTREG_HOME)/bin/jtdiff 96 97 # Problematic tests to be excluded 98 PROBLEM_LISTS=ProblemList.txt 99 100 # Create exclude list for this platform and arch 101 ifdef NO_EXCLUDES 102 JTREG_EXCLUSIONS = 103 else 104 JTREG_EXCLUSIONS = $(PROBLEM_LISTS:%=-exclude:%) 105 endif 106 107 # Default JCK to run 108 ifdef JPRT_JCK_HOME 109 JCK_HOME = $(JPRT_JCK_HOME) 110 else 111 JCK_HOME = $(SLASH_JAVA)/re/jck/8/promoted/latest/binaries 112 endif 113 114 # Default JDK for JTREG and JCK 115 # 116 # JT_JAVA is the version of java used to run jtreg/JCK. 117 # 118 ifndef JT_JAVA 119 ifdef JPRT_JAVA_HOME 120 JT_JAVA = $(JPRT_JAVA_HOME) 121 else 122 JT_JAVA = $(SLASH_JAVA)/re/jdk/1.9.0/archive/fcs/binaries/$(PLATFORM)-$(ARCH) 123 endif 124 endif 125 126 # Default JDK to test 127 ifdef JPRT_IMPORT_PRODUCT_HOME 128 TESTJAVA = $(JPRT_IMPORT_PRODUCT_HOME) 129 else 130 TESTJAVA = $(SLASH_JAVA)/re/jdk/1.9.0/promoted/latest/binaries/$(PLATFORM)-$(ARCH) 131 endif 132 133 # PRODUCT_HOME is a JPRT variable pointing to a directory containing the output from 134 # make/Makefile 135 # For langtools, this is a directory containing build and dist 136 # For a control build, this is build/$(PRODUCT)-$(ARCH)/XYZ-image 137 # (i.e, j2sdk-image or jdk-module-image) 138 ifdef PRODUCT_HOME 139 ifeq ($(shell [ -r $(PRODUCT_HOME)/dist/lib/classes.jar ]; echo $$?),0) 140 TESTBOOTCLASSPATH = $(PRODUCT_HOME)/dist/lib/classes.jar 141 endif 142 ifeq ($(shell [ -r $(PRODUCT_HOME)/bin/javac$(EXE_SUFFIX) ]; echo $$?),0) 143 TESTJAVA = $(PRODUCT_HOME) 144 endif 145 endif 146 147 ifdef TESTBOOTCLASSPATH 148 JTREG_OPTIONS += -Xbootclasspath/p:$(TESTBOOTCLASSPATH) 149 ### In the following, -refvmoptions is an undocumented option 150 ### The following does not work JCK 7 b30 2/6/2010. Awaiting b31. 151 JCK_OPTIONS += \ 152 -vmoptions:-Xbootclasspath/p:$(TESTBOOTCLASSPATH) \ 153 -refvmoptions:-Xbootclasspath/p:$(TESTBOOTCLASSPATH) 154 endif 155 156 ifdef EXTRA_JTREG_OPTIONS 157 JTREG_OPTIONS += $(EXTRA_JTREG_OPTIONS) 158 endif 159 160 # Concurrency is the number of tests that can execute at once. 161 # On an otherwise empty machine, suggest setting to (#cpus + 2) 162 # If unset, the default is (#cpus) 163 ### RFE: determine and use #cpus 164 ifdef CONCURRENCY 165 JTREG_OPTIONS += -agentvm -concurrency:$(CONCURRENCY) 166 else 167 JTREG_OPTIONS += -agentvm 168 endif 169 170 ifdef JCK_CONCURRENCY 171 JCK_OPTIONS += -concurrency:$(JCK_CONCURRENCY) 172 endif 173 174 # JCK is executed using "Multi-JVM Group Mode", which is a hybrid 175 # of otherVM and sameVM modes. New JVMs are created and reused for 176 # a number of tests, then eventually discarded and a new one started. 177 # This amortizes the JVM startup time. The "group size" defines 178 # how many tests are run in a JVM before it is replaced. 179 # If unset, the default is 100. 180 JCK_GROUP_SIZE = 1000 181 ifdef JCK_GROUP_SIZE 182 JCK_COMPILER_OPTIONS += \ 183 -jtoptions:-Ejck.env.compiler.testCompile.groupMode.groupSize=$(JCK_GROUP_SIZE) \ 184 -jtoptions:-Ejck.env.compiler.compRefExecute.groupMode.groupSize=$(JCK_GROUP_SIZE) 185 ### The following is not supported. Awaiting RFE 6924287 186 ### 6924287: Jck4Jdk: Allow to configure test group size for group mode via simple command line option 187 ### JCK_RUNTIME_OPTIONS += \ 188 ### -jtoptions:-Ejck.env.runtime.testCompile.groupMode.groupSize=$(JCK_GROUP_SIZE) 189 endif 190 191 # Timeouts -- by default, increase test timeouts when running on JPRT 192 ifdef JPRT_JOB_ID 193 ifndef JTREG_TIMEOUT_FACTOR 194 JTREG_TIMEOUT_FACTOR = 3 195 endif 196 endif 197 ifdef JTREG_TIMEOUT_FACTOR 198 JTREG_OPTIONS += -timeoutFactor:$(JTREG_TIMEOUT_FACTOR) 199 endif 200 201 ifdef JCK_TIMEOUT_FACTOR 202 JCK_OPTIONS += -timeout:$(JCK_TIMEOUT_FACTOR) 203 endif 204 205 # Default verbosity setting for jtreg 206 JTREG_VERBOSE = fail,error,nopass 207 208 # Default verbosity setting for jck 209 JCK_VERBOSE = non-pass 210 211 # Assertions: some tests show failures when assertions are enabled. 212 # Since javac is typically loaded via the bootclassloader (either via TESTJAVA 213 # or TESTBOOTCLASSPATH), you may need -esa to enable assertions in javac. 214 JTREG_OPTIONS += $(ASSERTION_OPTIONS) 215 JCK_OPTIONS += $(ASSERTION_OPTIONS:%=-vmoptions:%) 216 217 # Include shared options 218 JCK_COMPILER_OPTIONS += $(JCK_OPTIONS) 219 JCK_RUNTIME_OPTIONS += $(JCK_OPTIONS) 220 221 # Exit codes: 222 # jtreg, jck: 0: OK, 1: tests failed, 2: tests error; 3+: SERIOUS 223 FATAL_JTREG_EXIT = 3 224 FATAL_JCK_EXIT = 3 225 # jtdiff: 0: OK, 1: differences found; 2+: SERIOUS 226 FATAL_JTDIFF_EXIT = 2 227 # 228 # Exit -- used for final "normal" exit from "make". Redefine to "true" to avoid 229 # having make exit with non-zero return code. 230 EXIT = exit 231 # Function to exit shell if exit code of preceding command is greater than or equal 232 # to a given level. Redefine function or preceding FATAL_*_EXIT codes as needed. 233 EXIT_IF_FATAL = status=$$?; if [ $$status -ge $(1) ]; then exit $$status ; fi 234 235 # The test directories to run 236 DEFAULT_TESTDIRS = . 237 TESTDIRS = $(DEFAULT_TESTDIRS) 238 239 # Root of all test results 240 TEST_OUTPUT_DIR = $(TEST_ROOT)/../build/$(PLATFORM)-$(ARCH)/test/langtools 241 ABS_TEST_OUTPUT_DIR := \ 242 $(shell mkdir -p $(TEST_OUTPUT_DIR); \ 243 cd $(TEST_OUTPUT_DIR); \ 244 pwd $(CYGPATH)) 245 # Subdirectories for different test runs 246 JTREG_OUTPUT_DIR = $(ABS_TEST_OUTPUT_DIR)/jtreg 247 JCK_COMPILER_OUTPUT_DIR = $(ABS_TEST_OUTPUT_DIR)/jck-compiler 248 JCK_RUNTIME_OUTPUT_DIR = $(ABS_TEST_OUTPUT_DIR)/jck-runtime-Xcompile 249 250 # Default make rule -- warning, may take a while 251 all: $(JPRT_CLEAN) jtreg-tests jck-compiler-tests jck-runtime-tests $(JPRT_ARCHIVE_BUNDLE) all-summary 252 @echo "Testing completed successfully" 253 254 jtreg apt javac javadoc javah javap jdeps: $(JPRT_CLEAN) jtreg-tests $(JPRT_ARCHIVE_BUNDLE) jtreg-summary 255 @echo "Testing completed successfully" 256 257 jck-compiler: $(JPRT_CLEAN) jck-compiler-tests $(JPRT_ARCHIVE_BUNDLE) jck-compiler-summary 258 @echo "Testing completed successfully" 259 260 jck-runtime: $(JPRT_CLEAN) jck-runtime-tests $(JPRT_ARCHIVE_BUNDLE) jck-runtime-summary 261 @echo "Testing completed successfully" 262 263 # for use with JPRT -testrule 264 all: JTREG_TESTDIRS = . 265 jtreg: JTREG_TESTDIRS = . 266 apt: JTREG_TESTDIRS = tools/apt 267 javac: JTREG_TESTDIRS = tools/javac 268 javadoc: JTREG_TESTDIRS = tools/javadoc com/sun/javadoc 269 javah: JTREG_TESTDIRS = tools/javah 270 javap: JTREG_TESTDIRS = tools/javap 271 jdeps: JTREG_TESTDIRS = tools/jdeps 272 273 # a way to select jtreg tests from outside 274 ifdef TEST_SELECTION 275 JTREG_TESTDIRS = $(TEST_SELECTION) 276 endif 277 278 279 # Run jtreg tests 280 # 281 # JTREG_HOME 282 # Installed location of jtreg 283 # JT_JAVA 284 # Version of java used to run jtreg. Should normally be the same as TESTJAVA 285 # TESTJAVA 286 # Version of java to be tested. 287 # JTREG_VERBOSE 288 # Verbosity setting for jtreg 289 # JTREG_OPTIONS 290 # Additional options for jtreg 291 # JTREG_TESTDIRS 292 # Directories of tests to be run 293 # JTREG_OUTPUT_DIR 294 # Where to write the results 295 # JTREG_REFERENCE 296 # (Optional) reference results (e.g. work, report or summary.txt) 297 # 298 jtreg_tests: jtreg-tests 299 jtreg-tests: check-jtreg FRC 300 @rm -f -r $(JTREG_OUTPUT_DIR)/JTwork $(JTREG_OUTPUT_DIR)/JTreport \ 301 $(JTREG_OUTPUT_DIR)/diff.html $(JTREG_OUTPUT_DIR)/status.txt 302 @mkdir -p $(JTREG_OUTPUT_DIR) 303 JT_JAVA=$(JT_JAVA) $(JTREG) \ 304 -J-Xmx512m \ 305 -vmoption:-Xmx768m \ 306 -a -ignore:quiet $(if $(JTREG_VERBOSE),-v:$(JTREG_VERBOSE)) \ 307 -r:$(JTREG_OUTPUT_DIR)/JTreport \ 308 -w:$(JTREG_OUTPUT_DIR)/JTwork \ 309 -jdk:$(TESTJAVA) \ 310 $(JAVA_ARGS:%=-vmoption:%) \ 311 $(JTREG_EXCLUSIONS) \ 312 $(JTREG_OPTIONS) \ 313 $(JTREG_TESTDIRS) \ 314 || ( $(call EXIT_IF_FATAL,$(FATAL_JTREG_EXIT)) ; \ 315 echo $$status > $(JTREG_OUTPUT_DIR)/status.txt \ 316 ) 317 ifdef JTREG_REFERENCE 318 JT_JAVA=$(JT_JAVA) $(JTDIFF) -o $(JTREG_OUTPUT_DIR)/diff.html \ 319 $(JTREG_REFERENCE) $(JTREG_OUTPUT_DIR)/JTreport \ 320 || ( $(call EXIT_IF_FATAL,$(FATAL_JTDIFF_EXIT)) ) 321 endif 322 323 jtreg-summary: FRC 324 @if [ -r $(JTREG_OUTPUT_DIR)/status.txt ]; then \ 325 echo ; echo "Summary of jtreg test failures" ; \ 326 cat $(JTREG_OUTPUT_DIR)/JTreport/text/summary.txt | \ 327 grep -v 'Not run' | grep -v 'Passed' ; \ 328 echo ; \ 329 $(EXIT) `cat $(JTREG_OUTPUT_DIR)/status.txt` ; \ 330 fi 331 332 # Check to make sure these directories exist 333 check-jtreg: $(PRODUCT_HOME) $(JTREG) 334 335 336 # Run JCK-compiler tests 337 # 338 # JCK_HOME 339 # Installed location of JCK: should include JCK-compiler, and JCK-extras 340 # Default is JCK 8. 341 # JT_JAVA 342 # Version of java used to run JCK. Should normally be the same as TESTJAVA 343 # Default is JDK 7 344 # TESTJAVA 345 # Version of java to be tested. 346 # JCK_VERBOSE 347 # Verbosity setting for jtjck 348 # JCK_COMPILER_OPTIONS 349 # Additional options for JCK-compiler 350 # JCK_COMPILER_TESTDIRS 351 # Directories of tests to be run 352 # JCK_COMPILER_OUTPUT_DIR 353 # Where to write the results 354 # JCK_COMPILER_REFERENCE 355 # (Optional) reference results (e.g. work, report or summary.txt) 356 # 357 jck-compiler-tests: check-jck FRC 358 @rm -f -r $(JCK_COMPILER_OUTPUT_DIR)/work $(JCK_COMPILER_OUTPUT_DIR)/report \ 359 $(JCK_COMPILER_OUTPUT_DIR)/diff.html $(JCK_COMPILER_OUTPUT_DIR)/status.txt 360 @mkdir -p $(JCK_COMPILER_OUTPUT_DIR) 361 $(JT_JAVA)/bin/java -Xmx512m \ 362 -jar $(JCK_HOME)/JCK-compiler-9/lib/jtjck.jar \ 363 $(if $(JCK_VERBOSE),$(if $(filter $(JCK_VERBOSE),summary),-v,-v:$(JCK_VERBOSE))) \ 364 -r:$(JCK_COMPILER_OUTPUT_DIR)/report \ 365 -w:$(JCK_COMPILER_OUTPUT_DIR)/work \ 366 -jdk:$(TESTJAVA) \ 367 $(JCK_COMPILER_OPTIONS) \ 368 $(JCK_COMPILER_TESTDIRS) \ 369 || ( $(call EXIT_IF_FATAL,$(FATAL_JCK_EXIT)) ; \ 370 echo $$status > $(JCK_COMPILER_OUTPUT_DIR)/status.txt \ 371 ) 372 ifdef JCK_COMPILER_REFERENCE 373 JT_JAVA=$(JT_JAVA) $(JTDIFF) -o $(JCK_COMPILER_OUTPUT_DIR)/diff.html \ 374 $(JCK_COMPILER_REFERENCE) $(JCK_COMPILER_OUTPUT_DIR)/report \ 375 || ( $(call EXIT_IF_FATAL,$(FATAL_JTDIFF_EXIT)) ) 376 endif 377 378 jck-compiler-summary: FRC 379 @if [ -r $(JCK_COMPILER_OUTPUT_DIR)/status.txt ]; then \ 380 echo ; echo "Summary of JCK-compiler test failures" ; \ 381 cat $(JCK_COMPILER_OUTPUT_DIR)/report/text/summary.txt | \ 382 grep -v 'Not run' | grep -v 'Passed' ; \ 383 echo ; \ 384 $(EXIT) `cat $(JCK_COMPILER_OUTPUT_DIR)/status.txt` ; \ 385 fi 386 387 # Run JCK-runtime tests in -Xcompile mode 388 # This is a special mode to test javac by compiling the tests in the JCK-runtime test suite 389 # Normal JCK-runtime invocation belongs in the jdk/ repository. 390 # 391 # JCK_HOME 392 # Installed location of JCK: should include JCK-compiler, JCK-runtime and JCK-extras 393 # JT_JAVA 394 # Version of java used to run JCK. Should normally be the same as TESTJAVA 395 # TESTJAVA 396 # Version of java to be tested. 397 # JCK_VERBOSE 398 # Verbosity setting for jtjck 399 # JCK_RUNTIME_OPTIONS 400 # Additional options for JCK-runtime 401 # JCK_RUNTIME_TESTDIRS 402 # Directories of tests to be run 403 # JCK_RUNTIME_OUTPUT_DIR 404 # Where to write the results 405 # JCK_RUNTIME_REFERENCE 406 # (Optional) reference results (e.g. work, report or summary.txt) 407 # 408 jck-runtime-tests: check-jck FRC 409 @rm -f -r $(JCK_RUNTIME_OUTPUT_DIR)/work $(JCK_RUNTIME_OUTPUT_DIR)/report \ 410 $(JCK_RUNTIME_OUTPUT_DIR)/diff.html $(JCK_RUNTIME_OUTPUT_DIR)/status.txt 411 @mkdir -p $(JCK_RUNTIME_OUTPUT_DIR) 412 $(JT_JAVA)/bin/java -Xmx512m \ 413 -jar $(JCK_HOME)/JCK-runtime-9/lib/jtjck.jar \ 414 $(if $(JCK_VERBOSE),$(if $(filter $(JCK_VERBOSE),summary),-v,-v:$(JCK_VERBOSE))) \ 415 -r:$(JCK_RUNTIME_OUTPUT_DIR)/report \ 416 -w:$(JCK_RUNTIME_OUTPUT_DIR)/work \ 417 -jdk:$(TESTJAVA) \ 418 -Xcompile \ 419 $(JCK_RUNTIME_OPTIONS) \ 420 $(JCK_RUNTIME_TESTDIRS) \ 421 || ( $(call EXIT_IF_FATAL,$(FATAL_JCK_EXIT)) ; \ 422 echo $$status > $(JCK_RUNTIME_OUTPUT_DIR)/status.txt \ 423 ) 424 ifdef JCK_RUNTIME_REFERENCE 425 JT_JAVA=$(JT_JAVA) $(JTDIFF) -o $(JCK_RUNTIME_OUTPUT_DIR)/diff.html \ 426 $(JCK_RUNTIME_REFERENCE) $(JCK_RUNTIME_OUTPUT_DIR)/report \ 427 || ( $(call EXIT_IF_FATAL,$(FATAL_JTDIFF_EXIT)) ) 428 endif 429 430 jck-runtime-summary: FRC 431 @if [ -r $(JCK_RUNTIME_OUTPUT_DIR)/status.txt ]; then \ 432 echo ; echo "Summary of JCK-runtime test failures" ; \ 433 cat $(JCK_RUNTIME_OUTPUT_DIR)/report/text/summary.txt | \ 434 grep -v 'Not run' | grep -v 'Passed' ; \ 435 echo ; \ 436 $(EXIT) `cat $(JCK_RUNTIME_OUTPUT_DIR)/status.txt` ; \ 437 fi 438 439 # Check to make sure these directories exist 440 check-jck: 441 if [ ! -d '$(JCK_HOME)' ]; then \ 442 echo "JCK_HOME $(JCK_HOME) missing" ; \ 443 $(EXIT) 1 ; \ 444 fi 445 if [ ! -d '$(PRODUCT_HOME)' ]; then \ 446 echo "PRODUCT_HOME $(PRODUCT_HOME) missing" ; \ 447 $(EXIT) 1 ; \ 448 fi 449 450 all-summary: FRC 451 @if [ -n "`find $(TEST_OUTPUT_DIR) -name status.txt`" ]; then 452 echo ; echo "Summary of test failures" ; \ 453 cat `find $(TEST_OUTPUT_DIR) -name summary.txt` | \ 454 grep -v 'Not run' | grep -v 'Passed' ; \ 455 echo ; \ 456 $(EXIT) 1 457 fi 458 459 # Bundle up the results 460 $(JPRT_ARCHIVE_BUNDLE): FRC 461 @rm -f $@ 462 @mkdir -p $(@D) 463 ( cd $(TEST_OUTPUT_DIR) && zip -q -r $@ . ) 464 465 # Cleanup 466 clean: 467 rm -f $(JPRT_ARCHIVE_BUNDLE) 468 469 # Used to force a target rules to run 470 FRC: 471 472 # Phony targets (e.g. these are not filenames) 473 .PHONY: all clean \ 474 jtreg javac javadoc javah javap jdeps jtreg-tests jtreg-summary check-jtreg \ 475 jck-compiler jck-compiler-tests jck-compiler-summary \ 476 jck-runtime jck-runtime-tests jck-runtime-summary check-jck 477 478 # No use of suffix rules 479 .SUFFIXES: 480