1 # 2 # Makefile to run jtreg and other tests 3 # 4 5 # Product builds and langtools builds 6 # 7 # A full product build (or "control" build) creates a complete JDK image. 8 # To test a product build, set TESTJAVA to the path for the image. 9 # 10 # A langtools build just builds the langtools components of a JDK. 11 # To test a langtools build, set TESTJAVA to the path for a recent JDK 12 # build, and set TESTBOOTCLASSPATH to the compiled langtools classes -- 13 # for example build/classes or dist/lib/classes.jar. 14 15 # JPRT 16 # JPRT may invoke this Makefile directly, as part of a langtools build, 17 # or indirectly, via FOREST/test/Makefile, as part of a control build. 18 19 # Get OS/ARCH specifics 20 OSNAME = $(shell uname -s) 21 ifeq ($(OSNAME), SunOS) 22 SLASH_JAVA = /java 23 PLATFORM = solaris 24 ARCH = $(shell uname -p) 25 ifeq ($(ARCH), i386) 26 ARCH=i586 27 endif 28 endif 29 ifeq ($(OSNAME), Linux) 30 SLASH_JAVA = /java 31 PLATFORM = linux 32 ARCH = $(shell uname -m) 33 ifeq ($(ARCH), i386) 34 ARCH=i586 35 endif 36 endif 37 ifeq ($(OSNAME), Darwin) 38 PLATFORM = bsd 39 ARCH = $(shell uname -m) 40 ifeq ($(ARCH), i386) 41 ARCH=i586 42 endif 43 endif 44 ifeq ($(OSNAME), Windows_NT) 45 # MKS 46 PLATFORM=windows 47 endif 48 ifeq ($(PLATFORM),) 49 PLATFORM = windows 50 CYGPATH = | cygpath -m -s -f - 51 endif 52 53 ifeq ($(PLATFORM), windows) 54 SLASH_JAVA = J: 55 ifeq ($(word 1, $(PROCESSOR_IDENTIFIER)),ia64) 56 ARCH=ia64 57 else 58 ifeq ($(word 1, $(PROCESSOR_IDENTIFIER)),AMD64) 59 ARCH=x64 60 else 61 ifeq ($(word 1, $(PROCESSOR_IDENTIFIER)),EM64T) 62 ARCH=x64 63 else 64 ARCH=i586 65 endif 66 endif 67 endif 68 EXE_SUFFIX=.exe 69 endif 70 71 # Root of this test area (important to use full paths in some places) 72 TEST_ROOT := $(shell pwd $(CYGPATH) ) 73 74 # Default bundle of all test results (passed or not) (JPRT only) 75 ifdef JPRT_JOB_ID 76 JPRT_CLEAN = clean 77 JPRT_ARCHIVE_BUNDLE = $(TEST_ROOT)/JPRT_ARCHIVE_BUNDLE.zip 78 endif 79 80 ifeq ($(PLATFORM), windows) 81 SLASH_JAVA = J: 82 else 83 SLASH_JAVA = /java 84 endif 85 86 # Default JTREG to run 87 ifdef JPRT_JTREG_HOME 88 JTREG_HOME = $(JPRT_JTREG_HOME) 89 else 90 JTREG_HOME = $(SLASH_JAVA)/re/jtreg/4.1-jigsaw/nightly/binaries/jtreg/ 91 endif 92 JTREG = $(JTREG_HOME)/bin/jtreg 93 JTDIFF = $(JTREG_HOME)/bin/jtdiff 94 95 # Problematic tests to be excluded 96 PROBLEM_LISTS=ProblemList.txt 97 98 # Create exclude list for this platform and arch 99 ifdef NO_EXCLUDES 100 JTREG_EXCLUSIONS = 101 else 102 JTREG_EXCLUSIONS = $(PROBLEM_LISTS:%=-exclude:%) 103 endif 104 105 # Default JCK to run 106 ifdef JPRT_JCK_HOME 107 JCK_HOME = $(JPRT_JCK_HOME) 108 else 109 JCK_HOME = $(SLASH_JAVA)/re/jck/8/promoted/latest/binaries 110 endif 111 112 # Default JDK for JTREG and JCK 113 # 114 # JT_JAVA is the version of java used to run jtreg/JCK. 115 # 116 ifdef JPRT_JAVA_HOME 117 JT_JAVA = $(JPRT_JAVA_HOME) 118 else 119 JT_JAVA = $(SLASH_JAVA)/re/jdk/1.9.0/archive/fcs/binaries/$(PLATFORM)-$(ARCH) 120 endif 121 122 # Default JDK to test 123 ifdef JPRT_IMPORT_PRODUCT_HOME 124 TESTJAVA = $(JPRT_IMPORT_PRODUCT_HOME) 125 else 126 TESTJAVA = $(SLASH_JAVA)/re/jdk/1.9.0/promoted/latest/binaries/$(PLATFORM)-$(ARCH) 127 endif 128 129 # PRODUCT_HOME is a JPRT variable pointing to a directory containing the output from 130 # make/Makefile 131 # For langtools, this is a directory containing build and dist 132 # For a control build, this is build/$(PRODUCT)-$(ARCH)/XYZ-image 133 # (i.e, j2sdk-image or jdk-module-image) 134 ifdef PRODUCT_HOME 135 ifeq ($(shell [ -r $(PRODUCT_HOME)/dist/lib/classes.jar ]; echo $$?),0) 136 TESTBOOTCLASSPATH = $(PRODUCT_HOME)/dist/lib/classes.jar 137 endif 138 ifeq ($(shell [ -r $(PRODUCT_HOME)/bin/javac$(EXE_SUFFIX) ]; echo $$?),0) 139 TESTJAVA = $(PRODUCT_HOME) 140 endif 141 endif 142 143 ifdef TESTBOOTCLASSPATH 144 JTREG_OPTIONS += -Xbootclasspath/p:$(TESTBOOTCLASSPATH) 145 ### In the following, -refvmoptions is an undocumented option 146 ### The following does not work JCK 7 b30 2/6/2010. Awaiting b31. 147 JCK_OPTIONS += \ 148 -vmoptions:-Xbootclasspath/p:$(TESTBOOTCLASSPATH) \ 149 -refvmoptions:-Xbootclasspath/p:$(TESTBOOTCLASSPATH) 150 endif 151 152 # Concurrency is the number of tests that can execute at once. 153 # On an otherwise empty machine, suggest setting to (#cpus + 2) 154 # If unset, the default is (#cpus) 155 ### RFE: determine and use #cpus 156 ifdef CONCURRENCY 157 JTREG_OPTIONS += -agentvm -concurrency:$(CONCURRENCY) 158 else 159 JTREG_OPTIONS += -agentvm 160 endif 161 162 ifdef JCK_CONCURRENCY 163 JCK_OPTIONS += -concurrency:$(JCK_CONCURRENCY) 164 endif 165 166 # JCK is executed using "Multi-JVM Group Mode", which is a hybrid 167 # of otherVM and sameVM modes. New JVMs are created and reused for 168 # a number of tests, then eventually discarded and a new one started. 169 # This amortizes the JVM startup time. The "group size" defines 170 # how many tests are run in a JVM before it is replaced. 171 # If unset, the default is 100. 172 JCK_GROUP_SIZE = 1000 173 ifdef JCK_GROUP_SIZE 174 JCK_COMPILER_OPTIONS += \ 175 -jtoptions:-Ejck.env.compiler.testCompile.groupMode.groupSize=$(JCK_GROUP_SIZE) \ 176 -jtoptions:-Ejck.env.compiler.compRefExecute.groupMode.groupSize=$(JCK_GROUP_SIZE) 177 ### The following is not supported. Awaiting RFE 6924287 178 ### 6924287: Jck4Jdk: Allow to configure test group size for group mode via simple command line option 179 ### JCK_RUNTIME_OPTIONS += \ 180 ### -jtoptions:-Ejck.env.runtime.testCompile.groupMode.groupSize=$(JCK_GROUP_SIZE) 181 endif 182 183 # Timeouts -- by default, increase test timeouts when running on JPRT 184 ifdef JPRT_JOB_ID 185 ifndef JTREG_TIMEOUT_FACTOR 186 JTREG_TIMEOUT_FACTOR = 3 187 endif 188 endif 189 ifdef JTREG_TIMEOUT_FACTOR 190 JTREG_OPTIONS += -timeoutFactor:$(JTREG_TIMEOUT_FACTOR) 191 endif 192 193 ifdef JCK_TIMEOUT_FACTOR 194 JCK_OPTIONS += -timeout:$(JCK_TIMEOUT_FACTOR) 195 endif 196 197 # Default verbosity setting for jtreg 198 JTREG_VERBOSE = fail,error,nopass 199 200 # Default verbosity setting for jck 201 JCK_VERBOSE = non-pass 202 203 # Assertions: some tests show failures when assertions are enabled. 204 # Since javac is typically loaded via the bootclassloader (either via TESTJAVA 205 # or TESTBOOTCLASSPATH), you may need -esa to enable assertions in javac. 206 JTREG_OPTIONS += $(ASSERTION_OPTIONS) 207 JCK_OPTIONS += $(ASSERTION_OPTIONS:%=-vmoptions:%) 208 209 # Include shared options 210 JCK_COMPILER_OPTIONS += $(JCK_OPTIONS) 211 JCK_RUNTIME_OPTIONS += $(JCK_OPTIONS) 212 213 # Exit codes: 214 # jtreg, jck: 0: OK, 1: tests failed, 2: tests error; 3+: SERIOUS 215 FATAL_JTREG_EXIT = 3 216 FATAL_JCK_EXIT = 3 217 # jtdiff: 0: OK, 1: differences found; 2+: SERIOUS 218 FATAL_JTDIFF_EXIT = 2 219 # 220 # Exit -- used for final "normal" exit from "make". Redefine to "true" to avoid 221 # having make exit with non-zero return code. 222 EXIT = exit 223 # Function to exit shell if exit code of preceding command is greater than or equal 224 # to a given level. Redefine function or preceding FATAL_*_EXIT codes as needed. 225 EXIT_IF_FATAL = status=$$?; if [ $$status -ge $(1) ]; then exit $$status ; fi 226 227 # The test directories to run 228 DEFAULT_TESTDIRS = . 229 TESTDIRS = $(DEFAULT_TESTDIRS) 230 231 # Root of all test results 232 TEST_OUTPUT_DIR = $(TEST_ROOT)/../build/$(PLATFORM)-$(ARCH)/test/langtools 233 ABS_TEST_OUTPUT_DIR := \ 234 $(shell mkdir -p $(TEST_OUTPUT_DIR); \ 235 cd $(TEST_OUTPUT_DIR); \ 236 pwd $(CYGPATH)) 237 # Subdirectories for different test runs 238 JTREG_OUTPUT_DIR = $(ABS_TEST_OUTPUT_DIR)/jtreg 239 JCK_COMPILER_OUTPUT_DIR = $(ABS_TEST_OUTPUT_DIR)/jck-compiler 240 JCK_RUNTIME_OUTPUT_DIR = $(ABS_TEST_OUTPUT_DIR)/jck-runtime-Xcompile 241 242 # Default make rule -- warning, may take a while 243 all: $(JPRT_CLEAN) jtreg-tests jck-compiler-tests jck-runtime-tests $(JPRT_ARCHIVE_BUNDLE) all-summary 244 @echo "Testing completed successfully" 245 246 jtreg apt javac javadoc javah javap jdeps: $(JPRT_CLEAN) jtreg-tests $(JPRT_ARCHIVE_BUNDLE) jtreg-summary 247 @echo "Testing completed successfully" 248 249 jck-compiler: $(JPRT_CLEAN) jck-compiler-tests $(JPRT_ARCHIVE_BUNDLE) jck-compiler-summary 250 @echo "Testing completed successfully" 251 252 jck-runtime: $(JPRT_CLEAN) jck-runtime-tests $(JPRT_ARCHIVE_BUNDLE) jck-runtime-summary 253 @echo "Testing completed successfully" 254 255 # for use with JPRT -testrule 256 all: JTREG_TESTDIRS = . 257 jtreg: JTREG_TESTDIRS = . 258 apt: JTREG_TESTDIRS = tools/apt 259 javac: JTREG_TESTDIRS = tools/javac 260 javadoc: JTREG_TESTDIRS = tools/javadoc com/sun/javadoc 261 javah: JTREG_TESTDIRS = tools/javah 262 javap: JTREG_TESTDIRS = tools/javap 263 jdeps: JTREG_TESTDIRS = tools/jdeps 264 265 # Run jtreg tests 266 # 267 # JTREG_HOME 268 # Installed location of jtreg 269 # JT_JAVA 270 # Version of java used to run jtreg. Should normally be the same as TESTJAVA 271 # TESTJAVA 272 # Version of java to be tested. 273 # JTREG_VERBOSE 274 # Verbosity setting for jtreg 275 # JTREG_OPTIONS 276 # Additional options for jtreg 277 # JTREG_TESTDIRS 278 # Directories of tests to be run 279 # JTREG_OUTPUT_DIR 280 # Where to write the results 281 # JTREG_REFERENCE 282 # (Optional) reference results (e.g. work, report or summary.txt) 283 # 284 jtreg_tests: jtreg-tests 285 jtreg-tests: check-jtreg FRC 286 @rm -f -r $(JTREG_OUTPUT_DIR)/JTwork $(JTREG_OUTPUT_DIR)/JTreport \ 287 $(JTREG_OUTPUT_DIR)/diff.html $(JTREG_OUTPUT_DIR)/status.txt 288 @mkdir -p $(JTREG_OUTPUT_DIR) 289 JT_JAVA=$(JT_JAVA) $(JTREG) \ 290 -J-Xmx512m \ 291 -vmoption:-Xmx768m \ 292 -a -ignore:quiet $(if $(JTREG_VERBOSE),-v:$(JTREG_VERBOSE)) \ 293 -r:$(JTREG_OUTPUT_DIR)/JTreport \ 294 -w:$(JTREG_OUTPUT_DIR)/JTwork \ 295 -jdk:$(TESTJAVA) \ 296 $(JAVA_ARGS:%=-vmoption:%) \ 297 $(JTREG_EXCLUSIONS) \ 298 $(JTREG_OPTIONS) \ 299 $(JTREG_TESTDIRS) \ 300 || ( $(call EXIT_IF_FATAL,$(FATAL_JTREG_EXIT)) ; \ 301 echo $$status > $(JTREG_OUTPUT_DIR)/status.txt \ 302 ) 303 ifdef JTREG_REFERENCE 304 JT_JAVA=$(JT_JAVA) $(JTDIFF) -o $(JTREG_OUTPUT_DIR)/diff.html \ 305 $(JTREG_REFERENCE) $(JTREG_OUTPUT_DIR)/JTreport \ 306 || ( $(call EXIT_IF_FATAL,$(FATAL_JTDIFF_EXIT)) ) 307 endif 308 309 jtreg-summary: FRC 310 @if [ -r $(JTREG_OUTPUT_DIR)/status.txt ]; then \ 311 echo ; echo "Summary of jtreg test failures" ; \ 312 cat $(JTREG_OUTPUT_DIR)/JTreport/text/summary.txt | \ 313 grep -v 'Not run' | grep -v 'Passed' ; \ 314 echo ; \ 315 $(EXIT) `cat $(JTREG_OUTPUT_DIR)/status.txt` ; \ 316 fi 317 318 # Check to make sure these directories exist 319 check-jtreg: $(PRODUCT_HOME) $(JTREG) 320 321 322 # Run JCK-compiler tests 323 # 324 # JCK_HOME 325 # Installed location of JCK: should include JCK-compiler, and JCK-extras 326 # Default is JCK 8. 327 # JT_JAVA 328 # Version of java used to run JCK. Should normally be the same as TESTJAVA 329 # Default is JDK 7 330 # TESTJAVA 331 # Version of java to be tested. 332 # JCK_VERBOSE 333 # Verbosity setting for jtjck 334 # JCK_COMPILER_OPTIONS 335 # Additional options for JCK-compiler 336 # JCK_COMPILER_TESTDIRS 337 # Directories of tests to be run 338 # JCK_COMPILER_OUTPUT_DIR 339 # Where to write the results 340 # JCK_COMPILER_REFERENCE 341 # (Optional) reference results (e.g. work, report or summary.txt) 342 # 343 jck-compiler-tests: check-jck FRC 344 @rm -f -r $(JCK_COMPILER_OUTPUT_DIR)/work $(JCK_COMPILER_OUTPUT_DIR)/report \ 345 $(JCK_COMPILER_OUTPUT_DIR)/diff.html $(JCK_COMPILER_OUTPUT_DIR)/status.txt 346 @mkdir -p $(JCK_COMPILER_OUTPUT_DIR) 347 $(JT_JAVA)/bin/java -Xmx512m \ 348 -jar $(JCK_HOME)/JCK-compiler-9/lib/jtjck.jar \ 349 $(if $(JCK_VERBOSE),$(if $(filter $(JCK_VERBOSE),summary),-v,-v:$(JCK_VERBOSE))) \ 350 -r:$(JCK_COMPILER_OUTPUT_DIR)/report \ 351 -w:$(JCK_COMPILER_OUTPUT_DIR)/work \ 352 -jdk:$(TESTJAVA) \ 353 $(JCK_COMPILER_OPTIONS) \ 354 $(JCK_COMPILER_TESTDIRS) \ 355 || ( $(call EXIT_IF_FATAL,$(FATAL_JCK_EXIT)) ; \ 356 echo $$status > $(JCK_COMPILER_OUTPUT_DIR)/status.txt \ 357 ) 358 ifdef JCK_COMPILER_REFERENCE 359 JT_JAVA=$(JT_JAVA) $(JTDIFF) -o $(JCK_COMPILER_OUTPUT_DIR)/diff.html \ 360 $(JCK_COMPILER_REFERENCE) $(JCK_COMPILER_OUTPUT_DIR)/report \ 361 || ( $(call EXIT_IF_FATAL,$(FATAL_JTDIFF_EXIT)) ) 362 endif 363 364 jck-compiler-summary: FRC 365 @if [ -r $(JCK_COMPILER_OUTPUT_DIR)/status.txt ]; then \ 366 echo ; echo "Summary of JCK-compiler test failures" ; \ 367 cat $(JCK_COMPILER_OUTPUT_DIR)/report/text/summary.txt | \ 368 grep -v 'Not run' | grep -v 'Passed' ; \ 369 echo ; \ 370 $(EXIT) `cat $(JCK_COMPILER_OUTPUT_DIR)/status.txt` ; \ 371 fi 372 373 # Run JCK-runtime tests in -Xcompile mode 374 # This is a special mode to test javac by compiling the tests in the JCK-runtime test suite 375 # Normal JCK-runtime invocation belongs in the jdk/ repository. 376 # 377 # JCK_HOME 378 # Installed location of JCK: should include JCK-compiler, JCK-runtime and JCK-extras 379 # JT_JAVA 380 # Version of java used to run JCK. Should normally be the same as TESTJAVA 381 # TESTJAVA 382 # Version of java to be tested. 383 # JCK_VERBOSE 384 # Verbosity setting for jtjck 385 # JCK_RUNTIME_OPTIONS 386 # Additional options for JCK-runtime 387 # JCK_RUNTIME_TESTDIRS 388 # Directories of tests to be run 389 # JCK_RUNTIME_OUTPUT_DIR 390 # Where to write the results 391 # JCK_RUNTIME_REFERENCE 392 # (Optional) reference results (e.g. work, report or summary.txt) 393 # 394 jck-runtime-tests: check-jck FRC 395 @rm -f -r $(JCK_RUNTIME_OUTPUT_DIR)/work $(JCK_RUNTIME_OUTPUT_DIR)/report \ 396 $(JCK_RUNTIME_OUTPUT_DIR)/diff.html $(JCK_RUNTIME_OUTPUT_DIR)/status.txt 397 @mkdir -p $(JCK_RUNTIME_OUTPUT_DIR) 398 $(JT_JAVA)/bin/java -Xmx512m \ 399 -jar $(JCK_HOME)/JCK-runtime-9/lib/jtjck.jar \ 400 $(if $(JCK_VERBOSE),$(if $(filter $(JCK_VERBOSE),summary),-v,-v:$(JCK_VERBOSE))) \ 401 -r:$(JCK_RUNTIME_OUTPUT_DIR)/report \ 402 -w:$(JCK_RUNTIME_OUTPUT_DIR)/work \ 403 -jdk:$(TESTJAVA) \ 404 -Xcompile \ 405 $(JCK_RUNTIME_OPTIONS) \ 406 $(JCK_RUNTIME_TESTDIRS) \ 407 || ( $(call EXIT_IF_FATAL,$(FATAL_JCK_EXIT)) ; \ 408 echo $$status > $(JCK_RUNTIME_OUTPUT_DIR)/status.txt \ 409 ) 410 ifdef JCK_RUNTIME_REFERENCE 411 JT_JAVA=$(JT_JAVA) $(JTDIFF) -o $(JCK_RUNTIME_OUTPUT_DIR)/diff.html \ 412 $(JCK_RUNTIME_REFERENCE) $(JCK_RUNTIME_OUTPUT_DIR)/report \ 413 || ( $(call EXIT_IF_FATAL,$(FATAL_JTDIFF_EXIT)) ) 414 endif 415 416 jck-runtime-summary: FRC 417 @if [ -r $(JCK_RUNTIME_OUTPUT_DIR)/status.txt ]; then \ 418 echo ; echo "Summary of JCK-runtime test failures" ; \ 419 cat $(JCK_RUNTIME_OUTPUT_DIR)/report/text/summary.txt | \ 420 grep -v 'Not run' | grep -v 'Passed' ; \ 421 echo ; \ 422 $(EXIT) `cat $(JCK_RUNTIME_OUTPUT_DIR)/status.txt` ; \ 423 fi 424 425 # Check to make sure these directories exist 426 check-jck: 427 if [ ! -d '$(JCK_HOME)' ]; then \ 428 echo "JCK_HOME $(JCK_HOME) missing" ; \ 429 $(EXIT) 1 ; \ 430 fi 431 if [ ! -d '$(PRODUCT_HOME)' ]; then \ 432 echo "PRODUCT_HOME $(PRODUCT_HOME) missing" ; \ 433 $(EXIT) 1 ; \ 434 fi 435 436 all-summary: FRC 437 @if [ -n "`find $(TEST_OUTPUT_DIR) -name status.txt`" ]; then 438 echo ; echo "Summary of test failures" ; \ 439 cat `find $(TEST_OUTPUT_DIR) -name summary.txt` | \ 440 grep -v 'Not run' | grep -v 'Passed' ; \ 441 echo ; \ 442 $(EXIT) 1 443 fi 444 445 # Bundle up the results 446 $(JPRT_ARCHIVE_BUNDLE): FRC 447 @rm -f $@ 448 @mkdir -p $(@D) 449 ( cd $(TEST_OUTPUT_DIR) && zip -q -r $@ . ) 450 451 # Cleanup 452 clean: 453 rm -f $(JPRT_ARCHIVE_BUNDLE) 454 455 # Used to force a target rules to run 456 FRC: 457 458 # Phony targets (e.g. these are not filenames) 459 .PHONY: all clean \ 460 jtreg javac javadoc javah javap jdeps jtreg-tests jtreg-summary check-jtreg \ 461 jck-compiler jck-compiler-tests jck-compiler-summary \ 462 jck-runtime jck-runtime-tests jck-runtime-summary check-jck 463 464 # No use of suffix rules 465 .SUFFIXES: 466