1 #
2 # ----------------------------------------------------------------------------------------------------
3 #
4 # Copyright (c) 2007, 2015, Oracle and/or its affiliates. All rights reserved.
5 # DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
6 #
7 # This code is free software; you can redistribute it and/or modify it
8 # under the terms of the GNU General Public License version 2 only, as
9 # published by the Free Software Foundation.
10 #
11 # This code is distributed in the hope that it will be useful, but WITHOUT
12 # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 # FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
14 # version 2 for more details (a copy is included in the LICENSE file that
15 # accompanied this code).
16 #
17 # You should have received a copy of the GNU General Public License version
18 # 2 along with this work; if not, write to the Free Software Foundation,
19 # Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
20 #
21 # Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
22 # or visit www.oracle.com if you need additional information or have any
23 # questions.
24 #
25 # ----------------------------------------------------------------------------------------------------
26
27 import os
28 from os.path import join, dirname, basename, exists, abspath
29 from argparse import ArgumentParser
30 import sanitycheck
31 import re
32
33 import mx
34 from mx_gate import Task
35 from sanitycheck import _noneAsEmptyList
36
37 from mx_unittest import unittest
38 from mx_graal_bench import dacapo
39 import mx_gate
40 import mx_unittest
41
42 _suite = mx.suite('graal')
43
44 _jdk = mx.get_jdk(tag='default')
45 assert _jdk.javaCompliance >= "1.9"
46
47 def isJVMCIEnabled(vm):
48 return True
49
50 _jvmciModes = {
51 'hosted' : ['-XX:+UnlockExperimentalVMOptions', '-XX:+EnableJVMCI'],
52 'jit' : ['-XX:+UnlockExperimentalVMOptions', '-XX:+EnableJVMCI', '-XX:+UseJVMCICompiler'],
53 'disabled' : []
54 }
55
56 def get_vm():
57 """
58 Gets the name of the currently selected JVM variant.
59 """
60 return 'server'
61
62 class JVMCIMode:
63 """
64 A context manager for setting the current JVMCI mode.
65 """
66 def __init__(self, jvmciMode=None):
67 self.update(jvmciMode)
68
69 def update(self, jvmciMode=None):
70 assert jvmciMode is None or jvmciMode in _jvmciModes, jvmciMode
71 self.jvmciMode = jvmciMode or _vm.jvmciMode
72
73 def __enter__(self):
74 global _vm
75 self.previousVm = _vm
76 _vm = self
77
78 def __exit__(self, exc_type, exc_value, traceback):
79 global _vm
80 _vm = self.previousVm
81
82 _vm = JVMCIMode(jvmciMode='jit')
83
84 class BootClasspathDist(object):
85 """
86 Extra info for a Distribution that must be put onto the boot class path.
87 """
88 def __init__(self, name):
89 self._name = name
90
91 def dist(self):
92 return mx.distribution(self._name)
93
94 def get_classpath_repr(self):
95 return self.dist().classpath_repr()
96
97 _compilers = ['graal-economy', 'graal']
98 _bootClasspathDists = [
99 BootClasspathDist('GRAAL'),
100 ]
101
102 def add_compiler(compilerName):
103 _compilers.append(compilerName)
104
105 def add_boot_classpath_dist(dist):
106 _bootClasspathDists.append(dist)
107
108 mx_gate.add_jacoco_includes(['org.graalvm.compiler.*'])
109 mx_gate.add_jacoco_excluded_annotations(['@Snippet', '@ClassSubstitution'])
110
111 # This is different than the 'jmh' commmand in that it
112 # looks for internal JMH benchmarks (i.e. those that
113 # depend on the JMH library).
114 def microbench(args):
115 """run JMH microbenchmark projects"""
116 parser = ArgumentParser(prog='mx microbench', description=microbench.__doc__,
117 usage="%(prog)s [command options|VM options] [-- [JMH options]]")
118 parser.add_argument('--jar', help='Explicitly specify micro-benchmark location')
119 known_args, args = parser.parse_known_args(args)
120
121 vmArgs, jmhArgs = mx.extract_VM_args(args, useDoubleDash=True)
122
123 # look for -f in JMH arguments
124 forking = True
125 for i in range(len(jmhArgs)):
126 arg = jmhArgs[i]
127 if arg.startswith('-f'):
128 if arg == '-f' and (i+1) < len(jmhArgs):
129 arg += jmhArgs[i+1]
130 try:
131 if int(arg[2:]) == 0:
132 forking = False
133 except ValueError:
134 pass
135
136 if known_args.jar:
137 # use the specified jar
138 args = ['-jar', known_args.jar]
139 if not forking:
140 args += vmArgs
141 else:
142 # find all projects with a direct JMH dependency
143 jmhProjects = []
144 for p in mx.projects_opt_limit_to_suites():
145 if 'JMH' in [x.name for x in p.deps]:
146 jmhProjects.append(p.name)
147 cp = mx.classpath(jmhProjects)
148
149 # execute JMH runner
150 args = ['-cp', cp]
151 if not forking:
152 args += vmArgs
153 args += ['org.openjdk.jmh.Main']
154
155 if forking:
156 jvm = get_vm()
157 def quoteSpace(s):
158 if " " in s:
159 return '"' + s + '"'
160 return s
161
162 forkedVmArgs = map(quoteSpace, _parseVmArgs(_jdk, vmArgs))
163 args += ['--jvmArgsPrepend', ' '.join(['-' + jvm] + forkedVmArgs)]
164 run_vm(args + jmhArgs)
165
166 def ctw(args, extraVMarguments=None):
167 """run CompileTheWorld"""
168
169 defaultCtwopts = '-Inline'
170
171 parser = ArgumentParser(prog='mx ctw')
172 parser.add_argument('--ctwopts', action='store', help='space separated JVMCI options used for CTW compilations (default: --ctwopts="' + defaultCtwopts + '")', default=defaultCtwopts, metavar='<options>')
173 parser.add_argument('--cp', '--jar', action='store', help='jar or class path denoting classes to compile', metavar='<path>')
174
175 args, vmargs = parser.parse_known_args(args)
176
177 if args.ctwopts:
178 # Replace spaces with '#' since -G: options cannot contain spaces
179 vmargs.append('-G:CompileTheWorldConfig=' + re.sub(r'\s+', '#', args.ctwopts))
180
181 if args.cp:
182 cp = os.path.abspath(args.cp)
183 else:
184 cp = join(_jdk.home, 'lib', 'modules', 'bootmodules.jimage')
185 vmargs.append('-G:CompileTheWorldExcludeMethodFilter=sun.awt.X11.*.*')
186
187 # suppress menubar and dock when running on Mac; exclude x11 classes as they may cause vm crashes (on Solaris)
188 vmargs = ['-Djava.awt.headless=true'] + vmargs
189
190 if _vm.jvmciMode == 'disabled':
191 vmargs += ['-XX:+CompileTheWorld', '-Xbootclasspath/p:' + cp]
192 else:
193 if _vm.jvmciMode == 'jit':
194 vmargs += ['-XX:+BootstrapJVMCI']
195 vmargs += ['-G:CompileTheWorldClasspath=' + cp, 'org.graalvm.compiler.hotspot.CompileTheWorld']
196
197 run_vm(vmargs + _noneAsEmptyList(extraVMarguments))
198
199 class UnitTestRun:
200 def __init__(self, name, args):
201 self.name = name
202 self.args = args
203
204 def run(self, suites, tasks, extraVMarguments=None):
205 for suite in suites:
206 with Task(self.name + ': hosted-release ' + suite, tasks) as t:
207 if t: unittest(['--suite', suite, '--enable-timing', '--verbose', '--fail-fast'] + self.args + _noneAsEmptyList(extraVMarguments))
208
209 class BootstrapTest:
210 def __init__(self, name, args, suppress=None):
211 self.name = name
212 self.args = args
213 self.suppress = suppress
214
215 def run(self, tasks, extraVMarguments=None):
216 with JVMCIMode('jit'):
217 with Task(self.name, tasks) as t:
218 if t:
219 if self.suppress:
220 out = mx.DuplicateSuppressingStream(self.suppress).write
221 else:
222 out = None
223 run_vm(self.args + _noneAsEmptyList(extraVMarguments) + ['-XX:-TieredCompilation', '-XX:+BootstrapJVMCI', '-version'], out=out)
224
225 class MicrobenchRun:
226 def __init__(self, name, args):
227 self.name = name
228 self.args = args
229
230 def run(self, tasks, extraVMarguments=None):
231 with Task(self.name + ': hosted-product ', tasks) as t:
232 if t: microbench(_noneAsEmptyList(extraVMarguments) + ['--'] + self.args)
233
234 def compiler_gate_runner(suites, unit_test_runs, bootstrap_tests, tasks, extraVMarguments=None):
235
236 # Run unit tests in hosted mode
237 with JVMCIMode('hosted'):
238 for r in unit_test_runs:
239 r.run(suites, tasks, extraVMarguments)
240
241 # Run microbench in hosted mode (only for testing the JMH setup)
242 with JVMCIMode('hosted'):
243 for r in [MicrobenchRun('Microbench', ['TestJMH'])]:
244 r.run(tasks, extraVMarguments)
245
246 # Run ctw against rt.jar on server-hosted-jvmci
247 with JVMCIMode('hosted'):
248 with Task('CTW:hosted', tasks) as t:
249 if t: ctw(['--ctwopts', '-Inline +ExitVMOnException', '-esa', '-G:+CompileTheWorldMultiThreaded', '-G:-InlineDuringParsing', '-G:-CompileTheWorldVerbose', '-XX:ReservedCodeCacheSize=300m'], _noneAsEmptyList(extraVMarguments))
250
251 # bootstrap tests
252 for b in bootstrap_tests:
253 b.run(tasks, extraVMarguments)
254
255 # run dacapo sanitychecks
256 for test in sanitycheck.getDacapos(level=sanitycheck.SanityCheckLevel.Gate, gateBuildLevel='release', extraVmArguments=extraVMarguments) \
257 + sanitycheck.getScalaDacapos(level=sanitycheck.SanityCheckLevel.Gate, gateBuildLevel='release', extraVmArguments=extraVMarguments):
258 with Task(str(test) + ':' + 'release', tasks) as t:
259 if t and not test.test('jvmci'):
260 t.abort(test.name + ' Failed')
261
262 # ensure -Xbatch still works
263 with JVMCIMode('jit'):
264 with Task('DaCapo_pmd:BatchMode', tasks) as t:
265 if t: dacapo(_noneAsEmptyList(extraVMarguments) + ['-Xbatch', 'pmd'])
266
267 # ensure benchmark counters still work
268 with JVMCIMode('jit'):
269 with Task('DaCapo_pmd:BenchmarkCounters:product', tasks) as t:
270 if t: dacapo(_noneAsEmptyList(extraVMarguments) + ['-G:+LIRProfileMoves', '-G:+GenericDynamicCounters', '-XX:JVMCICounterSize=10', 'pmd'])
271
272 # ensure -Xcomp still works
273 with JVMCIMode('jit'):
274 with Task('XCompMode:product', tasks) as t:
275 if t: run_vm(_noneAsEmptyList(extraVMarguments) + ['-Xcomp', '-version'])
276
277
278 graal_unit_test_runs = [
279 UnitTestRun('UnitTests', []),
280 ]
281
282 _registers = 'o0,o1,o2,o3,f8,f9,d32,d34' if mx.get_arch() == 'sparcv9' else 'rbx,r11,r10,r14,xmm3,xmm11,xmm14'
283
284 graal_bootstrap_tests = [
285 BootstrapTest('BootstrapWithSystemAssertions', ['-esa']),
286 BootstrapTest('BootstrapWithSystemAssertionsNoCoop', ['-esa', '-XX:-UseCompressedOops', '-G:+ExitVMOnException']),
287 BootstrapTest('BootstrapWithGCVerification', ['-XX:+UnlockDiagnosticVMOptions', '-XX:+VerifyBeforeGC', '-XX:+VerifyAfterGC', '-G:+ExitVMOnException'], suppress=['VerifyAfterGC:', 'VerifyBeforeGC:']),
288 BootstrapTest('BootstrapWithG1GCVerification', ['-XX:+UnlockDiagnosticVMOptions', '-XX:-UseSerialGC', '-XX:+UseG1GC', '-XX:+VerifyBeforeGC', '-XX:+VerifyAfterGC', '-G:+ExitVMOnException'], suppress=['VerifyAfterGC:', 'VerifyBeforeGC:']),
289 BootstrapTest('BootstrapEconomyWithSystemAssertions', ['-esa', '-Djvmci.compiler=graal-economy', '-G:+ExitVMOnException']),
290 BootstrapTest('BootstrapWithExceptionEdges', ['-esa', '-G:+StressInvokeWithExceptionNode', '-G:+ExitVMOnException']),
291 BootstrapTest('BootstrapWithRegisterPressure', ['-esa', '-G:RegisterPressure=' + _registers, '-G:+ExitVMOnException', '-G:+LIRUnlockBackendRestart']),
292 BootstrapTest('BootstrapTraceRAWithRegisterPressure', ['-esa', '-G:+TraceRA', '-G:RegisterPressure=' + _registers, '-G:+ExitVMOnException', '-G:+LIRUnlockBackendRestart']),
293 BootstrapTest('BootstrapWithImmutableCode', ['-esa', '-G:+ImmutableCode', '-G:+VerifyPhases', '-G:+ExitVMOnException']),
294 ]
295
296 def _graal_gate_runner(args, tasks):
297 compiler_gate_runner(['graal'], graal_unit_test_runs, graal_bootstrap_tests, tasks, args.extra_vm_argument)
298
299 mx_gate.add_gate_runner(_suite, _graal_gate_runner)
300 mx_gate.add_gate_argument('--extra-vm-argument', action='append', help='add extra vm argument to gate tasks if applicable (multiple occurrences allowed)')
301
302 def _unittest_vm_launcher(vmArgs, mainClass, mainClassArgs):
303 run_vm(vmArgs + [mainClass] + mainClassArgs)
304
305 mx_unittest.set_vm_launcher('JDK9 VM launcher', _unittest_vm_launcher)
306
307 def _parseVmArgs(jdk, args, addDefaultArgs=True):
308 args = mx.expand_project_in_args(args, insitu=False)
309 jacocoArgs = mx_gate.get_jacoco_agent_args()
310 if jacocoArgs:
311 args = jacocoArgs + args
312
313 # Support for -G: options
314 def translateGOption(arg):
315 if arg.startswith('-G:+'):
316 if '=' in arg:
317 mx.abort('Mixing + and = in -G: option specification: ' + arg)
318 arg = '-Dgraal.' + arg[len('-G:+'):] + '=true'
319 elif arg.startswith('-G:-'):
320 if '=' in arg:
321 mx.abort('Mixing - and = in -G: option specification: ' + arg)
322 arg = '-Dgraal.' + arg[len('-G:+'):] + '=false'
323 elif arg.startswith('-G:'):
324 if '=' not in arg:
325 mx.abort('Missing "=" in non-boolean -G: option specification: ' + arg)
326 arg = '-Dgraal.' + arg[len('-G:'):]
327 return arg
328 args = map(translateGOption, args)
329
330 if '-G:+PrintFlags' in args and '-Xcomp' not in args:
331 mx.warn('Using -G:+PrintFlags may have no effect without -Xcomp as Graal initialization is lazy')
332
333 bcp = []
334 if _jvmciModes[_vm.jvmciMode]:
335 if _add_jvmci_to_bootclasspath:
336 bcp.append(mx.library('JVMCI').classpath_repr())
337 bcp.extend([d.get_classpath_repr() for d in _bootClasspathDists])
338 if bcp:
339 args = ['-Xbootclasspath/p:' + os.pathsep.join(bcp)] + args
340
341 # Remove JVMCI from class path. It's only there to support compilation.
342 cpIndex, cp = mx.find_classpath_arg(args)
343 if cp:
344 jvmciLib = mx.library('JVMCI').path
345 cp = os.pathsep.join([e for e in cp.split(os.pathsep) if e != jvmciLib])
346 args[cpIndex] = cp
347
348 # Set the default JVMCI compiler
349 jvmciCompiler = _compilers[-1]
350 args = ['-Djvmci.compiler=' + jvmciCompiler] + args
351
352 if '-version' in args:
353 ignoredArgs = args[args.index('-version') + 1:]
354 if len(ignoredArgs) > 0:
355 mx.log("Warning: The following options will be ignored by the vm because they come after the '-version' argument: " + ' '.join(ignoredArgs))
356 return jdk.processArgs(args, addDefaultArgs=addDefaultArgs)
357
358 def run_java(jdk, args, nonZeroIsFatal=True, out=None, err=None, cwd=None, timeout=None, env=None, addDefaultArgs=True):
359
360 args = _parseVmArgs(jdk, args, addDefaultArgs=addDefaultArgs)
361
362 jvmciModeArgs = _jvmciModes[_vm.jvmciMode]
363 cmd = [jdk.java] + ['-' + get_vm()] + jvmciModeArgs + args
364 return mx.run(cmd, nonZeroIsFatal=nonZeroIsFatal, out=out, err=err, cwd=cwd)
365
366 _JVMCI_JDK_TAG = 'jvmci'
367
368 class GraalJVMCI9JDKConfig(mx.JDKConfig):
369 def __init__(self, original):
370 self._original = original
371 mx.JDKConfig.__init__(self, original.home, tag=_JVMCI_JDK_TAG)
372
373 def run_java(self, args, **kwArgs):
374 run_java(self._original, args, **kwArgs)
375
376 class GraalJDKFactory(mx.JDKFactory):
377 def getJDKConfig(self):
378 return GraalJVMCI9JDKConfig(_jdk)
379
380 def description(self):
381 return "JVMCI JDK with Graal"
382
383 # This will override the 'generic' JVMCI JDK with a Graal JVMCI JDK that has
384 # support for -G style Graal options.
385 mx.addJDKFactory(_JVMCI_JDK_TAG, mx.JavaCompliance('9'), GraalJDKFactory())
386
387 def run_vm(args, vm=None, nonZeroIsFatal=True, out=None, err=None, cwd=None, timeout=None, debugLevel=None, vmbuild=None):
388 """run a Java program by executing the java executable in a JVMCI JDK"""
389
390 return run_java(_jdk, args, nonZeroIsFatal=nonZeroIsFatal, out=out, err=err, cwd=cwd, timeout=timeout)
391
392 class GraalArchiveParticipant:
393 def __init__(self, dist):
394 self.dist = dist
395
396 def __opened__(self, arc, srcArc, services):
397 self.services = services
398 self.arc = arc
399
400 def __add__(self, arcname, contents):
401 if arcname.startswith('META-INF/providers/'):
402 provider = arcname[len('META-INF/providers/'):]
403 for service in contents.strip().split(os.linesep):
404 assert service
405 self.services.setdefault(service, []).append(provider)
406 return True
407 elif arcname.endswith('_OptionDescriptors.class'):
408 # Need to create service files for the providers of the
409 # jdk.vm.ci.options.Options service created by
410 # jdk.vm.ci.options.processor.OptionProcessor.
411 provider = arcname[:-len('.class'):].replace('/', '.')
412 self.services.setdefault('org.graalvm.compiler.options.OptionDescriptors', []).append(provider)
413 return False
414
415 def __addsrc__(self, arcname, contents):
416 return False
417
418 def __closing__(self):
419 pass
420
421 mx.update_commands(_suite, {
422 'vm': [run_vm, '[-options] class [args...]'],
423 'ctw': [ctw, '[-vmoptions|noinline|nocomplex|full]'],
424 'microbench' : [microbench, '[VM options] [-- [JMH options]]'],
425 })
426
427 mx.add_argument('-M', '--jvmci-mode', action='store', choices=sorted(_jvmciModes.viewkeys()), help='the JVM variant type to build/run (default: ' + _vm.jvmciMode + ')')
428
429 def mx_post_parse_cmd_line(opts):
430 if opts.jvmci_mode is not None:
431 _vm.update(opts.jvmci_mode)
432 for dist in [d.dist() for d in _bootClasspathDists]:
433 dist.set_archiveparticipant(GraalArchiveParticipant(dist))
434
435 _add_jvmci_to_bootclasspath = False
436