1 /*
2 * Copyright (c) 2005, 2013, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation. Oracle designates this
8 * particular file as subject to the "Classpath" exception as provided
9 * by Oracle in the LICENSE file that accompanied this code.
10 *
11 * This code is distributed in the hope that it will be useful, but WITHOUT
12 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
14 * version 2 for more details (a copy is included in the LICENSE file that
15 * accompanied this code).
16 *
17 * You should have received a copy of the GNU General Public License version
18 * 2 along with this work; if not, write to the Free Software Foundation,
19 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
20 *
21 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
22 * or visit www.oracle.com if you need additional information or have any
23 * questions.
24 */
25 package org.openjdk.jmh.runner.options;
26
27 import joptsimple.OptionException;
28 import joptsimple.OptionParser;
29 import joptsimple.OptionSet;
30 import joptsimple.OptionSpec;
31 import org.openjdk.jmh.annotations.Mode;
32 import org.openjdk.jmh.annotations.Threads;
33 import org.openjdk.jmh.profile.Profiler;
34 import org.openjdk.jmh.profile.ProfilerFactory;
35 import org.openjdk.jmh.results.format.ResultFormatType;
36 import org.openjdk.jmh.util.HashMultimap;
37 import org.openjdk.jmh.util.Multimap;
38 import org.openjdk.jmh.util.Optional;
39
40 import java.io.IOException;
41 import java.util.ArrayList;
42 import java.util.Arrays;
43 import java.util.Collection;
44 import java.util.HashSet;
45 import java.util.List;
46 import java.util.Set;
47 import java.util.concurrent.TimeUnit;
48
49 /**
50 * Class that handles all the command line options.
51 */
52 public class CommandLineOptions implements Options {
53 private static final long serialVersionUID = 5565183446360224399L;
54
55 private final Optional<Integer> iterations;
56 private final Optional<TimeValue> runTime;
57 private final Optional<Integer> batchSize;
58 private final Optional<Integer> warmupIterations;
59 private final Optional<TimeValue> warmupTime;
60 private final Optional<Integer> warmupBatchSize;
61 private final List<Mode> benchMode = new ArrayList<Mode>();
62 private final Optional<Integer> threads;
63 private final List<Integer> threadGroups = new ArrayList<Integer>();
64 private final Optional<Boolean> synchIterations;
65 private final Optional<Boolean> gcEachIteration;
66 private final Optional<VerboseMode> verbose;
67 private final Optional<Boolean> failOnError;
68 private final Set<Class<? extends Profiler>> profilers = new HashSet<Class<? extends Profiler>>();
69 private final Optional<TimeUnit> timeUnit;
70 private final Optional<Integer> opsPerInvocation;
71 private final List<String> regexps = new ArrayList<String>();
72 private final Optional<Integer> fork;
73 private final Optional<Integer> warmupFork;
74 private final Optional<String> output;
75 private final Optional<String> result;
76 private final Optional<ResultFormatType> resultFormat;
77 private final Optional<String> jvm;
78 private final Optional<Collection<String>> jvmArgs;
79 private final Optional<Collection<String>> jvmArgsAppend;
80 private final Optional<Collection<String>> jvmArgsPrepend;
81 private final List<String> excludes = new ArrayList<String>();
82 private final Optional<WarmupMode> warmupMode;
83 private final List<String> warmupMicros = new ArrayList<String>();
84 private final Multimap<String, String> params = new HashMultimap<String, String>();
85 private final boolean list;
86 private final boolean listResultFormats;
87 private final boolean help;
88 private final boolean listProfilers;
89
90 private final transient OptionParser parser;
91
92 /**
93 * Parses the given command line.
94 * @param argv argument list
95 * @throws CommandLineOptionException if some options are misspelled
96 */
97 public CommandLineOptions(String... argv) throws CommandLineOptionException {
98 parser = new OptionParser();
99 parser.formatHelpWith(new OptionFormatter());
100
101 OptionSpec<Integer> optMeasureCount = parser.accepts("i", "Number of measurement iterations to do.")
102 .withRequiredArg().ofType(Integer.class).describedAs("int");
103
104 OptionSpec<Integer> optMeasureBatchSize = parser.accepts("bs", "Batch size: number of benchmark method calls per operation. " +
105 "(some benchmark modes can ignore this setting)")
106 .withRequiredArg().ofType(Integer.class).describedAs("int");
107
108 OptionSpec<String> optMeasureTime = parser.accepts("r", "Time to spend at each measurement iteration.")
109 .withRequiredArg().ofType(String.class).describedAs("time");
110
111 OptionSpec<Integer> optWarmupCount = parser.accepts("wi", "Number of warmup iterations to do.")
112 .withRequiredArg().ofType(Integer.class).describedAs("int");
113
114 OptionSpec<Integer> optWarmupBatchSize = parser.accepts("wbs", "Warmup batch size: number of benchmark method calls per operation. " +
115 "(some benchmark modes can ignore this setting)")
116 .withRequiredArg().ofType(Integer.class).describedAs("int");
117
118 OptionSpec<String> optWarmupTime = parser.accepts("w", "Time to spend at each warmup iteration.")
119 .withRequiredArg().ofType(String.class).describedAs("time");
120
121 OptionSpec<String> optThreads = parser.accepts("t", "Number of worker threads to run with.")
122 .withRequiredArg().ofType(String.class).describedAs("int");
123
124 OptionSpec<String> optBenchmarkMode = parser.accepts("bm", "Benchmark mode. Available modes are: " + Mode.getKnown())
125 .withRequiredArg().ofType(String.class).withValuesSeparatedBy(',').describedAs("mode");
126
127 OptionSpec<Boolean> optSyncIters = parser.accepts("si", "Synchronize iterations?")
128 .withOptionalArg().ofType(Boolean.class).describedAs("bool");
129
130 OptionSpec<Boolean> optGC = parser.accepts("gc", "Should JMH force GC between iterations?")
131 .withOptionalArg().ofType(Boolean.class).describedAs("bool");
132
133 OptionSpec<Boolean> optFOE = parser.accepts("foe", "Should JMH fail immediately if any benchmark had" +
134 " experienced the unrecoverable error?")
135 .withOptionalArg().ofType(Boolean.class).describedAs("bool");
136
137 OptionSpec<String> optVerboseMode = parser.accepts("v", "Verbosity mode. Available modes are: " + Arrays.toString(VerboseMode.values()))
138 .withRequiredArg().ofType(String.class).describedAs("mode");
139
140 OptionSpec<String> optArgs = parser.nonOptions("Benchmarks to run (regexp+).")
141 .describedAs("regexp+");
142
143 OptionSpec<Integer> optForks = parser.accepts("f", "How many times to forks a single benchmark." +
144 " Use 0 to disable forking altogether (WARNING: disabling forking may have detrimental" +
145 " impact on benchmark and infrastructure reliability, you might want to use different" +
146 " warmup mode instead).")
147 .withOptionalArg().ofType(Integer.class).describedAs("int");
148
149 OptionSpec<Integer> optWarmupForks = parser.accepts("wf", "How many warmup forks to make " +
150 "for a single benchmark. 0 to disable warmup forks.")
151 .withRequiredArg().ofType(Integer.class).describedAs("int");
152
153 OptionSpec<String> optOutput = parser.accepts("o", "Redirect human-readable output to file.")
154 .withRequiredArg().ofType(String.class).describedAs("filename");
155
156 OptionSpec<String> optOutputResults = parser.accepts("rff", "Write results to given file.")
157 .withRequiredArg().ofType(String.class).describedAs("filename");
158
159 OptionSpec<String> optProfilers = parser.accepts("prof", "Use profilers to collect additional data." +
160 " See the list of available profilers first.")
161 .withRequiredArg().withValuesSeparatedBy(',').ofType(String.class).describedAs("profiler+");
162
163 OptionSpec<Integer> optThreadGroups = parser.accepts("tg", "Override thread group distribution for asymmetric benchmarks.")
164 .withRequiredArg().withValuesSeparatedBy(',').ofType(Integer.class).describedAs("int+");
165
166 OptionSpec<String> optJvm = parser.accepts("jvm", "Custom JVM to use when forking (path to JVM executable).")
167 .withRequiredArg().ofType(String.class).describedAs("string");
168
169 OptionSpec<String> optJvmArgs = parser.accepts("jvmArgs", "Custom JVM args to use when forking.")
170 .withRequiredArg().ofType(String.class).describedAs("string");
171
172 OptionSpec<String> optJvmArgsAppend = parser.accepts("jvmArgsAppend", "Custom JVM args to use when forking (append these)")
173 .withRequiredArg().ofType(String.class).describedAs("string");
174
175 OptionSpec<String> optJvmArgsPrepend = parser.accepts("jvmArgsPrepend", "Custom JVM args to use when forking (prepend these)")
176 .withRequiredArg().ofType(String.class).describedAs("string");
177
178 OptionSpec<String> optTU = parser.accepts("tu", "Output time unit. Available time units are: [m, s, ms, us, ns].")
179 .withRequiredArg().ofType(String.class).describedAs("TU");
180
181 OptionSpec<Integer> optOPI = parser.accepts("opi", "Operations per invocation.")
182 .withRequiredArg().ofType(Integer.class).describedAs("int");
183
184 OptionSpec<String> optResultFormat = parser.accepts("rf", "Result format type. See the list of available result formats first.")
185 .withRequiredArg().ofType(String.class).describedAs("type");
186
187 OptionSpec<String> optWarmupMode = parser.accepts("wm", "Warmup mode for warming up selected benchmarks. Warmup modes are: " + Arrays.toString(WarmupMode.values()) + ".")
188 .withRequiredArg().ofType(String.class).describedAs("mode");
189
190 OptionSpec<String> optExcludes = parser.accepts("e", "Benchmarks to exclude from the run.")
191 .withRequiredArg().withValuesSeparatedBy(',').ofType(String.class).describedAs("regexp+");
192
193 OptionSpec<String> optParams = parser.accepts("p", "Benchmark parameters. This option is expected to be used once" +
194 " per parameter. Parameter name and parameter values should be separated with equals sign." +
195 " Parameter values should be separated with commas.")
196 .withRequiredArg().ofType(String.class).describedAs("param={v,}*");
197
198 OptionSpec<String> optWarmupBenchmarks = parser.accepts("wmb", "Warmup benchmarks to include in the run " +
199 "in addition to already selected. JMH will not measure these benchmarks, but only use them" +
200 " for the warmup.")
201 .withRequiredArg().withValuesSeparatedBy(',').ofType(String.class).describedAs("regexp+");
202
203 parser.accepts("l", "List matching benchmarks and exit.");
204 parser.accepts("lrf", "List result formats.");
205 parser.accepts("lprof", "List profilers.");
206 parser.accepts("h", "Display help.");
207
208 try {
209 OptionSet set = parser.parse(argv);
210
211 if (set.has(optExcludes)) {
212 excludes.addAll(optExcludes.values(set));
213 }
214
215 if (set.has(optWarmupBenchmarks)) {
216 warmupMicros.addAll(optWarmupBenchmarks.values(set));
217 }
218
219 if (set.has(optTU)) {
220 String va = optTU.value(set);
221 TimeUnit tu;
222 if (va.equalsIgnoreCase("ns")) {
223 tu = TimeUnit.NANOSECONDS;
224 } else if (va.equalsIgnoreCase("us")) {
225 tu = TimeUnit.MICROSECONDS;
226 } else if (va.equalsIgnoreCase("ms")) {
227 tu = TimeUnit.MILLISECONDS;
228 } else if (va.equalsIgnoreCase("s")) {
229 tu = TimeUnit.SECONDS;
230 } else if (va.equalsIgnoreCase("m")) {
231 tu = TimeUnit.MINUTES;
232 } else if (va.equalsIgnoreCase("h")) {
233 tu = TimeUnit.HOURS;
234 } else {
235 throw new CommandLineOptionException("Unknown time unit: " + va);
236 }
237 timeUnit = Optional.of(tu);
238 } else {
239 timeUnit = Optional.none();
240 }
241
242 opsPerInvocation = Optional.eitherOf(optOPI.value(set));
243
244 if (set.has(optWarmupMode)) {
245 try {
246 warmupMode = Optional.of(WarmupMode.valueOf(optWarmupMode.value(set)));
247 } catch (IllegalArgumentException iae) {
248 throw new CommandLineOptionException(iae.getMessage(), iae);
249 }
250 } else {
251 warmupMode = Optional.none();
252 }
253
254 if (set.has(optResultFormat)) {
255 try {
256 resultFormat = Optional.of(ResultFormatType.valueOf(optResultFormat.value(set).toUpperCase()));
257 } catch (IllegalArgumentException iae) {
258 throw new CommandLineOptionException(iae.getMessage(), iae);
259 }
260 } else {
261 resultFormat = Optional.none();
262 }
263
264 help = set.has("h");
265 list = set.has("l");
266 listResultFormats = set.has("lrf");
267 listProfilers = set.has("lprof");
268
269 iterations = Optional.eitherOf(optMeasureCount.value(set));
270
271 batchSize = Optional.eitherOf(optMeasureBatchSize.value(set));
272
273 if (set.has(optMeasureTime)) {
274 String value = optMeasureTime.value(set);
275 try {
276 runTime = Optional.of(TimeValue.fromString(value));
277 } catch (IllegalArgumentException iae) {
278 throw new CommandLineOptionException(iae.getMessage(), iae);
279 }
280 } else {
281 runTime = Optional.none();
282 }
283
284 warmupIterations = Optional.eitherOf(optWarmupCount.value(set));
285
286 warmupBatchSize = Optional.eitherOf(optWarmupBatchSize.value(set));
287
288 if (set.has(optWarmupTime)) {
289 String value = optWarmupTime.value(set);
290 try {
291 warmupTime = Optional.of(TimeValue.fromString(value));
292 } catch (IllegalArgumentException iae) {
293 throw new CommandLineOptionException(iae.getMessage(), iae);
294 }
295 } else {
296 warmupTime = Optional.none();
297 }
298
299 if (set.has(optThreads)) {
300 String v = optThreads.value(set);
301 if (v.equalsIgnoreCase("max")) {
302 threads = Optional.of(Threads.MAX);
303 } else {
304 try {
305 threads = Optional.of(Integer.valueOf(v));
306 } catch (IllegalArgumentException iae) {
307 throw new CommandLineOptionException(iae.getMessage(), iae);
308 }
309 }
310 } else {
311 threads = Optional.none();
312 }
313
314 if (set.has(optBenchmarkMode)) {
315 try {
316 List<Mode> modes = new ArrayList<Mode>();
317 for (String m : optBenchmarkMode.values(set)) {
318 modes.add(Mode.deepValueOf(m));
319 }
320 benchMode.addAll(modes);
321 } catch (IllegalArgumentException iae) {
322 throw new CommandLineOptionException(iae.getMessage(), iae);
323 }
324 }
325
326 if (set.has(optSyncIters)) {
327 if (set.hasArgument(optSyncIters)) {
328 synchIterations = Optional.of(optSyncIters.value(set));
329 } else {
330 synchIterations = Optional.of(true);
331 }
332 } else {
333 synchIterations = Optional.none();
334 }
335
336 if (set.has(optGC)) {
337 if (set.hasArgument(optGC)) {
338 gcEachIteration = Optional.of(optGC.value(set));
339 } else {
340 gcEachIteration = Optional.of(true);
341 }
342 } else {
343 gcEachIteration = Optional.none();
344 }
345
346 if (set.has(optFOE)) {
347 if (set.hasArgument(optFOE)) {
348 failOnError = Optional.of(optFOE.value(set));
349 } else {
350 failOnError = Optional.of(true);
351 }
352 } else {
353 failOnError = Optional.none();
354 }
355
356 if (set.has(optVerboseMode)) {
357 try {
358 if (set.hasArgument(optVerboseMode)) {
359 verbose = Optional.of(VerboseMode.valueOf(set.valueOf(optVerboseMode).toUpperCase()));
360 } else {
361 verbose = Optional.of(VerboseMode.EXTRA);
362 }
363 } catch (IllegalArgumentException iae) {
364 throw new CommandLineOptionException(iae.getMessage(), iae);
365 }
366 } else {
367 verbose = Optional.none();
368 }
369
370 regexps.addAll(set.valuesOf(optArgs));
371
372 if (set.has(optForks)) {
373 if (set.hasArgument(optForks)) {
374 fork = Optional.of(optForks.value(set));
375 } else {
376 fork = Optional.of(1);
377 }
378 } else {
379 fork = Optional.none();
380 }
381
382 warmupFork = Optional.eitherOf(optWarmupForks.value(set));
383 output = Optional.eitherOf(optOutput.value(set));
384 result = Optional.eitherOf(optOutputResults.value(set));
385
386 if (set.has(optProfilers)) {
387 try {
388 for (String m : optProfilers.values(set)) {
389 Class<? extends Profiler> pClass = ProfilerFactory.getProfilerByName(m);
390 if (pClass == null) {
391 throw new CommandLineOptionException("Unable to find profiler: " + m);
392 }
393 profilers.add(pClass);
394 }
395 } catch (IllegalArgumentException iae) {
396 throw new CommandLineOptionException(iae.getMessage(), iae);
397 }
398 }
399
400 if (set.has(optThreadGroups)) {
401 threadGroups.addAll(set.valuesOf(optThreadGroups));
402 }
403
404 jvm = Optional.eitherOf(optJvm.value(set));
405
406 if (set.hasArgument(optJvmArgs)) {
407 jvmArgs = Optional.<Collection<String>>of(Arrays.asList(optJvmArgs.value(set).trim().split("[ ]+")));
408 } else {
409 jvmArgs = Optional.none();
410 }
411
412 if (set.hasArgument(optJvmArgsAppend)) {
413 jvmArgsAppend = Optional.<Collection<String>>of(Arrays.asList(optJvmArgsAppend.value(set).trim().split("[ ]+")));
414 } else {
415 jvmArgsAppend = Optional.none();
416 }
417
418 if (set.hasArgument(optJvmArgsPrepend)) {
419 jvmArgsPrepend = Optional.<Collection<String>>of(Arrays.asList(optJvmArgsPrepend.value(set).trim().split("[ ]+")));
420 } else {
421 jvmArgsPrepend = Optional.none();
422 }
423
424 if (set.hasArgument(optParams)) {
425 for (String p : optParams.values(set)) {
426 String[] keys = p.split("=");
427 if (keys.length != 2) {
428 throw new CommandLineOptionException("Unable to parse parameter string \"" + p + "\"");
429 }
430 params.putAll(keys[0], Arrays.asList(keys[1].split(",")));
431 }
432 }
433
434 } catch (OptionException e) {
435 throw new CommandLineOptionException(e.getMessage(), e);
436 }
437 }
438
439 public void showHelp() throws IOException {
440 parser.printHelpOn(System.err);
441 }
442
443 public void listProfilers() {
444 StringBuilder supported = new StringBuilder();
445 StringBuilder unsupported = new StringBuilder();
446
447 List<Class<? extends Profiler>> discoveredProfilers = ProfilerFactory.getDiscoveredProfilers();
448
449 for (Class<? extends Profiler> s : ProfilerFactory.getAvailableProfilers()) {
450 List<String> initMessages = new ArrayList<String>();
451 if (ProfilerFactory.checkSupport(s, initMessages)) {
452 supported.append(String.format("%20s: %s %s\n",
453 ProfilerFactory.getLabel(s),
454 ProfilerFactory.getDescription(s),
455 discoveredProfilers.contains(s) ? "(discovered)" : ""));
456 } else {
457 unsupported.append(String.format("%20s: %s %s\n",
458 ProfilerFactory.getLabel(s),
459 ProfilerFactory.getDescription(s),
460 discoveredProfilers.contains(s) ? "(discovered)" : ""));
461 for (String im : initMessages) {
462 unsupported.append(String.format("%20s %s\n", "", im));
463 }
464 unsupported.append("\n");
465 }
466 }
467
468 if (!supported.toString().isEmpty()) {
469 System.out.println("Supported profilers:\n" + supported.toString());
470 }
471
472 if (!unsupported.toString().isEmpty()) {
473 System.out.println("Unsupported profilers:\n" + unsupported.toString());
474 }
475 }
476
477 public void listResultFormats() {
478 StringBuilder sb = new StringBuilder();
479
480 for (ResultFormatType f : ResultFormatType.values()) {
481 sb.append(f.toString().toLowerCase());
482 sb.append(", ");
483 }
484 sb.setLength(sb.length() - 2);
485
486 System.out.println("Available formats: " + sb.toString());
487 }
488
489 public boolean shouldList() {
490 return list;
491 }
492
493 public boolean shouldListResultFormats() {
494 return listResultFormats;
495 }
496
497 public boolean shouldHelp() {
498 return help;
499 }
500
501 public boolean shouldListProfilers() {
502 return listProfilers;
503 }
504
505 @Override
506 public Optional<WarmupMode> getWarmupMode() {
507 return warmupMode;
508 }
509
510 @Override
511 public List<String> getIncludes() {
512 return regexps;
513 }
514
515 @Override
516 public List<String> getExcludes() {
517 return excludes;
518 }
519
520 @Override
521 public List<String> getWarmupIncludes() {
522 return warmupMicros;
523 }
524
525 @Override
526 public Optional<String> getJvm() {
527 return jvm;
528 }
529
530 @Override
531 public Optional<Collection<String>> getJvmArgs() {
532 return jvmArgs;
533 }
534
535 @Override
536 public Optional<Collection<String>> getJvmArgsAppend() {
537 return jvmArgsAppend;
538 }
539
540 @Override
541 public Optional<Collection<String>> getJvmArgsPrepend() {
542 return jvmArgsPrepend;
543 }
544
545 @Override
546 public Optional<Collection<String>> getParameter(String name) {
547 Collection<String> list = params.get(name);
548 if (list == null || list.isEmpty()){
549 return Optional.none();
550 } else {
551 return Optional.of(list);
552 }
553 }
554
555 @Override
556 public Optional<Integer> getForkCount() {
557 return fork;
558 }
559
560 @Override
561 public Optional<Integer> getWarmupForkCount() {
562 return warmupFork;
563 }
564
565 @Override
566 public Optional<String> getOutput() {
567 return output;
568 }
569
570 @Override
571 public Optional<ResultFormatType> getResultFormat() {
572 return resultFormat;
573 }
574
575 @Override
576 public Optional<String> getResult() {
577 return result;
578 }
579
580 @Override
581 public Optional<Integer> getMeasurementIterations() {
582 return iterations;
583 }
584
585 @Override
586 public Optional<Integer> getMeasurementBatchSize() {
587 return batchSize;
588 }
589
590 @Override
591 public Optional<TimeValue> getMeasurementTime() {
592 return runTime;
593 }
594
595 @Override
596 public Optional<TimeValue> getWarmupTime() {
597 return warmupTime;
598 }
599
600 @Override
601 public Optional<Integer> getWarmupIterations() {
602 return warmupIterations;
603 }
604
605 @Override
606 public Optional<Integer> getWarmupBatchSize() {
607 return warmupBatchSize;
608 }
609
610 @Override
611 public Optional<Integer> getThreads() {
612 return threads;
613 }
614
615 @Override
616 public Optional<int[]> getThreadGroups() {
617 if (threadGroups.isEmpty()) {
618 return Optional.none();
619 } else {
620 int[] r = new int[threadGroups.size()];
621 for (int c = 0; c < r.length; c++) {
622 r[c] = threadGroups.get(c);
623 }
624 return Optional.of(r);
625 }
626 }
627
628 @Override
629 public Optional<Boolean> shouldDoGC() {
630 return gcEachIteration;
631 }
632
633 @Override
634 public Optional<Boolean> shouldSyncIterations() {
635 return synchIterations;
636 }
637
638 @Override
639 public Optional<VerboseMode> verbosity() {
640 return verbose;
641 }
642
643 @Override
644 public Optional<TimeUnit> getTimeUnit() {
645 return timeUnit;
646 }
647
648 @Override
649 public Optional<Integer> getOperationsPerInvocation() {
650 return opsPerInvocation;
651 }
652
653 @Override
654 public Optional<Boolean> shouldFailOnError() {
655 return failOnError;
656 }
657
658 @Override
659 public Set<Class<? extends Profiler>> getProfilers() {
660 return profilers;
661 }
662
663 @Override
664 public Collection<Mode> getBenchModes() {
665 return new HashSet<Mode>(benchMode);
666 }
667
668 }