1 /*
2 * Copyright (c) 2020, Red Hat Inc.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation. Oracle designates this
8 * particular file as subject to the "Classpath" exception as provided
9 * by Oracle in the LICENSE file that accompanied this code.
10 *
11 * This code is distributed in the hope that it will be useful, but WITHOUT
12 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
14 * version 2 for more details (a copy is included in the LICENSE file that
15 * accompanied this code).
16 *
17 * You should have received a copy of the GNU General Public License version
18 * 2 along with this work; if not, write to the Free Software Foundation,
19 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
20 *
21 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
22 * or visit www.oracle.com if you need additional information or have any
23 * questions.
24 */
25
26 package jdk.test.lib.containers.cgroup;
27
28 import java.io.IOException;
29 import java.nio.file.Files;
30 import java.nio.file.Path;
31 import java.nio.file.Paths;
32 import java.util.Arrays;
33 import java.util.List;
34 import java.util.concurrent.TimeUnit;
35 import java.util.stream.Collectors;
36
37 import jdk.internal.platform.CgroupSubsystem;
38 import jdk.internal.platform.Metrics;
39
40 public class MetricsTesterCgroupV2 implements CgroupMetricsTester {
41
42 private static final long UNLIMITED = -1;
43 private static final UnifiedController UNIFIED = new UnifiedController();
44 private static final String MAX = "max";
45 private static final int PER_CPU_SHARES = 1024;
46
47 private final long startSysVal;
48 private final long startUserVal;
49 private final long startUsage;
50
51 static class UnifiedController {
52
53 private static final String NAME = "unified";
54 private final String path;
55
56 UnifiedController() {
57 path = constructPath();
58 }
59
60 String getPath() {
61 return path;
62 }
63
64 private static String constructPath() {
65 String mountPath;
66 String cgroupPath;
67 try {
68 List<String> fifthTokens = Files.lines(Paths.get("/proc/self/mountinfo"))
69 .filter( l -> l.contains("- cgroup2"))
70 .map(UnifiedController::splitAndMountPath)
71 .collect(Collectors.toList());
72 if (fifthTokens.size() != 1) {
73 throw new AssertionError("Expected only one cgroup2 line");
74 }
75 mountPath = fifthTokens.get(0);
76
77 List<String> cgroupPaths = Files.lines(Paths.get("/proc/self/cgroup"))
78 .filter( l -> l.startsWith("0:"))
79 .map(UnifiedController::splitAndCgroupPath)
80 .collect(Collectors.toList());
81 if (cgroupPaths.size() != 1) {
82 throw new AssertionError("Expected only one unified controller line");
83 }
84 cgroupPath = cgroupPaths.get(0);
85 return Paths.get(mountPath, cgroupPath).toString();
86 } catch (IOException e) {
87 return null;
88 }
89 }
90
91 public static String splitAndMountPath(String input) {
92 String[] tokens = input.split("\\s+");
93 return tokens[4]; // fifth entry is the mount path
94 }
95
96 public static String splitAndCgroupPath(String input) {
97 String[] tokens = input.split(":");
98 return tokens[2];
99 }
100 }
101
102 private long getLongLimitValueFromFile(String file) {
103 String strVal = getStringVal(file);
104 if (MAX.equals(strVal)) {
105 return UNLIMITED;
106 }
107 return convertStringToLong(strVal);
108 }
109
110 public MetricsTesterCgroupV2() {
111 Metrics metrics = Metrics.systemMetrics();
112 // Initialize CPU usage metrics before we do any testing.
113 startSysVal = metrics.getCpuSystemUsage();
114 startUserVal = metrics.getCpuUserUsage();
115 startUsage = metrics.getCpuUsage();
116 }
117
118 private long getLongValueFromFile(String file) {
119 return convertStringToLong(getStringVal(file));
120 }
121
122 private long getLongValueEntryFromFile(String file, String metric) {
123 Path filePath = Paths.get(UNIFIED.getPath(), file);
124 try {
125 String strVal = Files.lines(filePath).filter(l -> l.startsWith(metric)).collect(Collectors.joining());
126 String[] keyValues = strVal.split("\\s+");
127 String value = keyValues[1];
128 return convertStringToLong(value);
129 } catch (IOException e) {
130 return 0;
131 }
132 }
133
134 private String getStringVal(String file) {
135 Path filePath = Paths.get(UNIFIED.getPath(), file);
136 try {
137 return Files.lines(filePath).collect(Collectors.joining());
138 } catch (IOException e) {
139 return null;
140 }
141 }
142
143 private void fail(String metric, long oldVal, long newVal) {
144 CgroupMetricsTester.fail(UnifiedController.NAME, metric, oldVal, newVal);
145 }
146
147 private void fail(String metric, String oldVal, String newVal) {
148 CgroupMetricsTester.fail(UnifiedController.NAME, metric, oldVal, newVal);
149 }
150
151 private void warn(String metric, long oldVal, long newVal) {
152 CgroupMetricsTester.warn(UnifiedController.NAME, metric, oldVal, newVal);
153 }
154
155 private long getCpuShares(String file) {
156 long rawVal = getLongValueFromFile(file);
157 if (rawVal == 0 || rawVal == 100) {
158 return UNLIMITED;
159 }
160 int shares = (int)rawVal;
161 // CPU shares (OCI) value needs to get translated into
162 // a proper Cgroups v2 value. See:
163 // https://github.com/containers/crun/blob/master/crun.1.md#cpu-controller
164 //
165 // Use the inverse of (x == OCI value, y == cgroupsv2 value):
166 // ((262142 * y - 1)/9999) + 2 = x
167 //
168 int x = 262142 * shares - 1;
169 double frac = x/9999.0;
170 x = ((int)frac) + 2;
171 if ( x <= PER_CPU_SHARES ) {
172 return PER_CPU_SHARES; // mimic cgroups v1
173 }
174 int f = x/PER_CPU_SHARES;
175 int lower_multiple = f * PER_CPU_SHARES;
176 int upper_multiple = (f + 1) * PER_CPU_SHARES;
177 int distance_lower = Math.max(lower_multiple, x) - Math.min(lower_multiple, x);
178 int distance_upper = Math.max(upper_multiple, x) - Math.min(upper_multiple, x);
179 x = distance_lower <= distance_upper ? lower_multiple : upper_multiple;
180 return x;
181 }
182
183 private long getCpuMaxValueFromFile(String file) {
184 return getCpuValueFromFile(file, 0 /* $MAX index */);
185 }
186
187 private long getCpuPeriodValueFromFile(String file) {
188 return getCpuValueFromFile(file, 1 /* $PERIOD index */);
189 }
190
191 private long getCpuValueFromFile(String file, int index) {
192 String maxPeriod = getStringVal(file);
193 if (maxPeriod == null) {
194 return UNLIMITED;
195 }
196 String[] tokens = maxPeriod.split("\\s+");
197 String val = tokens[index];
198 if (MAX.equals(val)) {
199 return UNLIMITED;
200 }
201 return convertStringToLong(val);
202 }
203
204 private long convertStringToLong(String val) {
205 return CgroupMetricsTester.convertStringToLong(val, UNLIMITED);
206 }
207
208 @Override
209 public void testMemorySubsystem() {
210 Metrics metrics = Metrics.systemMetrics();
211
212 // User Memory
213 long oldVal = metrics.getMemoryFailCount();
214 long newVal = getLongValueEntryFromFile("memory.events", "max");
215 if (!CgroupMetricsTester.compareWithErrorMargin(oldVal, newVal)) {
216 fail("memory.events[max]", oldVal, newVal);
217 }
218
219 oldVal = metrics.getMemoryLimit();
220 newVal = getLongLimitValueFromFile("memory.max");
221 if (!CgroupMetricsTester.compareWithErrorMargin(oldVal, newVal)) {
222 fail("memory.max", oldVal, newVal);
223 }
224
225 oldVal = metrics.getMemoryUsage();
226 newVal = getLongValueFromFile("memory.current");
227 if (!CgroupMetricsTester.compareWithErrorMargin(oldVal, newVal)) {
228 fail("memory.current", oldVal, newVal);
229 }
230
231 oldVal = metrics.getTcpMemoryUsage();
232 newVal = getLongValueEntryFromFile("memory.stat", "sock");
233 if (!CgroupMetricsTester.compareWithErrorMargin(oldVal, newVal)) {
234 fail("memory.stat[sock]", oldVal, newVal);
235 }
236
237 oldVal = metrics.getMemoryAndSwapLimit();
238 newVal = getLongLimitValueFromFile("memory.swap.max");
239 if (!CgroupMetricsTester.compareWithErrorMargin(oldVal, newVal)) {
240 fail("memory.swap.max", oldVal, newVal);
241 }
242
243 oldVal = metrics.getMemoryAndSwapUsage();
244 newVal = getLongValueFromFile("memory.swap.current");
245 if (!CgroupMetricsTester.compareWithErrorMargin(oldVal, newVal)) {
246 fail("memory.swap.current", oldVal, newVal);
247 }
248
249 oldVal = metrics.getMemorySoftLimit();
250 newVal = getLongLimitValueFromFile("memory.high");
251 if (!CgroupMetricsTester.compareWithErrorMargin(oldVal, newVal)) {
252 fail("memory.high", oldVal, newVal);
253 }
254
255 }
256
257 @Override
258 public void testCpuAccounting() {
259 Metrics metrics = Metrics.systemMetrics();
260 long oldVal = metrics.getCpuUsage();
261 long newVal = TimeUnit.MICROSECONDS.toNanos(getLongValueEntryFromFile("cpu.stat", "usage_usec"));
262
263 if (!CgroupMetricsTester.compareWithErrorMargin(oldVal, newVal)) {
264 warn("cpu.stat[usage_usec]", oldVal, newVal);
265 }
266
267 oldVal = metrics.getCpuUserUsage();
268 newVal = TimeUnit.MICROSECONDS.toNanos(getLongValueEntryFromFile("cpu.stat", "user_usec"));
269 if (!CgroupMetricsTester.compareWithErrorMargin(oldVal, newVal)) {
270 warn("cpu.stat[user_usec]", oldVal, newVal);
271 }
272
273 oldVal = metrics.getCpuSystemUsage();
274 newVal = TimeUnit.MICROSECONDS.toNanos(getLongValueEntryFromFile("cpu.stat", "system_usec"));
275 if (!CgroupMetricsTester.compareWithErrorMargin(oldVal, newVal)) {
276 warn("cpu.stat[system_usec]", oldVal, newVal);
277 }
278 }
279
280 @Override
281 public void testCpuSchedulingMetrics() {
282 Metrics metrics = Metrics.systemMetrics();
283 long oldVal = metrics.getCpuPeriod();
284 long newVal = getCpuPeriodValueFromFile("cpu.max");
285 if (!CgroupMetricsTester.compareWithErrorMargin(oldVal, newVal)) {
286 fail("cpu.max[$PERIOD]", oldVal, newVal);
287 }
288
289 oldVal = metrics.getCpuQuota();
290 newVal = getCpuMaxValueFromFile("cpu.max");
291 if (!CgroupMetricsTester.compareWithErrorMargin(oldVal, newVal)) {
292 fail("cpu.max[$MAX]", oldVal, newVal);
293 }
294
295 oldVal = metrics.getCpuShares();
296 newVal = getCpuShares("cpu.weight");
297 if (!CgroupMetricsTester.compareWithErrorMargin(oldVal, newVal)) {
298 fail("cpu.weight", oldVal, newVal);
299 }
300
301 oldVal = metrics.getCpuNumPeriods();
302 newVal = getLongValueEntryFromFile("cpu.stat", "nr_periods");
303 if (!CgroupMetricsTester.compareWithErrorMargin(oldVal, newVal)) {
304 fail("cpu.stat[nr_periods]", oldVal, newVal);
305 }
306
307 oldVal = metrics.getCpuNumThrottled();
308 newVal = getLongValueEntryFromFile("cpu.stat", "nr_throttled");
309 if (!CgroupMetricsTester.compareWithErrorMargin(oldVal, newVal)) {
310 fail("cpu.stat[nr_throttled]", oldVal, newVal);
311 }
312
313 oldVal = metrics.getCpuThrottledTime();
314 newVal = TimeUnit.MICROSECONDS.toNanos(getLongValueEntryFromFile("cpu.stat", "throttled_usec"));
315 if (!CgroupMetricsTester.compareWithErrorMargin(oldVal, newVal)) {
316 fail("cpu.stat[throttled_usec]", oldVal, newVal);
317 }
318 }
319
320 @Override
321 public void testCpuSets() {
322 Metrics metrics = Metrics.systemMetrics();
323 int[] cpus = mapNullToEmpty(metrics.getCpuSetCpus());
324 Integer[] oldVal = Arrays.stream(cpus).boxed().toArray(Integer[]::new);
325 Arrays.sort(oldVal);
326
327 String cpusstr = getStringVal("cpuset.cpus");
328 // Parse range string in the format 1,2-6,7
329 Integer[] newVal = CgroupMetricsTester.convertCpuSetsToArray(cpusstr);
330 Arrays.sort(newVal);
331 if (Arrays.compare(oldVal, newVal) != 0) {
332 fail("cpuset.cpus", Arrays.toString(oldVal),
333 Arrays.toString(newVal));
334 }
335
336 cpus = mapNullToEmpty(metrics.getEffectiveCpuSetCpus());
337 oldVal = Arrays.stream(cpus).boxed().toArray(Integer[]::new);
338 Arrays.sort(oldVal);
339 cpusstr = getStringVal("cpuset.cpus.effective");
340 newVal = CgroupMetricsTester.convertCpuSetsToArray(cpusstr);
341 Arrays.sort(newVal);
342 if (Arrays.compare(oldVal, newVal) != 0) {
343 fail("cpuset.cpus.effective", Arrays.toString(oldVal),
344 Arrays.toString(newVal));
345 }
346
347 cpus = mapNullToEmpty(metrics.getCpuSetMems());
348 oldVal = Arrays.stream(cpus).boxed().toArray(Integer[]::new);
349 Arrays.sort(oldVal);
350 cpusstr = getStringVal("cpuset.mems");
351 newVal = CgroupMetricsTester.convertCpuSetsToArray(cpusstr);
352 Arrays.sort(newVal);
353 if (Arrays.compare(oldVal, newVal) != 0) {
354 fail("cpuset.mems", Arrays.toString(oldVal),
355 Arrays.toString(newVal));
356 }
357
358 cpus = mapNullToEmpty(metrics.getEffectiveCpuSetMems());
359 oldVal = Arrays.stream(cpus).boxed().toArray(Integer[]::new);
360 Arrays.sort(oldVal);
361 cpusstr = getStringVal("cpuset.mems.effective");
362 newVal = CgroupMetricsTester.convertCpuSetsToArray(cpusstr);
363 Arrays.sort(newVal);
364 if (Arrays.compare(oldVal, newVal) != 0) {
365 fail("cpuset.mems.effective", Arrays.toString(oldVal),
366 Arrays.toString(newVal));
367 }
368 }
369
370 private int[] mapNullToEmpty(int[] cpus) {
371 if (cpus == null) {
372 // Not available. For sake of testing continue with an
373 // empty array.
374 cpus = new int[0];
375 }
376 return cpus;
377 }
378
379 @Override
380 public void testCpuConsumption() {
381 Metrics metrics = Metrics.systemMetrics();
382 // make system call
383 long newSysVal = metrics.getCpuSystemUsage();
384 long newUserVal = metrics.getCpuUserUsage();
385 long newUsage = metrics.getCpuUsage();
386
387 // system/user CPU usage counters may be slowly increasing.
388 // allow for equal values for a pass
389 if (newSysVal < startSysVal) {
390 fail("getCpuSystemUsage", newSysVal, startSysVal);
391 }
392
393 // system/user CPU usage counters may be slowly increasing.
394 // allow for equal values for a pass
395 if (newUserVal < startUserVal) {
396 fail("getCpuUserUsage", newUserVal, startUserVal);
397 }
398
399 if (newUsage <= startUsage) {
400 fail("getCpuUsage", newUsage, startUsage);
401 }
402 }
403
404 @Override
405 public void testMemoryUsage() {
406 Metrics metrics = Metrics.systemMetrics();
407 long memoryUsage = metrics.getMemoryUsage();
408 long newMemoryUsage = 0;
409
410 // allocate memory in a loop and check more than once for new values
411 // otherwise we might occasionally see the effect of decreasing new memory
412 // values. For example because the system could free up memory
413 byte[][] bytes = new byte[32][];
414 for (int i = 0; i < 32; i++) {
415 bytes[i] = new byte[8*1024*1024];
416 newMemoryUsage = metrics.getMemoryUsage();
417 if (newMemoryUsage > memoryUsage) {
418 break;
419 }
420 }
421
422 if (newMemoryUsage < memoryUsage) {
423 fail("getMemoryUsage", memoryUsage, newMemoryUsage);
424 }
425 }
426
427 @Override
428 public void testMisc() {
429 testIOStat();
430 }
431
432 private void testIOStat() {
433 Metrics metrics = Metrics.systemMetrics();
434 long oldVal = metrics.getBlkIOServiceCount();
435 long newVal = getIoStatAccumulate(new String[] { "rios", "wios" });
436 if (!CgroupMetricsTester.compareWithErrorMargin(oldVal, newVal)) {
437 fail("io.stat->rios/wios: ", oldVal, newVal);
438 }
439
440 oldVal = metrics.getBlkIOServiced();
441 newVal = getIoStatAccumulate(new String[] { "rbytes", "wbytes" });
442 if (!CgroupMetricsTester.compareWithErrorMargin(oldVal, newVal)) {
443 fail("io.stat->rbytes/wbytes: ", oldVal, newVal);
444 }
445 }
446
447 private long getIoStatAccumulate(String[] matchNames) {
448 try {
449 return Files.lines(Paths.get(UNIFIED.getPath(), "io.stat"))
450 .map(line -> {
451 long accumulator = 0;
452 String[] tokens = line.split("\\s+");
453 for (String t: tokens) {
454 String[] keyVal = t.split("=");
455 if (keyVal.length != 2) {
456 continue;
457 }
458 for (String match: matchNames) {
459 if (match.equals(keyVal[0])) {
460 accumulator += Long.parseLong(keyVal[1]);
461 }
462 }
463 }
464 return accumulator;
465 }).collect(Collectors.summingLong(e -> e));
466 } catch (IOException e) {
467 return CgroupSubsystem.LONG_RETVAL_UNLIMITED;
468 }
469 }
470 }